This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new edca3907eae [HUDI-9505] Adding table version 9 and fixing
upgrade/downgrade paths (#13520)
edca3907eae is described below
commit edca3907eae79dfcc51a6465dc0c62efe999b6fc
Author: Davis-Zhang-Onehouse
<[email protected]>
AuthorDate: Mon Jul 7 20:13:16 2025 -0700
[HUDI-9505] Adding table version 9 and fixing upgrade/downgrade paths
(#13520)
---
.../apache/hudi/client/BaseHoodieWriteClient.java | 3 +-
.../org/apache/hudi/config/HoodieWriteConfig.java | 3 +-
.../table/upgrade/EightToNineUpgradeHandler.java | 36 ++++++++++++++++
.../table/upgrade/NineToEightDowngradeHandler.java | 37 ++++++++++++++++
.../hudi/table/upgrade/UpgradeDowngrade.java | 4 ++
.../hudi/table/upgrade/UpgradeDowngradeUtils.java | 4 +-
.../org/apache/hudi/util/CommonClientUtils.java | 21 ++++++---
.../apache/hudi/config/TestHoodieWriteConfig.java | 2 +-
.../apache/hudi/utils/TestCommonClientUtils.java | 50 ++++++++++++++++++++++
.../hudi/common/table/HoodieTableVersion.java | 6 ++-
10 files changed, 151 insertions(+), 15 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
index 6496c89cdf9..4e9ee7c82d4 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java
@@ -46,7 +46,6 @@ import org.apache.hudi.common.model.TableServiceType;
import org.apache.hudi.common.model.WriteOperationType;
import org.apache.hudi.common.table.HoodieTableConfig;
import org.apache.hudi.common.table.HoodieTableMetaClient;
-import org.apache.hudi.common.table.HoodieTableVersion;
import org.apache.hudi.common.table.TableSchemaResolver;
import org.apache.hudi.common.table.timeline.HoodieActiveTimeline;
import org.apache.hudi.common.table.timeline.HoodieInstant;
@@ -1518,7 +1517,7 @@ public abstract class BaseHoodieWriteClient<T, I, K, O>
extends BaseHoodieClient
}
new UpgradeDowngrade(metaClient, config, context, upgradeDowngradeHelper)
- .run(HoodieTableVersion.current(), instantTime.orElse(null));
+ .run(config.getWriteVersion(), instantTime.orElse(null));
metaClient.reloadTableConfig();
metaClient.reloadActiveTimeline();
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index 289863cc3a4..fd590d1a9d0 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -146,7 +146,8 @@ public class HoodieWriteConfig extends HoodieConfig {
.defaultValue(HoodieTableVersion.current().versionCode())
.withValidValues(
String.valueOf(HoodieTableVersion.SIX.versionCode()),
- String.valueOf(HoodieTableVersion.current().versionCode())
+ String.valueOf(HoodieTableVersion.EIGHT.versionCode()),
+ String.valueOf(HoodieTableVersion.NINE.versionCode())
)
.sinceVersion("1.0.0")
.withDocumentation("The table version this writer is storing the table
in. This should match the current table version.");
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/EightToNineUpgradeHandler.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/EightToNineUpgradeHandler.java
new file mode 100644
index 00000000000..dfb429af9d1
--- /dev/null
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/EightToNineUpgradeHandler.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.table.upgrade;
+
+import org.apache.hudi.common.config.ConfigProperty;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import java.util.Collections;
+import java.util.Map;
+
+public class EightToNineUpgradeHandler implements UpgradeHandler {
+
+ @Override
+ public Map<ConfigProperty, String> upgrade(HoodieWriteConfig config,
HoodieEngineContext context,
+ String instantTime,
SupportsUpgradeDowngrade upgradeDowngradeHelper) {
+
+ return Collections.emptyMap();
+ }
+}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/NineToEightDowngradeHandler.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/NineToEightDowngradeHandler.java
new file mode 100644
index 00000000000..9d576125a1b
--- /dev/null
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/NineToEightDowngradeHandler.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.table.upgrade;
+
+import org.apache.hudi.common.config.ConfigProperty;
+import org.apache.hudi.common.engine.HoodieEngineContext;
+import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.config.HoodieWriteConfig;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class NineToEightDowngradeHandler implements DowngradeHandler {
+
+ @Override
+ public Pair<Map<ConfigProperty, String>, List<ConfigProperty>>
downgrade(HoodieWriteConfig config, HoodieEngineContext context, String
instantTime, SupportsUpgradeDowngrade upgradeDowngradeHelper) {
+ return Pair.of(Collections.emptyMap(), Collections.emptyList());
+ }
+}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java
index da391abe78f..b67065f127a 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngrade.java
@@ -245,6 +245,8 @@ public class UpgradeDowngrade {
return new SixToSevenUpgradeHandler().upgrade(config, context,
instantTime, upgradeDowngradeHelper);
} else if (fromVersion == HoodieTableVersion.SEVEN && toVersion ==
HoodieTableVersion.EIGHT) {
return new SevenToEightUpgradeHandler().upgrade(config, context,
instantTime, upgradeDowngradeHelper);
+ } else if (fromVersion == HoodieTableVersion.EIGHT && toVersion ==
HoodieTableVersion.NINE) {
+ return new EightToNineUpgradeHandler().upgrade(config, context,
instantTime, upgradeDowngradeHelper);
} else {
throw new HoodieUpgradeDowngradeException(fromVersion.versionCode(),
toVersion.versionCode(), true);
}
@@ -267,6 +269,8 @@ public class UpgradeDowngrade {
return new SevenToSixDowngradeHandler().downgrade(config, context,
instantTime, upgradeDowngradeHelper);
} else if (fromVersion == HoodieTableVersion.EIGHT && toVersion ==
HoodieTableVersion.SEVEN) {
return new EightToSevenDowngradeHandler().downgrade(config, context,
instantTime, upgradeDowngradeHelper);
+ } else if (fromVersion == HoodieTableVersion.NINE && toVersion ==
HoodieTableVersion.EIGHT) {
+ return new NineToEightDowngradeHandler().downgrade(config, context,
instantTime, upgradeDowngradeHelper);
} else {
throw new HoodieUpgradeDowngradeException(fromVersion.versionCode(),
toVersion.versionCode(), false);
}
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngradeUtils.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngradeUtils.java
index 3d39a47809b..d1a475883b3 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngradeUtils.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/table/upgrade/UpgradeDowngradeUtils.java
@@ -174,13 +174,13 @@ public class UpgradeDowngradeUtils {
properties.putAll(config.getProps());
// TimeGenerators are cached and re-used based on table base path. Since
here we are changing the lock configurations, avoiding the cache use
// for upgrade code block.
-
properties.put(HoodieTimeGeneratorConfig.TIME_GENERATOR_REUSE_ENABLE.key(),"false");
+
properties.put(HoodieTimeGeneratorConfig.TIME_GENERATOR_REUSE_ENABLE.key(),
"false");
// override w/ NoopLock Provider to avoid re-entrant locking. already
upgrade is happening within the table level lock.
// Below we do trigger rollback and compaction which might again try to
acquire the lock. So, here we are explicitly overriding to
// NoopLockProvider for just the upgrade code block.
properties.put(HoodieLockConfig.LOCK_PROVIDER_CLASS_NAME.key(),
NoopLockProvider.class.getName());
// if auto adjust it not disabled, chances that InProcessLockProvider
will get overridden for single writer use-cases.
- properties.put(HoodieWriteConfig.AUTO_ADJUST_LOCK_CONFIGS.key(),"false");
+ properties.put(HoodieWriteConfig.AUTO_ADJUST_LOCK_CONFIGS.key(),
"false");
HoodieWriteConfig rollbackWriteConfig = HoodieWriteConfig.newBuilder()
.withProps(properties)
.withWriteTableVersion(tableVersion.versionCode())
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/util/CommonClientUtils.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/util/CommonClientUtils.java
index c201f30aa93..7d0ab05c45c 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/util/CommonClientUtils.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/util/CommonClientUtils.java
@@ -52,14 +52,10 @@ public class CommonClientUtils {
public static void validateTableVersion(HoodieTableConfig tableConfig,
HoodieWriteConfig writeConfig) {
// mismatch of table versions.
- if (!tableConfig.getTableVersion().equals(writeConfig.getWriteVersion())) {
+ if (!isValidTableVersionWriteVersionPair(tableConfig.getTableVersion(),
writeConfig.getWriteVersion())) {
// if table version is greater than 6, while writer version is 6, we can
still allow it for upgrade
- if (tableConfig.getTableVersion().greaterThan(HoodieTableVersion.SIX) &&
writeConfig.getWriteVersion().equals(HoodieTableVersion.SIX)) {
- LOG.warn("Table version is greater than 6, while writer version is 6.
Allowing it for upgrade.");
- } else {
- throw new HoodieNotSupportedException(String.format("Table version
(%s) and Writer version (%s) do not match for table at: %s.",
- tableConfig.getTableVersion(), writeConfig.getWriteVersion(),
writeConfig.getBasePath()));
- }
+ throw new HoodieNotSupportedException(String.format("Table version (%s)
and Writer version (%s) do not match for table at: %s.",
+ tableConfig.getTableVersion(), writeConfig.getWriteVersion(),
writeConfig.getBasePath()));
}
// incompatible configurations.
if (tableConfig.getTableVersion().lesserThan(HoodieTableVersion.EIGHT) &&
writeConfig.shouldWritePartialUpdates()) {
@@ -72,6 +68,17 @@ public class CommonClientUtils {
}
}
+ public static boolean isValidTableVersionWriteVersionPair(HoodieTableVersion
tableVersion, HoodieTableVersion writeVersion) {
+ if (tableVersion.equals(writeVersion) ||
tableVersion.lesserThan(writeVersion)) {
+ return true;
+ }
+ if (tableVersion.greaterThan(HoodieTableVersion.SIX) &&
tableVersion.lesserThan(HoodieTableVersion.NINE) &&
writeVersion.equals(HoodieTableVersion.SIX)) {
+ LOG.warn("Table version is greater than 6 and lower than 9, while writer
version is 6. Allowing it for upgrade.");
+ return true;
+ }
+ return false;
+ }
+
/**
* Returns the base file format.
*/
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java
index c9cc16f1c04..690e8408a2c 100644
---
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java
+++
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/config/TestHoodieWriteConfig.java
@@ -96,7 +96,7 @@ public class TestHoodieWriteConfig {
@Test
public void testSupportedTableWriteVersions() {
Set<HoodieTableVersion> supportedVersions = CollectionUtils.createSet(
- HoodieTableVersion.SIX, HoodieTableVersion.EIGHT
+ HoodieTableVersion.SIX, HoodieTableVersion.EIGHT,
HoodieTableVersion.NINE
);
Arrays.stream(HoodieTableVersion.values())
.filter(version -> !supportedVersions.contains(version))
diff --git
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestCommonClientUtils.java
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestCommonClientUtils.java
index 9b3c56d5e79..21ee5300946 100644
---
a/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestCommonClientUtils.java
+++
b/hudi-client/hudi-client-common/src/test/java/org/apache/hudi/utils/TestCommonClientUtils.java
@@ -28,7 +28,13 @@ import org.apache.hudi.exception.HoodieNotSupportedException;
import org.apache.hudi.util.CommonClientUtils;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+import java.util.stream.Stream;
+
+import static
org.apache.hudi.util.CommonClientUtils.isValidTableVersionWriteVersionPair;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.Mockito.mock;
@@ -71,4 +77,48 @@ public class TestCommonClientUtils {
// when:
assertEquals("0-0-0",
CommonClientUtils.generateWriteToken(taskContextSupplier));
}
+
+ @ParameterizedTest(name = "Table version {0} with write version {1} should
be valid: {2}")
+ @MethodSource("provideValidTableVersionWriteVersionPairs")
+ public void testValidTableVersionWriteVersionPairs(
+ HoodieTableVersion tableVersion, HoodieTableVersion writeVersion,
boolean expectedResult) throws Exception {
+ boolean result = isValidTableVersionWriteVersionPair(tableVersion,
writeVersion);
+ assertEquals(expectedResult, result);
+ }
+
+ private static Stream<Arguments> provideValidTableVersionWriteVersionPairs()
{
+ return Stream.concat(
+ Stream.concat(
+ // Rule 1: writer > table version - always allowed
+ generateWriterGreaterThanTableCases(),
+ // Rule 2: same versions - always allowed
+ generateSameVersionCases()
+ ),
+ Stream.of(
+ // Rule 3: special case - upgrade scenario (table > 6, table < 9,
writer = 6)
+ Arguments.of(HoodieTableVersion.SEVEN, HoodieTableVersion.SIX,
true),
+ Arguments.of(HoodieTableVersion.EIGHT, HoodieTableVersion.SIX,
true),
+
+ // Rule 4: otherwise disallowed - table > writer (except special
case above)
+ Arguments.of(HoodieTableVersion.NINE, HoodieTableVersion.SIX,
false),
+ Arguments.of(HoodieTableVersion.NINE, HoodieTableVersion.EIGHT,
false),
+ Arguments.of(HoodieTableVersion.EIGHT, HoodieTableVersion.SEVEN,
false)
+ )
+ );
+ }
+
+ private static Stream<Arguments> generateWriterGreaterThanTableCases() {
+ HoodieTableVersion[] allVersions = HoodieTableVersion.values();
+ return Stream.of(allVersions)
+ .flatMap(tableVersion ->
+ Stream.of(allVersions)
+ .filter(writeVersion -> writeVersion.greaterThan(tableVersion))
+ .map(writeVersion -> Arguments.of(tableVersion, writeVersion,
true))
+ );
+ }
+
+ private static Stream<Arguments> generateSameVersionCases() {
+ return Stream.of(HoodieTableVersion.values())
+ .map(version -> Arguments.of(version, version, true));
+ }
}
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableVersion.java
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableVersion.java
index 2599a461f01..49a4b7215b1 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableVersion.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableVersion.java
@@ -47,8 +47,10 @@ public enum HoodieTableVersion {
// 0.16.0
SEVEN(7, CollectionUtils.createImmutableList("0.16.0"),
TimelineLayoutVersion.LAYOUT_VERSION_1),
// 1.0
- EIGHT(8, CollectionUtils.createImmutableList("1.0.0"),
TimelineLayoutVersion.LAYOUT_VERSION_2);
-
+ EIGHT(8, CollectionUtils.createImmutableList("1.0.0"),
TimelineLayoutVersion.LAYOUT_VERSION_2),
+ // 1.1
+ NINE(9, CollectionUtils.createImmutableList("1.1.0"),
TimelineLayoutVersion.LAYOUT_VERSION_2);
+
private final int versionCode;
private final List<String> releaseVersions;
