This is an automated email from the ASF dual-hosted git repository. danny0405 pushed a commit to branch release-1.1.2 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 7bf358e1526345811a6c746a465289537602a82f Author: danny0405 <[email protected]> AuthorDate: Sat Oct 10 16:22:12 2026 +0800 fix(ci): resolve release branch build and test failures Remove duplicate Maven dependencies and add the release script license header. Isolate Spark merge record contexts, repair Java 8 mocking and Trino test infrastructure, and stabilize containerized HDFS startup. Backport the Java metadata clustering retry test fixes from #19120 and #19434. --- hudi-aws/pom.xml | 6 -- hudi-client/hudi-java-client/pom.xml | 23 +++++ .../hudi/client/TestJavaHoodieBackedMetadata.java | 35 +++++-- hudi-common/pom.xml | 6 -- .../src/main/scala/org/apache/hudi/Iterators.scala | 5 +- hudi-trino-plugin/pom.xml | 12 +++ .../hudi/testing/MinioImageNameSubstitutor.java | 40 ++++++++ .../src/test/resources/testcontainers.properties | 14 +++ hudi-utilities/pom.xml | 6 -- packaging/bundle-validation/Dockerfile | 4 +- .../docker_java17/docker_java17_test.sh | 109 ++++++++++++--------- pom.xml | 6 -- scripts/release/download_staged_bundles.sh | 20 ++++ 13 files changed, 203 insertions(+), 83 deletions(-) diff --git a/hudi-aws/pom.xml b/hudi-aws/pom.xml index 4153d9ef66a7..1eef6497eac7 100644 --- a/hudi-aws/pom.xml +++ b/hudi-aws/pom.xml @@ -220,12 +220,6 @@ <version>${aws.sdk.httpcore.version}</version> </dependency> - <dependency> - <groupId>software.amazon.awssdk</groupId> - <artifactId>sts</artifactId> - <version>${aws.sdk.version}</version> - </dependency> - <!-- Test --> <dependency> <groupId>org.apache.hudi</groupId> diff --git a/hudi-client/hudi-java-client/pom.xml b/hudi-client/hudi-java-client/pom.xml index c301adda4b74..299d6893ad38 100644 --- a/hudi-client/hudi-java-client/pom.xml +++ b/hudi-client/hudi-java-client/pom.xml @@ -186,4 +186,27 @@ </resources> </build> + <profiles> + <profile> + <id>java8-test-attach</id> + <activation> + <jdk>[1.8,1.9)</jdk> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <!-- JOL and Mockito must share the Java 8 Attach API classloader. --> + <additionalClasspathElements> + <additionalClasspathElement>${java.home}/../lib/tools.jar</additionalClasspathElement> + </additionalClasspathElements> + </configuration> + </plugin> + </plugins> + </build> + </profile> + </profiles> + </project> diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java index d439189a0a63..9df7d4b8ae32 100644 --- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java +++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java @@ -1818,7 +1818,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase { HoodieJavaWriteClient client = getHoodieWriteClient(config); // Write 1 (Bulk insert) - String newCommitTime = "0000001"; + String newCommitTime = WriteClientTestUtils.createNewInstantTime(); List<HoodieRecord> records = dataGen.generateInserts(newCommitTime, 20); WriteClientTestUtils.startCommitWithTime(client, newCommitTime); List<WriteStatus> writeStatuses = client.insert(records, newCommitTime); @@ -1827,7 +1827,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase { validateMetadata(client); // Write 2 (inserts) - newCommitTime = "0000002"; + newCommitTime = WriteClientTestUtils.createNewInstantTime(); WriteClientTestUtils.startCommitWithTime(client, newCommitTime); records = dataGen.generateInserts(newCommitTime, 20); writeStatuses = client.insert(records, newCommitTime); @@ -1843,7 +1843,7 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase { .withClusteringExecutionStrategyClass(JavaSortAndSizeExecutionStrategy.class.getName()) .build(); - HoodieWriteConfig newWriteConfig = getConfigBuilder(TRIP_EXAMPLE_SCHEMA, HoodieIndex.IndexType.BLOOM, HoodieFailedWritesCleaningPolicy.EAGER) + HoodieWriteConfig newWriteConfig = getConfigBuilder(TRIP_EXAMPLE_SCHEMA, HoodieIndex.IndexType.BLOOM, HoodieFailedWritesCleaningPolicy.LAZY) .withClusteringConfig(clusteringConfig) .withRollbackUsingMarkers(false) .build(); @@ -1859,17 +1859,38 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase { partitionFiles.getValue().stream().forEach(file -> replacedFileIds.add(new HoodieFileGroupId(partitionFiles.getKey(), file)))); + // manually remove clustering completed instant from .hoodie folder to mimic clustering + // succeeded in metadata table, but failed before committing to data table. This must happen + // before any subsequent metadata-table read so that no reader ever sees the first attempt's + // instant as committed, matching the crash scenario this test simulates. + FileCreateUtilsLegacy.deleteReplaceCommit(basePath, clusteringCommitTime); + + // preconditions: clustering is pending on the data table while its deltacommit is completed + // in the metadata table. + metaClient.reloadActiveTimeline(); + assertFalse(metaClient.getActiveTimeline().filterCompletedInstants().containsInstant(clusteringCommitTime)); + assertTrue(metaClient.getActiveTimeline().filterPendingClusteringTimeline().containsInstant(clusteringCommitTime)); + HoodieTableMetaClient metadataMetaClient = HoodieTestUtils.createMetaClient( + metaClient.getStorageConf(), getMetadataTableBasePath(basePath)); + assertTrue(metadataMetaClient.getActiveTimeline().getDeltaCommitTimeline() + .filterCompletedInstants().containsInstant(clusteringCommitTime)); + // trigger new write to mimic other writes succeeding before re-attempt. - newCommitTime = "0000003"; + newCommitTime = WriteClientTestUtils.createNewInstantTime(); WriteClientTestUtils.startCommitWithTime(client, newCommitTime); records = dataGen.generateInserts(newCommitTime, 20); writeStatuses = client.insert(records, newCommitTime); client.commit(newCommitTime, writeStatuses); assertNoWriteErrors(writeStatuses); - validateMetadata(client); + // ignore the first clustering attempt's uncommitted files when comparing listings. + validateMetadata(client, Option.of(clusteringCommitTime)); + + // the new write must leave the pending clustering and its metadata-table deltacommit intact. + metaClient.reloadActiveTimeline(); + assertTrue(metaClient.getActiveTimeline().filterPendingClusteringTimeline().containsInstant(clusteringCommitTime)); + assertTrue(metadataMetaClient.reloadActiveTimeline().getDeltaCommitTimeline() + .filterCompletedInstants().containsInstant(clusteringCommitTime)); - // manually remove clustering completed instant from .hoodie folder and to mimic succeeded clustering in metadata table, but failed in data table. - FileCreateUtilsLegacy.deleteReplaceCommit(basePath, clusteringCommitTime); HoodieWriteMetadata<List<WriteStatus>> updatedClusterMetadata = newClient.cluster(clusteringCommitTime, true); metaClient.reloadActiveTimeline(); diff --git a/hudi-common/pom.xml b/hudi-common/pom.xml index 40b7e5ccfa82..19aa2fd9e220 100644 --- a/hudi-common/pom.xml +++ b/hudi-common/pom.xml @@ -139,12 +139,6 @@ <artifactId>jackson-module-afterburner</artifactId> </dependency> - - <dependency> - <groupId>com.fasterxml.jackson.module</groupId> - <artifactId>jackson-module-afterburner</artifactId> - </dependency> - <!-- Avro --> <dependency> <groupId>org.apache.avro</groupId> diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/Iterators.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/Iterators.scala index a6e158f1bc10..6b3a96d4c98d 100644 --- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/Iterators.scala +++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/Iterators.scala @@ -270,8 +270,9 @@ class RecordMergingFileIterator(logFiles: List[HoodieLogFile], private val recordMerger = HoodieRecordUtils.createRecordMerger(tableState.tablePath, EngineType.SPARK, tableState.recordMergeImplClasses.asJava, tableState.recordMergeStrategyId) - private val rowRecordContext: RecordContext[InternalRow] = SparkFileFormatInternalRecordContext.getFieldAccessorInstance - private val avroRecordContext: RecordContext[IndexedRecord] = AvroRecordContext.getFieldAccessorInstance + // Merging caches schema IDs and serializers; these contexts must not be shared across Spark tasks. + private val rowRecordContext: RecordContext[InternalRow] = SparkFileFormatInternalRecordContext() + private val avroRecordContext: RecordContext[IndexedRecord] = new AvroRecordContext() private val orderingFields: Array[String] = tableState.orderingFields.toArray override def doHasNext: Boolean = hasNextInternal diff --git a/hudi-trino-plugin/pom.xml b/hudi-trino-plugin/pom.xml index 45a3e8a3ff6d..b1703598b616 100644 --- a/hudi-trino-plugin/pom.xml +++ b/hudi-trino-plugin/pom.xml @@ -458,10 +458,22 @@ <artifactId>junit-jupiter-params</artifactId> <scope>test</scope> </dependency> + + <dependency> + <groupId>org.testcontainers</groupId> + <artifactId>testcontainers</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> <plugins> + <plugin> + <groupId>io.trino</groupId> + <artifactId>trino-maven-plugin</artifactId> + <!-- Version 17 resolves the SPI descriptor using the project's repositories. --> + <version>17</version> + </plugin> <plugin> <groupId>org.basepom.maven</groupId> <artifactId>duplicate-finder-maven-plugin</artifactId> diff --git a/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/MinioImageNameSubstitutor.java b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/MinioImageNameSubstitutor.java new file mode 100644 index 000000000000..5533fade45ce --- /dev/null +++ b/hudi-trino-plugin/src/test/java/io/trino/plugin/hudi/testing/MinioImageNameSubstitutor.java @@ -0,0 +1,40 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.hudi.testing; + +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.ImageNameSubstitutor; + +public class MinioImageNameSubstitutor + extends ImageNameSubstitutor +{ + // Use the pinned MinIO image from Trino's current test infrastructure. + private static final DockerImageName MINIO_IMAGE = DockerImageName.parse( + "cgr.dev/chainguard/minio@sha256:6a1d0b45c8669726bba580ced0bfa4cb9fdeed1ed636dfabd81d1577beb6937b"); + + @Override + public DockerImageName apply(DockerImageName original) + { + if (original.getRepository().equals("minio/minio")) { + return MINIO_IMAGE.asCompatibleSubstituteFor(original); + } + return original; + } + + @Override + protected String getDescription() + { + return "Use the available MinIO test image"; + } +} diff --git a/hudi-trino-plugin/src/test/resources/testcontainers.properties b/hudi-trino-plugin/src/test/resources/testcontainers.properties new file mode 100644 index 000000000000..2844ce763f9d --- /dev/null +++ b/hudi-trino-plugin/src/test/resources/testcontainers.properties @@ -0,0 +1,14 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +image.substitutor=io.trino.plugin.hudi.testing.MinioImageNameSubstitutor diff --git a/hudi-utilities/pom.xml b/hudi-utilities/pom.xml index 4e8c8c120a71..44d8932a8d8d 100644 --- a/hudi-utilities/pom.xml +++ b/hudi-utilities/pom.xml @@ -145,12 +145,6 @@ <artifactId>httpclient</artifactId> <version>${aws.sdk.httpclient.version}</version> </dependency> - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpcore</artifactId> - <version>${aws.sdk.httpcore.version}</version> - </dependency> - <dependency> <groupId>org.apache.hudi</groupId> <artifactId>hudi-gcp</artifactId> diff --git a/packaging/bundle-validation/Dockerfile b/packaging/bundle-validation/Dockerfile index 995e11ef828b..e917a1075ce8 100644 --- a/packaging/bundle-validation/Dockerfile +++ b/packaging/bundle-validation/Dockerfile @@ -54,5 +54,5 @@ ENV HDFS_SECONDARYNAMENODE_USER root ENV YARN_RESOURCEMANAGER_USER root ENV YARN_NODEMANAGER_USER root -# for RocksDb -RUN apk add --no-cache libstdc++ +# RocksDB needs libstdc++; Hadoop's daemon scripts need procps (ps -p). +RUN apk add --no-cache libstdc++ procps diff --git a/packaging/bundle-validation/docker_java17/docker_java17_test.sh b/packaging/bundle-validation/docker_java17/docker_java17_test.sh index 37aa745f3cd9..4de72e5e2176 100755 --- a/packaging/bundle-validation/docker_java17/docker_java17_test.sh +++ b/packaging/bundle-validation/docker_java17/docker_java17_test.sh @@ -47,63 +47,77 @@ use_default_java_runtime () { } start_datanode () { - DN=$1 - - echo "::warning::docker_test_java17.sh starting datanode:"$DN - - cat $HADOOP_HOME/hadoop/etc/hdfs-site.xml - cat $HADOOP_HOME/hadoop/etc/core-site.xml + local dn=$1 + local data_dir="$DOCKER_TEST_DIR/additional_datanode/$dn" + local pid_dir="$DOCKER_TEST_DIR/pid/datanode-$dn" + + echo "::warning::docker_test_java17.sh starting datanode:$dn" + mkdir -p "$data_dir" "$pid_dir" || return 1 + HADOOP_PID_DIR="$pid_dir" "$HADOOP_HOME/bin/hdfs" --daemon start datanode \ + -Dhadoop.tmp.dir="$data_dir" \ + -Ddfs.datanode.address="localhost:5001$dn" \ + -Ddfs.datanode.http.address="localhost:5008$dn" \ + -Ddfs.datanode.ipc.address="localhost:5002$dn" +} - DN_DIR_PREFIX=$DOCKER_TEST_DIR/additional_datanode/ - PID_DIR=$DOCKER_TEST_DIR/pid/$1 +setup_hdfs () { + # The base image's Java 8 cgroup metrics can crash on newer CI hosts. + # Disable container detection only for Hadoop and bound each daemon's heap explicitly. + export HADOOP_OPTS="${HADOOP_OPTS:-} -XX:-UseContainerSupport" + export HADOOP_HEAPSIZE_MAX=512 - if [ -z $DN_DIR_PREFIX ]; then - mkdir -p $DN_DIR_PREFIX - fi + echo "::warning::docker_test_java17.sh copying hadoop conf" + cp "$WORKDIR/tmp-conf-dir/hdfs-site.xml" "$HADOOP_HOME/etc/hadoop/hdfs-site.xml" || return 1 + cp "$WORKDIR/tmp-conf-dir/core-site.xml" "$HADOOP_HOME/etc/hadoop/core-site.xml" || return 1 - if [ -z $PID_DIR ]; then - mkdir -p $PID_DIR - fi + mkdir -p "$DOCKER_TEST_DIR/pid/namenode" || return 1 + "$HADOOP_HOME/bin/hdfs" namenode -format || return 1 + HADOOP_PID_DIR="$DOCKER_TEST_DIR/pid/namenode" "$HADOOP_HOME/bin/hdfs" --daemon start namenode || return 1 - export HADOOP_PID_DIR=$PID_PREFIX - DN_CONF_OPTS="\ - -Dhadoop.tmp.dir=$DN_DIR_PREFIX$DN\ - -Ddfs.datanode.address=localhost:5001$DN \ - -Ddfs.datanode.http.address=localhost:5008$DN \ - -Ddfs.datanode.ipc.address=localhost:5002$DN" - $HADOOP_HOME/bin/hdfs --daemon start datanode $DN_CONF_OPTS - $HADOOP_HOME/bin/hdfs dfsadmin -report -} + # All daemons run in this container. Avoid the SSH/su worker dispatch in start-dfs.sh. + local i + for i in 1 2 3; do + start_datanode "$i" || return 1 + done -setup_hdfs () { - echo "::warning::docker_test_java17.sh copying hadoop conf" - mv /opt/bundle-validation/tmp-conf-dir/hdfs-site.xml $HADOOP_HOME/etc/hadoop/hdfs-site.xml - mv /opt/bundle-validation/tmp-conf-dir/core-site.xml $HADOOP_HOME/etc/hadoop/core-site.xml - - $HADOOP_HOME/bin/hdfs namenode -format - $HADOOP_HOME/bin/hdfs --daemon start namenode - echo "::warning::docker_test_java17.sh starting hadoop hdfs" - $HADOOP_HOME/sbin/start-dfs.sh - - # start datanodes - for i in $(seq 1 3) - do - start_datanode $i + # Starting a daemon only forks it; wait for all replicas before running the tests. + local report + for i in $(seq 1 30); do + if report=$("$HADOOP_HOME/bin/hdfs" dfsadmin -Dipc.client.connect.max.retries=0 -report 2>&1) \ + && printf '%s\n' "$report" | grep -q 'Live datanodes (3)'; then + printf '%s\n' "$report" + "$HADOOP_HOME/bin/hdfs" dfsadmin -safemode wait || return 1 + "$HADOOP_HOME/bin/hdfs" dfs -mkdir -p /user/root || return 1 + "$HADOOP_HOME/bin/hdfs" dfs -ls /user/ + return $? + fi + sleep 2 done - echo "::warning::docker_test_java17.sh starting hadoop hdfs, hdfs report" - $HADOOP_HOME/bin/hdfs dfs -mkdir -p /user/root - $HADOOP_HOME/bin/hdfs dfs -ls /user/ - if [ "$?" -ne 0 ]; then - echo "::error::docker_test_java17.sh Failed setting up HDFS!" - exit 1 - fi + printf '%s\n' "$report" + echo "::error::docker_test_java17.sh Failed waiting for three live HDFS datanodes!" + tail -n 100 "$HADOOP_HOME"/logs/*.log "$HADOOP_HOME"/logs/*.out + return 1 } stop_hdfs() { use_default_java_runtime echo "::warning::docker_test_java17.sh stopping hadoop hdfs" - $HADOOP_HOME/sbin/stop-dfs.sh + local i + for i in 1 2 3; do + HADOOP_PID_DIR="$DOCKER_TEST_DIR/pid/datanode-$i" "$HADOOP_HOME/bin/hdfs" --daemon stop datanode + done + HADOOP_PID_DIR="$DOCKER_TEST_DIR/pid/namenode" "$HADOOP_HOME/bin/hdfs" --daemon stop namenode +} + +cleanup_hdfs() { + local exit_code=$? + if [ "$exit_code" -ne 0 ]; then + # Daemon startup errors are redirected to these files by Hadoop's launcher. + tail -n 100 "$HADOOP_HOME"/logs/*.log "$HADOOP_HOME"/logs/*.out + fi + stop_hdfs + return "$exit_code" } build_hudi () { @@ -171,7 +185,8 @@ echo "::warning::docker_test_java17.sh Building Hudi" build_hudi echo "::warning::docker_test_java17.sh Done building Hudi" -setup_hdfs +trap cleanup_hdfs EXIT +setup_hdfs || exit 1 echo "::warning::docker_test_java17.sh Running tests with Java 17" run_docker_tests @@ -179,5 +194,3 @@ if [ "$?" -ne 0 ]; then exit 1 fi echo "::warning::docker_test_java17.sh Done running tests with Java 17" - -stop_hdfs diff --git a/pom.xml b/pom.xml index 9feaebdb9290..06bd1518d9af 100644 --- a/pom.xml +++ b/pom.xml @@ -808,12 +808,6 @@ <version>${fasterxml.version}</version> </dependency> - <dependency> - <groupId>com.fasterxml.jackson.module</groupId> - <artifactId>jackson-module-afterburner</artifactId> - <version>${fasterxml.jackson.databind.version}</version> - </dependency> - <!-- Glassfish --> <dependency> <groupId>org.glassfish.jersey.core</groupId> diff --git a/scripts/release/download_staged_bundles.sh b/scripts/release/download_staged_bundles.sh index 9346d1bca94d..46e456240cb8 100755 --- a/scripts/release/download_staged_bundles.sh +++ b/scripts/release/download_staged_bundles.sh @@ -1,4 +1,24 @@ #!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + set -euo pipefail if [[ $# -ne 4 ]]; then
