This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch release-1.1.2
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/release-1.1.2 by this push:
new d36ea27b4301 fix(ci): repair Java 8 mocking, Trino SPI dependencies
and HDFS startup
d36ea27b4301 is described below
commit d36ea27b4301ba5bfe45aeb46b848a1f256bb99b
Author: danny0405 <[email protected]>
AuthorDate: Fri Oct 9 20:43:02 2026 +0800
fix(ci): repair Java 8 mocking, Trino SPI dependencies and HDFS startup
---
hudi-client/hudi-java-client/pom.xml | 23 ++++++
hudi-trino-plugin/pom.xml | 1 -
packaging/bundle-validation/Dockerfile | 4 +-
.../docker_java17/docker_java17_test.sh | 96 +++++++++++-----------
4 files changed, 72 insertions(+), 52 deletions(-)
diff --git a/hudi-client/hudi-java-client/pom.xml
b/hudi-client/hudi-java-client/pom.xml
index c301adda4b74..299d6893ad38 100644
--- a/hudi-client/hudi-java-client/pom.xml
+++ b/hudi-client/hudi-java-client/pom.xml
@@ -186,4 +186,27 @@
</resources>
</build>
+ <profiles>
+ <profile>
+ <id>java8-test-attach</id>
+ <activation>
+ <jdk>[1.8,1.9)</jdk>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <!-- JOL and Mockito must share the Java 8 Attach
API classloader. -->
+ <additionalClasspathElements>
+
<additionalClasspathElement>${java.home}/../lib/tools.jar</additionalClasspathElement>
+ </additionalClasspathElements>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
</project>
diff --git a/hudi-trino-plugin/pom.xml b/hudi-trino-plugin/pom.xml
index 45a3e8a3ff6d..3a30e4750d82 100644
--- a/hudi-trino-plugin/pom.xml
+++ b/hudi-trino-plugin/pom.xml
@@ -202,7 +202,6 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
- <scope>provided</scope>
</dependency>
<dependency>
diff --git a/packaging/bundle-validation/Dockerfile
b/packaging/bundle-validation/Dockerfile
index 995e11ef828b..e917a1075ce8 100644
--- a/packaging/bundle-validation/Dockerfile
+++ b/packaging/bundle-validation/Dockerfile
@@ -54,5 +54,5 @@ ENV HDFS_SECONDARYNAMENODE_USER root
ENV YARN_RESOURCEMANAGER_USER root
ENV YARN_NODEMANAGER_USER root
-# for RocksDb
-RUN apk add --no-cache libstdc++
+# RocksDB needs libstdc++; Hadoop's daemon scripts need procps (ps -p).
+RUN apk add --no-cache libstdc++ procps
diff --git a/packaging/bundle-validation/docker_java17/docker_java17_test.sh
b/packaging/bundle-validation/docker_java17/docker_java17_test.sh
index 37aa745f3cd9..dfbb427c6713 100755
--- a/packaging/bundle-validation/docker_java17/docker_java17_test.sh
+++ b/packaging/bundle-validation/docker_java17/docker_java17_test.sh
@@ -47,63 +47,62 @@ use_default_java_runtime () {
}
start_datanode () {
- DN=$1
-
- echo "::warning::docker_test_java17.sh starting datanode:"$DN
-
- cat $HADOOP_HOME/hadoop/etc/hdfs-site.xml
- cat $HADOOP_HOME/hadoop/etc/core-site.xml
-
- DN_DIR_PREFIX=$DOCKER_TEST_DIR/additional_datanode/
- PID_DIR=$DOCKER_TEST_DIR/pid/$1
-
- if [ -z $DN_DIR_PREFIX ]; then
- mkdir -p $DN_DIR_PREFIX
- fi
-
- if [ -z $PID_DIR ]; then
- mkdir -p $PID_DIR
- fi
-
- export HADOOP_PID_DIR=$PID_PREFIX
- DN_CONF_OPTS="\
- -Dhadoop.tmp.dir=$DN_DIR_PREFIX$DN\
- -Ddfs.datanode.address=localhost:5001$DN \
- -Ddfs.datanode.http.address=localhost:5008$DN \
- -Ddfs.datanode.ipc.address=localhost:5002$DN"
- $HADOOP_HOME/bin/hdfs --daemon start datanode $DN_CONF_OPTS
- $HADOOP_HOME/bin/hdfs dfsadmin -report
+ local dn=$1
+ local data_dir="$DOCKER_TEST_DIR/additional_datanode/$dn"
+ local pid_dir="$DOCKER_TEST_DIR/pid/datanode-$dn"
+
+ echo "::warning::docker_test_java17.sh starting datanode:$dn"
+ mkdir -p "$data_dir" "$pid_dir" || return 1
+ HADOOP_PID_DIR="$pid_dir" "$HADOOP_HOME/bin/hdfs" --daemon start datanode \
+ -Dhadoop.tmp.dir="$data_dir" \
+ -Ddfs.datanode.address="localhost:5001$dn" \
+ -Ddfs.datanode.http.address="localhost:5008$dn" \
+ -Ddfs.datanode.ipc.address="localhost:5002$dn"
}
setup_hdfs () {
echo "::warning::docker_test_java17.sh copying hadoop conf"
- mv /opt/bundle-validation/tmp-conf-dir/hdfs-site.xml
$HADOOP_HOME/etc/hadoop/hdfs-site.xml
- mv /opt/bundle-validation/tmp-conf-dir/core-site.xml
$HADOOP_HOME/etc/hadoop/core-site.xml
-
- $HADOOP_HOME/bin/hdfs namenode -format
- $HADOOP_HOME/bin/hdfs --daemon start namenode
- echo "::warning::docker_test_java17.sh starting hadoop hdfs"
- $HADOOP_HOME/sbin/start-dfs.sh
-
- # start datanodes
- for i in $(seq 1 3)
- do
- start_datanode $i
+ cp "$WORKDIR/tmp-conf-dir/hdfs-site.xml"
"$HADOOP_HOME/etc/hadoop/hdfs-site.xml" || return 1
+ cp "$WORKDIR/tmp-conf-dir/core-site.xml"
"$HADOOP_HOME/etc/hadoop/core-site.xml" || return 1
+
+ mkdir -p "$DOCKER_TEST_DIR/pid/namenode" || return 1
+ "$HADOOP_HOME/bin/hdfs" namenode -format || return 1
+ HADOOP_PID_DIR="$DOCKER_TEST_DIR/pid/namenode" "$HADOOP_HOME/bin/hdfs"
--daemon start namenode || return 1
+
+ # All daemons run in this container. Avoid the SSH/su worker dispatch in
start-dfs.sh.
+ local i
+ for i in 1 2 3; do
+ start_datanode "$i" || return 1
done
- echo "::warning::docker_test_java17.sh starting hadoop hdfs, hdfs report"
- $HADOOP_HOME/bin/hdfs dfs -mkdir -p /user/root
- $HADOOP_HOME/bin/hdfs dfs -ls /user/
- if [ "$?" -ne 0 ]; then
- echo "::error::docker_test_java17.sh Failed setting up HDFS!"
- exit 1
- fi
+ # Starting a daemon only forks it; wait for all replicas before running the
tests.
+ local report
+ for i in $(seq 1 30); do
+ if report=$("$HADOOP_HOME/bin/hdfs" dfsadmin
-Dipc.client.connect.max.retries=0 -report 2>&1) \
+ && printf '%s\n' "$report" | grep -q 'Live datanodes (3)'; then
+ printf '%s\n' "$report"
+ "$HADOOP_HOME/bin/hdfs" dfsadmin -safemode wait || return 1
+ "$HADOOP_HOME/bin/hdfs" dfs -mkdir -p /user/root || return 1
+ "$HADOOP_HOME/bin/hdfs" dfs -ls /user/
+ return $?
+ fi
+ sleep 2
+ done
+
+ printf '%s\n' "$report"
+ echo "::error::docker_test_java17.sh Failed waiting for three live HDFS
datanodes!"
+ tail -n 100 "$HADOOP_HOME"/logs/*.log "$HADOOP_HOME"/logs/*.out
+ return 1
}
stop_hdfs() {
use_default_java_runtime
echo "::warning::docker_test_java17.sh stopping hadoop hdfs"
- $HADOOP_HOME/sbin/stop-dfs.sh
+ local i
+ for i in 1 2 3; do
+ HADOOP_PID_DIR="$DOCKER_TEST_DIR/pid/datanode-$i" "$HADOOP_HOME/bin/hdfs"
--daemon stop datanode
+ done
+ HADOOP_PID_DIR="$DOCKER_TEST_DIR/pid/namenode" "$HADOOP_HOME/bin/hdfs"
--daemon stop namenode
}
build_hudi () {
@@ -171,7 +170,8 @@ echo "::warning::docker_test_java17.sh Building Hudi"
build_hudi
echo "::warning::docker_test_java17.sh Done building Hudi"
-setup_hdfs
+trap stop_hdfs EXIT
+setup_hdfs || exit 1
echo "::warning::docker_test_java17.sh Running tests with Java 17"
run_docker_tests
@@ -179,5 +179,3 @@ if [ "$?" -ne 0 ]; then
exit 1
fi
echo "::warning::docker_test_java17.sh Done running tests with Java 17"
-
-stop_hdfs
