This is an automated email from the ASF dual-hosted git repository.
yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new a17955528595 chore(ci): Add test jobs and Codecov integration in
GitHub Actions (#18225)
a17955528595 is described below
commit a17955528595e5df71e45159e5eed66fa31872cd
Author: Y Ethan Guo <[email protected]>
AuthorDate: Mon Mar 16 16:00:01 2026 -0700
chore(ci): Add test jobs and Codecov integration in GitHub Actions (#18225)
---
.github/workflows/bot.yml | 202 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 202 insertions(+)
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index 81279430e77a..f3a66d20d736 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -59,6 +59,208 @@ jobs:
- name: RAT check
run: ./scripts/release/validate_source_rat.sh
+ test-spark-client-and-hadoop-common:
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ strategy:
+ matrix:
+ include:
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark3.5"
+ flinkProfile: "flink1.18"
+
+ steps:
+ - uses: actions/checkout@v5
+ - name: Set up JDK 11
+ uses: actions/setup-java@v5
+ with:
+ java-version: '11'
+ distribution: 'temurin'
+ architecture: x64
+ cache: maven
+ - name: Build Project
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ FLINK_PROFILE: ${{ matrix.flinkProfile }}
+ run:
+ mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-D"$FLINK_PROFILE" -DskipTests=true -Phudi-platform-service $MVN_ARGS -am -pl
hudi-client/hudi-spark-client
+ - name: UT - hudi-hadoop-common
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ FLINK_PROFILE: ${{ matrix.flinkProfile }}
+ run:
+ mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-D"$FLINK_PROFILE" -fae -pl hudi-hadoop-common $MVN_ARGS -Djacoco.skip=false
+ - name: UT - hudi-client/hudi-spark-client
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ FLINK_PROFILE: ${{ matrix.flinkProfile }}
+ run:
+ mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-D"$FLINK_PROFILE" -fae -pl hudi-client/hudi-spark-client $MVN_ARGS
-Djacoco.skip=false
+ - name: FT - hudi-client/hudi-spark-client
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ FLINK_PROFILE: ${{ matrix.flinkProfile }}
+ run:
+ mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-D"$FLINK_PROFILE" -fae -pl hudi-client/hudi-spark-client $MVN_ARGS
-Djacoco.skip=false
+ - name: Generate merged coverage report
+ if: always()
+ run: ./scripts/jacoco/generate_merged_coverage_report.sh
$GITHUB_WORKSPACE
+ - name: Upload coverage to Codecov
+ if: always()
+ uses: codecov/codecov-action@v5
+ with:
+ files: ./jacoco-report.xml
+ disable_search: true
+ flags: spark-client-hadoop-common
+ token: ${{ secrets.CODECOV_TOKEN }}
+
+ test-utilities:
+ runs-on: ubuntu-latest
+ timeout-minutes: 60
+ strategy:
+ matrix:
+ include:
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark3.5"
+ flinkProfile: "flink1.18"
+
+ steps:
+ - uses: actions/checkout@v5
+ - name: Set up JDK 11
+ uses: actions/setup-java@v5
+ with:
+ java-version: '11'
+ distribution: 'temurin'
+ architecture: x64
+ cache: maven
+ - name: Build Project
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ FLINK_PROFILE: ${{ matrix.flinkProfile }}
+ run:
+ mvn clean install -T 2 -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-D"$FLINK_PROFILE" -DskipTests=true -Phudi-platform-service -Pthrift-gen-source
$MVN_ARGS -am -pl hudi-utilities
+ - name: UT - hudi-utilities (TestHoodie*)
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ FLINK_PROFILE: ${{ matrix.flinkProfile }}
+ run:
+ mvn test -Punit-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-D"$FLINK_PROFILE" -fae -Dtest="TestHoodie*"
-Dsurefire.failIfNoSpecifiedTests=false -pl hudi-utilities $MVN_ARGS
-Djacoco.skip=false
+ - name: FT - hudi-utilities
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ FLINK_PROFILE: ${{ matrix.flinkProfile }}
+ run:
+ mvn test -Pfunctional-tests -D"$SCALA_PROFILE" -D"$SPARK_PROFILE"
-D"$FLINK_PROFILE" -fae -Dsurefire.failIfNoSpecifiedTests=false -pl
hudi-utilities $MVN_ARGS -Djacoco.skip=false
+ - name: Generate merged coverage report
+ if: always()
+ run: ./scripts/jacoco/generate_merged_coverage_report.sh
$GITHUB_WORKSPACE
+ - name: Upload coverage to Codecov
+ if: always()
+ uses: codecov/codecov-action@v5
+ with:
+ files: ./jacoco-report.xml
+ disable_search: true
+ flags: utilities
+ token: ${{ secrets.CODECOV_TOKEN }}
+
+ test-common-and-other-modules:
+ runs-on: ubuntu-latest
+ timeout-minutes: 90
+ strategy:
+ matrix:
+ include:
+ - scalaProfile: "scala-2.12"
+ sparkProfile: "spark3.5"
+ flinkProfile: "flink1.18"
+
+ env:
+ UT_MODULES: >-
+
!hudi-hadoop-common,!hudi-hadoop-mr,!hudi-client/hudi-java-client,!hudi-client/hudi-spark-client,!hudi-cli,!hudi-examples,!hudi-examples/hudi-examples-common,!hudi-examples/hudi-examples-flink,!hudi-examples/hudi-examples-java,!hudi-examples/hudi-examples-spark,!hudi-spark-datasource,!hudi-spark-datasource/hudi-spark,!hudi-spark-datasource/hudi-spark3.5.x,!hudi-spark-datasource/hudi-spark3-common,!hudi-spark-datasource/hudi-spark-common,!hudi-utilities
+ FT_MODULES: >-
+
!hudi-client/hudi-spark-client,!hudi-cli,!hudi-examples,!hudi-examples/hudi-examples-common,!hudi-examples/hudi-examples-flink,!hudi-examples/hudi-examples-java,!hudi-examples/hudi-examples-spark,!hudi-spark-datasource/hudi-spark,!hudi-utilities
+
+ steps:
+ - uses: actions/checkout@v5
+ - name: Set up JDK 11
+ uses: actions/setup-java@v5
+ with:
+ java-version: '11'
+ distribution: 'temurin'
+ architecture: x64
+ cache: maven
+ - name: Build Docker image
+ run: docker build -t hudi-ci .
+ - name: Build Project
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ FLINK_PROFILE: ${{ matrix.flinkProfile }}
+ run: |
+ docker run --rm \
+ -v "$GITHUB_WORKSPACE:/hudi" \
+ -v "$HOME/.m2:/root/.m2" \
+ -v /var/run/docker.sock:/var/run/docker.sock \
+ -w /hudi \
+ hudi-ci \
+ /bin/bash -c "MAVEN_OPTS='-Xmx4g' mvn clean install -T 2
-D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -Phudi-platform-service
-Pthrift-gen-source -DskipTests=true -Dcheckstyle.skip=true -Drat.skip=true
$MVN_ARGS"
+ - name: UT - common and other modules
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ FLINK_PROFILE: ${{ matrix.flinkProfile }}
+ run: |
+ docker run --rm \
+ -v "$GITHUB_WORKSPACE:/hudi" \
+ -v "$HOME/.m2:/root/.m2" \
+ -v /var/run/docker.sock:/var/run/docker.sock \
+ -w /hudi \
+ hudi-ci \
+ /bin/bash -c "MAVEN_OPTS='-Xmx4g' mvn test -Punit-tests
-D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -fae -pl $UT_MODULES
-Dcheckstyle.skip=true -Drat.skip=true $MVN_ARGS -Djacoco.skip=false"
+ - name: UT - hudi-utilities (non-TestHoodie)
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ FLINK_PROFILE: ${{ matrix.flinkProfile }}
+ run: |
+ docker run --rm \
+ -v "$GITHUB_WORKSPACE:/hudi" \
+ -v "$HOME/.m2:/root/.m2" \
+ -v /var/run/docker.sock:/var/run/docker.sock \
+ -w /hudi \
+ hudi-ci \
+ /bin/bash -c "MAVEN_OPTS='-Xmx4g' mvn test -Punit-tests
-D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -fae -Dtest='!TestHoodie*'
-Dsurefire.failIfNoSpecifiedTests=false -pl hudi-utilities
-Dcheckstyle.skip=true -Drat.skip=true $MVN_ARGS -Djacoco.skip=false"
+ - name: FT - common and other modules
+ env:
+ SCALA_PROFILE: ${{ matrix.scalaProfile }}
+ SPARK_PROFILE: ${{ matrix.sparkProfile }}
+ FLINK_PROFILE: ${{ matrix.flinkProfile }}
+ run: |
+ docker run --rm \
+ -v "$GITHUB_WORKSPACE:/hudi" \
+ -v "$HOME/.m2:/root/.m2" \
+ -v /var/run/docker.sock:/var/run/docker.sock \
+ -w /hudi \
+ hudi-ci \
+ /bin/bash -c "MAVEN_OPTS='-Xmx4g' mvn test -Pfunctional-tests
-D$SCALA_PROFILE -D$SPARK_PROFILE -D$FLINK_PROFILE -fae -pl $FT_MODULES
-Dcheckstyle.skip=true -Drat.skip=true $MVN_ARGS -Djacoco.skip=false"
+ - name: Generate merged coverage report
+ if: always()
+ run: ./scripts/jacoco/generate_merged_coverage_report.sh
$GITHUB_WORKSPACE
+ - name: Upload coverage to Codecov
+ if: always()
+ uses: codecov/codecov-action@v5
+ with:
+ files: ./jacoco-report.xml
+ disable_search: true
+ flags: common-and-other-modules
+ token: ${{ secrets.CODECOV_TOKEN }}
+
test-spark-java-tests-part1:
runs-on: ubuntu-latest
strategy:
