This is an automated email from the ASF dual-hosted git repository.
roryqi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 1eb7f1b819 [MINOR] ci: orchestrate integration tests from Cron
Integration Test (#13050)
1eb7f1b819 is described below
commit 1eb7f1b819b9f42fd74c1d7e4297931e2203a2c6
Author: Danhua Wang <[email protected]>
AuthorDate: Thu Sep 10 16:01:45 2026 +0800
[MINOR] ci: orchestrate integration tests from Cron Integration Test
(#13050)
### What changes were proposed in this pull request?
Turn `cron-integration-test.yml` into an orchestrator that calls
existing IT reusable workflows, and extract `*-action.yml` for suites
that only had push/PR jobs so those workflows `uses` the same action (no
duplicated steps).
Phase-1 suites: backend, spark, flink, maintenance, python, trino,
access-control, iceberg-rest-trino, mcp, idp, trino-multi-version,
lance, multi-instance.
Out of scope: frontend, gvfs-fuse, web-ui, contrib-catalog.
Nightly `schedule` on `apache/gravitino` is kept. `workflow_dispatch` /
`workflow_call` allow selecting suites (for gravitino-test later).
Also fail fast if `nc` is missing in the MCP authz IT script, and
install netcat in the MCP action.
### Why are the changes needed?
One entrypoint to run ITs (nightly or remote), and a single copy of each
IT job so push/PR CI and cron stay in sync.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Workflow YAML only.
---------
Co-authored-by: Cursor <[email protected]>
---
.../access-control-integration-test-action.yml | 61 ++++
.../workflows/access-control-integration-test.yml | 50 +--
.github/workflows/cron-integration-test.yml | 380 ++++++++++++++++-----
.../iceberg-rest-trino-integration-test-action.yml | 65 ++++
.../iceberg-rest-trino-integration-test.yml | 52 +--
...dp-basic-test.yml => idp-basic-test-action.yml} | 57 +---
.github/workflows/idp-basic-test.yml | 64 +---
... => lance-compatibility-matrix-test-action.yml} | 23 +-
.../workflows/lance-compatibility-matrix-test.yml | 79 +----
...on-test.yml => mcp-integration-test-action.yml} | 45 +--
.github/workflows/mcp-integration-test.yml | 35 +-
.../multi-instance-consistency-test-action.yml | 73 ++++
.../workflows/multi-instance-consistency-test.yml | 118 +------
.../workflows/python-integration-test-action.yml | 61 ++++
.github/workflows/python-integration-test.yml | 50 +--
...-test.yml => trino-integration-test-action.yml} | 71 +---
.github/workflows/trino-integration-test.yml | 67 +---
...est.yml => trino-multi-version-test-action.yml} | 10 +-
.github/workflows/trino-multi-version-test.yml | 87 +----
mcp-server/dev/run_authz_integration_test.sh | 5 +
20 files changed, 642 insertions(+), 811 deletions(-)
diff --git a/.github/workflows/access-control-integration-test-action.yml
b/.github/workflows/access-control-integration-test-action.yml
new file mode 100644
index 0000000000..9b30e26773
--- /dev/null
+++ b/.github/workflows/access-control-integration-test-action.yml
@@ -0,0 +1,61 @@
+name: Access Control Integration Test Action
+
+on:
+ workflow_call:
+ inputs:
+ architecture:
+ required: true
+ description: 'Architecture of the platform'
+ type: string
+ java-version:
+ required: true
+ description: 'Java version'
+ type: string
+
+jobs:
+ access-control-IT:
+ name: JDK${{ inputs.java-version }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 90
+ env:
+ PLATFORM: ${{ inputs.architecture }}
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-java@v4
+ with:
+ java-version: ${{ inputs.java-version }}
+ distribution: 'temurin'
+ cache: 'gradle'
+
+ - name: Set up QEMU
+ uses:
docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
+
+ - name: Check required command
+ run: |
+ dev/ci/check_commands.sh
+
+ - name: Package Gravitino
+ run: |
+ ./gradlew compileDistribution -PskipWeb=true -x test
+
+ - name: Free up disk space
+ run: |
+ dev/ci/util_free_space.sh
+
+ - name: Authorization Integration Test (JDK${{ inputs.java-version }})
+ id: integrationTest
+ run: |
+ ./gradlew -PtestMode=embedded -PjdbcBackend=h2
-PskipDockerTests=false -PskipWeb=true :authorizations:test
+ ./gradlew -PtestMode=deploy -PjdbcBackend=mysql
-PskipDockerTests=false -PskipWeb=true :authorizations:test
+ ./gradlew -PtestMode=deploy -PjdbcBackend=postgresql
-PskipDockerTests=false -PskipWeb=true :authorizations:test
+
+ - name: Upload integrate tests reports
+ uses: actions/upload-artifact@v7
+ if: ${{ (failure() && steps.integrationTest.outcome == 'failure') ||
contains(github.event.pull_request.labels.*.name, 'upload log') }}
+ with:
+ name: authorizations-integrate-test-reports-${{ inputs.java-version
}}
+ path: |
+ build/reports
+ distribution/package/logs/gravitino-server.out
+ distribution/package/logs/gravitino-server.log
diff --git a/.github/workflows/access-control-integration-test.yml
b/.github/workflows/access-control-integration-test.yml
index 3600a2d0b3..1b5054ff49 100644
--- a/.github/workflows/access-control-integration-test.yml
+++ b/.github/workflows/access-control-integration-test.yml
@@ -52,53 +52,11 @@ jobs:
access-control-IT:
needs: changes
if: needs.changes.outputs.source_changes == 'true'
- runs-on: ubuntu-latest
- timeout-minutes: 90
strategy:
matrix:
- # Integration test for AMD64 architecture
architecture: [linux/amd64]
java-version: [ 17 ]
-
- env:
- PLATFORM: ${{ matrix.architecture }}
- steps:
- - uses: actions/checkout@v4
-
- - uses: actions/setup-java@v4
- with:
- java-version: ${{ matrix.java-version }}
- distribution: 'temurin'
- cache: 'gradle'
-
- - name: Set up QEMU
- uses:
docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
-
- - name: Check required command
- run: |
- dev/ci/check_commands.sh
-
- - name: Package Gravitino
- run: |
- ./gradlew compileDistribution -PskipWeb=true -x test
-
- - name: Free up disk space
- run: |
- dev/ci/util_free_space.sh
-
- - name: Authorization Integration Test (JDK${{ matrix.java-version }})
- id: integrationTest
- run: |
- ./gradlew -PtestMode=embedded -PjdbcBackend=h2
-PskipDockerTests=false -PskipWeb=true :authorizations:test
- ./gradlew -PtestMode=deploy -PjdbcBackend=mysql
-PskipDockerTests=false -PskipWeb=true :authorizations:test
- ./gradlew -PtestMode=deploy -PjdbcBackend=postgresql
-PskipDockerTests=false -PskipWeb=true :authorizations:test
-
- - name: Upload integrate tests reports
- uses: actions/upload-artifact@v7
- if: ${{ (failure() && steps.integrationTest.outcome == 'failure') ||
contains(github.event.pull_request.labels.*.name, 'upload log') }}
- with:
- name: authorizations-integrate-test-reports-${{ matrix.java-version
}}
- path: |
- build/reports
- distribution/package/logs/gravitino-server.out
- distribution/package/logs/gravitino-server.log
+ uses: ./.github/workflows/access-control-integration-test-action.yml
+ with:
+ architecture: ${{ matrix.architecture }}
+ java-version: ${{ matrix.java-version }}
diff --git a/.github/workflows/cron-integration-test.yml
b/.github/workflows/cron-integration-test.yml
index 51245b915d..654244770b 100644
--- a/.github/workflows/cron-integration-test.yml
+++ b/.github/workflows/cron-integration-test.yml
@@ -1,110 +1,302 @@
name: Cron Integration Test
-# Controls when the workflow will run
+# Orchestrates Apache Gravitino integration tests. Nightly schedule stays on
+# apache/gravitino. Manual / workflow_call runs can select suites (used later
+# by gravitino-test release-test).
+#
+# Phase-1 suites (gvfs-fuse, web-ui, frontend, contrib-catalog are excluded):
+# backend, spark, flink, maintenance, python, trino, access-control,
+# iceberg-rest-trino, mcp, idp, trino-multi-version, lance, multi-instance
+
+run-name: OSS IT (${{ inputs.suites || 'all' }}) caller=${{
inputs.caller_run_id || 'manual' }}
+
on:
- schedule: # Runs by default on main branch
- - cron: '0 19 * * *' # Runs every day at 19:00 PM UTC, equal to 03:00 AM
the next day in GMT+8 time zone
+ schedule:
+ - cron: '0 19 * * *' # 19:00 UTC; 03:00 GMT+8 next day
+ workflow_dispatch:
+ inputs:
+ suites:
+ description: 'Comma-separated suites, or all'
+ required: false
+ type: string
+ default: 'all'
+ java-version:
+ description: 'JDK version'
+ required: false
+ type: string
+ default: '17'
+ caller_run_id:
+ description: 'Optional caller GitHub run id'
+ required: false
+ type: string
+ default: ''
+ workflow_call:
+ inputs:
+ suites:
+ required: false
+ type: string
+ default: 'all'
+ java-version:
+ required: false
+ type: string
+ default: '17'
+ caller_run_id:
+ required: false
+ type: string
+ default: ''
concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: ${{ contains(github.ref, 'main') }}
+ group: cron-integration-test-${{ github.ref }}
+ cancel-in-progress: false
jobs:
- changes:
- if: github.repository == 'apache/gravitino'
+ select-suites:
+ name: Select suites
+ if: github.event_name != 'schedule' || github.repository ==
'apache/gravitino'
runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
- - name: Disable Git automatic maintenance
- run: git config --local maintenance.auto false
- - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d
- id: filter
- with:
- filters: |
- source_changes:
- - .github/**
- - api/**
- - bin/**
- - catalogs/**
- - clients/**
- - common/**
- - conf/**
- - core/**
- - dev/**
- - gradle/**
- - iceberg/**
- - integration-test/**
- - meta/**
- - server/**
- - server-common/**
- - spark-connector/**
- - trino-connector/**
- - web/**
- - docs/open-api/**
- - build.gradle.kts
- - gradle.properties
- - gradlew
- - settings.gradle.kts
outputs:
- source_changes: ${{ steps.filter.outputs.source_changes }}
-
- # Integration test for AMD64 architecture
- test-amd64-arch:
- needs: changes
- if: needs.changes.outputs.source_changes == 'true'
- runs-on: ubuntu-latest
- timeout-minutes: 120
- strategy:
- matrix:
- architecture: [linux/amd64]
- java-version: [ 17 ]
- test-mode: [ embedded, deploy ]
- env:
- DOCKER_RUN_NAME: hive-amd64
- PLATFORM: ${{ matrix.architecture }}
+ backend: ${{ steps.parse.outputs.backend }}
+ spark: ${{ steps.parse.outputs.spark }}
+ flink: ${{ steps.parse.outputs.flink }}
+ maintenance: ${{ steps.parse.outputs.maintenance }}
+ python: ${{ steps.parse.outputs.python }}
+ trino: ${{ steps.parse.outputs.trino }}
+ access-control: ${{ steps.parse.outputs.access-control }}
+ iceberg-rest-trino: ${{ steps.parse.outputs.iceberg-rest-trino }}
+ mcp: ${{ steps.parse.outputs.mcp }}
+ idp: ${{ steps.parse.outputs.idp }}
+ trino-multi-version: ${{ steps.parse.outputs.trino-multi-version }}
+ lance: ${{ steps.parse.outputs.lance }}
+ multi-instance: ${{ steps.parse.outputs.multi-instance }}
+ java-version: ${{ steps.parse.outputs.java-version }}
steps:
- - uses: actions/checkout@v4
+ - name: Parse suite selection
+ id: parse
+ env:
+ SUITES: ${{ inputs.suites || 'all' }}
+ JAVA_VERSION: ${{ inputs.java-version || '17' }}
+ CALLER_RUN_ID: ${{ inputs.caller_run_id || 'manual' }}
+ run: |
+ python3 << 'PY'
+ import os
- - uses: ./.github/actions/setup-java-toolchains
- with:
- java-version: ${{ matrix.java-version }}
+ known = [
+ "backend",
+ "spark",
+ "flink",
+ "maintenance",
+ "python",
+ "trino",
+ "access-control",
+ "iceberg-rest-trino",
+ "mcp",
+ "idp",
+ "trino-multi-version",
+ "lance",
+ "multi-instance",
+ ]
+ raw = os.environ["SUITES"].strip().lower()
+ if raw in ("all", ""):
+ selected = set(known)
+ else:
+ selected = {item.strip() for item in raw.split(",") if
item.strip()}
+ unknown = selected - set(known)
+ if unknown:
+ raise SystemExit(f"Unknown suites: {sorted(unknown)}. Known:
{known}")
- - name: Set up QEMU
- uses:
docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
+ output_path = os.environ["GITHUB_OUTPUT"]
+ with open(output_path, "a", encoding="utf-8") as handle:
+ for name in known:
+ value = "true" if name in selected else "false"
+ handle.write(f"{name}={value}\n")
+ print(f"{name}={value}")
+ handle.write(f"java-version={os.environ['JAVA_VERSION']}\n")
+ print(f"caller_run_id={os.environ['CALLER_RUN_ID']}")
+ PY
- - name: Package Gravitino
- run: |
- ./gradlew compileDistribution -PskipWeb=true -x test
+ backend:
+ name: backend
+ needs: select-suites
+ if: needs.select-suites.outputs.backend == 'true'
+ uses: ./.github/workflows/backend-integration-test-action.yml
+ with:
+ architecture: linux/amd64
+ java-version: ${{ needs.select-suites.outputs.java-version }}
+ backend: mysql
+ test-mode: deploy
- - name: Free up disk space
- run: |
- dev/ci/util_free_space.sh
+ spark:
+ name: spark
+ needs: select-suites
+ if: needs.select-suites.outputs.spark == 'true'
+ uses: ./.github/workflows/spark-integration-test-action.yml
+ with:
+ architecture: linux/amd64
+ java-version: ${{ needs.select-suites.outputs.java-version }}
+ scala-version: '2.12'
+ test-mode: deploy
+
+ flink:
+ name: flink
+ needs: select-suites
+ if: needs.select-suites.outputs.flink == 'true'
+ uses: ./.github/workflows/flink-integration-test-action.yml
+ with:
+ architecture: linux/amd64
+ java-version: ${{ needs.select-suites.outputs.java-version }}
+
+ maintenance:
+ name: maintenance
+ needs: select-suites
+ if: needs.select-suites.outputs.maintenance == 'true'
+ uses: ./.github/workflows/maintenance-integration-test-action.yml
+ with:
+ architecture: linux/amd64
+ java-version: ${{ needs.select-suites.outputs.java-version }}
+
+ python:
+ name: python
+ needs: select-suites
+ if: needs.select-suites.outputs.python == 'true'
+ uses: ./.github/workflows/python-integration-test-action.yml
+ with:
+ architecture: linux/amd64
+ java-version: ${{ needs.select-suites.outputs.java-version }}
+
+ trino:
+ name: trino
+ needs: select-suites
+ if: needs.select-suites.outputs.trino == 'true'
+ uses: ./.github/workflows/trino-integration-test-action.yml
+ with:
+ architecture: linux/amd64
+ java-version: ${{ needs.select-suites.outputs.java-version }}
- - name: Integration Test
- id: integrationTest
+ access-control:
+ name: access-control
+ needs: select-suites
+ if: needs.select-suites.outputs.access-control == 'true'
+ uses: ./.github/workflows/access-control-integration-test-action.yml
+ with:
+ architecture: linux/amd64
+ java-version: ${{ needs.select-suites.outputs.java-version }}
+
+ iceberg-rest-trino:
+ name: iceberg-rest-trino
+ needs: select-suites
+ if: needs.select-suites.outputs.iceberg-rest-trino == 'true'
+ uses: ./.github/workflows/iceberg-rest-trino-integration-test-action.yml
+ with:
+ architecture: linux/amd64
+ java-version: ${{ needs.select-suites.outputs.java-version }}
+
+ mcp:
+ name: mcp
+ needs: select-suites
+ if: needs.select-suites.outputs.mcp == 'true'
+ uses: ./.github/workflows/mcp-integration-test-action.yml
+ with:
+ java-version: ${{ needs.select-suites.outputs.java-version }}
+
+ idp:
+ name: idp
+ needs: select-suites
+ if: needs.select-suites.outputs.idp == 'true'
+ uses: ./.github/workflows/idp-basic-test-action.yml
+ with:
+ java-version: ${{ needs.select-suites.outputs.java-version }}
+
+ trino-multi-version:
+ name: trino-multi-version
+ needs: select-suites
+ if: needs.select-suites.outputs.trino-multi-version == 'true'
+ uses: ./.github/workflows/trino-multi-version-test-action.yml
+
+ lance:
+ name: lance
+ needs: select-suites
+ if: needs.select-suites.outputs.lance == 'true'
+ uses: ./.github/workflows/lance-compatibility-matrix-test-action.yml
+
+ multi-instance:
+ name: multi-instance
+ needs: select-suites
+ if: needs.select-suites.outputs.multi-instance == 'true'
+ uses: ./.github/workflows/multi-instance-consistency-test-action.yml
+
+ summary:
+ name: Integration test summary
+ needs:
+ - select-suites
+ - backend
+ - spark
+ - flink
+ - maintenance
+ - python
+ - trino
+ - access-control
+ - iceberg-rest-trino
+ - mcp
+ - idp
+ - trino-multi-version
+ - lance
+ - multi-instance
+ if: always()
+ runs-on: ubuntu-latest
+ steps:
+ - name: Write summary
+ env:
+ BACKEND: ${{ needs.backend.result }}
+ SPARK: ${{ needs.spark.result }}
+ FLINK: ${{ needs.flink.result }}
+ MAINTENANCE: ${{ needs.maintenance.result }}
+ PYTHON: ${{ needs.python.result }}
+ TRINO: ${{ needs.trino.result }}
+ ACCESS_CONTROL: ${{ needs.access-control.result }}
+ ICEBERG_REST_TRINO: ${{ needs.iceberg-rest-trino.result }}
+ MCP: ${{ needs.mcp.result }}
+ IDP: ${{ needs.idp.result }}
+ TRINO_MULTI_VERSION: ${{ needs.trino-multi-version.result }}
+ LANCE: ${{ needs.lance.result }}
+ MULTI_INSTANCE: ${{ needs.multi-instance.result }}
run: |
- # iceberg-rest-trino-it runs in its own dedicated workflow (Java 24
/ Trino, deploy
- # only); exclude it here so it is not run redundantly.
- ./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }}
-PskipDockerTests=false -PskipWeb=true \
- -x :iceberg:iceberg-rest-trino-it:test
-
- - name: Upload integrate tests reports
- uses: actions/upload-artifact@v7
- if: ${{ failure() && steps.integrationTest.outcome == 'failure' }}
- with:
- name: integrate test reports
- path: |
- build/reports
- iceberg/iceberg-rest-server/build/*.log
- integration-test/build/*.log
- integration-test/build/*.tar
- integration-test/build/trino-ci-container-log
- distribution/package/logs/*.out
- distribution/package/logs/*.log
- catalogs/**/*.log
- catalogs/**/*.tar
- distribution/**/*.log
- spark-connector/v3.5/spark/build/spark-3.5-integration-test.log
- spark-connector/v4.0/spark/build/spark-4.0-integration-test.log
- flink-connector/build/flink-connector-integration-test.log
- flink-connector/build/*.tar
+ cat >> "$GITHUB_STEP_SUMMARY" <<EOF
+ ## OSS Integration Test Summary
+
+ | | |
+ |---|---|
+ | **Ref** | \`${{ github.ref_name }} (${{ github.sha }})\` |
+ | **Suites** | \`${{ inputs.suites || 'all' }}\` |
+ | **Java** | \`${{ needs.select-suites.outputs.java-version }}\` |
+ | **Caller run** | \`${{ inputs.caller_run_id || 'manual' }}\` |
+
+ | Suite | Result |
+ |-------|--------|
+ | backend | ${BACKEND} |
+ | spark | ${SPARK} |
+ | flink | ${FLINK} |
+ | maintenance | ${MAINTENANCE} |
+ | python | ${PYTHON} |
+ | trino | ${TRINO} |
+ | access-control | ${ACCESS_CONTROL} |
+ | iceberg-rest-trino | ${ICEBERG_REST_TRINO} |
+ | mcp | ${MCP} |
+ | idp | ${IDP} |
+ | trino-multi-version | ${TRINO_MULTI_VERSION} |
+ | lance | ${LANCE} |
+ | multi-instance | ${MULTI_INSTANCE} |
+ EOF
+
+ failed=0
+ if [[ "${{ needs.select-suites.result }}" != "success" ]]; then
+ failed=1
+ fi
+ for result in "$BACKEND" "$SPARK" "$FLINK" "$MAINTENANCE" "$PYTHON"
"$TRINO" \
+ "$ACCESS_CONTROL" "$ICEBERG_REST_TRINO" "$MCP" "$IDP" \
+ "$TRINO_MULTI_VERSION" "$LANCE" "$MULTI_INSTANCE"
+ do
+ if [[ "$result" == "failure" || "$result" == "cancelled" ]]; then
+ failed=1
+ fi
+ done
+ exit "$failed"
diff --git a/.github/workflows/iceberg-rest-trino-integration-test-action.yml
b/.github/workflows/iceberg-rest-trino-integration-test-action.yml
new file mode 100644
index 0000000000..6d97a62fcb
--- /dev/null
+++ b/.github/workflows/iceberg-rest-trino-integration-test-action.yml
@@ -0,0 +1,65 @@
+name: Iceberg REST Trino Integration Test Action
+
+on:
+ workflow_call:
+ inputs:
+ architecture:
+ required: true
+ description: 'Architecture of the platform'
+ type: string
+ java-version:
+ required: true
+ description: 'Java version'
+ type: string
+
+jobs:
+ IcebergRestTrinoIT:
+ name: JDK${{ inputs.java-version }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 90
+ env:
+ PLATFORM: ${{ inputs.architecture }}
+ steps:
+ - uses: actions/checkout@v4
+
+ # Installs the primary JDK plus JDK 24 and registers JDK 24 with the
Gradle
+ # toolchain, which the trino-iceberg-rest module requires.
+ - uses: ./.github/actions/setup-java-toolchains
+ with:
+ java-version: ${{ inputs.java-version }}
+
+ - name: Set up QEMU
+ uses:
docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
+
+ - name: Check required command
+ run: |
+ dev/ci/check_commands.sh
+
+ - name: Package Gravitino
+ run: |
+ ./gradlew compileDistribution -PskipWeb=true -x test
+
+ - name: Free up disk space
+ run: |
+ dev/ci/util_free_space.sh
+
+ - name: Install dependencies
+ run: |
+ wget
https://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb
+ sudo dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb
+
+ - name: Iceberg REST Trino Integration Test
+ id: integrationTest
+ run: |
+ ./gradlew :iceberg:iceberg-rest-trino-it:test -PtestMode=deploy
-PskipDockerTests=false -PskipWeb=true
+
+ - name: Upload integrate tests reports
+ uses: actions/upload-artifact@v7
+ if: ${{ (failure() && steps.integrationTest.outcome == 'failure') ||
contains(github.event.pull_request.labels.*.name, 'upload log') }}
+ with:
+ name: iceberg-rest-trino-it-reports-${{ inputs.java-version }}
+ path: |
+ build/reports
+ iceberg/iceberg-rest-trino-it/build/reports
+ distribution/package/logs/*.out
+ distribution/package/logs/*.log
diff --git a/.github/workflows/iceberg-rest-trino-integration-test.yml
b/.github/workflows/iceberg-rest-trino-integration-test.yml
index 75f5c080df..6c0e1622df 100644
--- a/.github/workflows/iceberg-rest-trino-integration-test.yml
+++ b/.github/workflows/iceberg-rest-trino-integration-test.yml
@@ -46,55 +46,11 @@ jobs:
IcebergRestTrinoIT:
needs: changes
if: needs.changes.outputs.source_changes == 'true'
- runs-on: ubuntu-latest
- timeout-minutes: 90
strategy:
matrix:
architecture: [linux/amd64]
java-version: [ 17 ]
- env:
- PLATFORM: ${{ matrix.architecture }}
- steps:
- - uses: actions/checkout@v4
-
- # Installs the primary JDK plus JDK 24 and registers JDK 24 with the
Gradle
- # toolchain, which the trino-iceberg-rest module requires.
- - uses: ./.github/actions/setup-java-toolchains
- with:
- java-version: ${{ matrix.java-version }}
-
- - name: Set up QEMU
- uses:
docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
-
- - name: Check required command
- run: |
- dev/ci/check_commands.sh
-
- - name: Package Gravitino
- run: |
- ./gradlew compileDistribution -PskipWeb=true -x test
-
- - name: Free up disk space
- run: |
- dev/ci/util_free_space.sh
-
- - name: Install dependencies
- run: |
- wget
https://nz2.archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb
- sudo dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb
-
- - name: Iceberg REST Trino Integration Test
- id: integrationTest
- run: |
- ./gradlew :iceberg:iceberg-rest-trino-it:test -PtestMode=deploy
-PskipDockerTests=false -PskipWeb=true
-
- - name: Upload integrate tests reports
- uses: actions/upload-artifact@v7
- if: ${{ (failure() && steps.integrationTest.outcome == 'failure') ||
contains(github.event.pull_request.labels.*.name, 'upload log') }}
- with:
- name: iceberg-rest-trino-it-reports-${{ matrix.java-version }}
- path: |
- build/reports
- iceberg/iceberg-rest-trino-it/build/reports
- distribution/package/logs/*.out
- distribution/package/logs/*.log
+ uses: ./.github/workflows/iceberg-rest-trino-integration-test-action.yml
+ with:
+ architecture: ${{ matrix.architecture }}
+ java-version: ${{ matrix.java-version }}
diff --git a/.github/workflows/idp-basic-test.yml
b/.github/workflows/idp-basic-test-action.yml
similarity index 56%
copy from .github/workflows/idp-basic-test.yml
copy to .github/workflows/idp-basic-test-action.yml
index 610da15ddd..5f4a18c663 100644
--- a/.github/workflows/idp-basic-test.yml
+++ b/.github/workflows/idp-basic-test-action.yml
@@ -1,55 +1,16 @@
-name: IdP Basic Test
+name: IdP Basic Test Action
on:
- push:
- branches: ["main", "branch-*"]
- pull_request:
- branches: ["main", "branch-*"]
- workflow_dispatch:
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.event.pull_request.number ||
github.ref }}
- cancel-in-progress: true
+ workflow_call:
+ inputs:
+ java-version:
+ required: true
+ description: 'Java version'
+ type: string
jobs:
- changes:
- runs-on: ubuntu-22.04
- steps:
- - uses: actions/checkout@v4
- - name: Disable Git automatic maintenance
- run: git config --local maintenance.auto false
- - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d
- id: filter
- with:
- filters: |
- source_changes:
- - plugins/idp-basic/**
- - clients/client-java/**
- - clients/client-java-runtime/**
- - integration-test-common/**
- - core/**
- - gradle/**
- - gradlew
- - dev/**
- - build.gradle.kts
- - gradle.properties
- - settings.gradle.kts
- - .github/workflows/idp-basic-test.yml
- - name: Detect idp-basic module
- id: module
- run: |
- if [ -f plugins/idp-basic/build.gradle.kts ]; then
- echo "exists=true" >> "$GITHUB_OUTPUT"
- else
- echo "exists=false" >> "$GITHUB_OUTPUT"
- fi
- outputs:
- source_changes: ${{ steps.filter.outputs.source_changes }}
- module_exists: ${{ steps.module.outputs.exists }}
-
idp-basic-test:
- needs: changes
- if: needs.changes.outputs.source_changes == 'true' &&
needs.changes.outputs.module_exists == 'true'
+ name: JDK${{ inputs.java-version }}
runs-on: ubuntu-22.04
timeout-minutes: 30
steps:
@@ -57,7 +18,7 @@ jobs:
- uses: actions/setup-java@v4
with:
- java-version: 17
+ java-version: ${{ inputs.java-version }}
distribution: "temurin"
cache: "gradle"
diff --git a/.github/workflows/idp-basic-test.yml
b/.github/workflows/idp-basic-test.yml
index 610da15ddd..e7b07e34a1 100644
--- a/.github/workflows/idp-basic-test.yml
+++ b/.github/workflows/idp-basic-test.yml
@@ -35,6 +35,7 @@ jobs:
- gradle.properties
- settings.gradle.kts
- .github/workflows/idp-basic-test.yml
+ - .github/workflows/idp-basic-test-action.yml
- name: Detect idp-basic module
id: module
run: |
@@ -50,63 +51,6 @@ jobs:
idp-basic-test:
needs: changes
if: needs.changes.outputs.source_changes == 'true' &&
needs.changes.outputs.module_exists == 'true'
- runs-on: ubuntu-22.04
- timeout-minutes: 30
- steps:
- - uses: actions/checkout@v4
-
- - uses: actions/setup-java@v4
- with:
- java-version: 17
- distribution: "temurin"
- cache: "gradle"
-
- - name: Set up QEMU
- uses:
docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
-
- - name: Check required command
- run: |
- dev/ci/check_commands.sh
-
- - name: Package Gravitino
- run: |
- ./gradlew compileDistribution -PskipWeb=true -x test
-
- - name: Free up disk space
- run: |
- dev/ci/util_free_space.sh
-
- - name: Run idp-basic tests
- id: idpBasicTest
- env:
- dockerTest: true
- GRAVITINO_INITIAL_ADMIN_PASSWORD: Passw0rd-For-Admin1
- run: |
- set -euo pipefail
-
- # Unit and in-process tests (no Docker, no REST IT).
- ./gradlew :plugins:idp-basic:test -PskipITs -PskipDockerTests=true
-
- # REST IT: embedded and deploy, each against h2 / MySQL / PostgreSQL.
- for test_mode in embedded deploy; do
- for backend in h2 mysql postgresql; do
- ./gradlew :plugins:idp-basic:test \
- -PtestMode="${test_mode}" \
- -PjdbcBackend="${backend}" \
- -PskipDockerTests=false \
- --tests "org.apache.gravitino.idp.integration.test.**"
- done
- done
-
- - name: Upload idp-basic test reports
- uses: actions/upload-artifact@v7
- if: ${{ (failure() && steps.idpBasicTest.outcome == 'failure') ||
contains(github.event.pull_request.labels.*.name, 'upload log') }}
- with:
- name: idp-basic-test-reports
- path: |
- plugins/idp-basic/build/reports
- plugins/idp-basic/build/test-results
- distribution/package-all/logs/gravitino-server.out
- distribution/package-all/logs/gravitino-server.log
- distribution/package/logs/gravitino-server.out
- distribution/package/logs/gravitino-server.log
+ uses: ./.github/workflows/idp-basic-test-action.yml
+ with:
+ java-version: '17'
diff --git a/.github/workflows/lance-compatibility-matrix-test.yml
b/.github/workflows/lance-compatibility-matrix-test-action.yml
similarity index 78%
copy from .github/workflows/lance-compatibility-matrix-test.yml
copy to .github/workflows/lance-compatibility-matrix-test-action.yml
index cbd68565d1..7258faea0d 100644
--- a/.github/workflows/lance-compatibility-matrix-test.yml
+++ b/.github/workflows/lance-compatibility-matrix-test-action.yml
@@ -1,30 +1,23 @@
-name: Lance Compatibility Matrix Test
+name: Lance Compatibility Matrix Test Action
on:
- workflow_dispatch:
+ workflow_call:
inputs:
lance_spark_versions:
- description: "Comma-separated lance-spark-bundle versions"
required: false
- default: "0.2.0,0.4.0,0.5.1"
+ type: string
+ default: '0.2.0,0.4.0,0.5.1'
lance_ray_versions:
- description: "Comma-separated lance-ray versions"
required: false
- default: "0.4.2,0.3.0"
- schedule:
- # Run weekly on main so compatibility drift is caught outside PR
validation.
- - cron: "0 18 * * 0"
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.ref }}
- cancel-in-progress: true
+ type: string
+ default: '0.4.2,0.3.0'
jobs:
lance-spark-matrix:
runs-on: ubuntu-latest
timeout-minutes: 120
env:
- LANCE_SPARK_VERSIONS: ${{ github.event.inputs.lance_spark_versions ||
'0.2.0,0.4.0,0.5.1' }}
+ LANCE_SPARK_VERSIONS: ${{ inputs.lance_spark_versions }}
steps:
- uses: actions/checkout@v4
@@ -59,7 +52,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 120
env:
- LANCE_RAY_VERSIONS: ${{ github.event.inputs.lance_ray_versions ||
'0.4.2,0.3.0' }}
+ LANCE_RAY_VERSIONS: ${{ inputs.lance_ray_versions }}
steps:
- uses: actions/checkout@v4
diff --git a/.github/workflows/lance-compatibility-matrix-test.yml
b/.github/workflows/lance-compatibility-matrix-test.yml
index cbd68565d1..a64283e9ae 100644
--- a/.github/workflows/lance-compatibility-matrix-test.yml
+++ b/.github/workflows/lance-compatibility-matrix-test.yml
@@ -20,77 +20,8 @@ concurrency:
cancel-in-progress: true
jobs:
- lance-spark-matrix:
- runs-on: ubuntu-latest
- timeout-minutes: 120
- env:
- LANCE_SPARK_VERSIONS: ${{ github.event.inputs.lance_spark_versions ||
'0.2.0,0.4.0,0.5.1' }}
- steps:
- - uses: actions/checkout@v4
-
- - uses: actions/setup-java@v4
- with:
- java-version: 17
- distribution: "temurin"
- cache: "gradle"
-
- - name: Free up disk space
- run: dev/ci/util_free_space.sh
-
- - name: Run Lance Spark compatibility matrix
- id: lanceSparkMatrixTest
- run: |
- ./gradlew :lance:lance-rest-server:lanceSparkMatrixTest \
- -PlanceSparkBundleVersions="${LANCE_SPARK_VERSIONS}" \
- -PskipDockerTests=true \
- -PskipWeb=true
-
- - name: Upload Lance Spark matrix reports
- uses: actions/upload-artifact@v7
- if: ${{ failure() && steps.lanceSparkMatrixTest.outcome == 'failure' }}
- with:
- name: lance-spark-matrix-reports
- path: |
- lance/lance-rest-server/build/reports/lance-spark-matrix
- lance/lance-rest-server/build/test-results/lance-spark-matrix
- build/reports
-
- lance-ray-matrix:
- runs-on: ubuntu-latest
- timeout-minutes: 120
- env:
- LANCE_RAY_VERSIONS: ${{ github.event.inputs.lance_ray_versions ||
'0.4.2,0.3.0' }}
- steps:
- - uses: actions/checkout@v4
-
- - uses: actions/setup-java@v4
- with:
- java-version: 17
- distribution: "temurin"
- cache: "gradle"
-
- - name: Free up disk space
- run: dev/ci/util_free_space.sh
-
- - name: Package Gravitino
- run: ./gradlew compileDistribution -PskipWeb=true -x test
-
- - name: Run Lance Ray compatibility matrix
- id: lanceRayMatrixTest
- run: |
- ./gradlew :clients:client-python:lanceRayMatrixTest \
- -PlanceRayVersions="${LANCE_RAY_VERSIONS}" \
- -PskipDockerTests=true \
- -PskipWeb=true
-
- - name: Upload Lance Ray matrix reports
- uses: actions/upload-artifact@v7
- if: ${{ failure() && steps.lanceRayMatrixTest.outcome == 'failure' }}
- with:
- name: lance-ray-matrix-reports
- path: |
- clients/client-python/build/lance-ray-matrix
- build/reports
- integration-test/build/integration-test.log
- distribution/package/logs/gravitino-server.out
- distribution/package/logs/gravitino-server.log
+ lance-matrix:
+ uses: ./.github/workflows/lance-compatibility-matrix-test-action.yml
+ with:
+ lance_spark_versions: ${{ github.event.inputs.lance_spark_versions ||
'0.2.0,0.4.0,0.5.1' }}
+ lance_ray_versions: ${{ github.event.inputs.lance_ray_versions ||
'0.4.2,0.3.0' }}
diff --git a/.github/workflows/mcp-integration-test.yml
b/.github/workflows/mcp-integration-test-action.yml
similarity index 61%
copy from .github/workflows/mcp-integration-test.yml
copy to .github/workflows/mcp-integration-test-action.yml
index 05a839e9eb..283a8a44e0 100644
--- a/.github/workflows/mcp-integration-test.yml
+++ b/.github/workflows/mcp-integration-test-action.yml
@@ -16,53 +16,36 @@
# specific language governing permissions and limitations
# under the License.
#
-name: mcp-integration-test
+name: MCP Integration Test Action
on:
- push:
- branches: [ "main", "branch-*" ]
- pull_request:
- branches: [ "main", "branch-*" ]
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.event.pull_request.number ||
github.ref }}
- cancel-in-progress: true
+ workflow_call:
+ inputs:
+ java-version:
+ required: true
+ description: 'Java version'
+ type: string
jobs:
- changes:
- runs-on: ubuntu-latest
- outputs:
- mcp_or_authz_changes: ${{ steps.filter.outputs.mcp_or_authz_changes }}
- steps:
- - uses: actions/checkout@v4
- - name: Disable Git automatic maintenance
- run: git config --local maintenance.auto false
- - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d #
v4.0.1
- id: filter
- with:
- filters: |
- mcp_or_authz_changes:
- - 'mcp-server/**'
- - 'server/**'
- - 'core/src/main/java/org/apache/gravitino/authorization/**'
- -
'server-common/src/main/java/org/apache/gravitino/server/authentication/**'
- - '.github/workflows/mcp-integration-test.yml'
-
mcp-authz-integration-test:
+ name: JDK${{ inputs.java-version }}
runs-on: ubuntu-latest
timeout-minutes: 60
- needs: changes
- if: needs.changes.outputs.mcp_or_authz_changes == 'true'
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/setup-java-toolchains
with:
- java-version: 17
+ java-version: ${{ inputs.java-version }}
- name: Install uv
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 #
v8.2.0
+ - name: Install netcat
+ run: |
+ sudo apt-get update
+ sudo apt-get install -y netcat-openbsd
+
- name: Build Gravitino distribution
run: ./gradlew compileDistribution -x test -PskipWeb=true
diff --git a/.github/workflows/mcp-integration-test.yml
b/.github/workflows/mcp-integration-test.yml
index 05a839e9eb..ca510acda8 100644
--- a/.github/workflows/mcp-integration-test.yml
+++ b/.github/workflows/mcp-integration-test.yml
@@ -47,38 +47,11 @@ jobs:
- 'core/src/main/java/org/apache/gravitino/authorization/**'
-
'server-common/src/main/java/org/apache/gravitino/server/authentication/**'
- '.github/workflows/mcp-integration-test.yml'
+ - '.github/workflows/mcp-integration-test-action.yml'
mcp-authz-integration-test:
- runs-on: ubuntu-latest
- timeout-minutes: 60
needs: changes
if: needs.changes.outputs.mcp_or_authz_changes == 'true'
- steps:
- - uses: actions/checkout@v4
-
- - uses: ./.github/actions/setup-java-toolchains
- with:
- java-version: 17
-
- - name: Install uv
- uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 #
v8.2.0
-
- - name: Build Gravitino distribution
- run: ./gradlew compileDistribution -x test -PskipWeb=true
-
- - name: Run MCP authorization integration test
- env:
- GRAVITINO_HOME: ${{ github.workspace }}/distribution/package
- run: |
- ./mcp-server/dev/run_authz_integration_test.sh
-
- - name: Upload server logs on failure
- if: failure()
- uses: actions/upload-artifact@v4
- with:
- name: mcp-authz-it-logs
- path: |
- distribution/package/logs/**
- mcp-server/gravitino-mcp-audit.log
- mcp-server/gravitino-mcp.log
- if-no-files-found: ignore
+ uses: ./.github/workflows/mcp-integration-test-action.yml
+ with:
+ java-version: '17'
diff --git a/.github/workflows/multi-instance-consistency-test-action.yml
b/.github/workflows/multi-instance-consistency-test-action.yml
new file mode 100644
index 0000000000..0ba3ec2da0
--- /dev/null
+++ b/.github/workflows/multi-instance-consistency-test-action.yml
@@ -0,0 +1,73 @@
+name: Multi-Instance Consistency Test Action
+
+on:
+ workflow_call:
+
+jobs:
+ multi-instance-consistency:
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ services:
+ mysql:
+ image: mysql:8.0.33
+ env:
+ MYSQL_ROOT_PASSWORD: gravitino
+ MYSQL_DATABASE: gravitino
+ MYSQL_USER: gravitino
+ MYSQL_PASSWORD: gravitino
+ ports:
+ - 3306:3306
+ options: >-
+ --health-cmd="mysqladmin ping -uroot -pgravitino --silent"
+ --health-interval=5s
+ --health-timeout=3s
+ --health-retries=20
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-java@v4
+ with:
+ java-version: 17
+ distribution: 'temurin'
+ cache: 'gradle'
+
+ - name: Free disk space
+ run: dev/ci/util_free_space.sh
+
+ - name: Apply Gravitino MySQL schema
+ run: |
+ # Keep in sync with the latest schema file under scripts/mysql/ when
the version bumps.
+ mysql -h 127.0.0.1 -P 3306 -ugravitino -pgravitino gravitino \
+ < scripts/mysql/schema-2.0.0-mysql.sql
+
+ - name: Build Gravitino distribution
+ run: ./gradlew compileDistribution -PskipWeb=true -x test
+
+ - name: Stage and start two instances (A=8090, B=8190)
+ run: bash dev/ci/setup_multi_instance.sh
+
+ - name: Run multi-instance consistency tests
+ id: consistency
+ env:
+ INSTANCE_A: http://localhost:8090
+ INSTANCE_B: http://localhost:8190
+ ADMIN_USER: admin
+ POLL_WAIT_SECS: 6
+ run: bash dev/ci/test_multi_instance_consistency.sh
+
+ - name: Stop Gravitino servers
+ if: always()
+ run: |
+ distribution/package/bin/gravitino.sh stop || true
+ distribution/package-b/bin/gravitino.sh stop || true
+
+ - name: Upload logs on failure
+ if: ${{ failure() || cancelled() }}
+ uses: actions/upload-artifact@v7
+ with:
+ name: multi-instance-consistency-logs
+ path: |
+ distribution/package/logs/
+ distribution/package-b/logs/
+ distribution/package/conf/gravitino.conf
+ distribution/package-b/conf/gravitino.conf
diff --git a/.github/workflows/multi-instance-consistency-test.yml
b/.github/workflows/multi-instance-consistency-test.yml
index a7c854f23c..be0d50bba4 100644
--- a/.github/workflows/multi-instance-consistency-test.yml
+++ b/.github/workflows/multi-instance-consistency-test.yml
@@ -1,16 +1,12 @@
name: Multi-Instance Consistency Test
-# Verifies that entity and authorization caches propagate changes correctly
-# between two Gravitino instances backed by the same MySQL entity store. The
-# entity test warms one instance, mutates the other, and then re-reads the
-# target until it converges or the convergence deadline expires.
+# Verifies that authorization state (owners, roles, role privileges, user-role
+# grants) propagates correctly between two Gravitino instances backed by the
+# same MySQL entity store. Both instances run with the entity cache disabled
+# and authorization enabled, so the only consistency contracts under test are
+# the JcasbinAuthorizer caches and the JcasbinChangePoller invalidation paths.
#
-# The entity test opens with a stale-read control that requires the target to
-# serve a cached value before the poller runs. If that control fails, the cache
-# is not actually caching and the remaining assertions prove nothing, so treat
-# a failure there as a configuration problem rather than a propagation bug.
-#
-# The test plans and assertions live in the two test scripts under dev/ci/.
+# The test plan and assertions live in
dev/ci/test_multi_instance_consistency.sh.
on:
workflow_dispatch:
@@ -21,104 +17,4 @@ concurrency:
jobs:
multi-instance-consistency:
- runs-on: ubuntu-latest
- timeout-minutes: 30
- services:
- mysql:
- image: mysql:8.0.33
- env:
- MYSQL_ROOT_PASSWORD: gravitino
- MYSQL_DATABASE: gravitino
- MYSQL_USER: gravitino
- MYSQL_PASSWORD: gravitino
- ports:
- - 3306:3306
- options: >-
- --health-cmd="mysqladmin ping -uroot -pgravitino --silent"
- --health-interval=5s
- --health-timeout=3s
- --health-retries=20
- kafka:
- image: apache/kafka:3.9.1
- env:
- KAFKA_PROCESS_ROLES: broker,controller
- KAFKA_NODE_ID: 1
- KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093
- KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
- KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
- KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
- ports:
- - 9092:9092
- options: >-
- --health-cmd="/opt/kafka/bin/kafka-topics.sh --bootstrap-server
localhost:9092 --list"
- --health-interval=5s
- --health-timeout=10s
- --health-retries=20
- steps:
- - uses: actions/checkout@v4
-
- - uses: actions/setup-java@v4
- with:
- java-version: 17
- distribution: 'temurin'
- cache: 'gradle'
-
- - name: Free disk space
- run: dev/ci/util_free_space.sh
-
- - name: Apply Gravitino MySQL schema
- run: |
- # Keep in sync with the latest schema file under scripts/mysql/ when
the version bumps.
- mysql -h 127.0.0.1 -P 3306 -ugravitino -pgravitino gravitino \
- < scripts/mysql/schema-2.0.0-mysql.sql
-
- - name: Build Gravitino distribution
- run: ./gradlew compileDistribution -PskipWeb=true -x test
-
- - name: Stage and start two instances (A=8090, B=8190)
- run: bash dev/ci/setup_multi_instance.sh
-
- - name: Run multi-instance entity cache consistency tests
- id: entity-cache-consistency
- env:
- INSTANCE_A: http://localhost:8090
- INSTANCE_B: http://localhost:8190
- INSTANCE_B_HOME: distribution/package-b
- ADMIN_USER: admin
- KAFKA_BOOTSTRAP_SERVERS: localhost:9092
- # Convergence budget for every other case. A healthy node converges
in
- # about a second and never spends this, so a generous value buys
- # resilience to CI scheduling hiccups at no cost on green runs.
- AWAIT_TIMEOUT_SECS: 30
- # Every cacheable entity type must expose a warm stale value in at
- # least one round before its target converges through the change log.
- PER_TYPE_STALE_CONTROL_ROUNDS: 3
- RUN_RESTART_TEST: true
- run: bash dev/ci/test_multi_instance_entity_cache_consistency.sh
-
- - name: Run multi-instance authorization consistency tests
- id: authorization-consistency
- env:
- INSTANCE_A: http://localhost:8090
- INSTANCE_B: http://localhost:8190
- ADMIN_USER: admin
- POLL_WAIT_SECS: 6
- run: bash dev/ci/test_multi_instance_consistency.sh
-
- - name: Stop Gravitino servers
- if: always()
- run: |
- distribution/package/bin/gravitino.sh stop || true
- distribution/package-b/bin/gravitino.sh stop || true
-
- - name: Upload logs on failure
- if: ${{ failure() || cancelled() }}
- uses: actions/upload-artifact@v7
- with:
- name: multi-instance-consistency-logs
- path: |
- distribution/package/logs/
- distribution/package-b/logs/
- distribution/package/conf/gravitino.conf
- distribution/package-b/conf/gravitino.conf
+ uses: ./.github/workflows/multi-instance-consistency-test-action.yml
diff --git a/.github/workflows/python-integration-test-action.yml
b/.github/workflows/python-integration-test-action.yml
new file mode 100644
index 0000000000..c35d8e82ca
--- /dev/null
+++ b/.github/workflows/python-integration-test-action.yml
@@ -0,0 +1,61 @@
+name: Python Client Integration Test Action
+
+on:
+ workflow_call:
+ inputs:
+ architecture:
+ required: true
+ description: 'Architecture of the platform'
+ type: string
+ java-version:
+ required: true
+ description: 'Java version'
+ type: string
+
+jobs:
+ PythonIT:
+ name: JDK${{ inputs.java-version }}
+ runs-on: ubuntu-latest
+ timeout-minutes: 90
+ env:
+ PLATFORM: ${{ inputs.architecture }}
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-java@v4
+ with:
+ java-version: ${{ inputs.java-version }}
+ distribution: 'temurin'
+ cache: 'gradle'
+
+ - name: Set up QEMU
+ uses:
docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
+
+ - name: Free up disk space
+ run: |
+ dev/ci/util_free_space.sh
+
+ - name: Python Client Integration Test
+ id: integrationTest
+ run: |
+ ./gradlew compileDistribution -PskipWeb=true -x test
+
+ for pythonVersion in "3.10" "3.11" "3.12"
+ do
+ echo "Use Python version ${pythonVersion} to test the Python
client."
+ ./gradlew -PpythonVersion=${pythonVersion} -PskipDockerTests=false
-PskipWeb=true :clients:client-python:test
+ rm -rf ./distribution/package/data
+ done
+
+ - name: Upload integrate tests reports
+ uses: actions/upload-artifact@v7
+ if: ${{ failure() && steps.integrationTest.outcome == 'failure' }}
+ with:
+ name: python-integrate-test-reports-${{ inputs.java-version }}
+ path: |
+ build/reports
+ integration-test/build/integration-test.log
+ distribution/package/logs/gravitino-server.out
+ distribution/package/logs/gravitino-server.log
+ catalogs/**/*.log
+ catalogs/**/*.tar
diff --git a/.github/workflows/python-integration-test.yml
b/.github/workflows/python-integration-test.yml
index 0b41587be9..4ac4be7ab5 100644
--- a/.github/workflows/python-integration-test.yml
+++ b/.github/workflows/python-integration-test.yml
@@ -49,53 +49,11 @@ jobs:
PythonIT:
needs: changes
if: needs.changes.outputs.source_changes == 'true'
- runs-on: ubuntu-latest
- timeout-minutes: 90
strategy:
matrix:
- # Integration test for AMD64 architecture
architecture: [linux/amd64]
java-version: [ 17 ]
- env:
- PLATFORM: ${{ matrix.architecture }}
- steps:
- - uses: actions/checkout@v4
-
- - uses: actions/setup-java@v4
- with:
- java-version: ${{ matrix.java-version }}
- distribution: 'temurin'
- cache: 'gradle'
-
- - name: Set up QEMU
- uses:
docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
-
- - name: Free up disk space
- run: |
- dev/ci/util_free_space.sh
-
- - name: Python Client Integration Test
- id: integrationTest
- run: |
- ./gradlew compileDistribution -PskipWeb=true -x test
-
- for pythonVersion in "3.10" "3.11" "3.12"
- do
- echo "Use Python version ${pythonVersion} to test the Python
client."
- ./gradlew -PpythonVersion=${pythonVersion} -PskipDockerTests=false
-PskipWeb=true :clients:client-python:test
- # Clean Gravitino database to clean test data
- rm -rf ./distribution/package/data
- done
-
- - name: Upload integrate tests reports
- uses: actions/upload-artifact@v7
- if: ${{ failure() && steps.integrationTest.outcome == 'failure' }}
- with:
- name: integrate test reports
- path: |
- build/reports
- integration-test/build/integration-test.log
- distribution/package/logs/gravitino-server.out
- distribution/package/logs/gravitino-server.log
- catalogs/**/*.log
- catalogs/**/*.tar
+ uses: ./.github/workflows/python-integration-test-action.yml
+ with:
+ architecture: ${{ matrix.architecture }}
+ java-version: ${{ matrix.java-version }}
diff --git a/.github/workflows/trino-integration-test.yml
b/.github/workflows/trino-integration-test-action.yml
similarity index 59%
copy from .github/workflows/trino-integration-test.yml
copy to .github/workflows/trino-integration-test-action.yml
index bf528eca3e..043b8eb6f4 100644
--- a/.github/workflows/trino-integration-test.yml
+++ b/.github/workflows/trino-integration-test-action.yml
@@ -1,71 +1,30 @@
-name: Trino Integration Test
+name: Trino Integration Test Action
-# Controls when the workflow will run
on:
- push:
- branches: [ "main", "branch-*" ]
- pull_request:
- branches: [ "main", "branch-*" ]
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.event.pull_request.number ||
github.ref }}
- cancel-in-progress: true
+ workflow_call:
+ inputs:
+ architecture:
+ required: true
+ description: 'Architecture of the platform'
+ type: string
+ java-version:
+ required: true
+ description: 'Java version'
+ type: string
jobs:
- changes:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
- - name: Disable Git automatic maintenance
- run: git config --local maintenance.auto false
- - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d
- id: filter
- with:
- filters: |
- source_changes:
- - .github/**
- - api/**
- - bin/**
- - catalogs/**
- - clients/client-java/**
- - clients/client-java-runtime/**
- - common/**
- - conf/**
- - core/**
- - dev/**
- - gradle/**
- - iceberg/**
- - integration-test-common/**
- - meta/**
- - scripts/**
- - server/**
- - server-common/**
- - trino-connector/**
- - build.gradle.kts
- - gradle.properties
- - gradlew
- - settings.gradle.kts
- outputs:
- source_changes: ${{ steps.filter.outputs.source_changes }}
-
- # Integration test for AMD64 architecture
TrinoIT:
- needs: changes
- if: needs.changes.outputs.source_changes == 'true'
+ name: JDK${{ inputs.java-version }}
runs-on: ubuntu-latest
timeout-minutes: 60
- strategy:
- matrix:
- architecture: [linux/amd64]
- java-version: [ 17 ]
env:
- PLATFORM: ${{ matrix.architecture }}
+ PLATFORM: ${{ inputs.architecture }}
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/setup-java-toolchains
with:
- java-version: ${{ matrix.java-version }}
+ java-version: ${{ inputs.java-version }}
- name: Set up QEMU
uses:
docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
@@ -108,7 +67,7 @@ jobs:
uses: actions/upload-artifact@v7
if: ${{ (failure() && steps.integrationTest.outcome == 'failure') ||
contains(github.event.pull_request.labels.*.name, 'upload log') }}
with:
- name: trino-connector-integrate-test-reports-${{ matrix.java-version
}}
+ name: trino-connector-integrate-test-reports-${{ inputs.java-version
}}
path: |
build/reports
trino-connector/integration-test/build/*.log
diff --git a/.github/workflows/trino-integration-test.yml
b/.github/workflows/trino-integration-test.yml
index bf528eca3e..165e589743 100644
--- a/.github/workflows/trino-integration-test.yml
+++ b/.github/workflows/trino-integration-test.yml
@@ -52,70 +52,11 @@ jobs:
TrinoIT:
needs: changes
if: needs.changes.outputs.source_changes == 'true'
- runs-on: ubuntu-latest
- timeout-minutes: 60
strategy:
matrix:
architecture: [linux/amd64]
java-version: [ 17 ]
- env:
- PLATFORM: ${{ matrix.architecture }}
- steps:
- - uses: actions/checkout@v4
-
- - uses: ./.github/actions/setup-java-toolchains
- with:
- java-version: ${{ matrix.java-version }}
-
- - name: Set up QEMU
- uses:
docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
-
- - name: Check required command
- run: |
- dev/ci/check_commands.sh
-
- - name: Build Gravitino
- run: |
- ./gradlew build -PskipWeb=true -x test \
- -x :web:web:build -x :web:integration-test:build \
- -x :web-v2:web:build -x :web-v2:integration-test:build \
- -x :clients:client-python:build \
- -x :flink-connector:flink-common:build \
- -x :flink-connector:flink-1.18:build \
- -x :flink-connector:flink-runtime-1.18:build \
- -x :flink-connector:flink-1.19:build \
- -x :flink-connector:flink-runtime-1.19:build \
- -x :flink-connector:flink-1.20:build \
- -x :flink-connector:flink-runtime-1.20:build \
- -x :spark-connector:spark-3.5:build -x
:spark-connector:spark-4.0:build \
- -x :spark-connector:spark-runtime-3.5:build -x
:spark-connector:spark-runtime-4.0:build \
- -x :mcp-server:build -x :lineage:build \
- -x :maintenance:optimizer:build -x :maintenance:jobs:build \
- -x :lance:lance-common:build -x :lance:lance-rest-server:build
-
- - name: Free up disk space
- run: |
- dev/ci/util_free_space.sh
-
- - name: Trino Integration Test
- id: integrationTest
- run: |
-
trino-connector/integration-test/trino-test-tools/trino_integration_test.sh
- # Disable the Trino cascading query integration test, because the
connector jars are private now.
- #trino-connector/integration-test/trino-test-tools/run_test.sh
-
- - name: Upload integrate tests reports
- uses: actions/upload-artifact@v7
- if: ${{ (failure() && steps.integrationTest.outcome == 'failure') ||
contains(github.event.pull_request.labels.*.name, 'upload log') }}
- with:
- name: trino-connector-integrate-test-reports-${{ matrix.java-version
}}
- path: |
- build/reports
- trino-connector/integration-test/build/*.log
- trino-connector/integration-test/build/*.tar
- trino-connector/integration-test/build/trino-cascading-env
- integration-test-common/build/trino-ci-container-log
- distribution/package/logs/gravitino-server.out
- distribution/package/logs/gravitino-server.log
- catalogs/**/*.log
- catalogs/**/*.tar
+ uses: ./.github/workflows/trino-integration-test-action.yml
+ with:
+ architecture: ${{ matrix.architecture }}
+ java-version: ${{ matrix.java-version }}
diff --git a/.github/workflows/trino-multi-version-test.yml
b/.github/workflows/trino-multi-version-test-action.yml
similarity index 91%
copy from .github/workflows/trino-multi-version-test.yml
copy to .github/workflows/trino-multi-version-test-action.yml
index b5b79081bf..d435f46f0b 100644
--- a/.github/workflows/trino-multi-version-test.yml
+++ b/.github/workflows/trino-multi-version-test-action.yml
@@ -1,7 +1,7 @@
-name: Trino Multi-Version Integration Test
+name: Trino Multi-Version Integration Test Action
on:
- workflow_dispatch:
+ workflow_call:
jobs:
TrinoMultiVersionIT:
@@ -78,6 +78,12 @@ jobs:
--auto=all --auto_patch --trino_version=440 \
--trino_connector_dir=${{ env.CONNECTOR_DIR
}}/trino-connector-440-445/build/libs
+ - name: Test Trino 435
+ run: |
+
trino-connector/integration-test/trino-test-tools/trino_integration_test.sh \
+ --auto=all --auto_patch --trino_version=435 \
+ --trino_connector_dir=${{ env.CONNECTOR_DIR
}}/trino-connector-435-439/build/libs
+
- name: Upload test reports
uses: actions/upload-artifact@v7
if: failure()
diff --git a/.github/workflows/trino-multi-version-test.yml
b/.github/workflows/trino-multi-version-test.yml
index b5b79081bf..2cacdafde5 100644
--- a/.github/workflows/trino-multi-version-test.yml
+++ b/.github/workflows/trino-multi-version-test.yml
@@ -5,89 +5,4 @@ on:
jobs:
TrinoMultiVersionIT:
- runs-on: ubuntu-latest
- timeout-minutes: 240
- env:
- CONNECTOR_DIR: ${{ github.workspace }}/trino-connector
- steps:
- - uses: actions/checkout@v4
-
- - uses: ./.github/actions/setup-java-toolchains
-
- - name: Set up QEMU
- uses:
docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
-
- - name: Check required command
- run: dev/ci/check_commands.sh
-
- - name: Build Gravitino
- run: |
- ./gradlew build -PskipWeb=true -x test \
- -x :web:web:build -x :web:integration-test:build \
- -x :web-v2:web:build -x :web-v2:integration-test:build \
- -x :clients:client-python:build \
- -x :flink-connector:flink-common:build \
- -x :flink-connector:flink-1.18:build \
- -x :flink-connector:flink-runtime-1.18:build \
- -x :flink-connector:flink-1.19:build \
- -x :flink-connector:flink-runtime-1.19:build \
- -x :flink-connector:flink-1.20:build \
- -x :flink-connector:flink-runtime-1.20:build \
- -x :spark-connector:spark-3.5:build -x
:spark-connector:spark-4.0:build \
- -x :spark-connector:spark-runtime-3.5:build -x
:spark-connector:spark-runtime-4.0:build \
- -x :mcp-server:build -x :lineage:build \
- -x :maintenance:optimizer:build -x :maintenance:jobs:build \
- -x :lance:lance-common:build -x :lance:lance-rest-server:build
-
- - name: Free up disk space
- run: dev/ci/util_free_space.sh
-
- - name: Test Trino 478
- run: |
-
trino-connector/integration-test/trino-test-tools/trino_integration_test.sh \
- --auto=all --auto_patch --trino_version=478 \
- --trino_connector_dir=${{ env.CONNECTOR_DIR
}}/trino-connector-473-478/build/libs
-
- - name: Test Trino 473
- run: |
-
trino-connector/integration-test/trino-test-tools/trino_integration_test.sh \
- --auto=all --auto_patch --trino_version=473 \
- --trino_connector_dir=${{ env.CONNECTOR_DIR
}}/trino-connector-473-478/build/libs
-
- - name: Test Trino 469
- run: |
-
trino-connector/integration-test/trino-test-tools/trino_integration_test.sh \
- --auto=all --auto_patch --trino_version=469 \
- --trino_connector_dir=${{ env.CONNECTOR_DIR
}}/trino-connector-469-472/build/libs
-
- - name: Test Trino 452
- run: |
-
trino-connector/integration-test/trino-test-tools/trino_integration_test.sh \
- --auto=all --auto_patch --trino_version=452 \
- --trino_connector_dir=${{ env.CONNECTOR_DIR
}}/trino-connector-452-468/build/libs
-
- - name: Test Trino 446
- run: |
-
trino-connector/integration-test/trino-test-tools/trino_integration_test.sh \
- --auto=all --auto_patch --trino_version=446 \
- --trino_connector_dir=${{ env.CONNECTOR_DIR
}}/trino-connector-446-451/build/libs
-
- - name: Test Trino 440
- run: |
-
trino-connector/integration-test/trino-test-tools/trino_integration_test.sh \
- --auto=all --auto_patch --trino_version=440 \
- --trino_connector_dir=${{ env.CONNECTOR_DIR
}}/trino-connector-440-445/build/libs
-
- - name: Upload test reports
- uses: actions/upload-artifact@v7
- if: failure()
- with:
- name: trino-multi-version-test-reports
- path: |
- build/reports
- trino-connector/integration-test/build/*.log
- trino-connector/integration-test/build/*.tar
- integration-test-common/build/trino-ci-container-log
- distribution/package/logs/gravitino-server.out
- distribution/package/logs/gravitino-server.log
- catalogs/**/*.log
+ uses: ./.github/workflows/trino-multi-version-test-action.yml
diff --git a/mcp-server/dev/run_authz_integration_test.sh
b/mcp-server/dev/run_authz_integration_test.sh
index b0ca02faf5..58cc663270 100755
--- a/mcp-server/dev/run_authz_integration_test.sh
+++ b/mcp-server/dev/run_authz_integration_test.sh
@@ -150,6 +150,11 @@ rm -f "${MCP_AUDIT_LOG}"
) &
MCP_PID=$!
+if ! command -v nc >/dev/null 2>&1; then
+ log "ERROR: nc (netcat) is required to wait for the MCP server"
+ exit 1
+fi
+
log "Waiting for MCP server to become reachable..."
for i in $(seq 1 30); do
if nc -z localhost "${MCP_PORT}" 2>/dev/null; then
