This is an automated email from the ASF dual-hosted git repository.
zhonghongsheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new a9466bdb9c9 feat: add Stage 2 full dynamic matrix for e2e-sql CI
workflow (#38277)
a9466bdb9c9 is described below
commit a9466bdb9c96d5e7b471342b27d4547d2f620330
Author: Hongsheng Zhong <[email protected]>
AuthorDate: Sat Feb 28 23:16:11 2026 +0800
feat: add Stage 2 full dynamic matrix for e2e-sql CI workflow (#38277)
* feat: add Stage 2 full dynamic matrix for e2e-sql CI workflow
Co-authored-by: sandynz <[email protected]>
* Add Log Outputs
* E2E-SQL CI: two-matrix approach so smoke always runs fixed 6 scenarios
* Update Log Outputs
* Update generate-e2e-sql-matrix.sh log_counts() un-ended method
---------
Co-authored-by: copilot-swe-agent[bot]
<[email protected]>
---
.github/workflows/e2e-sql.yml | 79 +++++++++++++++++++++-
.../resources/scripts/generate-e2e-sql-matrix.sh | 69 +++++++++++++------
.../resources/scripts/generate-remaining-matrix.sh | 65 ++++++++++++++++++
3 files changed, 189 insertions(+), 24 deletions(-)
diff --git a/.github/workflows/e2e-sql.yml b/.github/workflows/e2e-sql.yml
index a44049abc67..b9b8e61be7f 100644
--- a/.github/workflows/e2e-sql.yml
+++ b/.github/workflows/e2e-sql.yml
@@ -58,9 +58,12 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
+ smoke-matrix: ${{ steps.generate-matrix.outputs.smoke-matrix }}
+ full-matrix: ${{ steps.generate-matrix.outputs.full-matrix }}
matrix: ${{ steps.generate-matrix.outputs.matrix }}
has-jobs: ${{ steps.generate-matrix.outputs.has-jobs }}
need-proxy-image: ${{ steps.generate-matrix.outputs.need-proxy-image }}
+ filter-json: ${{ steps.generate-matrix.outputs.filter-json }}
steps:
- uses: actions/[email protected]
- id: filter
@@ -109,6 +112,13 @@ jobs:
--arg pom_changes "$FILTER_POM_CHANGES" \
'$ARGS.named')
bash .github/workflows/resources/scripts/generate-e2e-sql-matrix.sh
"$filters"
+ echo "filter-json=$filters" >> "$GITHUB_OUTPUT"
+ - name: Log Outputs
+ run: |
+ echo "smoke-matrix=${{ steps.generate-matrix.outputs.smoke-matrix }}"
+ echo "has-jobs=${{ steps.generate-matrix.outputs.has-jobs }}"
+ echo "need-proxy-image=${{
steps.generate-matrix.outputs.need-proxy-image }}"
+ echo "filter-json=${{ steps.generate-matrix.outputs.filter-json }}"
build-e2e-image:
name: Build E2E Image
@@ -138,8 +148,8 @@ jobs:
path: /tmp/apache-shardingsphere-proxy-test.tar
retention-days: 10
- e2e-sql:
- name: E2E - SQL
+ e2e-sql-smoke:
+ name: E2E - SQL (Smoke)
needs: [ detect-and-generate-matrix, build-e2e-image, global-environment ]
if: always() && github.repository == 'apache/shardingsphere' &&
needs.detect-and-generate-matrix.outputs.has-jobs == 'true' &&
needs.detect-and-generate-matrix.result == 'success' &&
(needs.build-e2e-image.result == 'success' || needs.build-e2e-image.result ==
'skipped')
runs-on: ubuntu-latest
@@ -147,7 +157,70 @@ jobs:
strategy:
max-parallel: 15
fail-fast: false
- matrix: ${{ fromJSON(needs.detect-and-generate-matrix.outputs.matrix) }}
+ matrix: ${{
fromJSON(needs.detect-and-generate-matrix.outputs.smoke-matrix) }}
+ steps:
+ - uses: actions/[email protected]
+ - uses: ./.github/workflows/resources/actions/setup-build-environment
+ with:
+ cache-suffix: 'e2e-sql'
+ cache-save-enabled: 'false'
+ enable-docker-setup: 'true'
+ - name: Download E2E Image
+ if: matrix.adapter == 'proxy'
+ uses: actions/download-artifact@v4
+ with:
+ name: e2e-image
+ path: /tmp/
+ - name: Load E2E Image
+ if: matrix.adapter == 'proxy'
+ run: docker load -i /tmp/apache-shardingsphere-proxy-test.tar
+ - name: Run E2E Test
+ run: ./mvnw -nsu -B install -f test/e2e/sql/pom.xml
-Dspotless.apply.skip=true -De2e.run.type=DOCKER -De2e.artifact.modes=${{
matrix.mode }} -De2e.artifact.adapters=${{ matrix.adapter }}
-De2e.run.additional.cases=false -De2e.scenarios=${{ matrix.scenario }}
-De2e.artifact.databases=${{ matrix.database }} ${{ matrix.additional-options }}
+
+ detect-remaining-matrix:
+ name: Detect Remaining Matrix
+ needs: [ detect-and-generate-matrix, e2e-sql-smoke ]
+ if: >-
+ always()
+ && needs.detect-and-generate-matrix.outputs.has-jobs == 'true'
+ && needs.e2e-sql-smoke.result == 'success'
+ runs-on: ubuntu-latest
+ timeout-minutes: 5
+ outputs:
+ matrix: ${{ steps.generate.outputs.matrix }}
+ has-remaining-jobs: ${{ steps.generate.outputs.has-remaining-jobs }}
+ steps:
+ - uses: actions/[email protected]
+ - name: Detect Remaining Matrix
+ id: generate
+ env:
+ FILTER_JSON: ${{
needs.detect-and-generate-matrix.outputs.filter-json }}
+ FULL_MATRIX: ${{
needs.detect-and-generate-matrix.outputs.full-matrix }}
+ SMOKE_MATRIX: ${{
needs.detect-and-generate-matrix.outputs.smoke-matrix }}
+ run: |
+ bash
.github/workflows/resources/scripts/generate-remaining-matrix.sh \
+ "$FILTER_JSON" \
+ "$FULL_MATRIX" \
+ "$SMOKE_MATRIX"
+ - name: Log Outputs
+ run: |
+ echo "matrix=${{ steps.generate.outputs.matrix }}"
+ echo "has-remaining-jobs=${{
steps.generate.outputs.has-remaining-jobs }}"
+
+ e2e-sql-stage2:
+ name: E2E - SQL (Stage 2)
+ needs: [ detect-remaining-matrix, build-e2e-image, global-environment ]
+ if: >-
+ always()
+ && github.repository == 'apache/shardingsphere'
+ && needs.detect-remaining-matrix.outputs.has-remaining-jobs == 'true'
+ && (needs.build-e2e-image.result == 'success' ||
needs.build-e2e-image.result == 'skipped')
+ runs-on: ubuntu-latest
+ timeout-minutes: 40
+ strategy:
+ max-parallel: 15
+ fail-fast: false
+ matrix: ${{ fromJSON(needs.detect-remaining-matrix.outputs.matrix) }}
steps:
- uses: actions/[email protected]
- uses: ./.github/workflows/resources/actions/setup-build-environment
diff --git a/.github/workflows/resources/scripts/generate-e2e-sql-matrix.sh
b/.github/workflows/resources/scripts/generate-e2e-sql-matrix.sh
index abe9ba6842f..20c72cb397f 100755
--- a/.github/workflows/resources/scripts/generate-e2e-sql-matrix.sh
+++ b/.github/workflows/resources/scripts/generate-e2e-sql-matrix.sh
@@ -17,7 +17,7 @@
#
# Usage: generate-e2e-sql-matrix.sh '<json-with-all-18-filter-labels>'
-# Output: writes matrix=<JSON>, has-jobs=<true|false>, and
need-proxy-image=<true|false> to $GITHUB_OUTPUT
+# Output: writes smoke-matrix=<JSON>, full-matrix=<JSON>, matrix=<JSON>(alias
for full), has-jobs=<true|false>, and need-proxy-image=<true|false> to
$GITHUB_OUTPUT
set -euo pipefail
@@ -71,12 +71,14 @@ build_matrix() {
local modes="$2"
local databases="$3"
local scenarios="$4"
+ local include_extra_job="${5:-true}"
jq -cn \
--argjson adapters "$adapters" \
--argjson modes "$modes" \
--argjson databases "$databases" \
--argjson scenarios "$scenarios" \
+ --argjson include_extra_job "$include_extra_job" \
'
def should_exclude(adapter; mode; scenario):
(adapter == "jdbc" and scenario == "passthrough") or
@@ -96,7 +98,7 @@ build_matrix() {
([$base_jobs[] | select(.adapter == "proxy" and .mode == "Cluster")] |
length > 0) as $has_proxy_cluster |
($scenarios | map(select(. == "passthrough")) | length > 0) as
$has_passthrough |
- (if $has_proxy_cluster and $has_passthrough
+ (if $include_extra_job and $has_proxy_cluster and $has_passthrough
then [{adapter:"proxy", mode:"Cluster", database:"MySQL",
scenario:"passthrough",
"additional-options":"-Dmysql-connector-java.version=8.3.0"}]
else []
end) as $extra_job |
@@ -105,14 +107,13 @@ build_matrix() {
'
}
-# Full fallback: run the entire matrix
-if [ "$core_infra" = "true" ] || [ "$test_framework" = "true" ] || [
"$pom_changes" = "true" ]; then
- MATRIX=$(build_matrix "$ALL_ADAPTERS" "$ALL_MODES" "$ALL_DATABASES"
"$ALL_SCENARIOS")
- echo "matrix=$(echo "$MATRIX" | jq -c .)" >> "$GITHUB_OUTPUT"
- echo "has-jobs=true" >> "$GITHUB_OUTPUT"
- echo "need-proxy-image=true" >> "$GITHUB_OUTPUT"
- exit 0
-fi
+log_counts() {
+ local name="$1"
+ local matrix_json="$2"
+ local count
+ count=$(echo "$matrix_json" | jq '.include | length')
+ echo "::notice::$name count=$count"
+}
# Check whether any relevant dimension changed at all
any_relevant_change=false
@@ -130,6 +131,8 @@ if [ "$any_relevant_change" = "false" ]; then
echo "matrix={\"include\":[]}" >> "$GITHUB_OUTPUT"
echo "has-jobs=false" >> "$GITHUB_OUTPUT"
echo "need-proxy-image=false" >> "$GITHUB_OUTPUT"
+ echo "smoke-matrix={\"include\":[]}" >> "$GITHUB_OUTPUT"
+ echo "full-matrix={\"include\":[]}" >> "$GITHUB_OUTPUT"
exit 0
fi
@@ -229,26 +232,50 @@ if [ "$feature_broadcast" = "true" ]; then
add_scenario "empty_rules"
fi
-# If no feature triggered, use core smoke scenario set
-if [ "$any_feature_triggered" = "false" ]; then
- scenarios_json="$SMOKE_SCENARIOS"
+echo "::notice::filters core_infra=$core_infra test_framework=$test_framework
pom_changes=$pom_changes"
+echo "::notice::dimensions adapters=$adapters modes=$modes
databases=$databases"
+
+# Always generate smoke-matrix (fixed 6 scenarios), and DO NOT add the extra
passthrough job
+SMOKE_MATRIX=$(build_matrix "$adapters" "$modes" "$databases"
"$SMOKE_SCENARIOS" false)
+log_counts "smoke-matrix" "$SMOKE_MATRIX"
+
+# Build full-matrix scenarios
+if [ "$core_infra" = "true" ] || [ "$test_framework" = "true" ] || [
"$pom_changes" = "true" ]; then
+ full_scenarios_json="$ALL_SCENARIOS"
+ echo "::notice::full-matrix reason=full-fallback"
else
- scenarios_json=$(printf '%s\n' "${scenarios_set[@]}" | jq -R . | jq -sc .)
+ if [ "$any_feature_triggered" = "false" ]; then
+ # When no feature triggered, full-matrix is limited to smoke scenario set
+ full_scenarios_json="$SMOKE_SCENARIOS"
+ echo "::notice::full-matrix reason=no-feature-triggered (use smoke
scenarios)"
+ else
+ full_scenarios_json=$(printf '%s\n' "${scenarios_set[@]}" | jq -R . | jq
-sc .)
+ echo "::notice::full-matrix reason=feature-triggered"
+ fi
fi
-MATRIX=$(build_matrix "$adapters" "$modes" "$databases" "$scenarios_json")
+FULL_MATRIX=$(build_matrix "$adapters" "$modes" "$databases"
"$full_scenarios_json" true)
+log_counts "full-matrix" "$FULL_MATRIX"
-JOB_COUNT=$(echo "$MATRIX" | jq '.include | length')
-
-if [ "$JOB_COUNT" -eq 0 ]; then
+# Determine whether there are any jobs at all (based on full-matrix)
+FULL_JOB_COUNT=$(echo "$FULL_MATRIX" | jq '.include | length')
+if [ "$FULL_JOB_COUNT" -eq 0 ]; then
+ echo "smoke-matrix=$(echo "$SMOKE_MATRIX" | jq -c .)" >> "$GITHUB_OUTPUT"
+ echo "full-matrix={\"include\":[]}" >> "$GITHUB_OUTPUT"
echo "matrix={\"include\":[]}" >> "$GITHUB_OUTPUT"
echo "has-jobs=false" >> "$GITHUB_OUTPUT"
echo "need-proxy-image=false" >> "$GITHUB_OUTPUT"
exit 0
fi
-HAS_PROXY=$(echo "$MATRIX" | jq '[.include[] | select(.adapter == "proxy")] |
length > 0')
+HAS_PROXY_SMOKE=$(echo "$SMOKE_MATRIX" | jq '[.include[] | select(.adapter ==
"proxy")] | length > 0')
+HAS_PROXY_FULL=$(echo "$FULL_MATRIX" | jq '[.include[] | select(.adapter ==
"proxy")] | length > 0')
+NEED_PROXY_IMAGE=$(jq -cn --argjson a "$HAS_PROXY_SMOKE" --argjson b
"$HAS_PROXY_FULL" '$a or $b')
-echo "matrix=$(echo "$MATRIX" | jq -c .)" >> "$GITHUB_OUTPUT"
+echo "smoke-matrix=$(echo "$SMOKE_MATRIX" | jq -c .)" >> "$GITHUB_OUTPUT"
+echo "full-matrix=$(echo "$FULL_MATRIX" | jq -c .)" >> "$GITHUB_OUTPUT"
+echo "matrix=$(echo "$FULL_MATRIX" | jq -c .)" >> "$GITHUB_OUTPUT"
echo "has-jobs=true" >> "$GITHUB_OUTPUT"
-echo "need-proxy-image=$HAS_PROXY" >> "$GITHUB_OUTPUT"
+echo "need-proxy-image=$NEED_PROXY_IMAGE" >> "$GITHUB_OUTPUT"
+
+exit 0
diff --git a/.github/workflows/resources/scripts/generate-remaining-matrix.sh
b/.github/workflows/resources/scripts/generate-remaining-matrix.sh
new file mode 100755
index 00000000000..578c0f85da3
--- /dev/null
+++ b/.github/workflows/resources/scripts/generate-remaining-matrix.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Usage: generate-remaining-matrix.sh '<filter-json>' '<full-matrix-json>'
'<smoke-matrix-json>'
+# Generates the Stage 2 matrix by subtracting smoke combinations from the full
matrix.
+# Output: writes matrix=<JSON>, has-remaining-jobs=<true|false> to
$GITHUB_OUTPUT
+
+set -euo pipefail
+
+FILTER_JSON="$1"
+FULL_MATRIX="$2"
+SMOKE_MATRIX="$3"
+
+# Parse completed jobs from smoke matrix
+COMPLETED_JOBS=$(echo "$SMOKE_MATRIX" | jq -c '.include // []')
+
+if [ -z "$FULL_MATRIX" ]; then
+ echo "matrix={\"include\":[]}" >> "$GITHUB_OUTPUT"
+ echo "has-remaining-jobs=false" >> "$GITHUB_OUTPUT"
+ exit 0
+fi
+
+# Subtract Stage 1 completed jobs from the full matrix
+REMAINING_JOBS=$(jq -cn \
+ --argjson full "$FULL_MATRIX" \
+ --argjson completed "$COMPLETED_JOBS" \
+ '[
+ $full.include[] as $job |
+ select(
+ [$completed[] | select(
+ .adapter == $job.adapter and
+ .mode == $job.mode and
+ .database == $job.database and
+ .scenario == $job.scenario and
+ .["additional-options"] == $job["additional-options"]
+ )] | length == 0
+ ) |
+ $job
+ ]')
+
+JOB_COUNT=$(echo "$REMAINING_JOBS" | jq 'length')
+
+if [ "$JOB_COUNT" -eq 0 ]; then
+ echo "matrix={\"include\":[]}" >> "$GITHUB_OUTPUT"
+ echo "has-remaining-jobs=false" >> "$GITHUB_OUTPUT"
+ exit 0
+fi
+
+echo "matrix=$(echo "{\"include\":$REMAINING_JOBS}" | jq -c .)" >>
"$GITHUB_OUTPUT"
+echo "has-remaining-jobs=true" >> "$GITHUB_OUTPUT"
