This is an automated email from the ASF dual-hosted git repository.
gkoszyk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/master by this push:
new 5a2939a17 feat(ci): dependency-DAG-based test scoping for Rust CI
(#3095)
5a2939a17 is described below
commit 5a2939a17c465612808a174f8ac4cc689dbcc5eb
Author: Krishna Vishal <[email protected]>
AuthorDate: Mon Apr 13 20:08:24 2026 +0530
feat(ci): dependency-DAG-based test scoping for Rust CI (#3095)
## Which issue does this PR close?
Closes #3094
## Summary
- Use `cargo-rail` to compute affected crates via the workspace
dependency DAG, then scope cargo build and `cargo nextest` run to only
those crates
- Fix two detection blind spots: `core/iobuf and `core/server-ng` were
not covered by any component in `components.yml`
- Split `rust-simulator` out of `rust-cluster` so simulator-only changes
don't trigger Python/Node/Go/Java/C# SDK tests
---
.config/rail.toml | 28 +++++++++
.github/actions/rust/pre-merge/action.yml | 95 +++++++++++++++++++++++++++++--
.github/config/components.yml | 21 +++++++
3 files changed, 140 insertions(+), 4 deletions(-)
diff --git a/.config/rail.toml b/.config/rail.toml
new file mode 100644
index 000000000..14114221d
--- /dev/null
+++ b/.config/rail.toml
@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# cargo-rail change detection configuration
+# See: https://github.com/loadingalias/cargo-rail
+
+[change-detection]
+# Changes to these paths trigger a full workspace rebuild/retest
+infrastructure = [
+ "Cargo.lock",
+ "rust-toolchain.toml",
+ ".cargo/**",
+ ".github/**",
+]
diff --git a/.github/actions/rust/pre-merge/action.yml
b/.github/actions/rust/pre-merge/action.yml
index 0daa91b5d..742d489f9 100644
--- a/.github/actions/rust/pre-merge/action.yml
+++ b/.github/actions/rust/pre-merge/action.yml
@@ -59,6 +59,50 @@ runs:
esac
shell: bash
+ # DAG-based test scoping: use cargo-rail to compute affected crates from
the
+ # workspace dependency graph + git diff, avoiding the full test suite when
only
+ # a subset of crates changed.
+ # Safety: cargo check/clippy run on the full workspace separately,
catching all
+ # compilation errors. This only scopes test BUILD and EXECUTION.
+ - name: Fetch base branch for DAG analysis
+ if: startsWith(inputs.task, 'test-')
+ run: git fetch origin master --depth=1 2>/dev/null || true
+ shell: bash
+
+ - name: Install cargo-rail
+ if: startsWith(inputs.task, 'test-')
+ uses: taiki-e/install-action@v2
+ with:
+ tool: cargo-rail
+
+ - name: Compute affected crates (cargo-rail)
+ if: startsWith(inputs.task, 'test-')
+ run: |
+ TOTAL_CRATES=$(cargo metadata --format-version 1 --no-deps 2>/dev/null
| jq '.workspace_members | length' 2>/dev/null || echo "?")
+ echo "$TOTAL_CRATES" > /tmp/total-crates.txt
+
+ PLAN_JSON=$(cargo rail plan --since origin/master -f json
2>/tmp/affected-stderr.txt || echo "")
+
+ if [[ -n "$PLAN_JSON" ]]; then
+ MODE=$(echo "$PLAN_JSON" | jq -r '.scope.mode')
+ if [[ "$MODE" == "crates" ]]; then
+ CRATES=$(echo "$PLAN_JSON" | jq -r '.scope.crates[]')
+ CRATE_COUNT=$(echo "$CRATES" | wc -l)
+ # Build -p flags for cargo build/test
+ echo "$CRATES" | sed 's/^/-p /' | tr '\n' ' ' > /tmp/packages.txt
+ # Build nextest filter expression for cargo nextest run
+ echo "$CRATES" | sed 's/^/package(/; s/$/)/' | paste -sd ' | ' >
/tmp/nextest-filter.txt
+ echo "::notice::DAG analysis: ${CRATE_COUNT} affected crates (of
${TOTAL_CRATES} total)"
+ else
+ echo "::notice::Full workspace affected (${TOTAL_CRATES} crates)"
+ fi
+ else
+ STDERR=$(cat /tmp/affected-stderr.txt 2>/dev/null || echo "")
+ echo "::warning::Could not compute affected crates, running full
test suite. ${STDERR}"
+ rm -f /tmp/nextest-filter.txt /tmp/packages.txt
+ fi
+ shell: bash
+
# Individual lint tasks for parallel execution
- name: Cargo check
if: inputs.task == 'check'
@@ -117,16 +161,45 @@ runs:
echo "::notice::Running test partition ${PARTITION_INDEX}/2"
fi
+ # Read DAG-based affected crate filter (computed in earlier step)
+ NEXTEST_FILTER=""
+ PACKAGE_FLAGS=""
+ TOTAL_CRATES="?"
+ if [[ -f /tmp/nextest-filter.txt ]]; then
+ NEXTEST_FILTER=$(cat /tmp/nextest-filter.txt)
+ fi
+ if [[ -f /tmp/packages.txt ]]; then
+ PACKAGE_FLAGS=$(cat /tmp/packages.txt)
+ fi
+ if [[ -f /tmp/total-crates.txt ]]; then
+ TOTAL_CRATES=$(cat /tmp/total-crates.txt)
+ fi
+
+ if [[ -n "$PACKAGE_FLAGS" ]]; then
+ CRATE_COUNT=$(echo "$NEXTEST_FILTER" | grep -o 'package(' | wc -l)
+ echo "::notice::DAG-scoped build: ${CRATE_COUNT} crates (cargo
check/clippy cover full workspace separately)"
+ else
+ echo "::notice::Full workspace build (no DAG filter available)"
+ fi
+
source <(cargo llvm-cov show-env --export-prefix)
bins_start=$(date +%s)
- cargo build --locked
+ if [[ -n "$PACKAGE_FLAGS" ]]; then
+ cargo build --locked $PACKAGE_FLAGS
+ else
+ cargo build --locked
+ fi
bins_end=$(date +%s)
bins_duration=$((bins_end - bins_start))
echo "::notice::Binaries and libraries built in ${bins_duration}s
($(date -ud @${bins_duration} +'%M:%S'))"
compile_start=$(date +%s)
- cargo test --locked --no-run
+ if [[ -n "$PACKAGE_FLAGS" ]]; then
+ cargo test --locked --no-run $PACKAGE_FLAGS
+ else
+ cargo test --locked --no-run
+ fi
compile_end=$(date +%s)
compile_duration=$((compile_end - compile_start))
echo "::notice::Tests compiled in ${compile_duration}s ($(date -ud
@${compile_duration} +'%M:%S'))"
@@ -144,12 +217,20 @@ runs:
test_start=$(date +%s)
if command -v cargo-nextest &> /dev/null; then
- cargo nextest run --locked --no-fail-fast --profile ci
$PARTITION_FLAG
+ if [[ -n "$NEXTEST_FILTER" ]]; then
+ cargo nextest run --locked --no-fail-fast --profile ci
$PARTITION_FLAG -E "$NEXTEST_FILTER"
+ else
+ cargo nextest run --locked --no-fail-fast --profile ci
$PARTITION_FLAG
+ fi
else
if [[ -n "$PARTITION_FLAG" ]]; then
echo "::error::cargo-nextest not found, falling back to cargo test
without partitioning (all tests will run on every partition)"
fi
- cargo test --locked --no-fail-fast
+ if [[ -n "$PACKAGE_FLAGS" ]]; then
+ cargo test --locked --no-fail-fast $PACKAGE_FLAGS
+ else
+ cargo test --locked --no-fail-fast
+ fi
fi
test_end=$(date +%s)
test_duration=$((test_end - test_start))
@@ -159,6 +240,12 @@ runs:
total_duration=$((build_duration + test_duration))
echo ""
echo "========================================="
+ if [[ -n "$PACKAGE_FLAGS" ]]; then
+ CRATE_COUNT=$(echo "$NEXTEST_FILTER" | grep -o 'package(' | wc -l)
+ echo "DAG scope: ${CRATE_COUNT}/${TOTAL_CRATES}
crates"
+ else
+ echo "DAG scope: full workspace (${TOTAL_CRATES}
crates)"
+ fi
echo "All targets build: ${bins_duration}s ($(date -ud
@${bins_duration} +'%M:%S'))"
echo "Tests compile: ${compile_duration}s ($(date -ud
@${compile_duration} +'%M:%S'))"
echo "Tests execute: ${test_duration}s ($(date -ud
@${test_duration} +'%M:%S'))"
diff --git a/.github/config/components.yml b/.github/config/components.yml
index 7585a91b9..a898f30db 100644
--- a/.github/config/components.yml
+++ b/.github/config/components.yml
@@ -51,9 +51,17 @@ components:
paths:
- "core/common/**"
+ # Leaf crate: zero-copy I/O buffer, depended on by binary_protocol and
cluster
+ rust-iobuf:
+ depends_on:
+ - "rust-workspace"
+ paths:
+ - "core/iobuf/**"
+
rust-binary-protocol:
depends_on:
- "rust-workspace" # Protocol is affected by workspace changes
+ - "rust-iobuf" # binary_protocol depends on iobuf
- "ci-infrastructure" # CI changes trigger full regression
paths:
- "core/binary_protocol/**"
@@ -67,10 +75,13 @@ components:
- "rust-cluster"
paths:
- "core/server/**"
+ - "core/server-ng/**"
rust-cluster:
depends_on:
- "rust-workspace"
+ - "rust-iobuf" # cluster crates depend on iobuf
+ - "rust-binary-protocol" # cluster crates depend on binary_protocol
paths:
- "core/clock/**"
- "core/consensus/**"
@@ -79,18 +90,28 @@ components:
- "core/metadata/**"
- "core/message_bus/**"
- "core/partitions/**"
+
+ # Standalone simulation tool, does NOT affect server binary or foreign SDKs.
+ # Split from rust-cluster to avoid triggering SDK tests on simulator-only
changes.
+ rust-simulator:
+ depends_on:
+ - "rust-workspace"
+ - "rust-cluster"
+ paths:
- "core/simulator/**"
# Main Rust workspace testing
rust:
depends_on:
- "rust-workspace"
+ - "rust-iobuf"
- "rust-configs"
- "rust-sdk"
- "rust-common"
- "rust-binary-protocol"
- "rust-server"
- "rust-cluster"
+ - "rust-simulator"
- "rust-tools"
- "rust-cli"
- "rust-bench"
