This is an automated email from the ASF dual-hosted git repository.
hgruszecki pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iggy.git
The following commit(s) were added to refs/heads/master by this push:
new 93d7f8970 ci: split post-merge monolith and harden Codecov uploads
(#3006)
93d7f8970 is described below
commit 93d7f8970862cdf729aa49b5baa14f6839a6f363
Author: Hubert Gruszecki <[email protected]>
AuthorDate: Wed Mar 25 09:31:56 2026 +0100
ci: split post-merge monolith and harden Codecov uploads (#3006)
---
.github/actions/rust/pre-merge/action.yml | 3 +
.github/workflows/_test.yml | 1 +
.../{post-merge.yml => coverage-baseline.yml} | 262 ++---------
.github/workflows/edge-release.yml | 123 +++++
.github/workflows/post-merge.yml | 506 +--------------------
.github/workflows/pre-merge.yml | 4 +
codecov.yml | 29 +-
7 files changed, 199 insertions(+), 729 deletions(-)
diff --git a/.github/actions/rust/pre-merge/action.yml
b/.github/actions/rust/pre-merge/action.yml
index 4edcd1f91..d1d07138d 100644
--- a/.github/actions/rust/pre-merge/action.yml
+++ b/.github/actions/rust/pre-merge/action.yml
@@ -146,6 +146,9 @@ runs:
if command -v cargo-nextest &> /dev/null; then
cargo nextest run --locked --no-fail-fast $PARTITION_FLAG
else
+ if [[ -n "$PARTITION_FLAG" ]]; then
+ echo "::error::cargo-nextest not found, falling back to cargo test
without partitioning (all tests will run on every partition)"
+ fi
cargo test --locked --no-fail-fast
fi
test_end=$(date +%s)
diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml
index 6fd50003e..ac3eb6ca7 100644
--- a/.github/workflows/_test.yml
+++ b/.github/workflows/_test.yml
@@ -68,6 +68,7 @@ jobs:
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: codecov.json
+ disable_search: true
flags: rust
fail_ci_if_error: false
verbose: true
diff --git a/.github/workflows/post-merge.yml
b/.github/workflows/coverage-baseline.yml
similarity index 62%
copy from .github/workflows/post-merge.yml
copy to .github/workflows/coverage-baseline.yml
index 18c16eeba..8fe98cde7 100644
--- a/.github/workflows/post-merge.yml
+++ b/.github/workflows/coverage-baseline.yml
@@ -15,166 +15,27 @@
# specific language governing permissions and limitations
# under the License.
-name: Post-merge
+# Full coverage baseline for all 6 languages (Rust, Java, C#, Python,
+# Node, Go). Runs on every push to master so Codecov has complete data
+# for carryforward on PR builds where only a subset of SDKs is tested.
+
+name: Coverage baseline
on:
push:
branches: [master]
permissions:
- contents: write
- packages: write
- id-token: write
+ contents: read
concurrency:
- group: post-merge-${{ github.ref }}
- cancel-in-progress: false
+ group: coverage-baseline-${{ github.ref }}
+ cancel-in-progress: true
env:
IGGY_CI_BUILD: true
jobs:
- check-auto-publish:
- name: Check auto-publish
- runs-on: ubuntu-latest
- if: ${{ !github.event.repository.fork }}
- outputs:
- docker_components: ${{ steps.check.outputs.docker_components }}
- crates_to_publish: ${{ steps.check.outputs.crates_to_publish }}
- sdks_to_publish: ${{ steps.check.outputs.sdks_to_publish }}
- steps:
- - uses: actions/checkout@v4
- with:
- fetch-depth: 0
-
- - name: Setup yq
- run: |
- if ! command -v yq >/dev/null 2>&1; then
- YQ_VERSION="v4.47.1"
-
YQ_CHECKSUM="0fb28c6680193c41b364193d0c0fc4a03177aecde51cfc04d506b1517158c2fb"
- curl -sSL -o /usr/local/bin/yq
https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64
- echo "${YQ_CHECKSUM} /usr/local/bin/yq" | sha256sum -c - || exit 1
- chmod +x /usr/local/bin/yq
- fi
-
- - name: Check all components
- id: check
- run: |
- chmod +x scripts/extract-version.sh
-
- # Get all Docker components (always publish :edge)
- DOCKER_COMPONENTS=$(yq -r '.components | to_entries | .[] |
select(.value.registry == "dockerhub") | .key' .github/config/publish.yml | tr
'\n' ',' | sed 's/,$//')
- echo "docker_components=$DOCKER_COMPONENTS" >> "$GITHUB_OUTPUT"
- echo "Docker components: $DOCKER_COMPONENTS"
-
- # Check Rust crates for pre-release versions without tags
- CRATES_TO_PUBLISH=""
- for crate in rust-common rust-binary-protocol rust-sdk rust-cli; do
- VERSION=$(scripts/extract-version.sh "$crate")
- TAG=$(scripts/extract-version.sh "$crate" --tag)
-
- echo "Checking $crate: version=$VERSION, tag=$TAG"
-
- if [[ ! "$VERSION" =~ -(edge|rc) ]]; then
- echo " ⏭️ Stable version - skipping"
- continue
- fi
-
- if git rev-parse "$TAG" >/dev/null 2>&1; then
- echo " ⏭️ Tag exists - skipping"
- continue
- fi
-
- echo " ✅ Will publish"
- CRATES_TO_PUBLISH="${CRATES_TO_PUBLISH:+$CRATES_TO_PUBLISH,}$crate"
- done
- echo "crates_to_publish=$CRATES_TO_PUBLISH" >> "$GITHUB_OUTPUT"
- echo "Crates to publish: ${CRATES_TO_PUBLISH:-<none>}"
-
- # Check SDKs for pre-release versions without tags
- SDKS_TO_PUBLISH=""
- for sdk in sdk-python sdk-node sdk-java sdk-csharp sdk-go; do
- VERSION=$(scripts/extract-version.sh "$sdk")
- TAG=$(scripts/extract-version.sh "$sdk" --tag)
-
- echo "Checking $sdk: version=$VERSION, tag=$TAG"
-
- # Maven SNAPSHOT versions are mutable -- republish when SDK
sources change,
- # skip tag gate since no tag will be created for SNAPSHOTs
- if [[ "$VERSION" =~ -SNAPSHOT$ ]]; then
- SDK_NAME="${sdk#sdk-}"
- SDK_DIR="foreign/$SDK_NAME"
- if ! git diff --name-only HEAD~1 HEAD -- "$SDK_DIR/" | grep -q
.; then
- echo " ⏭️ SNAPSHOT - no changes in $SDK_DIR/"
- continue
- fi
- echo " ✅ SNAPSHOT version - will publish"
- SDKS_TO_PUBLISH="${SDKS_TO_PUBLISH:+$SDKS_TO_PUBLISH,}$SDK_NAME"
- continue
- fi
-
- if [[ ! "$VERSION" =~ -(edge|rc) ]] && [[ ! "$VERSION" =~
(\.dev|rc)[0-9]+$ ]]; then
- echo " ⏭️ Stable version - skipping"
- continue
- fi
-
- if git rev-parse "$TAG" >/dev/null 2>&1; then
- echo " ⏭️ Tag exists - skipping"
- continue
- fi
-
- echo " ✅ Will publish"
- # Convert sdk-csharp to csharp for publish.yml input format
- SDK_NAME="${sdk#sdk-}"
- SDKS_TO_PUBLISH="${SDKS_TO_PUBLISH:+$SDKS_TO_PUBLISH,}$SDK_NAME"
- done
- echo "sdks_to_publish=$SDKS_TO_PUBLISH" >> "$GITHUB_OUTPUT"
- echo "SDKs to publish: ${SDKS_TO_PUBLISH:-<none>}"
-
- call-publish:
- name: Publish components
- needs: check-auto-publish
- uses: ./.github/workflows/publish.yml
- with:
- commit: ${{ github.sha }}
- dry_run: false
- use_latest_ci: false
- skip_tag_creation: false
- publish_crates: ${{ needs.check-auto-publish.outputs.crates_to_publish }}
- publish_dockerhub: ${{
needs.check-auto-publish.outputs.docker_components }}
- publish_other: ${{ needs.check-auto-publish.outputs.sdks_to_publish }}
- create_edge_docker_tag: true
- secrets: inherit
-
- warm-cache:
- name: Warm cache (${{ matrix.runner }})
- runs-on: ${{ matrix.runner }}
- strategy:
- fail-fast: false
- matrix:
- # Don't cache ubuntu-24.04-arm due to size
- runner: [ubuntu-latest, macos-14]
- steps:
- - uses: actions/checkout@v4
-
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
- with:
- save-cache: "true"
-
- - name: Build
- run: cargo build --locked
-
- - name: Compile tests
- run: cargo test --no-run --locked
-
- build-artifacts:
- name: Build artifacts
- uses: ./.github/workflows/_build_rust_artifacts.yml
- with:
- version: edge
- upload_artifacts: true
-
rust-coverage:
name: Rust coverage baseline
runs-on: ubuntu-latest
@@ -194,6 +55,7 @@ jobs:
- name: Setup Rust with cache
uses: ./.github/actions/utils/setup-rust-with-cache
with:
+ # Also warms the GitHub Actions build cache for subsequent PR builds
save-cache: "true"
- name: Install cargo-llvm-cov
@@ -216,11 +78,7 @@ jobs:
eval $(echo -n "test" | gnome-keyring-daemon --unlock
--components=secrets)
echo -n "warmup" | secret-tool store --label="ci-warmup" ci-test
warmup
- if command -v cargo-nextest &> /dev/null; then
- cargo nextest run --locked --no-fail-fast
- else
- cargo test --locked --no-fail-fast
- fi
+ cargo test --locked --no-fail-fast
shell: bash
- name: Generate coverage report
@@ -234,6 +92,7 @@ jobs:
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: codecov.json
+ disable_search: true
flags: rust
fail_ci_if_error: false
@@ -547,11 +406,24 @@ jobs:
- name: Merge coverage profiles
run: |
mkdir -p reports
+ if [[ ! -s reports/go-coverage-unit.out ]]; then
+ echo "::warning::Unit coverage profile is empty or missing"
+ if [[ -s reports/go-coverage-bdd.out ]]; then
+ cp reports/go-coverage-bdd.out reports/go-coverage.out
+ else
+ echo "::warning::Both unit and BDD coverage profiles are empty
or missing, no coverage will be uploaded"
+ fi
+ exit 0
+ fi
# Start with the mode line from the unit profile
head -1 reports/go-coverage-unit.out > reports/go-coverage.out
# Append data lines (skip mode line) from both profiles
tail -n +2 reports/go-coverage-unit.out >> reports/go-coverage.out
- tail -n +2 reports/go-coverage-bdd.out >> reports/go-coverage.out
+ if [[ -s reports/go-coverage-bdd.out ]]; then
+ tail -n +2 reports/go-coverage-bdd.out >> reports/go-coverage.out
+ else
+ echo "::warning::BDD coverage profile is empty or missing, using
unit coverage only"
+ fi
- name: Stop Iggy server
if: always()
@@ -569,82 +441,20 @@ jobs:
flags: go
fail_ci_if_error: false
- create-prerelease:
- name: Create edge pre-release
- runs-on: ubuntu-latest
- needs: build-artifacts
- if: needs.build-artifacts.result == 'success' &&
!github.event.repository.fork
- permissions:
- contents: write
+ warm-cache-macos:
+ name: Warm macOS cache
+ runs-on: macos-14
+ timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- - name: Get server version
- id: meta
- run: |
- chmod +x scripts/extract-version.sh
- server_version=$(scripts/extract-version.sh rust-server)
- echo "server_version=${server_version}" >> "$GITHUB_OUTPUT"
-
- - name: Download all artifacts
- uses: actions/download-artifact@v4
+ - name: Setup Rust with cache
+ uses: ./.github/actions/utils/setup-rust-with-cache
with:
- name: rust-artifacts-all
- path: ./artifacts
+ save-cache: "true"
- - name: Delete existing edge release and tag
- env:
- GH_TOKEN: ${{ github.token }}
- run: |
- if gh release view edge &>/dev/null; then
- echo "Deleting existing edge release and tag..."
- gh release delete edge --cleanup-tag --yes || echo "Delete failed,
continuing..."
- fi
+ - name: Build
+ run: cargo build --locked
- - name: Create edge pre-release
- uses: softprops/action-gh-release@v2
- with:
- tag_name: edge
- name: edge
- draft: false
- prerelease: true
- make_latest: false
- files: artifacts/*.tar.gz
- body: |
- Rolling edge build of Apache Iggy binaries and connector plugins.
-
- **This release is automatically updated on every push to master.**
-
- ## Binaries included
- - `iggy-server` - The server binary
- - `iggy` - The command-line interface
- - `iggy-bench` - The benchmarking tool
- - `iggy-connectors` - The connectors runtime
-
- ## Connector plugins included (.so)
- - `iggy_connector_elasticsearch_sink`
- - `iggy_connector_elasticsearch_source`
- - `iggy_connector_iceberg_sink`
- - `iggy_connector_postgres_sink`
- - `iggy_connector_postgres_source`
- - `iggy_connector_quickwit_sink`
- - `iggy_connector_random_source`
- - `iggy_connector_stdout_sink`
-
- ## Downloads
-
- ### Binaries
- - `iggy-x86_64-unknown-linux-gnu-edge.tar.gz` - Linux x86_64
(glibc)
- - `iggy-x86_64-unknown-linux-musl-edge.tar.gz` - Linux x86_64
(musl, static)
- - `iggy-aarch64-unknown-linux-gnu-edge.tar.gz` - Linux ARM64
(glibc)
- - `iggy-aarch64-unknown-linux-musl-edge.tar.gz` - Linux ARM64
(musl, static)
-
- ### Connector plugins
- - `iggy-connectors-x86_64-unknown-linux-gnu-edge.tar.gz` - Linux
x86_64 (glibc)
- - `iggy-connectors-aarch64-unknown-linux-gnu-edge.tar.gz` - Linux
ARM64 (glibc)
-
- ## Build info
- - Server version: ${{ steps.meta.outputs.server_version }}
- - Commit: ${{ github.sha }}
-
- **Not an official ASF release** - for development/testing only.
+ - name: Compile tests
+ run: cargo test --no-run --locked
diff --git a/.github/workflows/edge-release.yml
b/.github/workflows/edge-release.yml
new file mode 100644
index 000000000..0cbc6e151
--- /dev/null
+++ b/.github/workflows/edge-release.yml
@@ -0,0 +1,123 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Builds cross-platform binaries and connector plugins, then publishes
+# them as a rolling "edge" GitHub pre-release updated on every master push.
+
+name: Edge release
+
+on:
+ push:
+ branches: [master]
+
+permissions:
+ contents: read
+
+concurrency:
+ group: edge-release-${{ github.ref }}
+ cancel-in-progress: true
+
+env:
+ IGGY_CI_BUILD: true
+
+jobs:
+ build-artifacts:
+ name: Build artifacts
+ uses: ./.github/workflows/_build_rust_artifacts.yml
+ with:
+ version: edge
+ upload_artifacts: true
+
+ create-prerelease:
+ name: Create edge pre-release
+ runs-on: ubuntu-latest
+ needs: build-artifacts
+ if: needs.build-artifacts.result == 'success' &&
!github.event.repository.fork
+ permissions:
+ contents: write
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Get server version
+ id: meta
+ run: |
+ chmod +x scripts/extract-version.sh
+ server_version=$(scripts/extract-version.sh rust-server)
+ echo "server_version=${server_version}" >> "$GITHUB_OUTPUT"
+
+ - name: Download all artifacts
+ uses: actions/download-artifact@v4
+ with:
+ name: rust-artifacts-all
+ path: ./artifacts
+
+ - name: Delete existing edge release and tag
+ env:
+ GH_TOKEN: ${{ github.token }}
+ run: |
+ if gh release view edge &>/dev/null; then
+ echo "Deleting existing edge release and tag..."
+ gh release delete edge --cleanup-tag --yes || echo "Delete failed,
continuing..."
+ fi
+
+ - name: Create edge pre-release
+ uses:
softprops/action-gh-release@153bb8e04406b158c6c84fc1615b65b24149a1fe # v2
+ with:
+ tag_name: edge
+ name: edge
+ draft: false
+ prerelease: true
+ make_latest: false
+ files: artifacts/*.tar.gz
+ body: |
+ Rolling edge build of Apache Iggy binaries and connector plugins.
+
+ **This release is automatically updated on every push to master.**
+
+ ## Binaries included
+ - `iggy-server` - The server binary
+ - `iggy` - The command-line interface
+ - `iggy-bench` - The benchmarking tool
+ - `iggy-connectors` - The connectors runtime
+
+ ## Connector plugins included (.so)
+ - `iggy_connector_elasticsearch_sink`
+ - `iggy_connector_elasticsearch_source`
+ - `iggy_connector_iceberg_sink`
+ - `iggy_connector_postgres_sink`
+ - `iggy_connector_postgres_source`
+ - `iggy_connector_quickwit_sink`
+ - `iggy_connector_random_source`
+ - `iggy_connector_stdout_sink`
+
+ ## Downloads
+
+ ### Binaries
+ - `iggy-x86_64-unknown-linux-gnu-edge.tar.gz` - Linux x86_64
(glibc)
+ - `iggy-x86_64-unknown-linux-musl-edge.tar.gz` - Linux x86_64
(musl, static)
+ - `iggy-aarch64-unknown-linux-gnu-edge.tar.gz` - Linux ARM64
(glibc)
+ - `iggy-aarch64-unknown-linux-musl-edge.tar.gz` - Linux ARM64
(musl, static)
+
+ ### Connector plugins
+ - `iggy-connectors-x86_64-unknown-linux-gnu-edge.tar.gz` - Linux
x86_64 (glibc)
+ - `iggy-connectors-aarch64-unknown-linux-gnu-edge.tar.gz` - Linux
ARM64 (glibc)
+
+ ## Build info
+ - Server version: ${{ steps.meta.outputs.server_version }}
+ - Commit: ${{ github.sha }}
+
+ **Not an official ASF release** - for development/testing only.
diff --git a/.github/workflows/post-merge.yml b/.github/workflows/post-merge.yml
index 18c16eeba..9f49334f0 100644
--- a/.github/workflows/post-merge.yml
+++ b/.github/workflows/post-merge.yml
@@ -15,6 +15,10 @@
# specific language governing permissions and limitations
# under the License.
+# Auto-publish: detects pre-release crate/SDK versions without tags
+# and publishes them to crates.io, Docker Hub, PyPI, npm, Maven, NuGet.
+# Runs on every push to master.
+
name: Post-merge
on:
@@ -146,505 +150,3 @@ jobs:
create_edge_docker_tag: true
secrets: inherit
- warm-cache:
- name: Warm cache (${{ matrix.runner }})
- runs-on: ${{ matrix.runner }}
- strategy:
- fail-fast: false
- matrix:
- # Don't cache ubuntu-24.04-arm due to size
- runner: [ubuntu-latest, macos-14]
- steps:
- - uses: actions/checkout@v4
-
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
- with:
- save-cache: "true"
-
- - name: Build
- run: cargo build --locked
-
- - name: Compile tests
- run: cargo test --no-run --locked
-
- build-artifacts:
- name: Build artifacts
- uses: ./.github/workflows/_build_rust_artifacts.yml
- with:
- version: edge
- upload_artifacts: true
-
- rust-coverage:
- name: Rust coverage baseline
- runs-on: ubuntu-latest
- timeout-minutes: 45
- steps:
- - uses: actions/checkout@v4
-
- - name: Cleanup disk space
- run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
/opt/hostedtoolcache/CodeQL
-
- - name: Install system dependencies
- run: |
- sudo apt-get update --yes
- sudo apt-get install --yes gnome-keyring keyutils dbus-x11
libsecret-tools
- rm -f $HOME/.local/share/keyrings/*
-
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
- with:
- save-cache: "true"
-
- - name: Install cargo-llvm-cov
- uses: taiki-e/install-action@v2
- with:
- tool: cargo-llvm-cov
-
- - name: Build and test with coverage
- run: |
- source <(cargo llvm-cov show-env --export-prefix)
- cargo build --locked
- cargo test --locked --no-run
-
- # Start D-Bus and unlock keyring right before test execution to avoid
- # gnome-keyring auto-locking the collection during the build phase.
- # Previously this ran before `cargo build`, leaving a 7+ minute idle
- # window that triggered org.freedesktop.Secret.Error.IsLocked ~10%
of runs.
- eval $(dbus-launch --sh-syntax)
- export DBUS_SESSION_BUS_ADDRESS
- eval $(echo -n "test" | gnome-keyring-daemon --unlock
--components=secrets)
- echo -n "warmup" | secret-tool store --label="ci-warmup" ci-test
warmup
-
- if command -v cargo-nextest &> /dev/null; then
- cargo nextest run --locked --no-fail-fast
- else
- cargo test --locked --no-fail-fast
- fi
- shell: bash
-
- - name: Generate coverage report
- run: |
- source <(cargo llvm-cov show-env --export-prefix)
- cargo llvm-cov report --codecov --output-path codecov.json
- shell: bash
-
- - name: Upload to Codecov
- uses: codecov/codecov-action@v5
- with:
- token: ${{ secrets.CODECOV_TOKEN }}
- files: codecov.json
- flags: rust
- fail_ci_if_error: false
-
- java-coverage:
- name: Java coverage baseline
- runs-on: ubuntu-latest
- timeout-minutes: 30
- steps:
- - uses: actions/checkout@v4
-
- - name: Setup Java with cache
- uses: ./.github/actions/utils/setup-java-with-cache
-
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
- with:
- save-cache: "false"
-
- - name: Start Iggy server
- id: iggy
- uses: ./.github/actions/utils/server-start
-
- - name: Run tests and generate coverage
- working-directory: foreign/java
- env:
- USE_EXTERNAL_SERVER: true
- run: |
- ./gradlew test
- ./gradlew jacocoAggregatedReport
-
- - name: Stop Iggy server
- if: always()
- uses: ./.github/actions/utils/server-stop
- with:
- pid-file: ${{ steps.iggy.outputs.pid_file }}
- log-file: ${{ steps.iggy.outputs.log_file }}
-
- - name: Upload to Codecov
- uses: codecov/codecov-action@v5
- with:
- token: ${{ secrets.CODECOV_TOKEN }}
- files:
foreign/java/build/reports/jacoco/aggregate/jacocoAggregated.xml
- disable_search: true
- flags: java
- fail_ci_if_error: false
-
- csharp-coverage:
- name: C# coverage baseline
- runs-on: ubuntu-latest
- timeout-minutes: 30
- steps:
- - uses: actions/checkout@v4
-
- - name: Setup .NET
- uses: actions/setup-dotnet@v4
- with:
- dotnet-version: "10.0.x"
-
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
- with:
- save-cache: "false"
-
- - name: Build Iggy server Docker image
- id: docker_build
- uses: ./.github/actions/utils/docker-build-test-server
- with:
- image-tag: "iggy-server:test"
- libc: "glibc"
- profile: "debug"
-
- - name: Restore and build
- working-directory: foreign/csharp
- run: |
- dotnet restore Iggy_SDK.sln
- dotnet build Iggy_SDK.sln --no-restore
-
- - name: Run unit tests with coverage
- working-directory: foreign/csharp
- run: |
- dotnet test \
- --project Iggy_SDK_Tests \
- --no-build \
- --verbosity normal \
- --coverage \
- --coverage-output-format cobertura \
- --coverage-output coverage.cobertura.xml \
- --results-directory ./reports/unit
-
- - name: Run integration tests with coverage
- working-directory: foreign/csharp
- env:
- IGGY_SERVER_DOCKER_IMAGE: ${{
steps.docker_build.outputs.docker_image }}
- run: |
- dotnet test \
- --project Iggy_SDK.Tests.Integration \
- --no-build \
- --verbosity normal \
- --coverage \
- --coverage-output-format cobertura \
- --coverage-output coverage.cobertura.xml \
- --results-directory ./reports/integration
-
- - name: Merge coverage reports
- working-directory: foreign/csharp
- run: |
- dotnet tool install --global dotnet-coverage
- dotnet-coverage merge ./reports/**/*.cobertura.xml -f cobertura -o
./reports/coverage.cobertura.xml
-
- - name: Upload to Codecov
- uses: codecov/codecov-action@v5
- with:
- token: ${{ secrets.CODECOV_TOKEN }}
- files: foreign/csharp/reports/coverage.cobertura.xml
- disable_search: true
- flags: csharp
- fail_ci_if_error: false
-
- python-coverage:
- name: Python coverage baseline
- runs-on: ubuntu-latest
- timeout-minutes: 30
- steps:
- - uses: actions/checkout@v4
-
- - name: Setup Python
- uses: actions/setup-python@v5
- with:
- python-version: "3.10"
-
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
- with:
- save-cache: "false"
-
- - name: Install cargo-llvm-cov
- uses: taiki-e/install-action@v2
- with:
- tool: cargo-llvm-cov
-
- - name: Install uv
- uses: astral-sh/setup-uv@5a095e7a2014a4212f075830d4f7277575a9d098 #
v7.3.1
-
- - name: Install dependencies
- run: |
- cd foreign/python
- uv sync --frozen --extra dev --extra testing --extra testing-docker
-
- - name: Build Python wheel with coverage instrumentation
- run: |
- cd foreign/python
- source <(cargo llvm-cov show-env --sh)
- export CARGO_TARGET_DIR=$CARGO_LLVM_COV_TARGET_DIR
-
- # Propagate LLVM_PROFILE_FILE so the test step can write profraw data
- echo "LLVM_PROFILE_FILE=${LLVM_PROFILE_FILE}" >> $GITHUB_ENV
-
- cargo llvm-cov clean --workspace
- echo "Building Python wheel with coverage instrumentation..."
- uv run --no-sync maturin develop
- shell: bash
-
- - name: Start Iggy server
- id: iggy
- uses: ./.github/actions/utils/server-start
-
- - name: Run tests
- run: |
- cd foreign/python
- IGGY_SERVER_HOST=127.0.0.1 \
- IGGY_SERVER_TCP_PORT=8090 \
- uv run --no-sync pytest tests/ -v \
- --junitxml=../../reports/python-junit.xml \
- --tb=short \
- --capture=no
-
- - name: Stop Iggy server
- if: always()
- uses: ./.github/actions/utils/server-stop
- with:
- pid-file: ${{ steps.iggy.outputs.pid_file }}
- log-file: ${{ steps.iggy.outputs.log_file }}
-
- - name: Generate coverage report
- run: |
- mkdir -p reports
- cd foreign/python
- source <(cargo llvm-cov show-env --sh)
- export CARGO_TARGET_DIR=$CARGO_LLVM_COV_TARGET_DIR
-
- cargo llvm-cov report --lcov \
- --ignore-filename-regex='(\.cargo/|/rustc/|/core/)' \
- --output-path ../../reports/python-coverage.lcov
-
- # Fix paths: cargo-llvm-cov outputs paths relative to the crate root
(src/...)
- # but Codecov expects paths relative to the repo root
(foreign/python/src/...)
- sed -i 's|^SF:src/|SF:foreign/python/src/|'
../../reports/python-coverage.lcov
-
- echo "Coverage report generated: $(wc -l <
../../reports/python-coverage.lcov) lines"
- shell: bash
-
- - name: Upload to Codecov
- uses: codecov/codecov-action@v5
- with:
- token: ${{ secrets.CODECOV_TOKEN }}
- files: reports/python-coverage.lcov
- disable_search: true
- flags: python
- fail_ci_if_error: false
-
- node-coverage:
- name: Node coverage baseline
- runs-on: ubuntu-latest
- timeout-minutes: 30
- steps:
- - uses: actions/checkout@v4
-
- - name: Setup Node.js
- uses: actions/setup-node@v4
- with:
- node-version: "23"
- cache: "npm"
- cache-dependency-path: foreign/node/package-lock.json
-
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
- with:
- save-cache: "false"
-
- - name: Install netcat
- run: sudo apt-get update && sudo apt-get install -y netcat-openbsd
-
- - name: Install dependencies
- run: |
- cd foreign/node
- npm ci --ignore-scripts
-
- - name: Start Iggy server
- id: iggy
- uses: ./.github/actions/utils/server-start
- env:
- IGGY_CLUSTER_ENABLED: true
-
- - name: Run unit tests with coverage
- run: |
- cd foreign/node
- mkdir -p ../../reports/node-coverage/unit
- npx c8 --reporter=lcov
--reports-dir=../../reports/node-coverage/unit npm run test:unit
-
- - name: Run e2e tests with coverage
- run: |
- cd foreign/node
- mkdir -p ../../reports/node-coverage/e2e
- npx c8 --reporter=lcov --reports-dir=../../reports/node-coverage/e2e
npm run test:e2e
- env:
- IGGY_SERVER_HOST: 127.0.0.1
- IGGY_SERVER_TCP_PORT: 8090
-
- - name: Stop Iggy server
- if: always()
- uses: ./.github/actions/utils/server-stop
- with:
- pid-file: ${{ steps.iggy.outputs.pid_file }}
- log-file: ${{ steps.iggy.outputs.log_file }}
-
- - name: Upload to Codecov
- uses: codecov/codecov-action@v5
- with:
- token: ${{ secrets.CODECOV_TOKEN }}
- files:
reports/node-coverage/unit/lcov.info,reports/node-coverage/e2e/lcov.info
- disable_search: true
- flags: node
- fail_ci_if_error: false
-
- go-coverage:
- name: Go coverage baseline
- runs-on: ubuntu-latest
- timeout-minutes: 30
- steps:
- - uses: actions/checkout@v4
-
- - name: Setup Go
- uses: ./.github/actions/utils/setup-go-with-cache
-
- - name: Setup Rust with cache
- uses: ./.github/actions/utils/setup-rust-with-cache
- with:
- save-cache: "false"
-
- - name: Start Iggy server
- id: iggy
- uses: ./.github/actions/utils/server-start
-
- - name: Run unit tests with coverage
- run: |
- cd foreign/go
- mkdir -p ../../reports
- go test -v -race -coverprofile=../../reports/go-coverage-unit.out
./...
-
- - name: Run BDD tests with coverage
- run: |
- cd bdd/go
- mkdir -p ../../reports
- # Run only tcp_test suite (ginkgo); godog leader_redirection
- # tests require a multi-node cluster not available in CI.
- go test -v -race \
- -coverpkg=github.com/apache/iggy/foreign/go/... \
- -coverprofile=../../reports/go-coverage-bdd.out \
- ./tests/tcp_test/...
-
- - name: Merge coverage profiles
- run: |
- mkdir -p reports
- # Start with the mode line from the unit profile
- head -1 reports/go-coverage-unit.out > reports/go-coverage.out
- # Append data lines (skip mode line) from both profiles
- tail -n +2 reports/go-coverage-unit.out >> reports/go-coverage.out
- tail -n +2 reports/go-coverage-bdd.out >> reports/go-coverage.out
-
- - name: Stop Iggy server
- if: always()
- uses: ./.github/actions/utils/server-stop
- with:
- pid-file: ${{ steps.iggy.outputs.pid_file }}
- log-file: ${{ steps.iggy.outputs.log_file }}
-
- - name: Upload to Codecov
- uses: codecov/codecov-action@v5
- with:
- token: ${{ secrets.CODECOV_TOKEN }}
- files: reports/go-coverage.out
- disable_search: true
- flags: go
- fail_ci_if_error: false
-
- create-prerelease:
- name: Create edge pre-release
- runs-on: ubuntu-latest
- needs: build-artifacts
- if: needs.build-artifacts.result == 'success' &&
!github.event.repository.fork
- permissions:
- contents: write
- steps:
- - uses: actions/checkout@v4
-
- - name: Get server version
- id: meta
- run: |
- chmod +x scripts/extract-version.sh
- server_version=$(scripts/extract-version.sh rust-server)
- echo "server_version=${server_version}" >> "$GITHUB_OUTPUT"
-
- - name: Download all artifacts
- uses: actions/download-artifact@v4
- with:
- name: rust-artifacts-all
- path: ./artifacts
-
- - name: Delete existing edge release and tag
- env:
- GH_TOKEN: ${{ github.token }}
- run: |
- if gh release view edge &>/dev/null; then
- echo "Deleting existing edge release and tag..."
- gh release delete edge --cleanup-tag --yes || echo "Delete failed,
continuing..."
- fi
-
- - name: Create edge pre-release
- uses: softprops/action-gh-release@v2
- with:
- tag_name: edge
- name: edge
- draft: false
- prerelease: true
- make_latest: false
- files: artifacts/*.tar.gz
- body: |
- Rolling edge build of Apache Iggy binaries and connector plugins.
-
- **This release is automatically updated on every push to master.**
-
- ## Binaries included
- - `iggy-server` - The server binary
- - `iggy` - The command-line interface
- - `iggy-bench` - The benchmarking tool
- - `iggy-connectors` - The connectors runtime
-
- ## Connector plugins included (.so)
- - `iggy_connector_elasticsearch_sink`
- - `iggy_connector_elasticsearch_source`
- - `iggy_connector_iceberg_sink`
- - `iggy_connector_postgres_sink`
- - `iggy_connector_postgres_source`
- - `iggy_connector_quickwit_sink`
- - `iggy_connector_random_source`
- - `iggy_connector_stdout_sink`
-
- ## Downloads
-
- ### Binaries
- - `iggy-x86_64-unknown-linux-gnu-edge.tar.gz` - Linux x86_64
(glibc)
- - `iggy-x86_64-unknown-linux-musl-edge.tar.gz` - Linux x86_64
(musl, static)
- - `iggy-aarch64-unknown-linux-gnu-edge.tar.gz` - Linux ARM64
(glibc)
- - `iggy-aarch64-unknown-linux-musl-edge.tar.gz` - Linux ARM64
(musl, static)
-
- ### Connector plugins
- - `iggy-connectors-x86_64-unknown-linux-gnu-edge.tar.gz` - Linux
x86_64 (glibc)
- - `iggy-connectors-aarch64-unknown-linux-gnu-edge.tar.gz` - Linux
ARM64 (glibc)
-
- ## Build info
- - Server version: ${{ steps.meta.outputs.server_version }}
- - Commit: ${{ github.sha }}
-
- **Not an official ASF release** - for development/testing only.
diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml
index a3a976780..abbaa30ed 100644
--- a/.github/workflows/pre-merge.yml
+++ b/.github/workflows/pre-merge.yml
@@ -15,6 +15,10 @@
# specific language governing permissions and limitations
# under the License.
+# PR gate: detects changed components, builds test matrices, and runs
+# lint/test/build/compat/BDD/examples jobs only for affected languages.
+# All jobs must pass before merge.
+
name: Pre-merge
on:
pull_request:
diff --git a/codecov.yml b/codecov.yml
index 6320624d0..77a1dfc53 100644
--- a/codecov.yml
+++ b/codecov.yml
@@ -66,8 +66,35 @@ flag_management:
paths:
- foreign/go/
+component_management:
+ individual_components:
+ - component_id: rust
+ name: Rust Core
+ paths:
+ - core/
+ - component_id: java
+ name: Java SDK
+ paths:
+ - foreign/java/
+ - component_id: csharp
+ name: "C# SDK"
+ paths:
+ - foreign/csharp/
+ - component_id: python
+ name: Python SDK
+ paths:
+ - foreign/python/
+ - component_id: node
+ name: Node SDK
+ paths:
+ - foreign/node/
+ - component_id: go
+ name: Go SDK
+ paths:
+ - foreign/go/
+
comment:
- layout: "header, diff, flags, files"
+ layout: "header, diff, components, files"
behavior: default
require_changes: true
require_base: true
