From d5b45c416eb5cc3d6860e406d4a9d4d512882233 Mon Sep 17 00:00:00 2001 From: ndk Date: Thu, 12 Feb 2026 11:20:59 +0100 Subject: [PATCH 1/5] Use GitHub Actions matrix for integration tests Run Westend parachain and Polkadot solochain tests in parallel instead of sequentially, eliminating ~75 lines of duplication and cutting CI wall time roughly in half. Each runtime gets its own isolated runner, avoiding flaky cross-runtime state issues (#237). --- .github/workflows/integration-test.yml | 76 +++++++------------------- 1 file changed, 21 insertions(+), 55 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index eece8eabb..4c86142ac 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -78,9 +78,17 @@ jobs: integration-tests: needs: [setup] - name: Integration Tests + name: Integration Tests (${{ matrix.runtime.name }}) runs-on: ubuntu-latest timeout-minutes: 200 + strategy: + fail-fast: false + matrix: + runtime: + - name: "Westend parachain" + package: "bulletin-westend-runtime" + - name: "Polkadot solochain" + package: "bulletin-polkadot-runtime" steps: - name: Checkout sources @@ -138,80 +146,38 @@ jobs: echo "${ZOMBIENET_BIN_DIR}" >> "$GITHUB_PATH" echo "ZOMBIENET_BINARY=zombienet-linux-x64" >> "$GITHUB_ENV" - # ======================================================================== - # Westend parachain tests - # ======================================================================== - - name: Start services (Westend parachain) + - name: Start services working-directory: examples run: | TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test" echo "TEST_DIR=$TEST_DIR" >> "$GITHUB_ENV" - just start-services "$TEST_DIR" "bulletin-westend-runtime" + just start-services "$TEST_DIR" "${{ matrix.runtime.package }}" - - name: Test authorize-and-store ws (Westend parachain) + - name: Test authorize-and-store ws working-directory: examples - run: just run-test-authorize-and-store "${{ env.TEST_DIR }}" "bulletin-westend-runtime" "ws" + run: just run-test-authorize-and-store "${{ env.TEST_DIR }}" "${{ matrix.runtime.package }}" "ws" - - name: Test authorize-and-store smoldot (Westend parachain) + - name: Test authorize-and-store smoldot working-directory: examples - run: just run-test-authorize-and-store "${{ env.TEST_DIR }}" "bulletin-westend-runtime" "smoldot" + run: just run-test-authorize-and-store "${{ env.TEST_DIR }}" "${{ matrix.runtime.package }}" "smoldot" - - name: Test store-chunked-data (Westend parachain) + - name: Test store-chunked-data working-directory: examples run: just run-test-store-chunked-data "${{ env.TEST_DIR }}" - - name: Test store-big-data (Westend parachain) + - name: Test store-big-data working-directory: examples run: just run-test-store-big-data "${{ env.TEST_DIR }}" "big32" - - name: Test authorize-preimage-and-store (Westend parachain) + - name: Test authorize-preimage-and-store working-directory: examples run: just run-test-authorize-preimage-and-store "${{ env.TEST_DIR }}" - - name: Test chopsticks compatibility (Westend parachain) + - name: Test chopsticks compatibility working-directory: examples run: just run-test-chopsticks "ws://localhost:10000" - - name: Stop services (Westend parachain) - if: always() - working-directory: examples - run: just stop-services "${{ env.TEST_DIR }}" - - # ======================================================================== - # Polkadot solochain tests - # ======================================================================== - - name: Start services (Polkadot solochain) - working-directory: examples - run: | - TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test" - echo "TEST_DIR=$TEST_DIR" >> "$GITHUB_ENV" - just start-services "$TEST_DIR" "bulletin-polkadot-runtime" - - - name: Test authorize-and-store ws (Polkadot solochain) - working-directory: examples - run: just run-test-authorize-and-store "${{ env.TEST_DIR }}" "bulletin-polkadot-runtime" "ws" - - - name: Test authorize-and-store smoldot (Polkadot solochain) - working-directory: examples - run: just run-test-authorize-and-store "${{ env.TEST_DIR }}" "bulletin-polkadot-runtime" "smoldot" - - - name: Test store-chunked-data (Polkadot solochain) - working-directory: examples - run: just run-test-store-chunked-data "${{ env.TEST_DIR }}" - - - name: Test store-big-data (Polkadot solochain) - working-directory: examples - run: just run-test-store-big-data "${{ env.TEST_DIR }}" "big32" - - - name: Test authorize-preimage-and-store (Polkadot solochain) - working-directory: examples - run: just run-test-authorize-preimage-and-store "${{ env.TEST_DIR }}" - - - name: Test chopsticks compatibility (Polkadot solochain) - working-directory: examples - run: just run-test-chopsticks "ws://localhost:10000" - - - name: Stop services (Polkadot solochain) + - name: Stop services if: always() working-directory: examples run: just stop-services "${{ env.TEST_DIR }}" @@ -221,6 +187,6 @@ jobs: if: failure() uses: actions/upload-artifact@v4 with: - name: failed-zombienet-logs + name: failed-zombienet-logs-${{ matrix.runtime.name }} path: | ${{ env.TEST_DIR }}/*.log From 4fd3615e7fcccc8342328e35772b3a8d790ca9a9 Mon Sep 17 00:00:00 2001 From: ndk Date: Thu, 12 Feb 2026 12:28:46 +0100 Subject: [PATCH 2/5] Reuse runtimes-matrix.json for integration test matrix Instead of hardcoding the runtime list in the workflow, read from scripts/runtimes-matrix.json and filter by a new `integration_tests` flag. This keeps the single source of truth for runtime definitions. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/integration-test.yml | 26 ++++++++++++++++++++------ scripts/runtimes-matrix.json | 2 ++ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 7cd601d23..4db5b9bb9 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -72,19 +72,33 @@ jobs: "https://github.com/paritytech/zombienet/releases/download/${ZOMBIENET_VERSION}/zombienet-linux-x64" chmod +x zombienet-linux-x64 + runtime-matrix: + runs-on: ubuntu-latest + outputs: + runtime: ${{ steps.runtime.outputs.runtime }} + name: Extract tasks from matrix + steps: + - uses: actions/checkout@v6 + - id: runtime + run: | + TASKS=$(jq '[.[] | select(.integration_tests == true)]' scripts/runtimes-matrix.json) + SKIPPED=$(jq '[.[] | select(.integration_tests != true)]' scripts/runtimes-matrix.json) + echo "--- Running integration tests for ---" + echo "$TASKS" + echo "--- Skipping integration tests for ---" + echo "$SKIPPED" + TASKS=$(echo "$TASKS" | jq -c .) + echo "runtime=$TASKS" >> $GITHUB_OUTPUT + integration-tests: - needs: [setup] + needs: [setup, runtime-matrix] name: Integration Tests (${{ matrix.runtime.name }}) runs-on: ubuntu-latest timeout-minutes: 200 strategy: fail-fast: false matrix: - runtime: - - name: "Westend parachain" - package: "bulletin-westend-runtime" - - name: "Polkadot solochain" - package: "bulletin-polkadot-runtime" + runtime: ${{ fromJSON(needs.runtime-matrix.outputs.runtime) }} steps: - name: Checkout sources diff --git a/scripts/runtimes-matrix.json b/scripts/runtimes-matrix.json index daffd6b32..e457be268 100644 --- a/scripts/runtimes-matrix.json +++ b/scripts/runtimes-matrix.json @@ -4,6 +4,7 @@ "package": "bulletin-polkadot-runtime", "path": "runtimes/bulletin-polkadot", "blocktime": 6000, + "integration_tests": true, "benchmarks_templates": { "pallet_xcm_benchmarks::generic": "templates/xcm-bench-template.hbs", "pallet_xcm_benchmarks::fungible": "templates/xcm-bench-template.hbs" @@ -20,6 +21,7 @@ "wss://westend-bulletin-rpc.polkadot.io" ], "blocktime": 6000, + "integration_tests": true, "benchmarks_templates": { "pallet_xcm_benchmarks::generic": "templates/xcm-bench-template.hbs", "pallet_xcm_benchmarks::fungible": "templates/xcm-bench-template.hbs" From 07d1b29d1ad7a4f67217ed4a8b2d75fa43c7d558 Mon Sep 17 00:00:00 2001 From: Branislav Kontur Date: Thu, 12 Feb 2026 15:00:59 +0100 Subject: [PATCH 3/5] Add summary job for Integration Tests status check The matrix job reports per-runtime check names (e.g. "Integration Tests (bulletin-polkadot)") which don't match the branch protection rule expecting a single "Integration Tests" status. Add a summary job that aggregates matrix results under that exact name. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/integration-test.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 4db5b9bb9..ba046a5ea 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -203,3 +203,16 @@ jobs: name: failed-zombienet-logs-${{ matrix.runtime.name }} path: | ${{ env.TEST_DIR }}/*.log + + integration-tests-complete: + name: Integration Tests + needs: [integration-tests] + if: always() + runs-on: ubuntu-latest + steps: + - name: Check integration test results + run: | + if [ "${{ needs.integration-tests.result }}" != "success" ]; then + echo "Integration tests failed or were cancelled" + exit 1 + fi From 330ee7b994073f3302aff210f25321f7a2ac830f Mon Sep 17 00:00:00 2001 From: Branislav Kontur Date: Thu, 12 Feb 2026 15:28:25 +0100 Subject: [PATCH 4/5] Load env vars in Setup job to fix cache key resolution The Setup job uses POLKADOT_SDK_VERSION and ZOMBIENET_VERSION in cache keys, but these are defined in .github/env. Without loading that file, the cache keys resolve to empty strings causing cache misses every run. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/integration-test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index ba046a5ea..5b42fae99 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -25,12 +25,18 @@ jobs: name: Setup runs-on: ubuntu-latest steps: + - name: Checkout sources + uses: actions/checkout@v6 + - name: Free Disk Space (Ubuntu) if: ${{ runner.environment == 'github-hosted' }} uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 with: tool-cache: false + - name: Load common environment variables via env file + run: cat .github/env >> $GITHUB_ENV + # Cache the Polkadot SDK binaries - name: Cache Polkadot SDK binaries uses: actions/cache@v5 From 5659c6e5bb9528e49896a42c487685cb1a7e5c60 Mon Sep 17 00:00:00 2001 From: ndk Date: Fri, 13 Feb 2026 09:16:47 +0100 Subject: [PATCH 5/5] Use GITHUB_ENV for RUNTIME_PACKAGE instead of per-step matrix refs Set RUNTIME_PACKAGE once in $GITHUB_ENV alongside TEST_DIR, then reference both as plain env vars in all subsequent steps. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/integration-test.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 5b42fae99..c6764f318 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -170,27 +170,28 @@ jobs: run: | TEST_DIR="$(mktemp -d $GITHUB_WORKSPACE/bulletin-tests-run-XXXXX)/test" echo "TEST_DIR=$TEST_DIR" >> "$GITHUB_ENV" + echo "RUNTIME_PACKAGE=${{ matrix.runtime.package }}" >> "$GITHUB_ENV" just start-services "$TEST_DIR" "${{ matrix.runtime.package }}" - name: Test authorize-and-store ws working-directory: examples - run: just run-test-authorize-and-store "${{ env.TEST_DIR }}" "${{ matrix.runtime.package }}" "ws" + run: just run-test-authorize-and-store "$TEST_DIR" "$RUNTIME_PACKAGE" "ws" - name: Test authorize-and-store smoldot working-directory: examples - run: just run-test-authorize-and-store "${{ env.TEST_DIR }}" "${{ matrix.runtime.package }}" "smoldot" + run: just run-test-authorize-and-store "$TEST_DIR" "$RUNTIME_PACKAGE" "smoldot" - name: Test store-chunked-data working-directory: examples - run: just run-test-store-chunked-data "${{ env.TEST_DIR }}" + run: just run-test-store-chunked-data "$TEST_DIR" - name: Test store-big-data working-directory: examples - run: just run-test-store-big-data "${{ env.TEST_DIR }}" "big32" + run: just run-test-store-big-data "$TEST_DIR" "big32" - name: Test authorize-preimage-and-store working-directory: examples - run: just run-test-authorize-preimage-and-store "${{ env.TEST_DIR }}" + run: just run-test-authorize-preimage-and-store "$TEST_DIR" - name: Test chopsticks compatibility working-directory: examples @@ -199,7 +200,7 @@ jobs: - name: Stop services if: always() working-directory: examples - run: just stop-services "${{ env.TEST_DIR }}" + run: just stop-services "$TEST_DIR" # Collects logs from the last failed zombienet run. - name: Upload Zombienet logs (on failure)