perf(ci): build common artifacts once, run the two test suites in parallel

.github/workflows/integration-tests.yml

@@ -33,17 +33,17 @@ jobs:
           TASKS=$(echo "$TASKS" | jq -c .)
           echo "runtime=$TASKS" >> $GITHUB_OUTPUT
 
-  integration-tests:
-    name: Integration Tests (${{ matrix.runtime.name }})
+  # ── Build once ────────────────────────────────────────────────────────────
+  # Build the shared artifacts once (provider binary + one compressed wasm per
+  # tested runtime) so both test jobs download instead of rebuilding. The test
+  # jobs still compile the fs/s3 examples and sc contracts via the shared cache.
+  build:
+    name: Build
     runs-on: parity-large
     timeout-minutes: 60
-    needs: [set-image, runtime-matrix]
+    needs: [set-image]
     container:
       image: ${{ needs.set-image.outputs.CI_IMAGE }}
-    strategy:
-      fail-fast: false
-      matrix:
-        runtime: ${{ fromJSON(needs.runtime-matrix.outputs.runtime) }}
     steps:
       - name: Checkout sources
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
@@ -54,12 +54,62 @@ jobs:
       - name: Rust cache
         uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
         with:
-          shared-key: "integration-tests-${{ matrix.runtime.name }}"
+          shared-key: "integration-tests-build"
           save-if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }}
 
       - name: Install just
         run: cargo install just --locked || true
 
+      # One wasm per runtime the test jobs fan out on.
+      - name: Build all runtimes
+        run: |
+          for cmd in $(jq -r '.[] | select(.integration_tests == true) | .build_command' scripts/runtimes-matrix.json); do
+            echo "::group::just $cmd"
+            just "$cmd"
+            echo "::endgroup::"
+          done
+
+      - name: Build provider
+        run: just build-provider
+
+      # Upload only what the test jobs consume: provider binary + the final
+      # compressed wasm per runtime. NOT the full wbuild tree (~900 MiB Cargo
+      # scratch). Both paths sit under target/release, so the artifact is rooted
+      # there (LCA) and the download steps extract back into target/release.
+      - name: Upload build artifacts
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        with:
+          name: build
+          retention-days: 1
+          if-no-files-found: error
+          path: |
+            target/release/storage-provider-node
+            target/release/wbuild/*/*.compact.compressed.wasm
+
+  # ── L0 / file-system / S3 / PAPI demos ──────────────────────────────────────
+  # Own chain + inmemory/disk providers; demos sequential on the shared chain.
+  # Runs in parallel with sc-integration-tests off the prebuilt artifact.
+  integration-tests:
+    name: Integration Tests (${{ matrix.runtime.name }})
+    runs-on: parity-large
+    timeout-minutes: 30
+    needs: [set-image, runtime-matrix, build]
+    container:
+      image: ${{ needs.set-image.outputs.CI_IMAGE }}
+    strategy:
+      fail-fast: false
+      matrix:
+        runtime: ${{ fromJSON(needs.runtime-matrix.outputs.runtime) }}
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+
+      - name: Load common environment variables via env file
+        run: cat .github/env >> $GITHUB_ENV
+
+      - name: Install just
+        run: cargo install just --locked || true
+
       - name: Cache Polkadot SDK binaries
         uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
         id: sdk-cache
@@ -87,19 +137,29 @@ jobs:
         if: steps.zombienet-cache.outputs.cache-hit != 'true'
         run: just zombienet_version=${{ env.ZOMBIENET_VERSION }} download-zombienet
 
-      - name: Log binary versions
-        run: |
-          .bin/zombienet --version
-          .bin/polkadot --version
-          .bin/polkadot-omni-node --version
-
       - name: Setup Node.js
         uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6
         with:
           node-version: ${{ env.NODE_VERSION }}
 
-      - name: Build runtime and provider
-        run: just ${{ matrix.runtime.build_command }} && just build-provider
+      # Before the download so the cached target/ can't clobber it. save-if
+      # false — only the build job writes this cache.
+      - name: Rust cache
+        uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
+        with:
+          shared-key: "integration-tests-build"
+          save-if: false
+
+      # Extract into target/release so the artifact's contents land back at
+      # their original paths (chmod, chain_spec_command scripts, demos).
+      - name: Download build artifacts
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v6
+        with:
+          name: build
+          path: target/release
+
+      - name: Restore executable bits
+        run: chmod +x target/release/storage-provider-node
 
       - name: Start chain
         run: |
@@ -199,15 +259,15 @@ jobs:
             /tmp/zombie-*/*-plain.json
             /tmp/zombie-*/*-raw.json
 
-  # Smart-contract demos run in a parallel job with their own chain +
-  # provider — the sequential time of L0 demos + fs/s3 lifecycles + sc demos
-  # was bumping against the 60-minute job timeout. Parallel splits the
-  # wall-clock roughly in half; SC tests only need //Alice as a provider.
+  # ── Smart-contract demos ─────────────────────────────────────────────────────
+  # Own chain + inmemory provider, parallel with integration-tests off the
+  # prebuilt artifact. Still installs solc/resolc and builds the example
+  # contracts here (cheap, not in the shared artifact).
   sc-integration-tests:
     name: SC Integration Tests (${{ matrix.runtime.name }})
     runs-on: parity-large
-    timeout-minutes: 60
-    needs: [set-image, runtime-matrix]
+    timeout-minutes: 30
+    needs: [set-image, runtime-matrix, build]
     container:
       image: ${{ needs.set-image.outputs.CI_IMAGE }}
     strategy:
@@ -221,12 +281,6 @@ jobs:
       - name: Load common environment variables via env file
         run: cat .github/env >> $GITHUB_ENV
 
-      - name: Rust cache
-        uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2
-        with:
-          shared-key: "sc-integration-tests-${{ matrix.runtime.name }}"
-          save-if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }}
-
       - name: Install just
         run: cargo install just --locked || true
 
@@ -262,6 +316,17 @@ jobs:
         with:
           node-version: ${{ env.NODE_VERSION }}
 
+      # Extract into target/release so the artifact's target/release-rooted
+      # contents land back at their original paths (see integration-tests job).
+      - name: Download build artifacts
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v6
+        with:
+          name: build
+          path: target/release
+
+      - name: Restore executable bits
+        run: chmod +x target/release/storage-provider-node
+
       - name: Install solc and resolc
         run: |
           for bin_url in \
@@ -286,9 +351,6 @@ jobs:
           solc --version
           resolc --version
 
-      - name: Build runtime and provider
-        run: just ${{ matrix.runtime.build_command }} && just build-provider
-
       - name: Build example contracts
         run: just build-contracts
 
@@ -362,7 +424,7 @@ jobs:
 
   integration-tests-complete:
     name: Integration Tests
-    needs: [integration-tests, sc-integration-tests]
+    needs: [build, integration-tests, sc-integration-tests]
     if: always()
     runs-on: ubuntu-latest
     steps:

.github/workflows/ui-e2e.yml

@@ -1,39 +1,50 @@
 name: UI E2E
 
+# No top-level `paths:` filter: the required gate must always report. The
+# expensive e2e job skips (via `changes`) when nothing relevant is touched; a
+# workflow-level filter would instead leave the required check pending forever.
 on:
   push:
     branches: [main, dev]
-    paths:
-      - "user-interfaces/**"
-      - "pallet/**"
-      - "runtime/**"
-      - "storage-interfaces/**"
-      - "provider-node/**"
-      - ".github/workflows/ui-e2e.yml"
   pull_request:
     branches: [main, dev]
-    paths:
-      - "user-interfaces/**"
-      - "pallet/**"
-      - "runtime/**"
-      - "storage-interfaces/**"
-      - "provider-node/**"
-      - ".github/workflows/ui-e2e.yml"
   workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
 jobs:
+  changes:
+    name: Detect e2e-relevant changes
+    runs-on: ubuntu-latest
+    outputs:
+      e2e: ${{ steps.filter.outputs.e2e }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3
+        id: filter
+        with:
+          filters: |
+            e2e:
+              - 'user-interfaces/**'
+              - 'pallet/**'
+              - 'runtime/**'
+              - 'storage-interfaces/**'
+              - 'provider-node/**'
+              - '.github/workflows/ui-e2e.yml'
+
   set-image:
+    needs: changes
+    if: needs.changes.outputs.e2e == 'true'
     uses: ./.github/workflows/set-image.yml
 
   ui-e2e:
-    name: UI E2E Tests (chain + provider + UIs)
+    name: UI E2E run
     runs-on: parity-large
     timeout-minutes: 60
-    needs: [set-image]
+    needs: [changes, set-image]
+    if: needs.changes.outputs.e2e == 'true'
     container:
       image: ${{ needs.set-image.outputs.CI_IMAGE }}
     steps:
@@ -187,3 +198,17 @@ jobs:
             /tmp/provider.log
             /tmp/zombie-*/*.log
             /tmp/zombie-*/**/*.log
+
+  # Aggregate gate — the check to require in branch protection. Always reports:
+  # green when the e2e job is skipped, red on real failure/cancellation.
+  ui-e2e-gate:
+    name: UI E2E Tests (chain + provider + UIs)
+    needs: [changes, set-image, ui-e2e]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - name: Decide outcome
+        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
+        run: |
+          echo "UI E2E failed or was cancelled"
+          exit 1

examples/papi/api.js

@@ -3,8 +3,10 @@ import { blake2b256 } from "@polkadot-labs/hdkd-helpers";
 import {
   hexToBytes,
   providerFetch,
+  READ_OPTS,
   requireOneEvent,
   submitTx,
+  submitTxFinalized,
   toHex,
 } from "./common.js";
 
@@ -134,7 +136,9 @@ export async function submitClientCheckpoint(api, client, provider, bucketId, ck
 }
 
 export async function challengeOffchain(api, client, provider, bucketId, upload) {
-  const result = await submitTx(
+  // Finalized: the challenge_id must survive to the respond that references it
+  // (a best-block reorg would invalidate it -> ChallengeNotFound).
+  const result = await submitTxFinalized(
     api.tx.StorageProvider.challenge_offchain({
       bucket_id: bucketId,
       provider: provider.address,
@@ -158,7 +162,8 @@ export async function challengeOffchain(api, client, provider, bucketId, upload)
 }
 
 export async function challengeCheckpoint(api, client, provider, bucketId, leafIndex) {
-  const result = await submitTx(
+  // Finalized: see challengeOffchain.
+  const result = await submitTxFinalized(
     api.tx.StorageProvider.challenge_checkpoint({
       bucket_id: bucketId,
       provider: provider.address,
@@ -547,8 +552,10 @@ export async function signCheckpointProposal(providerUrl, bucketId, duty, window
  * from chain state and fetching MMR + chunk proofs from the provider node.
  */
 export async function fetchChallengeProof(api, providerUrl, challengeId) {
+  // Best block: a finalized read would lag the just-created challenge.
   const challenges = await api.query.StorageProvider.Challenges.getValue(
-    challengeId.deadline
+    challengeId.deadline,
+    READ_OPTS
   );
   if (!challenges)
     throw new Error("No challenges at deadline " + challengeId.deadline);

examples/papi/bucket-membership.js

@@ -22,6 +22,7 @@ import {
   connect,
   makeSigner,
   printBucketMembers,
+  READ_OPTS,
   waitForBlockProduction,
   waitForChainReady,
   waitForNextBlock,
@@ -34,7 +35,8 @@ const READER_SEED = process.argv[5] || "//Charlie";
 
 async function verifyReverseIndex(api, member, bucketId, shouldContain) {
   const buckets = await api.query.StorageProvider.MemberBuckets.getValue(
-    member.address
+    member.address,
+    READ_OPTS
   );
   console.log("  MemberBuckets[%s] = %o", member.address, buckets);
   const has = buckets.some((id) => id === bucketId);