diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index b501c6c4..e6559441 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -33,17 +33,17 @@ jobs: TASKS=$(echo "$TASKS" | jq -c .) echo "runtime=$TASKS" >> $GITHUB_OUTPUT - integration-tests: - name: Integration Tests (${{ matrix.runtime.name }}) + # ── Build once ──────────────────────────────────────────────────────────── + # Build the shared artifacts once (provider binary + one compressed wasm per + # tested runtime) so both test jobs download instead of rebuilding. The test + # jobs still compile the fs/s3 examples and sc contracts via the shared cache. + build: + name: Build runs-on: parity-large timeout-minutes: 60 - needs: [set-image, runtime-matrix] + needs: [set-image] container: image: ${{ needs.set-image.outputs.CI_IMAGE }} - strategy: - fail-fast: false - matrix: - runtime: ${{ fromJSON(needs.runtime-matrix.outputs.runtime) }} steps: - name: Checkout sources uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -54,12 +54,62 @@ jobs: - name: Rust cache uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 with: - shared-key: "integration-tests-${{ matrix.runtime.name }}" + shared-key: "integration-tests-build" save-if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} - name: Install just run: cargo install just --locked || true + # One wasm per runtime the test jobs fan out on. + - name: Build all runtimes + run: | + for cmd in $(jq -r '.[] | select(.integration_tests == true) | .build_command' scripts/runtimes-matrix.json); do + echo "::group::just $cmd" + just "$cmd" + echo "::endgroup::" + done + + - name: Build provider + run: just build-provider + + # Upload only what the test jobs consume: provider binary + the final + # compressed wasm per runtime. NOT the full wbuild tree (~900 MiB Cargo + # scratch). Both paths sit under target/release, so the artifact is rooted + # there (LCA) and the download steps extract back into target/release. + - name: Upload build artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: build + retention-days: 1 + if-no-files-found: error + path: | + target/release/storage-provider-node + target/release/wbuild/*/*.compact.compressed.wasm + + # ── L0 / file-system / S3 / PAPI demos ────────────────────────────────────── + # Own chain + inmemory/disk providers; demos sequential on the shared chain. + # Runs in parallel with sc-integration-tests off the prebuilt artifact. + integration-tests: + name: Integration Tests (${{ matrix.runtime.name }}) + runs-on: parity-large + timeout-minutes: 30 + needs: [set-image, runtime-matrix, build] + container: + image: ${{ needs.set-image.outputs.CI_IMAGE }} + strategy: + fail-fast: false + matrix: + runtime: ${{ fromJSON(needs.runtime-matrix.outputs.runtime) }} + steps: + - name: Checkout sources + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + + - name: Load common environment variables via env file + run: cat .github/env >> $GITHUB_ENV + + - name: Install just + run: cargo install just --locked || true + - name: Cache Polkadot SDK binaries uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 id: sdk-cache @@ -87,19 +137,29 @@ jobs: if: steps.zombienet-cache.outputs.cache-hit != 'true' run: just zombienet_version=${{ env.ZOMBIENET_VERSION }} download-zombienet - - name: Log binary versions - run: | - .bin/zombienet --version - .bin/polkadot --version - .bin/polkadot-omni-node --version - - name: Setup Node.js uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 with: node-version: ${{ env.NODE_VERSION }} - - name: Build runtime and provider - run: just ${{ matrix.runtime.build_command }} && just build-provider + # Before the download so the cached target/ can't clobber it. save-if + # false — only the build job writes this cache. + - name: Rust cache + uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + shared-key: "integration-tests-build" + save-if: false + + # Extract into target/release so the artifact's contents land back at + # their original paths (chmod, chain_spec_command scripts, demos). + - name: Download build artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v6 + with: + name: build + path: target/release + + - name: Restore executable bits + run: chmod +x target/release/storage-provider-node - name: Start chain run: | @@ -199,15 +259,15 @@ jobs: /tmp/zombie-*/*-plain.json /tmp/zombie-*/*-raw.json - # Smart-contract demos run in a parallel job with their own chain + - # provider — the sequential time of L0 demos + fs/s3 lifecycles + sc demos - # was bumping against the 60-minute job timeout. Parallel splits the - # wall-clock roughly in half; SC tests only need //Alice as a provider. + # ── Smart-contract demos ───────────────────────────────────────────────────── + # Own chain + inmemory provider, parallel with integration-tests off the + # prebuilt artifact. Still installs solc/resolc and builds the example + # contracts here (cheap, not in the shared artifact). sc-integration-tests: name: SC Integration Tests (${{ matrix.runtime.name }}) runs-on: parity-large - timeout-minutes: 60 - needs: [set-image, runtime-matrix] + timeout-minutes: 30 + needs: [set-image, runtime-matrix, build] container: image: ${{ needs.set-image.outputs.CI_IMAGE }} strategy: @@ -221,12 +281,6 @@ jobs: - name: Load common environment variables via env file run: cat .github/env >> $GITHUB_ENV - - name: Rust cache - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 - with: - shared-key: "sc-integration-tests-${{ matrix.runtime.name }}" - save-if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} - - name: Install just run: cargo install just --locked || true @@ -262,6 +316,17 @@ jobs: with: node-version: ${{ env.NODE_VERSION }} + # Extract into target/release so the artifact's target/release-rooted + # contents land back at their original paths (see integration-tests job). + - name: Download build artifacts + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v6 + with: + name: build + path: target/release + + - name: Restore executable bits + run: chmod +x target/release/storage-provider-node + - name: Install solc and resolc run: | for bin_url in \ @@ -286,9 +351,6 @@ jobs: solc --version resolc --version - - name: Build runtime and provider - run: just ${{ matrix.runtime.build_command }} && just build-provider - - name: Build example contracts run: just build-contracts @@ -362,7 +424,7 @@ jobs: integration-tests-complete: name: Integration Tests - needs: [integration-tests, sc-integration-tests] + needs: [build, integration-tests, sc-integration-tests] if: always() runs-on: ubuntu-latest steps: diff --git a/.github/workflows/ui-e2e.yml b/.github/workflows/ui-e2e.yml index 06d6ed26..2f0789de 100644 --- a/.github/workflows/ui-e2e.yml +++ b/.github/workflows/ui-e2e.yml @@ -1,24 +1,13 @@ name: UI E2E +# No top-level `paths:` filter: the required gate must always report. The +# expensive e2e job skips (via `changes`) when nothing relevant is touched; a +# workflow-level filter would instead leave the required check pending forever. on: push: branches: [main, dev] - paths: - - "user-interfaces/**" - - "pallet/**" - - "runtime/**" - - "storage-interfaces/**" - - "provider-node/**" - - ".github/workflows/ui-e2e.yml" pull_request: branches: [main, dev] - paths: - - "user-interfaces/**" - - "pallet/**" - - "runtime/**" - - "storage-interfaces/**" - - "provider-node/**" - - ".github/workflows/ui-e2e.yml" workflow_dispatch: concurrency: @@ -26,14 +15,36 @@ concurrency: cancel-in-progress: true jobs: + changes: + name: Detect e2e-relevant changes + runs-on: ubuntu-latest + outputs: + e2e: ${{ steps.filter.outputs.e2e }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3 + id: filter + with: + filters: | + e2e: + - 'user-interfaces/**' + - 'pallet/**' + - 'runtime/**' + - 'storage-interfaces/**' + - 'provider-node/**' + - '.github/workflows/ui-e2e.yml' + set-image: + needs: changes + if: needs.changes.outputs.e2e == 'true' uses: ./.github/workflows/set-image.yml ui-e2e: - name: UI E2E Tests (chain + provider + UIs) + name: UI E2E run runs-on: parity-large timeout-minutes: 60 - needs: [set-image] + needs: [changes, set-image] + if: needs.changes.outputs.e2e == 'true' container: image: ${{ needs.set-image.outputs.CI_IMAGE }} steps: @@ -187,3 +198,17 @@ jobs: /tmp/provider.log /tmp/zombie-*/*.log /tmp/zombie-*/**/*.log + + # Aggregate gate — the check to require in branch protection. Always reports: + # green when the e2e job is skipped, red on real failure/cancellation. + ui-e2e-gate: + name: UI E2E Tests (chain + provider + UIs) + needs: [changes, set-image, ui-e2e] + if: always() + runs-on: ubuntu-latest + steps: + - name: Decide outcome + if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: | + echo "UI E2E failed or was cancelled" + exit 1 diff --git a/examples/papi/api.js b/examples/papi/api.js index b559b362..0731932b 100644 --- a/examples/papi/api.js +++ b/examples/papi/api.js @@ -3,8 +3,10 @@ import { blake2b256 } from "@polkadot-labs/hdkd-helpers"; import { hexToBytes, providerFetch, + READ_OPTS, requireOneEvent, submitTx, + submitTxFinalized, toHex, } from "./common.js"; @@ -134,7 +136,9 @@ export async function submitClientCheckpoint(api, client, provider, bucketId, ck } export async function challengeOffchain(api, client, provider, bucketId, upload) { - const result = await submitTx( + // Finalized: the challenge_id must survive to the respond that references it + // (a best-block reorg would invalidate it -> ChallengeNotFound). + const result = await submitTxFinalized( api.tx.StorageProvider.challenge_offchain({ bucket_id: bucketId, provider: provider.address, @@ -158,7 +162,8 @@ export async function challengeOffchain(api, client, provider, bucketId, upload) } export async function challengeCheckpoint(api, client, provider, bucketId, leafIndex) { - const result = await submitTx( + // Finalized: see challengeOffchain. + const result = await submitTxFinalized( api.tx.StorageProvider.challenge_checkpoint({ bucket_id: bucketId, provider: provider.address, @@ -547,8 +552,10 @@ export async function signCheckpointProposal(providerUrl, bucketId, duty, window * from chain state and fetching MMR + chunk proofs from the provider node. */ export async function fetchChallengeProof(api, providerUrl, challengeId) { + // Best block: a finalized read would lag the just-created challenge. const challenges = await api.query.StorageProvider.Challenges.getValue( - challengeId.deadline + challengeId.deadline, + READ_OPTS ); if (!challenges) throw new Error("No challenges at deadline " + challengeId.deadline); diff --git a/examples/papi/bucket-membership.js b/examples/papi/bucket-membership.js index a6129342..8862ae44 100644 --- a/examples/papi/bucket-membership.js +++ b/examples/papi/bucket-membership.js @@ -22,6 +22,7 @@ import { connect, makeSigner, printBucketMembers, + READ_OPTS, waitForBlockProduction, waitForChainReady, waitForNextBlock, @@ -34,7 +35,8 @@ const READER_SEED = process.argv[5] || "//Charlie"; async function verifyReverseIndex(api, member, bucketId, shouldContain) { const buckets = await api.query.StorageProvider.MemberBuckets.getValue( - member.address + member.address, + READ_OPTS ); console.log(" MemberBuckets[%s] = %o", member.address, buckets); const has = buckets.some((id) => id === bucketId); diff --git a/examples/papi/checkpoint-missed.js b/examples/papi/checkpoint-missed.js index 67262e20..22b19950 100644 --- a/examples/papi/checkpoint-missed.js +++ b/examples/papi/checkpoint-missed.js @@ -33,6 +33,7 @@ import { ensureSoleAcceptingProvider, makeSigner, parseProviderClientArgs, + READ_OPTS, sameAddress, waitForAgreementAcceptance, waitForBlock, @@ -86,7 +87,10 @@ async function main() { await waitForAgreementAcceptance(api, provider.address, bucketId); console.log(" Agreement accepted"); - const bucket = await api.query.StorageProvider.Buckets.getValue(bucketId); + const bucket = await api.query.StorageProvider.Buckets.getValue( + bucketId, + READ_OPTS + ); assert.ok( bucket.primary_providers.some((p) => sameAddress(p, provider.address)), "Provider should be primary after accept" @@ -104,7 +108,7 @@ async function main() { ); console.log("\n=== Step 3: Pick a window and let it elapse without a checkpoint ==="); - const head = Number(await api.query.System.Number.getValue()); + const head = Number(await api.query.System.Number.getValue(READ_OPTS)); const missedWindow = BigInt(Math.floor(head / WINDOW_INTERVAL)); // window_end = (missedWindow + 1) * interval ; need current_block > window_end const windowEnd = (Number(missedWindow) + 1) * WINDOW_INTERVAL; @@ -119,10 +123,12 @@ async function main() { console.log("\n=== Step 4: Record balances, then report_missed_checkpoint ==="); const providerBefore = await api.query.StorageProvider.Providers.getValue( - provider.address + provider.address, + READ_OPTS ); const reporterAcctBefore = await api.query.System.Account.getValue( - client.address + client.address, + READ_OPTS ); console.log(" Provider stake before: %s", providerBefore.stake.toString()); console.log( @@ -145,7 +151,8 @@ async function main() { console.log("\n=== Step 5: Verify slashing + reporter reward ==="); const providerAfter = await api.query.StorageProvider.Providers.getValue( - provider.address + provider.address, + READ_OPTS ); const stakeDelta = providerBefore.stake - providerAfter.stake; console.log(" Provider stake delta: %s", stakeDelta.toString()); @@ -157,7 +164,10 @@ async function main() { // LastCheckpointWindow is updated to prevent re-reporting. const lastWindow = - await api.query.StorageProvider.LastCheckpointWindow.getValue(bucketId); + await api.query.StorageProvider.LastCheckpointWindow.getValue( + bucketId, + READ_OPTS + ); assert.strictEqual( lastWindow, missedWindow, diff --git a/examples/papi/checkpoint-rewards.js b/examples/papi/checkpoint-rewards.js index fb9aa063..1dda0b66 100644 --- a/examples/papi/checkpoint-rewards.js +++ b/examples/papi/checkpoint-rewards.js @@ -37,6 +37,7 @@ import { ensureSoleAcceptingProvider, makeSigner, parseProviderClientArgs, + READ_OPTS, sameAddress, waitForAgreementAcceptance, waitForBlock, @@ -70,7 +71,7 @@ async function runProviderCheckpoint(api, papi, provider, bucketId) { // the call fails with InvalidCheckpointWindow. Require enough headroom // (a few blocks for inclusion under load) before submitting. const HEADROOM_BLOCKS = 15; - let currentBlock = Number(await api.query.System.Number.getValue()); + let currentBlock = Number(await api.query.System.Number.getValue(READ_OPTS)); let windowNum = Math.floor(currentBlock / WINDOW_INTERVAL); let nextWindowStart = (windowNum + 1) * WINDOW_INTERVAL; if (nextWindowStart - currentBlock < HEADROOM_BLOCKS) { @@ -81,7 +82,7 @@ async function runProviderCheckpoint(api, papi, provider, bucketId) { windowNum + 1 ); await waitForBlock(papi, nextWindowStart - 1); - currentBlock = Number(await api.query.System.Number.getValue()); + currentBlock = Number(await api.query.System.Number.getValue(READ_OPTS)); windowNum = Math.floor(currentBlock / WINDOW_INTERVAL); } const window = BigInt(windowNum); @@ -128,7 +129,8 @@ async function claimAndVerify(api, provider, bucketId, expectedReward) { // can drain a provider's pending rewards without scanning every bucket. const pending = await api.query.StorageProvider.CheckpointRewards.getValue( provider.address, - bucketId + bucketId, + READ_OPTS ); console.log(" Pending rewards before claim: %s", pending.toString()); assert.strictEqual( @@ -147,7 +149,8 @@ async function claimAndVerify(api, provider, bucketId, expectedReward) { const after = await api.query.StorageProvider.CheckpointRewards.getValue( provider.address, - bucketId + bucketId, + READ_OPTS ); assert.strictEqual( after, @@ -190,7 +193,10 @@ async function main() { await waitForAgreementAcceptance(api, provider.address, bucketId); console.log(" Agreement accepted (auto by provider node)"); - const bucket = await api.query.StorageProvider.Buckets.getValue(bucketId); + const bucket = await api.query.StorageProvider.Buckets.getValue( + bucketId, + READ_OPTS + ); assert.ok( bucket.primary_providers.some((p) => sameAddress(p, provider.address)), "Provider should be in primary_providers after agreement" @@ -224,7 +230,8 @@ async function main() { fundEvent.amount.toString() ); const balance = await api.query.StorageProvider.CheckpointPool.getValue( - bucketId + bucketId, + READ_OPTS ); console.log(" CheckpointPool[%s] = %s", bucketId, balance.toString()); assert.ok( diff --git a/examples/papi/common.js b/examples/papi/common.js index e638d1b1..2f5c61f8 100644 --- a/examples/papi/common.js +++ b/examples/papi/common.js @@ -113,7 +113,7 @@ export async function waitForBlockProduction(api, { timeoutSec = 300 } = {}) { const deadline = Date.now() + timeoutSec * 1_000; while (Date.now() < deadline) { try { - const n = await api.query.System.Number.getValue(); + const n = await api.query.System.Number.getValue(READ_OPTS); const blockNumber = typeof n === "bigint" ? Number(n) : Number(n); if (blockNumber > 0) { console.log(`✅ Chain producing blocks (head=#${blockNumber})`); @@ -149,7 +149,8 @@ export class NonceManager { } /** Read the on-chain nonce for `address` and build a manager from it. */ static async forAccount(api, address) { - const account = await api.query.System.Account.getValue(address); + // Best block: a finalized read lags and returns a stale (too-low) nonce. + const account = await api.query.System.Account.getValue(address, READ_OPTS); return new NonceManager(account.nonce); } } @@ -209,6 +210,14 @@ export async function waitForBlock(papi, target, { logEvery = 5 } = {}) { */ export const DEFAULT_TX_TIMEOUT_MS = 180_000; +/** + * Options for storage reads (`getValue` / `getEntries`). `submitTx` returns at + * in-block inclusion, so reads must target the best block to see a just-written + * value (the default finalized head lags by ~6 blocks). Single switch for the + * suite: flip to "finalized" here to trade read-your-writes for reorg-safety. + */ +export const READ_OPTS = { at: "best" }; + /** * Transaction "doneness" thresholds for `waitForTransaction`. Pick the * loosest one your caller can tolerate — earlier modes return faster but @@ -334,16 +343,14 @@ export async function waitForTransaction( } /** - * Sign + submit a transaction, assert it dispatched successfully, and return - * the PAPI result (`{ ok, events, block, ... }`). + * Sign + submit a tx, assert it dispatched OK, return the in-block event + * (`{ ok, events, ... }`). Resolves at inclusion, not finalization — ~6x faster + * per tx, and the demos read what they need from the events. PAPI doesn't throw + * on a dispatch error (only on an invalid tx), so the `ok === false` check in + * `waitForTransaction` is what surfaces failures here. * - * PAPI's bare `signAndSubmit` resolves with `{ ok, events, dispatchError }` - * and does NOT throw when dispatch fails — only when the tx is invalid (bad - * signature, low nonce, etc). Without this helper, a failed extrinsic looks - * indistinguishable from a successful one with no events, and the failure - * surfaces later as a confusing "Expected exactly 1 X event, got 0". - * - * Bounded by `timeoutMs` so a stuck mempool can't hang the example. + * Best blocks can be reorged, so for a tx whose effect a LATER tx references by + * id (e.g. create a challenge, then respond to it) use `submitTxFinalized`. */ export async function submitTx( tx, @@ -351,28 +358,28 @@ export async function submitTx( label, timeoutMs = DEFAULT_TX_TIMEOUT_MS ) { - let timer; - const timeoutPromise = new Promise((_, reject) => { - timer = setTimeout( - () => - reject( - new Error(`${label}: signAndSubmit timed out after ${timeoutMs}ms`) - ), - timeoutMs - ); - }); - let result; - try { - result = await Promise.race([tx.signAndSubmit(signer), timeoutPromise]); - } finally { - clearTimeout(timer); - } - if (!result.ok) { - throw new Error( - `${label} dispatch failed: ${formatDispatchError(result.dispatchError)}` - ); - } - return result; + return waitForTransaction(tx, signer, label, TX_MODE_IN_BLOCK, timeoutMs); +} + +/** + * Like `submitTx` but waits for FINALIZATION (~6 blocks slower). Use only when + * a later tx references this one's effect by id: a challenge id embeds its + * creation block, so an in-block creation that gets reorged makes the response + * fail with `ChallengeNotFound`. Finalizing pins the id. + */ +export async function submitTxFinalized( + tx, + signer, + label, + timeoutMs = DEFAULT_TX_TIMEOUT_MS +) { + return waitForTransaction( + tx, + signer, + label, + TX_MODE_FINALIZED_BLOCK, + timeoutMs + ); } export async function providerFetch(providerUrl, path, opts = {}) { @@ -404,7 +411,8 @@ export async function ensureProviderRegistered(api, provider, providerUrl, { // modules finish initialization before this function ever runs. const { registerProvider, updateProviderSettings } = await import("./api.js"); const existing = await api.query.StorageProvider.Providers.getValue( - provider.address + provider.address, + READ_OPTS ); if (!existing) { console.log(" Registering provider", provider.address); @@ -468,7 +476,8 @@ export async function waitForAgreementAcceptance( while (Date.now() < deadline) { const req = await api.query.StorageProvider.AgreementRequests.getValue( bucketId, - providerAddress + providerAddress, + READ_OPTS ); if (!req) return; await new Promise((r) => setTimeout(r, pollMs)); @@ -481,7 +490,10 @@ export async function waitForAgreementAcceptance( } export async function printBucketMembers(api, bucketId, label = "members") { - const bucket = await api.query.StorageProvider.Buckets.getValue(bucketId); + const bucket = await api.query.StorageProvider.Buckets.getValue( + bucketId, + READ_OPTS + ); console.log(` [${label}] bucket ${bucketId}:`); for (const m of bucket.members) { console.log(` - ${m.account} role=${fmtRole(m.role)}`); @@ -549,7 +561,7 @@ const KNOWN_DEV_SEEDS = [ */ export async function ensureSoleAcceptingProvider(api, keep) { const toggled = []; - const others = await api.query.StorageProvider.Providers.getEntries(); + const others = await api.query.StorageProvider.Providers.getEntries(READ_OPTS); for (const { keyArgs, value: info } of others) { const account = keyArgs[0]; if (sameAddress(account, keep.address)) continue; diff --git a/examples/papi/drive-lifecycle.js b/examples/papi/drive-lifecycle.js index e6a46f54..b104938d 100644 --- a/examples/papi/drive-lifecycle.js +++ b/examples/papi/drive-lifecycle.js @@ -29,6 +29,7 @@ import { fmtRole, makeSigner, printBucketMembers, + READ_OPTS, sameAddress, waitForAgreementAcceptance, waitForBlockProduction, @@ -43,7 +44,7 @@ const OWNER_SEED = process.argv[5] || "//Bob"; const MEMBER_SEED = process.argv[6] || "//Charlie"; async function printDriveInfo(api, owner, driveId, bucketId) { - const drive = await api.query.DriveRegistry.Drives.getValue(driveId); + const drive = await api.query.DriveRegistry.Drives.getValue(driveId, READ_OPTS); console.log(" owner =", drive.owner); console.log(" name =", drive.name ? drive.name.asText() : "(none)"); console.log(" max_capacity =", drive.max_capacity); @@ -52,7 +53,8 @@ async function printDriveInfo(api, owner, driveId, bucketId) { console.log(" payment locked =", drive.payment); const userDrives = await api.query.DriveRegistry.UserDrives.getValue( - owner.address + owner.address, + READ_OPTS ); console.log(" UserDrives[owner] =", userDrives); assert.ok( @@ -61,7 +63,8 @@ async function printDriveInfo(api, owner, driveId, bucketId) { ); const driveIdForBucket = await api.query.DriveRegistry.BucketToDrive.getValue( - bucketId + bucketId, + READ_OPTS ); assert.strictEqual( driveIdForBucket, @@ -71,7 +74,7 @@ async function printDriveInfo(api, owner, driveId, bucketId) { } async function getFree(api, who) { - const acc = await api.query.System.Account.getValue(who.address); + const acc = await api.query.System.Account.getValue(who.address, READ_OPTS); return acc.data.free; } @@ -159,7 +162,10 @@ async function main() { console.log(" owner free delta =", ownerAfter - ownerBefore); console.log(" provider free delta =", providerAfter - providerBefore); - const driveAfter = await api.query.DriveRegistry.Drives.getValue(driveId); + const driveAfter = await api.query.DriveRegistry.Drives.getValue( + driveId, + READ_OPTS + ); assert.strictEqual(driveAfter, undefined, "Drive should be gone after delete"); console.log("PASSED: drive lifecycle complete"); } catch (err) { diff --git a/examples/papi/full-flow.js b/examples/papi/full-flow.js index 7e161170..47043437 100644 --- a/examples/papi/full-flow.js +++ b/examples/papi/full-flow.js @@ -36,6 +36,8 @@ import { ensureProviderRegistered, makeSigner, parseProviderClientArgs, + READ_OPTS, + requireOneEvent, waitForBlock, waitForBlockProduction, waitForChainReady, @@ -52,14 +54,15 @@ const { async function setupAgreement(api, client, provider, bucketId) { const existing = await api.query.StorageProvider.StorageAgreements.getValue( bucketId, - provider.address + provider.address, + READ_OPTS ); if (existing) { console.log(" Agreement already exists"); return; } const maxBytes = 1_073_741_824n; // 1 GiB - const duration = 50; + const duration = 15; console.log( " Requesting agreement (%s), duration=%d blocks...", client.seed, @@ -99,21 +102,24 @@ async function uploadAndVerify(bucketId) { async function claimPaymentAfterExpiry(api, papi, provider, client, bucketId) { const agreement = await api.query.StorageProvider.StorageAgreements.getValue( bucketId, - provider.address + provider.address, + READ_OPTS ); const expiresAt = Number(agreement.expires_at); console.log(" Agreement expires at block:", expiresAt); - const freeBefore = (await api.query.System.Account.getValue(provider.address)) - .data.free; + const freeBefore = ( + await api.query.System.Account.getValue(provider.address, READ_OPTS) + ).data.free; console.log(" Provider balance before:", freeBefore.toString()); console.log(" Waiting for agreement to expire..."); await waitForBlock(papi, expiresAt); await endAgreement(api, client, provider, bucketId, "Pay"); - const freeAfter = (await api.query.System.Account.getValue(provider.address)) - .data.free; + const freeAfter = ( + await api.query.System.Account.getValue(provider.address, READ_OPTS) + ).data.free; const earned = freeAfter - freeBefore; console.log(" Provider balance after:", freeAfter.toString()); console.log(" Earned from agreement:", earned.toString()); @@ -121,18 +127,20 @@ async function claimPaymentAfterExpiry(api, papi, provider, client, bucketId) { console.log("PASSED: Provider received payment!"); } -function watchDefendedEvents(api) { - const events = []; - const sub = api.event.StorageProvider.ChallengeDefended.watch().subscribe( - (event) => { - console.log(" >> ChallengeDefended event:", { - deadline: event.payload.challenge_id.deadline, - index: event.payload.challenge_id.index, - }); - events.push(event); - } +// Read ChallengeDefended from the respond tx's own in-block events. A +// background `api.event...watch()` only sees finalized blocks, so a count taken +// right after responding would read 0. +function recordDefended(api, result, label) { + const event = requireOneEvent( + result.events, + api.event.StorageProvider.ChallengeDefended, + label ); - return { events, unsubscribe: () => sub.unsubscribe() }; + console.log(" >> ChallengeDefended event:", { + deadline: event.challenge_id.deadline, + index: event.challenge_id.index, + }); + return event; } async function main() { @@ -148,7 +156,7 @@ async function main() { await waitForChainReady(api); await waitForBlockProduction(api); await waitForNextBlock(papi); - const defended = watchDefendedEvents(api); + const defendedEvents = []; try { console.log("\n=== Step 1: Setup ==="); @@ -176,7 +184,15 @@ async function main() { console.log("\n=== Step 4: Respond to off-chain challenge ==="); const offchainProof = await fetchChallengeProof(api, PROVIDER_URL, offchainId); - await respondToChallenge(api, provider, offchainId, offchainProof); + const offchainResp = await respondToChallenge( + api, + provider, + offchainId, + offchainProof + ); + defendedEvents.push( + recordDefended(api, offchainResp, "ChallengeDefended (offchain)") + ); console.log(" Challenge defended"); console.log("\n=== Step 5: Submit checkpoint ==="); @@ -206,16 +222,26 @@ async function main() { PROVIDER_URL, checkpointId ); - await respondToChallenge(api, provider, checkpointId, checkpointProof); + const checkpointResp = await respondToChallenge( + api, + provider, + checkpointId, + checkpointProof + ); + defendedEvents.push( + recordDefended(api, checkpointResp, "ChallengeDefended (checkpoint)") + ); console.log(" Challenge defended"); console.log("\n=== Verifying challenge results ==="); - await new Promise((r) => setTimeout(r, 3000)); - console.log("ChallengeDefended events: %d (expected: 2)", defended.events.length); + console.log( + "ChallengeDefended events: %d (expected: 2)", + defendedEvents.length + ); assert.strictEqual( - defended.events.length, + defendedEvents.length, 2, - `Expected 2 ChallengeDefended events, got ${defended.events.length}` + `Expected 2 ChallengeDefended events, got ${defendedEvents.length}` ); console.log("PASSED: Both challenges were defended!"); @@ -226,7 +252,6 @@ async function main() { if (err.stack) console.error(err.stack); process.exitCode = 1; } finally { - defended.unsubscribe(); papi.destroy(); } } diff --git a/examples/papi/provider-discovery.js b/examples/papi/provider-discovery.js index 79197723..81d7653d 100644 --- a/examples/papi/provider-discovery.js +++ b/examples/papi/provider-discovery.js @@ -23,6 +23,7 @@ import { connect, + READ_OPTS, waitForBlockProduction, waitForChainReady, waitForNextBlock, @@ -70,7 +71,7 @@ function scoreProvider(info, req) { } async function fetchAndRankProviders(api, req) { - const entries = await api.query.StorageProvider.Providers.getEntries(); + const entries = await api.query.StorageProvider.Providers.getEntries(READ_OPTS); const ranked = entries.map(({ keyArgs, value }) => ({ address: keyArgs[0], info: value, diff --git a/examples/papi/s3-lifecycle.js b/examples/papi/s3-lifecycle.js index 3ae7ac5f..c25868ab 100644 --- a/examples/papi/s3-lifecycle.js +++ b/examples/papi/s3-lifecycle.js @@ -33,6 +33,7 @@ import { ensureSoleAcceptingProvider, makeSigner, parseProviderClientArgs, + READ_OPTS, sameAddress, toHex, waitForAgreementAcceptance, @@ -56,7 +57,10 @@ const OBJECT_KEYS = { }; async function listObjects(api, s3BucketId) { - const entries = await api.query.S3Registry.Objects.getEntries(s3BucketId); + const entries = await api.query.S3Registry.Objects.getEntries( + s3BucketId, + READ_OPTS + ); console.log(" bucket contains %d object(s):", entries.length); for (const { keyArgs, value } of entries) { const key = new TextDecoder().decode(keyArgs[1].asBytes()); @@ -68,7 +72,10 @@ async function listObjects(api, s3BucketId) { new TextDecoder().decode(value.content_type.asBytes()) ); } - const bucketInfo = await api.query.S3Registry.S3Buckets.getValue(s3BucketId); + const bucketInfo = await api.query.S3Registry.S3Buckets.getValue( + s3BucketId, + READ_OPTS + ); console.log( " bucket stats: object_count=%s total_size=%s", bucketInfo.object_count, @@ -182,7 +189,10 @@ async function main() { for (const key of Object.values(OBJECT_KEYS)) { await deleteObjectMetadata(api, client, s3BucketId, key); } - const afterDelete = await api.query.S3Registry.Objects.getEntries(s3BucketId); + const afterDelete = await api.query.S3Registry.Objects.getEntries( + s3BucketId, + READ_OPTS + ); assert.strictEqual(afterDelete.length, 0, "Expected empty bucket after delete"); console.log(" Bucket is empty"); diff --git a/examples/papi/sc-api.js b/examples/papi/sc-api.js index 39fe13eb..f0041a3d 100644 --- a/examples/papi/sc-api.js +++ b/examples/papi/sc-api.js @@ -13,7 +13,13 @@ import { Binary } from "@polkadot-api/substrate-bindings"; import { decodeEventLog, encodeFunctionData, keccak256 } from "viem"; -import { hexToBytes, requireOneEvent, submitTx, toHex } from "./common.js"; +import { + hexToBytes, + requireOneEvent, + submitTx, + submitTxFinalized, + toHex, +} from "./common.js"; /** * Compute the EVM-side H160 of a substrate account via `AccountId32Mapper`'s @@ -113,7 +119,14 @@ export async function callContract( signer, contractAddressBytes, data, - { value = 0n, gasLimit = DEFAULT_GAS_LIMIT, storageDepositLimit = DEFAULT_STORAGE_DEPOSIT_LIMIT } = {} + { + value = 0n, + gasLimit = DEFAULT_GAS_LIMIT, + storageDepositLimit = DEFAULT_STORAGE_DEPOSIT_LIMIT, + // Wait for finalization when a later tx references this call's effect by id + // (e.g. a precompile that creates a challenge). See submitTxFinalized. + finalized = false, + } = {} ) { const tx = api.tx.Revive.call({ dest: Binary.fromBytes(contractAddressBytes), @@ -122,7 +135,8 @@ export async function callContract( storage_deposit_limit: storageDepositLimit, data: Binary.fromBytes(data), }); - return submitTx(tx, signer.signer, "Revive.call"); + const submit = finalized ? submitTxFinalized : submitTx; + return submit(tx, signer.signer, "Revive.call"); } /** diff --git a/examples/papi/sc-coverage.js b/examples/papi/sc-coverage.js index 6329dd29..d905f4fd 100644 --- a/examples/papi/sc-coverage.js +++ b/examples/papi/sc-coverage.js @@ -37,6 +37,7 @@ import { hexToBytes, makeSigner, parseProviderClientArgs, + READ_OPTS, sameAddress, toHex, waitForBlockProduction, @@ -118,7 +119,8 @@ async function main() { // 1. createBucket ----------------------------------------------------- console.log("\n[1] IWeb3Storage.createBucket(1)"); - let nextBucketBefore = await api.query.StorageProvider.NextBucketId.getValue(); + let nextBucketBefore = + await api.query.StorageProvider.NextBucketId.getValue(READ_OPTS); let r = await callPrecompile(api, client, WEB3_STORAGE_ADDR, iWeb3, "createBucket", [1]); const created = assertEvent(r.events, "StorageProvider", "BucketCreated", "createBucket"); const bucketA = created.bucket_id; @@ -133,7 +135,10 @@ async function main() { 1, // Writer ]); assertEvent(r.events, "StorageProvider", "MemberSet", "setMember"); - let bucket = await api.query.StorageProvider.Buckets.getValue(bucketA); + let bucket = await api.query.StorageProvider.Buckets.getValue( + bucketA, + READ_OPTS + ); assert.ok( bucket.members.some((m) => sameAddress(m.account, member.address)), "Charlie should be in bucket members after setMember" @@ -200,7 +205,8 @@ async function main() { // MILLIUNIT = 1e9 atomic). 10% of `1MiB × 100k blocks × 1` ≈ 1e10 atomic, // comfortably above ED. console.log("\n[8] IWeb3Storage.createBucketWithStorage(1MiB, 100k blocks, maxPrice=10) [burn-sized]"); - nextBucketBefore = await api.query.StorageProvider.NextBucketId.getValue(); + nextBucketBefore = + await api.query.StorageProvider.NextBucketId.getValue(READ_OPTS); r = await callPrecompile( api, client, @@ -230,7 +236,8 @@ async function main() { // challenge. The agreement is left open and is not ended; settlement // happens through chain-driven expiry, not this test. console.log("\n[10] IWeb3Storage.createBucketWithStorage(2KiB, 100, maxPrice=10) [freeze/challenge target]"); - nextBucketBefore = await api.query.StorageProvider.NextBucketId.getValue(); + nextBucketBefore = + await api.query.StorageProvider.NextBucketId.getValue(READ_OPTS); r = await callPrecompile( api, client, @@ -251,12 +258,16 @@ async function main() { await submitClientCheckpoint(api, client, provider, bucketC, ck); console.log("\n[10a] IWeb3Storage.challengeCheckpoint(bucketC, provider, leafIdx, chunkIdx=0)"); - r = await callPrecompile(api, client, WEB3_STORAGE_ADDR, iWeb3, "challengeCheckpoint", [ - bucketC, - toHex(providerBytes32), - BigInt(upload.commit.leaf_indices[0]), - 0n, - ]); + // Finalized: respondToChallenge below references the challenge_id. + r = await callPrecompile( + api, + client, + WEB3_STORAGE_ADDR, + iWeb3, + "challengeCheckpoint", + [bucketC, toHex(providerBytes32), BigInt(upload.commit.leaf_indices[0]), 0n], + { finalized: true } + ); const challenge = assertEvent(r.events, "StorageProvider", "ChallengeCreated", "challengeCheckpoint"); console.log(" [substrate] respondToChallenge"); @@ -275,7 +286,8 @@ async function main() { // 12. createDrive ----------------------------------------------------- console.log("\n[12] IDriveRegistry.createDrive(\"cov\", 1MiB, 50 blocks, 1 UNIT, default-providers)"); - const nextDriveBefore = await api.query.DriveRegistry.NextDriveId.getValue(); + const nextDriveBefore = + await api.query.DriveRegistry.NextDriveId.getValue(READ_OPTS); r = await callPrecompile(api, client, DRIVE_REGISTRY_ADDR, iDrive, "createDrive", [ "cov", 1n << 20n, // 1 MiB diff --git a/examples/papi/sc-flow.js b/examples/papi/sc-flow.js index 113701e6..9c6ecfaa 100644 --- a/examples/papi/sc-flow.js +++ b/examples/papi/sc-flow.js @@ -39,6 +39,7 @@ import { ensureSoleAcceptingProvider, makeSigner, parseProviderClientArgs, + READ_OPTS, requireOneEvent, toHex, waitForBlockProduction, @@ -182,16 +183,18 @@ async function main() { bucketId, toHex(providerBytes32), ]); - const freeBefore = (await api.query.System.Account.getValue(provider.address)) - .data.free; + const freeBefore = ( + await api.query.System.Account.getValue(provider.address, READ_OPTS) + ).data.free; const endResult = await callContract( api, client, deployed.addressBytes, endData ); - const freeAfter = (await api.query.System.Account.getValue(provider.address)) - .data.free; + const freeAfter = ( + await api.query.System.Account.getValue(provider.address, READ_OPTS) + ).data.free; const earned = freeAfter - freeBefore; console.log(" provider earned:", earned.toString(), "atomic units"); assert.ok(earned > 0n, `expected provider to earn tokens, got ${earned}`); diff --git a/examples/papi/sc-token-gated.js b/examples/papi/sc-token-gated.js index fdc1c212..ce4fd858 100644 --- a/examples/papi/sc-token-gated.js +++ b/examples/papi/sc-token-gated.js @@ -32,6 +32,7 @@ import { hexToBytes, makeSigner, parseProviderClientArgs, + READ_OPTS, requireOneEvent, waitForBlockProduction, waitForChainReady, @@ -106,7 +107,7 @@ async function main() { // Bucket name: 3-63 chars, lowercase alphanumeric + hyphens. Append the // block number so the name is unique across reruns on the same chain // (`pallet_s3_registry` enforces global name uniqueness). - const blockHead = await api.query.System.Number.getValue(); + const blockHead = await api.query.System.Number.getValue(READ_OPTS); const bucketName = `cov-bucket-${Number(blockHead)}`; console.log(`\n[2/6] initialize{value: 5 UNIT}('${bucketName}', 1MiB, 50 blocks, …)`); const initData = encodeCall(abi, "initialize", [bucketName, 1n << 20n, 50, 2n * UNIT]);