From 96d71c5f7023deebc22d748f67432cca9c03e079 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Mon, 27 Jul 2026 15:50:51 -0700 Subject: [PATCH 01/15] test(e2e): consolidate priority 2 live seams Signed-off-by: Prekshi Vyas --- .github/workflows/e2e.yaml | 193 +------------ test/e2e-recommendations.test.ts | 10 +- test/e2e/README.md | 25 +- test/e2e/live/sandbox-rebuild.test.ts | 263 ----------------- .../e2e/live/upgrade-stale-sandbox-helpers.ts | 268 ------------------ test/e2e/live/upgrade-stale-sandbox.test.ts | 178 ------------ test/e2e/mock-parity.json | 14 - test/e2e/support/e2e-workflow.test.ts | 7 +- ...inference-switch-workflow-boundary.test.ts | 24 +- .../retired-selector-compatibility.test.ts | 9 +- ...unner-comparison-workflow-boundary.test.ts | 31 +- ...sted-hermes-swap-workflow-boundary.test.ts | 5 - test/pr-e2e-gate-shards.test.ts | 4 +- test/pr-e2e-gate-signal-shards.test.ts | 4 +- test/pr-e2e-gate.test.ts | 4 +- ...review-advisor-security-boundaries.test.ts | 2 +- test/pr-review-advisor.test.ts | 12 +- test/pr-risk-plan.test.ts | 6 +- tools/advisors/risk-plan.mts | 4 +- .../inference-switch-workflow-boundary.mts | 33 +-- tools/e2e/retired-selector-compatibility.mts | 29 ++ .../runner-comparison-workflow-boundary.mts | 15 +- .../trusted-hermes-swap-workflow-boundary.mts | 1 - tools/e2e/workflow-boundary.mts | 180 +----------- 24 files changed, 120 insertions(+), 1201 deletions(-) delete mode 100644 test/e2e/live/sandbox-rebuild.test.ts delete mode 100644 test/e2e/live/upgrade-stale-sandbox-helpers.ts delete mode 100644 test/e2e/live/upgrade-stale-sandbox.test.ts diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index e96a33edf0c..c72fed07d3d 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -390,7 +390,7 @@ jobs: retired-selector-compatibility: needs: generate-matrix - if: ${{ inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',credential-migration,') || contains(format(',{0},', inputs.jobs), ',credential-sanitization,') || contains(format(',{0},', inputs.jobs), ',diagnostics,') || contains(format(',{0},', inputs.jobs), ',docs-validation,') || contains(format(',{0},', inputs.jobs), ',gateway-drift-preflight,') || contains(format(',{0},', inputs.jobs), ',gateway-health-honest,') || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths,') || contains(format(',{0},', inputs.jobs), ',openshell-version-pin,') || contains(format(',{0},', inputs.jobs), ',ubuntu-repo-cli-smoke,')) }} + if: ${{ inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',credential-migration,') || contains(format(',{0},', inputs.jobs), ',credential-sanitization,') || contains(format(',{0},', inputs.jobs), ',diagnostics,') || contains(format(',{0},', inputs.jobs), ',docs-validation,') || contains(format(',{0},', inputs.jobs), ',gateway-drift-preflight,') || contains(format(',{0},', inputs.jobs), ',gateway-health-honest,') || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths,') || contains(format(',{0},', inputs.jobs), ',openshell-version-pin,') || contains(format(',{0},', inputs.jobs), ',sandbox-rebuild,') || contains(format(',{0},', inputs.jobs), ',ubuntu-repo-cli-smoke,') || contains(format(',{0},', inputs.jobs), ',upgrade-stale-sandbox,')) }} runs-on: ubuntu-latest timeout-minutes: 15 env: @@ -1704,12 +1704,6 @@ jobs: fail-fast: false matrix: include: - - mode: hosted - sandbox_name: e2e-hermes-inference-switch - switch_provider: nvidia-prod - switch_model: nvidia/nemotron-3-super-120b-a12b - switch_inference_api: openai-completions - switch_mock_anthropic: "0" - mode: anthropic sandbox_name: e2e-hermes-anthropic-inference-switch switch_provider: compatible-anthropic-endpoint @@ -1722,7 +1716,6 @@ jobs: E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-inference-switch/${{ matrix.mode }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" - NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_AGENT: "hermes" @@ -1771,9 +1764,6 @@ jobs: - name: Install OpenShell CLI run: bash scripts/install-openshell.sh - name: Run Hermes inference switch live Vitest test - env: - NVIDIA_API_KEY: ${{ matrix.mode == 'hosted' && secrets.NVIDIA_API_KEY || '' }} - NVIDIA_INFERENCE_API_KEY: ${{ matrix.mode == 'hosted' && secrets.NVIDIA_INFERENCE_API_KEY || '' }} run: | set -euo pipefail export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" @@ -3174,68 +3164,6 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh - sandbox-rebuild: - needs: generate-matrix - if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',sandbox-rebuild,') || contains(format(',{0},', inputs.targets), ',sandbox-rebuild,') }} - runs-on: ubuntu-latest - timeout-minutes: 90 - env: - E2E_JOB: "1" - E2E_TARGET_ID: "sandbox-rebuild" - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/sandbox-rebuild - NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js - NEMOCLAW_RUN_LIVE_E2E: "1" - NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" - NEMOCLAW_NON_INTERACTIVE: "1" - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" - OPENSHELL_GATEWAY: nemoclaw - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.checkout_repository || github.repository }} - ref: ${{ inputs.checkout_sha || github.sha }} - persist-credentials: false - - - *dockerhub-auth - - - name: Prepare E2E workspace - uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 - - - name: Install OpenShell - env: - NEMOCLAW_NON_INTERACTIVE: "1" - run: | - set -euo pipefail - env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh - - - name: Run sandbox rebuild live test - env: - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} - run: | - set -euo pipefail - export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" - if command -v openshell >/dev/null 2>&1; then - OPENSHELL_BIN="$(command -v openshell)" - elif [ -x "$HOME/.local/bin/openshell" ]; then - OPENSHELL_BIN="$HOME/.local/bin/openshell" - else - echo "::error::OpenShell CLI not found after install" - ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true - exit 1 - fi - export OPENSHELL_BIN - "$OPENSHELL_BIN" --version - npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/sandbox-rebuild.test.ts - - - name: Upload sandbox rebuild artifacts - if: always() - uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 - - - name: Clean up Docker auth - if: always() - shell: bash - run: bash .github/scripts/docker-auth-cleanup.sh - sandbox-rlimits-connect: needs: generate-matrix if: ${{ contains(format(',{0},', inputs.jobs), ',sandbox-rlimits-connect,') || contains(format(',{0},', inputs.targets), ',sandbox-rlimits-connect,') }} @@ -3408,68 +3336,6 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh - upgrade-stale-sandbox: - needs: generate-matrix - if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',upgrade-stale-sandbox,') || contains(format(',{0},', inputs.targets), ',upgrade-stale-sandbox,') }} - runs-on: ubuntu-latest - timeout-minutes: 55 - env: - E2E_JOB: "1" - E2E_TARGET_ID: "upgrade-stale-sandbox" - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/upgrade-stale-sandbox - NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js - NEMOCLAW_RUN_LIVE_E2E: "1" - NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" - NEMOCLAW_NON_INTERACTIVE: "1" - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" - NEMOCLAW_SANDBOX_NAME: "e2e-upgrade-stale" - OPENSHELL_GATEWAY: "nemoclaw" - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ inputs.checkout_repository || github.repository }} - ref: ${{ inputs.checkout_sha || github.sha }} - persist-credentials: false - - - *dockerhub-auth - - - name: Prepare E2E workspace - uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 - - - name: Install OpenShell CLI - run: | - set -euo pipefail - env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN \ - bash scripts/install-openshell.sh - - - name: Run upgrade stale sandbox live Vitest test - env: - NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} - run: | - set -euo pipefail - export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" - if command -v openshell >/dev/null 2>&1; then - OPENSHELL_BIN="$(command -v openshell)" - elif [ -x "$HOME/.local/bin/openshell" ]; then - OPENSHELL_BIN="$HOME/.local/bin/openshell" - else - echo "::error::OpenShell CLI not found after install" - ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true - exit 1 - fi - export OPENSHELL_BIN - "$OPENSHELL_BIN" --version - npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/upgrade-stale-sandbox.test.ts - - - name: Upload upgrade stale sandbox artifacts - if: always() - uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 - - - name: Clean up Docker auth - if: always() - shell: bash - run: bash .github/scripts/docker-auth-cleanup.sh - double-onboard: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',double-onboard,') || contains(format(',{0},', inputs.targets), ',double-onboard,') }} @@ -4973,12 +4839,6 @@ jobs: fail-fast: false matrix: include: - - mode: hosted - sandbox_name: e2e-openclaw-inference-switch - switch_provider: nvidia-prod - switch_model: nvidia/nemotron-3-super-120b-a12b - switch_inference_api: openai-completions - switch_mock_anthropic: "0" - mode: anthropic sandbox_name: e2e-openclaw-anthropic-inference-switch switch_provider: compatible-anthropic-endpoint @@ -4991,7 +4851,6 @@ jobs: E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-inference-switch/${{ matrix.mode }} NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_LIVE_E2E: "1" - NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" NEMOCLAW_AGENT: "openclaw" @@ -5015,13 +4874,9 @@ jobs: uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 - name: Run OpenClaw inference switch live test - # Preserves - # the real install.sh/OpenShell/Docker/live-provider boundary while + # Preserves the real install.sh/OpenShell/Docker/managed-proxy boundary while # Vitest owns the route, config, registry, inference.local, and agent # assertions. - env: - NVIDIA_API_KEY: ${{ matrix.mode == 'hosted' && secrets.NVIDIA_API_KEY || '' }} - NVIDIA_INFERENCE_API_KEY: ${{ matrix.mode == 'hosted' && secrets.NVIDIA_INFERENCE_API_KEY || '' }} run: | set -euo pipefail npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/openclaw-inference-switch.test.ts @@ -5047,7 +4902,7 @@ jobs: strategy: fail-fast: false matrix: - agent: [openclaw, hermes] + agent: [openclaw] env: E2E_JOB: "1" E2E_TARGET_ID: "bedrock-runtime-compatible-anthropic" @@ -5062,26 +4917,6 @@ jobs: NEMOCLAW_SANDBOX_NAME: e2e-bedrock-${{ matrix.agent }} OPENSHELL_GATEWAY: "nemoclaw" steps: - - id: trusted_hermes_swap - name: Provision trusted Hermes E2E swap - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.agent == 'hermes' }} - shell: /bin/bash --noprofile --norc -e -o pipefail {0} - env: - BASH_ENV: /dev/null - CHECKOUT_SHA: ${{ inputs.checkout_sha }} - DISPATCH_SHA: ${{ github.sha }} - ENV: /dev/null - EVENT_NAME: ${{ github.event_name }} - EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }} - LC_ALL: C - REF: ${{ github.ref }} - REPOSITORY: ${{ github.repository }} - RUNNER_ARCH_KIND: ${{ runner.arch }} - RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }} - RUNNER_OS_KIND: ${{ runner.os }} - WORKFLOW_SHA: ${{ github.workflow_sha }} - run: *trusted-hermes-e2e-swap - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ inputs.checkout_repository || github.repository }} @@ -5093,28 +4928,14 @@ jobs: - name: Prepare E2E workspace uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 - - name: Initialize runner comparison telemetry - if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && matrix.agent == 'hermes' }} - continue-on-error: true - shell: bash - run: npx tsx tools/e2e/runner-comparison.mts initialize - - name: Run Bedrock Runtime compatible Anthropic live test - # Direct Vitest coverage for - # Preserves the - # fake Bedrock Runtime endpoint, /etc/hosts mapping, source CLI - # onboard, OpenShell adapter route, agent-specific runtime probes, and - # leak-scan contract for both OpenClaw and Hermes. + # Preserves the fake Bedrock Runtime endpoint, /etc/hosts mapping, + # source CLI onboard, OpenShell adapter route, OpenClaw runtime probe, + # and leak-scan contract. run: | set -euo pipefail npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts - - name: Finalize runner comparison telemetry - if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && matrix.agent == 'hermes' }} - continue-on-error: true - shell: bash - run: npx tsx tools/e2e/runner-comparison.mts finalize - - name: Upload Bedrock Runtime compatible Anthropic artifacts if: always() uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 @@ -5794,11 +5615,9 @@ jobs: rebuild-openclaw, rebuild-hermes, rebuild-hermes-stale-base, - sandbox-rebuild, sandbox-rlimits-connect, overlayfs-autofix, state-backup-restore, - upgrade-stale-sandbox, openshell-gateway-upgrade, token-rotation, messaging-compatible-endpoint, diff --git a/test/e2e-recommendations.test.ts b/test/e2e-recommendations.test.ts index ad02fc75f77..142059c4390 100644 --- a/test/e2e-recommendations.test.ts +++ b/test/e2e-recommendations.test.ts @@ -130,8 +130,8 @@ describe("E2E recommendation normalizer", () => { ); expect(normalized.requiredTests.map((item) => item.id)).toEqual([ + "rebuild-openclaw", "state-backup-restore", - "upgrade-stale-sandbox", ]); expect(JSON.stringify(normalized)).not.toMatch( /forged|evil\.yaml|gh workflow run|--ref attacker/u, @@ -296,8 +296,8 @@ describe("E2E recommendation normalizer", () => { ); expect(normalized.required.map((item) => item.id)).toEqual([ + "rebuild-openclaw", "state-backup-restore", - "upgrade-stale-sandbox", ]); expect(normalized.required.every((item) => item.required)).toBe(true); expect(normalized.noTargetE2eReason).toBeNull(); @@ -330,7 +330,7 @@ describe("E2E recommendation normalizer", () => { required: [], optional: [ { - id: "upgrade-stale-sandbox", + id: "rebuild-openclaw", workflow: E2E_WORKFLOW, selectorType: "job", reason: "model called the regression optional", @@ -341,8 +341,8 @@ describe("E2E recommendation normalizer", () => { metadata({ changedFiles: ["src/lib/actions/upgrade-sandboxes.ts"] }), ); - expect(normalized.required.map((item) => item.id)).toContain("upgrade-stale-sandbox"); - expect(normalized.optional.map((item) => item.id)).not.toContain("upgrade-stale-sandbox"); + expect(normalized.required.map((item) => item.id)).toContain("rebuild-openclaw"); + expect(normalized.optional.map((item) => item.id)).not.toContain("rebuild-openclaw"); }); it("preserves indirectly selected Hermes jobs in the deterministic risk floor", () => { diff --git a/test/e2e/README.md b/test/e2e/README.md index df82ef5825b..19a1c79c722 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -83,6 +83,11 @@ The retired `hermes-dashboard` selector remains a compatibility alias for the manually selected `mock`, `internal-nvidia`, or `public-nvidia` inference mode. +PR-gate requests using the retired `sandbox-rebuild` and +`upgrade-stale-sandbox` job selectors run focused replacement tests through the +compatibility controller. `rebuild-openclaw` is the canonical live rebuild and +upgrade seam. + ## Larger-runner routing The larger-runner experiment is inactive while the configuration variable @@ -122,7 +127,7 @@ Cleanup removes it only after `swapoff` succeeds. Successful state is discarded with the ephemeral runner. The fallback covers agent-turn latency, Hermes inference switch and shields, -the Hermes Bedrock and stable MCP shards, the Hermes common-egress and channel +the Hermes stable MCP shard, the Hermes common-egress and channel stop/start shards, the dashboard-bearing `hermes-e2e` lane, `hermes-discord`, and Hermes security-posture tests. Rebuild lanes with workflow-managed swap, dedicated-runner lanes, `mcp-bridge-dev`, and non-Hermes shards do not use it. @@ -142,7 +147,7 @@ lanes: - `common-egress-agent`; - `hermes-e2e`, including dashboard coverage, and `hermes-discord`; -- both `hermes-inference-switch` modes; +- the Anthropic-compatible `hermes-inference-switch` mode; - `hermes-shields-config`; - the Hermes shards of `security-posture` and `channels-stop-start`; - `rebuild-hermes`; @@ -266,11 +271,10 @@ window. ### Runner comparison telemetry Trusted `main` runs without an alternate checkout SHA record runner-comparison -telemetry for 13 routed workflow lane identities / 16 +telemetry for 12 routed workflow lane identities / 14 concrete job executions. - `agent-turn-latency`, spanning its sequential OpenClaw and Hermes setup -- `bedrock-runtime-compatible-anthropic` with the `hermes` shard - `common-egress-agent` with the `openclaw-balanced-weather`, `openclaw-open-reference`, and `hermes-open-reference` shards - `rebuild-hermes` @@ -280,19 +284,18 @@ concrete job executions. - `channels-stop-start` with the `hermes` shard - `hermes-discord` - `hermes-e2e`, including dashboard coverage -- `hermes-inference-switch` with the `hosted` and `anthropic` modes +- `hermes-inference-switch` with the `anthropic` mode - `hermes-shields-config` - `security-posture` with the `hermes` shard -The three extra executions come from `common-egress-agent`, which runs three -scenario shards, and `hermes-inference-switch`, which runs both listed modes. +The two extra executions come from `common-egress-agent`, which runs three +scenario shards. The OpenClaw matrix entries for `mcp-bridge`, -`channels-stop-start`, `security-posture`, and -`bedrock-runtime-compatible-anthropic` are not instrumented. +`channels-stop-start`, and `security-posture` are not instrumented. The #7145 standard-versus-larger-runner cohort compares the same lane and equivalent workload while varying the runner class. The newly instrumented -`agent-turn-latency` and Bedrock Hermes lanes extend diagnostic coverage; this -change does not route them to a larger runner. +`agent-turn-latency` extends diagnostic coverage; this does not route it to a +larger runner. Each execution writes one bounded, ordered v2 time series to the canonical `runner-comparison.jsonl` ledger. It contains: diff --git a/test/e2e/live/sandbox-rebuild.test.ts b/test/e2e/live/sandbox-rebuild.test.ts deleted file mode 100644 index dc32f56a2db..00000000000 --- a/test/e2e/live/sandbox-rebuild.test.ts +++ /dev/null @@ -1,263 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; -import { resultText } from "../fixtures/clients/command.ts"; -import { expect, test } from "../fixtures/e2e-test.ts"; -import { - latestRebuildBackupDir, - listCredentialLeakPaths, - patchRegistrySandboxEntry, - readRegistrySandboxEntry, - restoreRegistryAndSession, - snapshotRegistryAndSession, -} from "../fixtures/phases/state-validation.ts"; -import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; - -// This dependent migration reuses the rebuild/state helper shape seeded by the -// OpenClaw rebuild anchor while keeping the contract focused: onboard a real -// sandbox, mark workspace state, force stale registry metadata, run the real -// `nemoclaw rebuild --yes`, then verify state preservation, registry -// refresh, and backup credential hygiene. - -const MARKER_FILE = "/sandbox/.openclaw/workspace/rebuild-marker.txt"; -const STALE_AGENT_VERSION = "0.0.1"; -const TEST_SANDBOX_PREFIX = "e2e-sandbox-rebuild"; -const SANDBOX_NAME = - process.env.NEMOCLAW_SANDBOX_NAME ?? - [TEST_SANDBOX_PREFIX, process.env.GITHUB_RUN_ID, process.env.GITHUB_RUN_ATTEMPT, process.pid] - .filter(Boolean) - .join("-"); -const TEST_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_TIMEOUT_SECONDS ?? 1_200) * 1_000; -const STATUS_TIMEOUT_MS = 60_000; -const ONBOARD_TIMEOUT_MS = TEST_TIMEOUT_MS; -const REBUILD_TIMEOUT_MS = TEST_TIMEOUT_MS; -const MARKER_CONTENT = `REBUILD_E2E_${Date.now()}`; - -function sandboxRebuildEnv(apiKey: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { - return { - ...buildAvailabilityProbeEnv(), - ...extra, - NVIDIA_INFERENCE_API_KEY: apiKey, - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, - }; -} - -function assertTestOwnedSandboxName(): void { - if (!SANDBOX_NAME.startsWith(TEST_SANDBOX_PREFIX)) { - throw new Error( - `sandbox-rebuild live test is destructive and only accepts sandbox names with prefix ${TEST_SANDBOX_PREFIX}; got ${SANDBOX_NAME}`, - ); - } -} - -async function bestEffortPreclean(run: () => Promise): Promise { - try { - await run(); - } catch { - // Cleanup remains best-effort so earlier lifecycle failures stay visible. - } -} - -// biome-ignore format: preserve legacy live-test body formatting so phase-only changes stay reviewable. -test( - "sandbox-rebuild: rebuild preserves marker state and refreshes registry metadata", - { - timeout: TEST_TIMEOUT_MS * 3, - meta: { - e2ePhases: [ - "confirm Docker and environment readiness", - "onboard the OpenClaw rebuild sandbox", - "write state and mark the registry stale", - "rebuild the stale sandbox", - "confirm marker preservation and registry refresh", - "scan the rebuild backup for credential leaks", - ], - }, - }, - async ({ - artifacts, - cleanup, - environment, - host, - lifecycle, - onboard, - progress, - sandbox, - secrets, - skip, - stateValidation, - }) => { - assertTestOwnedSandboxName(); - const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); - const dockerInfo = await host.command("docker", ["info"], { - artifactName: "prereq-docker-info", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, - }); - if (dockerInfo.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error( - `Docker is required for sandbox-rebuild live coverage: ${resultText(dockerInfo)}`, - ); - } - skip("Docker is required for sandbox-rebuild live coverage"); - } - - const ready = await environment.assertReady({ - platform: "ubuntu-local", - install: "repo-current", - runtime: "docker-running", - onboarding: "cloud-openclaw", - }); - - await artifacts.writeJson("contract.json", { - sandboxName: SANDBOX_NAME, - markerFile: MARKER_FILE, - staleAgentVersion: STALE_AGENT_VERSION, - preservedBoundaries: [ - "real nemoclaw onboard with Docker/OpenShell", - "openshell sandbox exec marker write/read", - "local registry stale agentVersion mutation", - "real nemoclaw rebuild --yes", - "backup credential leak scan under ~/.nemoclaw/rebuild-backups", - ], - }); - - const stateSnapshot = snapshotRegistryAndSession(); - const backupRoot = path.join( - process.env.HOME ?? os.homedir(), - ".nemoclaw", - "rebuild-backups", - SANDBOX_NAME, - ); - cleanup.trackDisposable(`restore NemoClaw state files for ${SANDBOX_NAME}`, () => { - restoreRegistryAndSession(stateSnapshot); - fs.rmSync(backupRoot, { recursive: true, force: true }); - }); - if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX !== "1") { - cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => - sandbox.cleanupSandbox(SANDBOX_NAME, { - artifactName: "cleanup-openshell-sandbox-delete", - env: buildAvailabilityProbeEnv(), - timeoutMs: 60_000, - }), - ); - cleanup.trackSandbox(host, SANDBOX_NAME, { - artifactName: "cleanup-nemoclaw-destroy", - env: buildAvailabilityProbeEnv(), - timeoutMs: 15 * 60_000, - }); - } - - await bestEffortPreclean(() => - onboard.destroySandbox(SANDBOX_NAME, "pre-cleanup-nemoclaw-destroy"), - ); - await bestEffortPreclean(() => - sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { - artifactName: "pre-cleanup-openshell-sandbox-delete", - env: buildAvailabilityProbeEnv(), - timeoutMs: 60_000, - }), - ); - - progress.phase("onboard the OpenClaw rebuild sandbox"); - const instance = await onboard.from(ready, { - sandboxName: SANDBOX_NAME, - timeoutMs: ONBOARD_TIMEOUT_MS, - }); - - const status = await host.nemoclaw([SANDBOX_NAME, "status"], { - artifactName: "phase-2-status-version-detection", - env: sandboxRebuildEnv(apiKey), - redactionValues: [apiKey], - timeoutMs: STATUS_TIMEOUT_MS, - }); - await artifacts.writeText("phase-2-status-output.txt", resultText(status)); - if (/Agent:.*v?\d+\.\d+/i.test(resultText(status))) { - await artifacts.writeJson("phase-2-status-version-summary.json", { versionVisible: true }); - } else { - await artifacts.writeJson("phase-2-status-version-summary.json", { - versionVisible: false, - note: "Legacy shell accepted first-run status output without cached version.", - }); - } - - progress.phase("write state and mark the registry stale"); - await stateValidationWriteMarker(); - - patchRegistrySandboxEntry(SANDBOX_NAME, { agentVersion: STALE_AGENT_VERSION }); - await artifacts.writeJson("phase-4-stale-registry-summary.json", { - sandboxName: SANDBOX_NAME, - agentVersion: readRegistrySandboxEntry(SANDBOX_NAME).agentVersion, - }); - - const staleStatus = await host.nemoclaw([SANDBOX_NAME, "status"], { - artifactName: "phase-4-status-stale-warning", - env: sandboxRebuildEnv(apiKey), - redactionValues: [apiKey], - timeoutMs: STATUS_TIMEOUT_MS, - }); - expect(staleStatus.exitCode, resultText(staleStatus)).toBe(0); - expect(resultText(staleStatus)).toMatch(/rebuild/i); - - progress.phase("rebuild the stale sandbox"); - await lifecycle.rebuildSandbox(instance, { - artifactName: "phase-5-nemoclaw-rebuild", - env: sandboxRebuildEnv(apiKey), - redactionValues: [apiKey], - timeoutMs: REBUILD_TIMEOUT_MS, - }); - await lifecycle.assertSandboxReadyAfterRebuild(instance, { - artifactNamePrefix: "phase-5-sandbox-ready-after-rebuild", - env: buildAvailabilityProbeEnv(), - attempts: 12, - delayMs: 5_000, - }); - - progress.phase("confirm marker preservation and registry refresh"); - await stateValidation.expectMarkerFileContent(instance, MARKER_FILE, MARKER_CONTENT, { - artifactName: "phase-6-read-marker-after-rebuild", - env: buildAvailabilityProbeEnv(), - timeoutMs: 60_000, - }); - - const updatedVersion = stateValidation.expectRegistryAgentVersionUpdated( - SANDBOX_NAME, - STALE_AGENT_VERSION, - ); - await artifacts.writeJson("phase-7-registry-version-summary.json", { - sandboxName: SANDBOX_NAME, - staleVersion: STALE_AGENT_VERSION, - updatedVersion, - }); - - progress.phase("scan the rebuild backup for credential leaks"); - const backupDir = latestRebuildBackupDir(SANDBOX_NAME); - const leaks = listCredentialLeakPaths(backupDir, { extraSecrets: [apiKey] }); - await artifacts.writeJson("phase-8-backup-credential-scan.json", { - backupDir: backupDir ?? null, - leaks, - note: backupDir ? undefined : "No backup directory found; former shell skipped this check.", - }); - expect(leaks, "backup files must not contain credential-shaped values").toEqual([]); - - async function stateValidationWriteMarker(): Promise { - await stateValidation.writeMarkerFile(instance, MARKER_FILE, MARKER_CONTENT, { - artifactName: "phase-3-write-marker", - env: buildAvailabilityProbeEnv(), - timeoutMs: 60_000, - }); - await stateValidation.expectMarkerFileContent(instance, MARKER_FILE, MARKER_CONTENT, { - artifactName: "phase-3-read-marker-before-rebuild", - env: buildAvailabilityProbeEnv(), - timeoutMs: 60_000, - }); - } - }, -); diff --git a/test/e2e/live/upgrade-stale-sandbox-helpers.ts b/test/e2e/live/upgrade-stale-sandbox-helpers.ts deleted file mode 100644 index e6981ec7aaf..00000000000 --- a/test/e2e/live/upgrade-stale-sandbox-helpers.ts +++ /dev/null @@ -1,268 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; -import type { CleanupRegistry } from "../fixtures/cleanup.ts"; -import type { HostCliClient } from "../fixtures/clients/host.ts"; -import { assertExitZero, resultText } from "../fixtures/clients/index.ts"; -import { type SandboxClient, validateSandboxName } from "../fixtures/clients/sandbox.ts"; -import { expect } from "../fixtures/e2e-test.ts"; -import { - readJsonFileOrFallback, - restoreFile, - snapshotFile, - writeJsonFile, -} from "../fixtures/file-state.ts"; -import { REPO_ROOT } from "../fixtures/paths.ts"; -import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; -import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts"; -import { createOldBaseBuildContext } from "./rebuild-openclaw-old-base-context.ts"; - -export { REPO_ROOT }; - -const TEST_SANDBOX_PREFIX = "e2e-upgrade-stale"; -export const SANDBOX_NAME = - process.env.NEMOCLAW_SANDBOX_NAME ?? - [TEST_SANDBOX_PREFIX, process.env.GITHUB_RUN_ID, process.env.GITHUB_RUN_ATTEMPT, process.pid] - .filter(Boolean) - .join("-"); -validateSandboxName(SANDBOX_NAME); -assertSafeSandboxName(); -export const OLD_OPENCLAW_VERSION = "2026.3.11"; -export const OLD_BASE_TAG = `nemoclaw-old-base:${SANDBOX_NAME.toLowerCase().replace(/[^a-z0-9_.-]+/g, "-")}`; -const REGISTRY_FILE = path.join(os.homedir(), ".nemoclaw", "sandboxes.json"); -const SESSION_FILE = path.join(os.homedir(), ".nemoclaw", "onboard-session.json"); -const INSTALL_ATTEMPTS = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true" ? 3 : 1; - -function assertSafeSandboxName(): void { - if (!SANDBOX_NAME.startsWith(TEST_SANDBOX_PREFIX)) { - throw new Error( - `upgrade-stale-sandbox live test is destructive and only accepts sandbox names with prefix ${TEST_SANDBOX_PREFIX}; got ${SANDBOX_NAME}`, - ); - } -} - -export function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { - return { - ...buildAvailabilityProbeEnv(), - ...extra, - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_REBUILD_VERBOSE: "1", - NEMOCLAW_RECREATE_SANDBOX: "1", - NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, - OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", - }; -} - -async function bestEffortPreclean(run: () => Promise): Promise { - try { - await run(); - } catch { - // Cleanup must not mask the primary assertion failure. - } -} - -export function writeStaleRegistryEntry(): void { - const session = readJsonFileOrFallback>(SESSION_FILE, {}); - const envProvider = - process.env.NEMOCLAW_PROVIDER === "custom" - ? "compatible-endpoint" - : process.env.NEMOCLAW_PROVIDER; - const provider = - typeof session.provider === "string" && session.provider - ? session.provider - : envProvider || "compatible-endpoint"; - const model = - (typeof session.model === "string" && session.model) || - process.env.NEMOCLAW_MODEL || - process.env.NEMOCLAW_COMPAT_MODEL || - "nvidia/nvidia/nemotron-3-ultra"; - const registry = readJsonFileOrFallback<{ - sandboxes?: Record>; - defaultSandbox?: string; - }>(REGISTRY_FILE, {}); - const dashboardPort = registry.sandboxes?.[SANDBOX_NAME]?.dashboardPort; - expect( - typeof dashboardPort === "number" && - Number.isInteger(dashboardPort) && - dashboardPort > 0 && - dashboardPort <= 65535, - "initial onboard must persist the dashboard port used by authoritative rebuild", - ).toBe(true); - registry.sandboxes = registry.sandboxes ?? {}; - registry.sandboxes[SANDBOX_NAME] = { - name: SANDBOX_NAME, - createdAt: new Date().toISOString(), - model, - provider, - gpuEnabled: false, - policies: [], - policyTier: null, - fromDockerfile: null, - dashboardPort, - agent: null, - agentVersion: OLD_OPENCLAW_VERSION, - }; - registry.defaultSandbox = SANDBOX_NAME; - writeJsonFile(REGISTRY_FILE, registry); - writeJsonFile(SESSION_FILE, { ...session, sandboxName: SANDBOX_NAME, status: "complete" }); -} - -export function assertDockerAvailable( - result: ShellProbeResult, - skip: (note?: string) => never, -): void { - result.exitCode === 0 || process.env.GITHUB_ACTIONS === "true" - ? undefined - : skip(`Docker is required for stale sandbox upgrade E2E: ${resultText(result)}`); - result.exitCode === 0 || - process.env.GITHUB_ACTIONS !== "true" || - (() => { - throw new Error(`Docker is required for stale sandbox upgrade E2E: ${resultText(result)}`); - })(); -} - -export function registerStateRestore(cleanup: Pick): void { - const registrySnapshot = snapshotFile(REGISTRY_FILE); - const sessionSnapshot = snapshotFile(SESSION_FILE); - cleanup.trackDisposable(`restore NemoClaw state files for ${SANDBOX_NAME}`, () => { - restoreFile(REGISTRY_FILE, registrySnapshot); - restoreFile(SESSION_FILE, sessionSnapshot); - }); -} - -export async function precleanStaleSandbox( - host: HostCliClient, - sandbox: SandboxClient, -): Promise { - await bestEffortPreclean(() => - host.nemoclaw([SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "cleanup-nemoclaw-destroy-upgrade-stale", - env: commandEnv(), - timeoutMs: 120_000, - }), - ); - await bestEffortPreclean(() => - sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { - artifactName: "cleanup-openshell-delete-upgrade-stale", - env: commandEnv(), - timeoutMs: 60_000, - }), - ); -} - -export async function cleanupOldImage(host: HostCliClient): Promise { - const result = await host.command("docker", ["image", "rm", "-f", OLD_BASE_TAG], { - artifactName: "cleanup-docker-image-upgrade-stale", - env: buildAvailabilityProbeEnv(), - timeoutMs: 60_000, - }); - if (result.exitCode === 0 || /No such image|image[^\n]*not found/i.test(resultText(result))) - return; - assertExitZero(result, `cleanup Docker image ${OLD_BASE_TAG}`); -} - -export async function installCurrentNemoclaw( - host: HostCliClient, - hosted: { apiKey: string; env: NodeJS.ProcessEnv }, -): Promise { - let install: ShellProbeResult | undefined; - for (let attempt = 1; attempt <= INSTALL_ATTEMPTS; attempt += 1) { - install = await host.command("bash", ["install.sh", "--non-interactive"], { - artifactName: - attempt === 1 - ? "phase-1-install-current-nemoclaw" - : `phase-1-install-current-nemoclaw-attempt-${attempt}`, - cwd: REPO_ROOT, - env: commandEnv(hosted.env), - redactionValues: [hosted.apiKey], - timeoutMs: 20 * 60_000, - }); - const retry = - install.exitCode !== 0 && - isTransientProviderValidationFailure(install) && - attempt < INSTALL_ATTEMPTS; - install.exitCode === 0 && (attempt = INSTALL_ATTEMPTS + 1); - retry && (await new Promise((resolve) => setTimeout(resolve, 10_000 * attempt))); - !retry && install.exitCode !== 0 && (attempt = INSTALL_ATTEMPTS + 1); - } - if (!install) throw new Error("install command did not run"); - return install; -} - -export function assertDeleteInstalledSandboxAllowed(result: ShellProbeResult): void { - result.exitCode === 0 || expect(result.exitCode, resultText(result)).toBe(1); - result.exitCode === 0 || - expect(resultText(result)).toMatch(/not found|does not exist|no sandbox/i); -} - -export async function buildOldOpenClawBase(host: HostCliClient): Promise { - const oldBaseBuildContext = createOldBaseBuildContext(); - try { - return await host.command( - "docker", - [ - "build", - "--build-arg", - `OPENCLAW_VERSION=${OLD_OPENCLAW_VERSION}`, - "--build-arg", - "NEMOCLAW_E2E_FIXTURE_LEGACY_OPENCLAW=1", - "-f", - path.join(REPO_ROOT, "Dockerfile.base"), - "-t", - OLD_BASE_TAG, - oldBaseBuildContext, - ], - { - artifactName: "phase-2-build-old-openclaw-base", - env: buildAvailabilityProbeEnv(), - timeoutMs: 20 * 60_000, - }, - ); - } finally { - fs.rmSync(oldBaseBuildContext, { recursive: true, force: true }); - } -} - -export function createFixtureDockerfile(cleanup: Pick): string { - const fixtureDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-old-openclaw-")); - cleanup.trackDisposable("remove stale sandbox fixture Dockerfile", () => - fs.rmSync(fixtureDir, { recursive: true, force: true }), - ); - const fixtureDockerfile = path.join(fixtureDir, "Dockerfile"); - fs.writeFileSync( - fixtureDockerfile, - [ - `FROM ${OLD_BASE_TAG}`, - "USER sandbox", - "WORKDIR /sandbox", - "RUN mkdir -p /sandbox/.openclaw/workspace /sandbox/.openclaw && echo '{}' > /sandbox/.openclaw/openclaw.json", - 'CMD ["/bin/bash"]', - "", - ].join("\n"), - ); - return fixtureDockerfile; -} - -export async function waitSandboxReady( - host: HostCliClient, - artifactName: string, -): Promise { - return await host.command( - "bash", - [ - "-lc", - `for _i in $(seq 1 30); do openshell sandbox list 2>/dev/null | grep -q '${SANDBOX_NAME}.*Ready' && exit 0; sleep 5; done; openshell sandbox list >&2; exit 1`, - ], - { artifactName, env: commandEnv(), timeoutMs: 180_000 }, - ); -} - -export function registeredStaleSandboxJson(): string { - return fs.readFileSync(REGISTRY_FILE, "utf8"); -} diff --git a/test/e2e/live/upgrade-stale-sandbox.test.ts b/test/e2e/live/upgrade-stale-sandbox.test.ts deleted file mode 100644 index 64392c7c5dd..00000000000 --- a/test/e2e/live/upgrade-stale-sandbox.test.ts +++ /dev/null @@ -1,178 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -/** - * Preserves the #1904 contract with real Docker/OpenShell/NemoClaw - * boundaries: onboard current NemoClaw, create an old OpenClaw sandbox from a - * real image, register stale sandbox metadata, prove upgrade-sandboxes detects - * the stale sandbox, rebuild it, and prove the stale version is gone. - */ - -import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; -import { resultText } from "../fixtures/clients/index.ts"; -import { expect, test } from "../fixtures/e2e-test.ts"; -import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; -import { - assertDeleteInstalledSandboxAllowed, - assertDockerAvailable, - buildOldOpenClawBase, - cleanupOldImage, - commandEnv, - createFixtureDockerfile, - installCurrentNemoclaw, - OLD_OPENCLAW_VERSION, - precleanStaleSandbox, - registeredStaleSandboxJson, - registerStateRestore, - SANDBOX_NAME, - waitSandboxReady, - writeStaleRegistryEntry, -} from "./upgrade-stale-sandbox-helpers.ts"; - -const LIVE_TIMEOUT_MS = 45 * 60_000; - -test("upgrade-sandboxes detects and rebuilds stale OpenClaw sandboxes (#1904)", { - timeout: LIVE_TIMEOUT_MS, - meta: { - e2ePhases: [ - "confirm Docker and install current NemoClaw", - "construct an old OpenClaw sandbox", - "register stale sandbox metadata", - "detect the stale sandbox", - "rebuild to the current OpenClaw runtime", - "confirm the upgrade check is clean", - ], - }, -}, async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { - const hosted = requireHostedInferenceConfig(secrets); - - await artifacts.target.declare({ - id: "upgrade-stale-sandbox", - boundary: "install.sh + Docker old base image + OpenShell sandbox create + NemoClaw rebuild", - sandboxName: SANDBOX_NAME, - oldOpenClawVersion: OLD_OPENCLAW_VERSION, - contracts: [ - "current NemoClaw install/onboard succeeds before stale fixture creation", - "an old OpenClaw base image can be created with the legacy version", - "a sandbox registered with old agentVersion is reported stale by upgrade-sandboxes --check", - "nemoclaw rebuild --yes upgrades the sandbox away from the old OpenClaw version", - "upgrade-sandboxes --check reports up-to-date after rebuild", - ], - }); - - const dockerInfo = await host.command("docker", ["info"], { - artifactName: "phase-0-docker-info", - env: buildAvailabilityProbeEnv(), - timeoutMs: 30_000, - }); - assertDockerAvailable(dockerInfo, skip); - - registerStateRestore(cleanup); - cleanup.trackDisposable("remove stale OpenClaw test image", () => cleanupOldImage(host)); - cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => - sandbox.cleanupSandbox(SANDBOX_NAME, { - artifactName: "cleanup-openshell-delete-upgrade-stale", - env: commandEnv(), - timeoutMs: 60_000, - }), - ); - cleanup.trackSandbox(host, SANDBOX_NAME, { - artifactName: "cleanup-nemoclaw-destroy-upgrade-stale", - env: commandEnv(), - timeoutMs: 120_000, - }); - await precleanStaleSandbox(host, sandbox); - - const install = await installCurrentNemoclaw(host, hosted); - expect(install.exitCode, resultText(install)).toBe(0); - - progress.phase("construct an old OpenClaw sandbox"); - const deleteInstalledSandbox = await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { - artifactName: "phase-2-delete-installed-sandbox", - env: commandEnv(), - timeoutMs: 120_000, - }); - assertDeleteInstalledSandboxAllowed(deleteInstalledSandbox); - - const buildOldBase = await buildOldOpenClawBase(host); - expect(buildOldBase.exitCode, resultText(buildOldBase)).toBe(0); - - const fixtureDockerfile = createFixtureDockerfile(cleanup); - const createOldSandbox = await sandbox.openshell( - [ - "sandbox", - "create", - "--name", - SANDBOX_NAME, - "--from", - fixtureDockerfile, - "--gateway", - "nemoclaw", - "--no-tty", - "--", - "true", - ], - { - artifactName: "phase-3-create-old-openclaw-sandbox", - env: commandEnv(), - timeoutMs: 15 * 60_000, - }, - ); - expect(createOldSandbox.exitCode, resultText(createOldSandbox)).toBe(0); - - const waitReady = await waitSandboxReady(host, "phase-3-wait-old-sandbox-ready"); - expect(waitReady.exitCode, resultText(waitReady)).toBe(0); - - const oldVersion = await sandbox.exec(SANDBOX_NAME, ["openclaw", "--version"], { - artifactName: "phase-3-old-openclaw-version", - env: commandEnv(), - timeoutMs: 60_000, - }); - expect(oldVersion.exitCode, resultText(oldVersion)).toBe(0); - expect(resultText(oldVersion)).toContain(OLD_OPENCLAW_VERSION); - - progress.phase("register stale sandbox metadata"); - writeStaleRegistryEntry(); - await artifacts.writeText("registered-stale-sandbox.json", registeredStaleSandboxJson()); - - progress.phase("detect the stale sandbox"); - const staleCheck = await host.nemoclaw(["upgrade-sandboxes", "--check"], { - artifactName: "phase-5-upgrade-sandboxes-check-stale", - env: commandEnv(hosted.env), - redactionValues: [hosted.apiKey], - timeoutMs: 120_000, - }); - expect(staleCheck.exitCode, resultText(staleCheck)).toBe(0); - expect(resultText(staleCheck)).toMatch(/stale|need upgrading/i); - expect(resultText(staleCheck)).not.toMatch(/up to date/i); - - progress.phase("rebuild to the current OpenClaw runtime"); - const rebuild = await host.nemoclaw([SANDBOX_NAME, "rebuild", "--yes"], { - artifactName: "phase-6-rebuild-stale-sandbox", - env: commandEnv(hosted.env), - redactionValues: [hosted.apiKey], - timeoutMs: 25 * 60_000, - }); - expect(rebuild.exitCode, resultText(rebuild)).toBe(0); - - const waitRebuiltReady = await waitSandboxReady(host, "phase-6-wait-rebuilt-sandbox-ready"); - expect(waitRebuiltReady.exitCode, resultText(waitRebuiltReady)).toBe(0); - - const newVersion = await sandbox.exec(SANDBOX_NAME, ["openclaw", "--version"], { - artifactName: "phase-6-new-openclaw-version", - env: commandEnv(), - timeoutMs: 60_000, - }); - expect(newVersion.exitCode, resultText(newVersion)).toBe(0); - expect(resultText(newVersion)).not.toContain(OLD_OPENCLAW_VERSION); - - progress.phase("confirm the upgrade check is clean"); - const cleanCheck = await host.nemoclaw(["upgrade-sandboxes", "--check"], { - artifactName: "phase-7-upgrade-sandboxes-check-clean", - env: commandEnv(hosted.env), - redactionValues: [hosted.apiKey], - timeoutMs: 120_000, - }); - expect(cleanCheck.exitCode, resultText(cleanCheck)).toBe(0); - expect(resultText(cleanCheck)).toMatch(/up to date/i); -}); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index d0162ee1f2b..b8464b31466 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -409,13 +409,6 @@ "test/e2e/support/e2e-clients.test.ts" ] }, - { - "live": "test/e2e/live/sandbox-rebuild.test.ts", - "fast": [ - "test/e2e/support/e2e-cleanup-resources.test.ts", - "test/e2e/support/e2e-clients.test.ts" - ] - }, { "live": "test/e2e/live/sandbox-survival.test.ts", "fast": [ @@ -472,13 +465,6 @@ "test/e2e/support/e2e-clients.test.ts" ] }, - { - "live": "test/e2e/live/upgrade-stale-sandbox.test.ts", - "fast": [ - "test/e2e/support/e2e-cleanup-resources.test.ts", - "test/e2e/support/e2e-clients.test.ts" - ] - }, { "live": "test/e2e/live/state-dir-guard-metadata.test.ts", "liveOnlyReason": "Runs the installed guard from both production images against a real bind-mounted Linux filesystem; mocks cannot reproduce kernel xattr copying, POSIX ACL mask synchronization, inode replacement, or allocated-byte behavior." diff --git a/test/e2e/support/e2e-workflow.test.ts b/test/e2e/support/e2e-workflow.test.ts index 52448406ace..31d22166db3 100644 --- a/test/e2e/support/e2e-workflow.test.ts +++ b/test/e2e/support/e2e-workflow.test.ts @@ -1087,12 +1087,15 @@ jobs: ); fs.writeFileSync( workflowPath, - workflow.replace(" || contains(format(',{0},', inputs.targets), ',sandbox-rebuild,')", ""), + workflow.replace( + " || contains(format(',{0},', inputs.targets), ',state-backup-restore,')", + "", + ), ); try { expect(validateE2eWorkflowBoundary(workflowPath)).toContain( - "free-standing inventory mapping sandbox-rebuild:sandbox-rebuild must match the workflow job selector", + "free-standing inventory mapping state-backup-restore:state-backup-restore must match the workflow job selector", ); } finally { fs.rmSync(tmp, { recursive: true, force: true }); diff --git a/test/e2e/support/inference-switch-workflow-boundary.test.ts b/test/e2e/support/inference-switch-workflow-boundary.test.ts index 327f154d7fb..b27328c51ed 100644 --- a/test/e2e/support/inference-switch-workflow-boundary.test.ts +++ b/test/e2e/support/inference-switch-workflow-boundary.test.ts @@ -14,7 +14,7 @@ import { import { validateE2eWorkflowBoundary } from "../../../tools/e2e/workflow-boundary.mts"; describe("inference switch workflow boundary", () => { - it("accepts hosted and Anthropic-compatible modes for both agents", () => { + it("accepts the canonical Anthropic-compatible mode for both agents", () => { expect(validateInferenceSwitchWorkflowBoundary()).toEqual([]); expect(validateE2eWorkflowBoundary()).toEqual([]); }); @@ -23,7 +23,7 @@ describe("inference switch workflow boundary", () => { const missingMode = readInferenceSwitchWorkflow(); missingMode.jobs["hermes-inference-switch"].strategy?.matrix?.include?.pop(); expect(validateInferenceSwitchWorkflow(missingMode)).toContain( - "hermes-inference-switch must run the exact hosted and Anthropic-compatible modes", + "hermes-inference-switch must run the canonical Anthropic-compatible mode", ); const failFast = readInferenceSwitchWorkflow(); @@ -71,32 +71,32 @@ describe("inference switch workflow boundary", () => { ); }); - it("uses a healthy hosted switch target and scopes its credentials to hosted mode", () => { + it("pins the local Anthropic switch target without hosted credentials", () => { const wrongTarget = readInferenceSwitchWorkflow(); - const hosted = wrongTarget.jobs["hermes-inference-switch"].strategy?.matrix?.include?.find( - (entry) => entry.mode === "hosted", + const anthropic = wrongTarget.jobs["hermes-inference-switch"].strategy?.matrix?.include?.find( + (entry) => entry.mode === "anthropic", ); - hosted!.switch_model = "nvidia/nvidia/nemotron-3-super-v3"; + anthropic!.switch_model = "nvidia/nvidia/nemotron-3-super-v3"; expect(validateInferenceSwitchWorkflow(wrongTarget)).toContain( - "hermes-inference-switch must run the exact hosted and Anthropic-compatible modes", + "hermes-inference-switch must run the canonical Anthropic-compatible mode", ); const unscopedSecret = readInferenceSwitchWorkflow(); const runStep = unscopedSecret.jobs["openclaw-inference-switch"].steps!.find( (step) => step.name === "Run OpenClaw inference switch live test", )!; - runStep.env!.NVIDIA_INFERENCE_API_KEY = "${{ secrets.NVIDIA_INFERENCE_API_KEY }}"; + runStep.env = { NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}" }; expect(validateInferenceSwitchWorkflow(unscopedSecret)).toContain( - "openclaw-inference-switch must expose NVIDIA_INFERENCE_API_KEY only to its hosted run step", + "openclaw-inference-switch must not expose NVIDIA_INFERENCE_API_KEY in its Anthropic-compatible mode", ); const unscopedPublicKey = readInferenceSwitchWorkflow(); const publicRunStep = unscopedPublicKey.jobs["hermes-inference-switch"].steps!.find( (step) => step.name === "Run Hermes inference switch live Vitest test", )!; - publicRunStep.env!.NVIDIA_API_KEY = "${{ secrets.NVIDIA_API_KEY }}"; + publicRunStep.env = { NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}" }; expect(validateInferenceSwitchWorkflow(unscopedPublicKey)).toContain( - "hermes-inference-switch must expose NVIDIA_API_KEY only to its hosted run step", + "hermes-inference-switch must not expose NVIDIA_API_KEY in its Anthropic-compatible mode", ); const publicKey = readInferenceSwitchWorkflow(); @@ -130,7 +130,7 @@ describe("inference switch workflow boundary", () => { try { writeFileSync(workflowPath, YAML.stringify(workflow)); expect(validateE2eWorkflowBoundary(workflowPath)).toContain( - "openclaw-inference-switch must run the exact hosted and Anthropic-compatible modes", + "openclaw-inference-switch must run the canonical Anthropic-compatible mode", ); } finally { rmSync(directory, { force: true, recursive: true }); diff --git a/test/e2e/support/retired-selector-compatibility.test.ts b/test/e2e/support/retired-selector-compatibility.test.ts index 1b51c56a9e4..28c3172e35d 100644 --- a/test/e2e/support/retired-selector-compatibility.test.ts +++ b/test/e2e/support/retired-selector-compatibility.test.ts @@ -17,7 +17,11 @@ const EXPECTED_SHA = "a".repeat(40); const PLAN_HASH = "b".repeat(64); const CORRELATION_ID = "123e4567-e89b-42d3-a456-426614174000"; const REPLACEMENT_FILES = [ + "src/lib/actions/sandbox/rebuild-finalization.test.ts", + "src/lib/actions/sandbox/rebuild-flow-helpers.test.ts", + "src/lib/sandbox/version.test.ts", "src/lib/security/credential-filter-secret-patterns.test.ts", + "test/cli/list-share-live-inference.test.ts", "test/credential-migration-reconciliation.test.ts", "test/package-contract/cli/debug-cli-command.test.ts", "test/package-contract/cli/public-cli-contracts.test.ts", @@ -26,6 +30,7 @@ const REPLACEMENT_FILES = [ "test/credentials.test.ts", "test/package-contract/onboard/invalid-nvidia-key.test.ts", "test/install-openshell-version-pin.test.ts", + "test/rebuild-stale-recovery.test.ts", ] as const; function workspace(): { artifactRoot: string; output: string; root: string } { @@ -92,8 +97,8 @@ describe("retired E2E selector compatibility", () => { expect(selected).toEqual([...RETIRED_CONTROLLER_SELECTOR_IDS].sort()); expect(commands).toEqual([ - "npx vitest run --project cli src/lib/security/credential-filter-secret-patterns.test.ts", - "npx vitest run --project integration test/credential-migration-reconciliation.test.ts test/credentials.test.ts test/gateway-drift-preflight.test.ts test/gateway-health-honest.test.ts", + "npx vitest run --project cli src/lib/actions/sandbox/rebuild-finalization.test.ts src/lib/actions/sandbox/rebuild-flow-helpers.test.ts src/lib/sandbox/version.test.ts src/lib/security/credential-filter-secret-patterns.test.ts", + "npx vitest run --project integration test/cli/list-share-live-inference.test.ts test/credential-migration-reconciliation.test.ts test/credentials.test.ts test/gateway-drift-preflight.test.ts test/gateway-health-honest.test.ts test/rebuild-stale-recovery.test.ts", "npx vitest run --project installer-integration test/install-openshell-version-pin.test.ts", "npx vitest run --project package-contract test/package-contract/cli/debug-cli-command.test.ts test/package-contract/cli/public-cli-contracts.test.ts test/package-contract/onboard/invalid-nvidia-key.test.ts", ]); diff --git a/test/e2e/support/runner-comparison-workflow-boundary.test.ts b/test/e2e/support/runner-comparison-workflow-boundary.test.ts index b0d67e558eb..926b4108d41 100644 --- a/test/e2e/support/runner-comparison-workflow-boundary.test.ts +++ b/test/e2e/support/runner-comparison-workflow-boundary.test.ts @@ -30,7 +30,6 @@ type Workflow = { const JOBS = [ "agent-turn-latency", - "bedrock-runtime-compatible-anthropic", "channels-stop-start", "common-egress-agent", "hermes-discord", @@ -61,7 +60,7 @@ function telemetrySteps(workflow: Workflow, jobId: string): WorkflowStep[] { } describe("runner comparison E2E workflow boundary (#7140)", () => { - it("accepts 13 routed workflow lane identities / 16 concrete job executions", () => { + it("accepts 12 routed workflow lane identities / 14 concrete job executions", () => { const workflow = loadWorkflow(); expect(validateRunnerComparisonWorkflowBoundary(workflow)).toEqual([]); @@ -83,20 +82,16 @@ describe("runner comparison E2E workflow boundary (#7140)", () => { const inferenceSwitchModes = workflow.jobs[ "hermes-inference-switch" ]!.strategy!.matrix!.include?.map((entry) => entry.mode); - expect(inferenceSwitchModes).toEqual(["hosted", "anthropic"]); + expect(inferenceSwitchModes).toEqual(["anthropic"]); const routedLanes = JOBS.length - 1 + mcpLanes; const concreteExecutions = routedLanes + commonEgressScenarios!.length - 1 + inferenceSwitchModes!.length - 1; - expect(routedLanes).toBe(13); - expect(concreteExecutions).toBe(16); + expect(routedLanes).toBe(12); + expect(concreteExecutions).toBe(14); }); - it("locks the matrix topology that produces sixteen concrete executions", () => { + it("locks the matrix topology that produces fourteen concrete executions", () => { const workflow = loadWorkflow(); - workflow.jobs["bedrock-runtime-compatible-anthropic"]!.strategy!.matrix!.agent = [ - "openclaw", - "openclaw", - ]; workflow.jobs["mcp-bridge"]!.strategy!.matrix!.agent = ["openclaw", "hermes", "hermes"]; workflow.jobs["channels-stop-start"]!.strategy!.matrix!.agent = ["openclaw", "openclaw"]; workflow.jobs["common-egress-agent"]!.strategy!.matrix!.include = [ @@ -109,23 +104,22 @@ describe("runner comparison E2E workflow boundary (#7140)", () => { { agent: "openclaw" }, ]; workflow.jobs["hermes-inference-switch"]!.strategy!.matrix!.include = [ - { mode: "hosted" }, - { mode: "hosted" }, + { mode: "anthropic" }, + { mode: "anthropic" }, ]; expect(validateRunnerComparisonWorkflow(workflow)).toEqual( expect.arrayContaining([ - "bedrock-runtime-compatible-anthropic matrix must contain exactly openclaw, hermes for runner comparison telemetry", "channels-stop-start matrix must contain exactly openclaw, hermes for runner comparison telemetry", "common-egress-agent matrix must contain exactly openclaw-balanced-weather, openclaw-open-reference, hermes-open-reference for runner comparison telemetry", "mcp-bridge matrix must contain exactly openclaw, hermes, deepagents for runner comparison telemetry", "security-posture matrix must contain exactly openclaw, hermes for runner comparison telemetry", - "hermes-inference-switch matrix must contain exactly hosted, anthropic for runner comparison telemetry", + "hermes-inference-switch matrix must contain exactly anthropic for runner comparison telemetry", ]), ); }); - it("rejects runner comparison consumers outside the twelve comparison jobs", () => { + it("rejects runner comparison consumers outside the eleven comparison jobs", () => { const workflow = loadWorkflow(); workflow.jobs["shields-config"]!.steps.push( structuredClone(telemetrySteps(workflow, "common-egress-agent")[0]!), @@ -242,11 +236,6 @@ describe("runner comparison E2E workflow boundary (#7140)", () => { "(matrix.agent == 'openclaw' || matrix.agent == 'hermes')", ); } - const bedrockComparison = step(workflow, "bedrock-runtime-compatible-anthropic", name); - bedrockComparison.if = bedrockComparison.if!.replace( - "matrix.agent == 'hermes'", - "(matrix.agent == 'openclaw' || matrix.agent == 'hermes')", - ); } expect(validateRunnerComparisonWorkflow(workflow)).toEqual( @@ -257,8 +246,6 @@ describe("runner comparison E2E workflow boundary (#7140)", () => { "channels-stop-start must use the exact always-run trusted finalize telemetry step", "security-posture must use the exact trusted initialize telemetry step", "security-posture must use the exact always-run trusted finalize telemetry step", - "bedrock-runtime-compatible-anthropic must use the exact trusted initialize telemetry step", - "bedrock-runtime-compatible-anthropic must use the exact always-run trusted finalize telemetry step", ]), ); }); diff --git a/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts b/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts index 9e2bc4bee18..a5535db4c23 100644 --- a/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts +++ b/test/e2e/support/trusted-hermes-swap-workflow-boundary.test.ts @@ -33,7 +33,6 @@ type SwapWorkflow = { const PROTECTED_JOBS = [ "agent-turn-latency", - "bedrock-runtime-compatible-anthropic", "channels-stop-start", "common-egress-agent", "hermes-discord", @@ -525,9 +524,6 @@ describe("trusted Hermes swap workflow boundary", () => { securityProvision.if = securityProvision.if!.replace(" && matrix.agent == 'hermes'", ""); securityProvision.env!.NVIDIA_INFERENCE_API_KEY = "${{ secrets.NVIDIA_INFERENCE_API_KEY }}"; - const bedrockProvision = trustedSwapStep(workflow, "bedrock-runtime-compatible-anthropic"); - bedrockProvision.run = "sudo bash tools/e2e/live-vitest-invocation.mts"; - const channelsProvision = trustedSwapStep(workflow, "channels-stop-start"); channelsProvision.if = channelsProvision.if!.replace(" && matrix.agent == 'hermes'", ""); @@ -557,7 +553,6 @@ describe("trusted Hermes swap workflow boundary", () => { "common-egress-agent trusted Hermes swap step must preserve the trusted main guard", "security-posture trusted Hermes swap step must preserve the trusted main guard", "security-posture trusted Hermes swap step must bind only trusted workflow, checkout, and runner identity", - "bedrock-runtime-compatible-anthropic trusted Hermes swap step must preserve the fixed privileged program", "mcp-bridge-dev job must not provision trusted Hermes swap", ]), ); diff --git a/test/pr-e2e-gate-shards.test.ts b/test/pr-e2e-gate-shards.test.ts index ecac915a4b3..a4870704397 100644 --- a/test/pr-e2e-gate-shards.test.ts +++ b/test/pr-e2e-gate-shards.test.ts @@ -27,7 +27,7 @@ function temporaryE2eWorkflow(source: string): string { describe("PR E2E shard policy", () => { it("derives Bedrock signal shards from the agent matrix (#6938)", () => { expect(expectedSignalShards(["bedrock-runtime-compatible-anthropic"])).toEqual({ - "bedrock-runtime-compatible-anthropic": ["openclaw", "hermes"], + "bedrock-runtime-compatible-anthropic": ["openclaw"], }); }); @@ -58,7 +58,7 @@ describe("PR E2E shard policy", () => { const nonStringField = temporaryE2eWorkflow( workflow.replace( - " - mode: hosted\n sandbox_name: e2e-hermes-inference-switch", + " - mode: anthropic\n sandbox_name: e2e-hermes-anthropic-inference-switch", " - mode: 1\n sandbox_name: e2e-hermes-inference-switch", ), ); diff --git a/test/pr-e2e-gate-signal-shards.test.ts b/test/pr-e2e-gate-signal-shards.test.ts index 71b87ded314..34e8d2338ac 100644 --- a/test/pr-e2e-gate-signal-shards.test.ts +++ b/test/pr-e2e-gate-signal-shards.test.ts @@ -32,8 +32,8 @@ describe("PR E2E signal shard policy", () => { "vllm-docker-storage": ["default"], }); expect(expectedSignalShards(["hermes-inference-switch", "openclaw-inference-switch"])).toEqual({ - "hermes-inference-switch": ["hosted", "anthropic"], - "openclaw-inference-switch": ["hosted", "anthropic"], + "hermes-inference-switch": ["anthropic"], + "openclaw-inference-switch": ["anthropic"], }); expect(expectedSignalShards(["openshell-gateway-upgrade"], undefined, [DCODE_TARGET])).toEqual({ "openshell-gateway-upgrade": [ diff --git a/test/pr-e2e-gate.test.ts b/test/pr-e2e-gate.test.ts index 0c0ea1a9313..4c6de108729 100644 --- a/test/pr-e2e-gate.test.ts +++ b/test/pr-e2e-gate.test.ts @@ -71,8 +71,8 @@ const BROAD_JOBS = [ "network-policy", "onboard-repair", "onboard-resume", + "rebuild-openclaw", "state-backup-restore", - "upgrade-stale-sandbox", ] as const; afterEach(() => { @@ -1397,7 +1397,7 @@ describe("PR E2E controller", () => { status: "in_progress", output: { title: "Running 13 E2E checks", - summary: expect.stringContaining("upgrade-stale-sandbox"), + summary: expect.stringContaining("rebuild-openclaw"), }, }); expect(checkUpdates[2]?.body).toMatchObject({ diff --git a/test/pr-review-advisor-security-boundaries.test.ts b/test/pr-review-advisor-security-boundaries.test.ts index 7e818bb62d3..3be278c9e5a 100644 --- a/test/pr-review-advisor-security-boundaries.test.ts +++ b/test/pr-review-advisor-security-boundaries.test.ts @@ -269,8 +269,8 @@ describe("PR review advisor security boundaries", () => { ); expect(result.e2e.coverage.requiredTests.map((item) => item.id)).toEqual([ + "rebuild-openclaw", "state-backup-restore", - "upgrade-stale-sandbox", ]); const normalized = JSON.stringify(result); const summary = renderSummary(result); diff --git a/test/pr-review-advisor.test.ts b/test/pr-review-advisor.test.ts index c706b105e6b..be20e52fdd2 100644 --- a/test/pr-review-advisor.test.ts +++ b/test/pr-review-advisor.test.ts @@ -228,7 +228,7 @@ describe("PR review advisor", () => { requiredTests: [], optionalTests: [ { - id: "upgrade-stale-sandbox", + id: "rebuild-openclaw", reason: "The model tried to downgrade the deterministic job.", }, ], @@ -245,7 +245,7 @@ describe("PR review advisor", () => { ], optional: [ { - id: "upgrade-stale-sandbox", + id: "rebuild-openclaw", workflow: "e2e.yaml", selectorType: "job", reason: "The model tried to downgrade the deterministic job.", @@ -259,13 +259,13 @@ describe("PR review advisor", () => { ); expect(result.e2e.coverage.requiredTests.map((test) => test.id)).toEqual([ + "rebuild-openclaw", "state-backup-restore", - "upgrade-stale-sandbox", ]); expect(result.e2e.coverage.optionalTests).toEqual([]); expect(result.e2e.targets.required.map((target) => target.id)).toEqual([ + "rebuild-openclaw", "state-backup-restore", - "upgrade-stale-sandbox", ]); expect(result.e2e.targets.optional).toEqual([]); expect(result.e2e.targets.required[1]).not.toHaveProperty("dispatchCommand"); @@ -276,9 +276,9 @@ describe("PR review advisor", () => { const comment = buildComment({ summary: renderSummary(result), result }); expect(comment).toContain("### E2E guidance"); expect(comment).toContain("Advisory only. E2E / PR Gate selects and runs jobs independently."); - expect(comment).toContain("upgrade-stale-sandbox"); + expect(comment).toContain("rebuild-openclaw"); expect(comment).toContain("**Recommended E2E:**"); - expect(comment.match(/upgrade-stale-sandbox<\/code>/gu)).toHaveLength(1); + expect(comment.match(/rebuild-openclaw<\/code>/gu)).toHaveLength(1); expect(comment).not.toContain("Recommended coverage"); expect(comment).not.toContain("Recommended selectors"); expect(comment).not.toContain("rm -rf"); diff --git a/test/pr-risk-plan.test.ts b/test/pr-risk-plan.test.ts index 636ca5db373..7cfe464ee94 100644 --- a/test/pr-risk-plan.test.ts +++ b/test/pr-risk-plan.test.ts @@ -28,7 +28,7 @@ describe("deterministic PR risk plan", () => { const second = plan("src/lib/onboard.ts", "src/lib/state/registry.ts"); expect(first).toEqual(second); - expect(first.version).toBe(7); + expect(first.version).toBe(8); expect(first.headSha).toBe(HEAD_SHA); expect(first.planHash).toMatch(/^[a-f0-9]{64}$/u); expect(first.changedFiles).toEqual(["src/lib/onboard.ts", "src/lib/state/registry.ts"]); @@ -233,7 +233,7 @@ describe("deterministic PR risk plan", () => { { file: "src/lib/actions/upgrade-sandboxes.ts", family: "upgrade-rebuild", - jobs: ["state-backup-restore", "upgrade-stale-sandbox"], + jobs: ["rebuild-openclaw", "state-backup-restore"], }, { file: "src/lib/actions/sandbox/agents/apply.ts", @@ -421,8 +421,8 @@ describe("deterministic PR risk plan", () => { "network-policy", "onboard-repair", "onboard-resume", + "rebuild-openclaw", "state-backup-restore", - "upgrade-stale-sandbox", ]); }); diff --git a/tools/advisors/risk-plan.mts b/tools/advisors/risk-plan.mts index 6adbad8c721..6fc6a6d22a7 100644 --- a/tools/advisors/risk-plan.mts +++ b/tools/advisors/risk-plan.mts @@ -3,7 +3,7 @@ import { createHash } from "node:crypto"; -export const RISK_PLAN_VERSION = 7 as const; +export const RISK_PLAN_VERSION = 8 as const; export const PR_E2E_TYPED_TARGET_IDS = ["ubuntu-repo-cloud-langchain-deepagents-code"] as const; @@ -162,7 +162,7 @@ export const RISK_RULES: readonly RiskRule[] = [ summary: "Upgrade, rebuild, snapshot, and restore operations must preserve user state while replacing stale runtime state.", tier: 2, - requiredJobs: ["upgrade-stale-sandbox", "state-backup-restore"], + requiredJobs: ["rebuild-openclaw", "state-backup-restore"], invariants: [ "host and in-sandbox runtime versions agree after mutation", "credentials, policy, messaging, and workspace state survive intended preservation paths", diff --git a/tools/e2e/inference-switch-workflow-boundary.mts b/tools/e2e/inference-switch-workflow-boundary.mts index 662b75b1fce..3ca24e262fa 100644 --- a/tools/e2e/inference-switch-workflow-boundary.mts +++ b/tools/e2e/inference-switch-workflow-boundary.mts @@ -36,7 +36,6 @@ export type InferenceSwitchWorkflow = { type JobSpec = { agent: "hermes" | "openclaw"; job: string; - runStep: string; scenario: string; uploadStep: string; }; @@ -45,14 +44,12 @@ const JOBS: JobSpec[] = [ { agent: "hermes", job: "hermes-inference-switch", - runStep: "Run Hermes inference switch live Vitest test", scenario: "hermes-inference-switch", uploadStep: "Upload Hermes inference switch artifacts", }, { agent: "openclaw", job: "openclaw-inference-switch", - runStep: "Run OpenClaw inference switch live test", scenario: "openclaw-inference-switch", uploadStep: "Upload OpenClaw inference switch artifacts", }, @@ -60,14 +57,6 @@ const JOBS: JobSpec[] = [ function expectedModes(agent: JobSpec["agent"]): Array> { return [ - { - mode: "hosted", - sandbox_name: `e2e-${agent}-inference-switch`, - switch_provider: "nvidia-prod", - switch_model: "nvidia/nemotron-3-super-120b-a12b", - switch_inference_api: "openai-completions", - switch_mock_anthropic: "0", - }, { mode: "anthropic", sandbox_name: `e2e-${agent}-anthropic-inference-switch`, @@ -87,7 +76,7 @@ function validateJob(errors: string[], spec: JobSpec, job: WorkflowJob): void { errors.push(`${spec.job} mode matrix must not fail fast`); } if (!isDeepStrictEqual(job.strategy?.matrix?.include, expectedModes(spec.agent))) { - errors.push(`${spec.job} must run the exact hosted and Anthropic-compatible modes`); + errors.push(`${spec.job} must run the canonical Anthropic-compatible mode`); } const requiredEnv: Record = { @@ -110,21 +99,17 @@ function validateJob(errors: string[], spec: JobSpec, job: WorkflowJob): void { if (job.env?.NVIDIA_API_KEY !== undefined) { errors.push(`${spec.job} must not expose NVIDIA_API_KEY at job scope`); } - const runStep = job.steps?.find((step) => step.name === spec.runStep); - const hostedSecret = "${{ matrix.mode == 'hosted' && secrets.NVIDIA_INFERENCE_API_KEY || '' }}"; - if (runStep?.env?.NVIDIA_INFERENCE_API_KEY !== hostedSecret) { - errors.push(`${spec.job} must expose NVIDIA_INFERENCE_API_KEY only to its hosted run step`); - } - const hostedPublicSecret = "${{ matrix.mode == 'hosted' && secrets.NVIDIA_API_KEY || '' }}"; - if (runStep?.env?.NVIDIA_API_KEY !== hostedPublicSecret) { - errors.push(`${spec.job} must expose NVIDIA_API_KEY only to its hosted run step`); + if (job.env?.NEMOCLAW_E2E_USE_HOSTED_INFERENCE !== undefined) { + errors.push(`${spec.job} must not enable hosted inference for its Anthropic-compatible mode`); } for (const step of job.steps ?? []) { - if (step !== runStep && step.env?.NVIDIA_INFERENCE_API_KEY !== undefined) { - errors.push(`${spec.job} must expose NVIDIA_INFERENCE_API_KEY only to its run step`); + if (step.env?.NVIDIA_INFERENCE_API_KEY !== undefined) { + errors.push( + `${spec.job} must not expose NVIDIA_INFERENCE_API_KEY in its Anthropic-compatible mode`, + ); } - if (step !== runStep && step.env?.NVIDIA_API_KEY !== undefined) { - errors.push(`${spec.job} must expose NVIDIA_API_KEY only to its run step`); + if (step.env?.NVIDIA_API_KEY !== undefined) { + errors.push(`${spec.job} must not expose NVIDIA_API_KEY in its Anthropic-compatible mode`); } } diff --git a/tools/e2e/retired-selector-compatibility.mts b/tools/e2e/retired-selector-compatibility.mts index 26042ce84c4..6ecdd8d32ce 100644 --- a/tools/e2e/retired-selector-compatibility.mts +++ b/tools/e2e/retired-selector-compatibility.mts @@ -100,6 +100,22 @@ const REPLACEMENTS: Readonly> = }, ], }, + "sandbox-rebuild": { + legacyFile: "test/e2e/live/sandbox-rebuild.test.ts", + tests: [ + { + files: [ + "src/lib/actions/sandbox/rebuild-finalization.test.ts", + "src/lib/actions/sandbox/rebuild-flow-helpers.test.ts", + ], + project: "cli", + }, + { + files: ["test/rebuild-stale-recovery.test.ts"], + project: "integration", + }, + ], + }, "ubuntu-repo-cli-smoke": { legacyFile: "test/e2e/live/ubuntu-repo-cli-smoke.test.ts", tests: [ @@ -109,6 +125,19 @@ const REPLACEMENTS: Readonly> = }, ], }, + "upgrade-stale-sandbox": { + legacyFile: "test/e2e/live/upgrade-stale-sandbox.test.ts", + tests: [ + { + files: ["src/lib/sandbox/version.test.ts"], + project: "cli", + }, + { + files: ["test/cli/list-share-live-inference.test.ts"], + project: "integration", + }, + ], + }, }; export function selectedRetiredControllerJobs(options: { diff --git a/tools/e2e/runner-comparison-workflow-boundary.mts b/tools/e2e/runner-comparison-workflow-boundary.mts index a212cd1d34f..1b5a4bcacc0 100644 --- a/tools/e2e/runner-comparison-workflow-boundary.mts +++ b/tools/e2e/runner-comparison-workflow-boundary.mts @@ -26,10 +26,6 @@ const COMPARISON_JOBS: ReadonlyMap"}'`; - const stepEnv = asRecord(step.env); - if (step.name !== "Run sandbox rebuild live test") { - requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_INFERENCE_API_KEY"); - } - if (step.name !== "Authenticate to Docker Hub") { - requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); - requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); - requireNoDockerHubAuthInRun(errors, stepName, stringValue(step.run)); - } - requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "GITHUB_TOKEN"); - } - - const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); - if (!checkout) errors.push("sandbox-rebuild job missing checkout step"); - requireFullShaAction(errors, checkout, "sandbox-rebuild checkout"); - if (asRecord(checkout?.with)["persist-credentials"] !== false) { - errors.push("sandbox-rebuild checkout step must set persist-credentials=false"); - } - - const installOpenShell = requireJobStep(errors, jobName, steps, "Install OpenShell"); - requireRunContains(errors, installOpenShell, "bash scripts/install-openshell.sh"); - requireRunContains(errors, installOpenShell, "env -u DOCKER_CONFIG"); - requireRunContains(errors, installOpenShell, "-u DOCKERHUB_USERNAME"); - requireRunContains(errors, installOpenShell, "-u DOCKERHUB_TOKEN"); - requireRunContains(errors, installOpenShell, "-u NVIDIA_INFERENCE_API_KEY"); - requireRunContains(errors, installOpenShell, "-u GITHUB_TOKEN"); - - const runVitest = requireJobStep(errors, jobName, steps, "Run sandbox rebuild live test"); - const runVitestEnv = asRecord(runVitest?.env); - if (runVitestEnv.NVIDIA_INFERENCE_API_KEY !== "${{ secrets.NVIDIA_INFERENCE_API_KEY }}") { - errors.push("sandbox-rebuild step must receive NVIDIA_INFERENCE_API_KEY from secrets"); - } - requireRunContains(errors, runVitest, "OPENSHELL_BIN"); - requireRunContains(errors, runVitest, "tools/e2e/live-vitest-invocation.mts run --test-path"); - requireRunContains(errors, runVitest, "test/e2e/live/sandbox-rebuild.test.ts"); -} - function validateStateBackupRestoreJob(errors: string[], jobs: WorkflowRecord): void { const jobName = "state-backup-restore"; const targetName = "state-backup-restore"; @@ -2150,97 +2071,6 @@ function validateStateBackupRestoreJob(errors: string[], jobs: WorkflowRecord): requireRunContains(errors, runVitest, "test/e2e/live/state-backup-restore.test.ts"); } -function validateUpgradeStaleSandboxJob(errors: string[], jobs: WorkflowRecord): void { - const jobName = "upgrade-stale-sandbox"; - const targetName = "upgrade-stale-sandbox"; - const job = asRecord(jobs[jobName]); - if (Object.keys(job).length === 0) { - errors.push("workflow missing upgrade-stale-sandbox job"); - return; - } - - if (job["runs-on"] !== "ubuntu-latest") { - errors.push("upgrade-stale-sandbox job must run on ubuntu-latest"); - } - validateFreeStandingJobSelector(errors, jobs, jobName, targetName); - if (job["timeout-minutes"] !== 55) { - errors.push("upgrade-stale-sandbox job must keep the legacy 55 minute timeout"); - } - - const jobEnv = asRecord(job.env); - if (jobEnv.NEMOCLAW_RUN_LIVE_E2E !== "1") { - errors.push("upgrade-stale-sandbox job must set NEMOCLAW_RUN_LIVE_E2E=1"); - } - if ( - jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/live/upgrade-stale-sandbox" - ) { - errors.push( - "upgrade-stale-sandbox job must write artifacts under e2e-artifacts/live/upgrade-stale-sandbox", - ); - } - if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") { - errors.push("upgrade-stale-sandbox job must point NEMOCLAW_CLI_BIN at the repo CLI"); - } - if (jobEnv.OPENSHELL_GATEWAY !== "nemoclaw") { - errors.push("upgrade-stale-sandbox job must force OPENSHELL_GATEWAY=nemoclaw"); - } - if (jobEnv.NEMOCLAW_SANDBOX_NAME !== "e2e-upgrade-stale") { - errors.push("upgrade-stale-sandbox job must set NEMOCLAW_SANDBOX_NAME=e2e-upgrade-stale"); - } - if ("DOCKER_CONFIG" in jobEnv) { - errors.push("upgrade-stale-sandbox job must not set DOCKER_CONFIG at job level"); - } - for (const secret of [...COMMON_SECRET_ENV_NAMES]) { - requireEnvDoesNotExposeSecret(errors, "upgrade-stale-sandbox job", jobEnv, secret); - } - - const steps = asSteps(job.steps); - requireNoDispatchInputInterpolation(errors, steps); - for (const step of steps) { - const stepName = `upgrade-stale-sandbox step '${step.name ?? step.uses ?? ""}'`; - const stepEnv = asRecord(step.env); - if (step.name !== "Run upgrade stale sandbox live Vitest test") { - requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_INFERENCE_API_KEY"); - requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_API_KEY"); - } - if (step.name !== "Authenticate to Docker Hub") { - requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); - requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); - requireNoDockerHubAuthInRun(errors, stepName, stringValue(step.run)); - } - requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "GITHUB_TOKEN"); - } - - const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); - if (!checkout) errors.push("upgrade-stale-sandbox job missing checkout step"); - requireFullShaAction(errors, checkout, "upgrade-stale-sandbox checkout"); - if (asRecord(checkout?.with)["persist-credentials"] !== false) { - errors.push("upgrade-stale-sandbox checkout step must set persist-credentials=false"); - } - - const installOpenShell = requireJobStep(errors, jobName, steps, "Install OpenShell CLI"); - requireRunContains(errors, installOpenShell, "bash scripts/install-openshell.sh"); - requireRunContains(errors, installOpenShell, "env -u DOCKER_CONFIG"); - requireRunContains(errors, installOpenShell, "-u DOCKERHUB_USERNAME"); - requireRunContains(errors, installOpenShell, "-u DOCKERHUB_TOKEN"); - requireRunContains(errors, installOpenShell, "-u NVIDIA_INFERENCE_API_KEY"); - requireRunContains(errors, installOpenShell, "-u GITHUB_TOKEN"); - - const runVitest = requireJobStep( - errors, - jobName, - steps, - "Run upgrade stale sandbox live Vitest test", - ); - const runVitestEnv = asRecord(runVitest?.env); - if (runVitestEnv.NVIDIA_INFERENCE_API_KEY !== "${{ secrets.NVIDIA_INFERENCE_API_KEY }}") { - errors.push("upgrade-stale-sandbox step must receive NVIDIA_INFERENCE_API_KEY from secrets"); - } - requireRunContains(errors, runVitest, "OPENSHELL_BIN"); - requireRunContains(errors, runVitest, "tools/e2e/live-vitest-invocation.mts run --test-path"); - requireRunContains(errors, runVitest, "test/e2e/live/upgrade-stale-sandbox.test.ts"); -} - function validateTokenRotationJob(errors: string[], jobs: WorkflowRecord): void { const jobName = "token-rotation"; const job = asRecord(jobs[jobName]); @@ -3826,8 +3656,8 @@ function validateBedrockRuntimeCompatibleAnthropicJob( errors.push("bedrock-runtime-compatible-anthropic strategy.fail-fast must be false"); } const matrix = asRecord(strategy.matrix); - if (!Array.isArray(matrix.agent) || matrix.agent.join(",") !== "openclaw,hermes") { - errors.push("bedrock-runtime-compatible-anthropic matrix.agent must be openclaw,hermes"); + if (!Array.isArray(matrix.agent) || matrix.agent.join(",") !== "openclaw") { + errors.push("bedrock-runtime-compatible-anthropic matrix.agent must be openclaw"); } const jobEnv = asRecord(job.env); @@ -4833,9 +4663,7 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { validateRebuildOpenClawJob(errors, jobs); validateRebuildHermesJob(errors, jobs, { staleBase: false }); validateRebuildHermesJob(errors, jobs, { staleBase: true }); - validateSandboxRebuildJob(errors, jobs); validateStateBackupRestoreJob(errors, jobs); - validateUpgradeStaleSandboxJob(errors, jobs); validateTokenRotationJob(errors, jobs); validateMessagingCompatibleEndpointJob(errors, jobs); validateFreeStandingJobSelector(errors, jobs, "gateway-guard-recovery", "gateway-guard-recovery"); From a773b523f2b513b7f2f210e09184c71fcb417a45 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Mon, 27 Jul 2026 16:35:05 -0700 Subject: [PATCH 02/15] test(e2e): guard retired selector execution Signed-off-by: Prekshi Vyas --- ...or-compatibility-workflow-boundary.test.ts | 62 +++++++++++++++++++ tools/e2e/workflow-boundary.mts | 41 +++++++++++- 2 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 test/e2e/support/retired-selector-compatibility-workflow-boundary.test.ts diff --git a/test/e2e/support/retired-selector-compatibility-workflow-boundary.test.ts b/test/e2e/support/retired-selector-compatibility-workflow-boundary.test.ts new file mode 100644 index 00000000000..fd362845fab --- /dev/null +++ b/test/e2e/support/retired-selector-compatibility-workflow-boundary.test.ts @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { expect, it } from "vitest"; + +import { validateE2eWorkflow } from "../../../tools/e2e/workflow-boundary.mts"; +import { readWorkflow } from "../../helpers/e2e-workflow-contract"; + +type WorkflowStep = Record; + +function compatibilitySteps(): { + steps: WorkflowStep[]; + workflow: ReturnType; +} { + const workflow = readWorkflow() as ReturnType & { + jobs: Record; + }; + const steps = workflow.jobs["retired-selector-compatibility"]?.steps; + if (!steps) throw new Error("retired-selector-compatibility steps are required"); + return { steps, workflow }; +} + +const DRIFT_CASES = [ + { + name: "candidate checkout", + mutate: (steps: WorkflowStep[]) => { + const index = steps.findIndex((step) => String(step.uses).startsWith("actions/checkout@")); + steps.splice(index, 1); + }, + error: "retired-selector-compatibility job must check out the candidate revision", + }, + { + name: "replacement helper", + mutate: (steps: WorkflowStep[]) => { + const step = steps.find( + (candidate) => candidate.name === "Verify retired selector replacements", + ); + if (step) step.run = "echo skipped"; + }, + error: "retired-selector-compatibility job must invoke the replacement helper", + }, + { + name: "compatibility artifact upload", + mutate: (steps: WorkflowStep[]) => { + const step = steps.find( + (candidate) => candidate.name === "Upload retired selector compatibility evidence", + ); + if (step) step.uses = "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a"; + }, + error: "retired-selector-compatibility job must upload compatibility evidence", + }, +] as const; + +it.each(DRIFT_CASES)("rejects retired-selector compatibility drift in $name (#7615)", ({ + mutate, + error, +}) => { + const { steps, workflow } = compatibilitySteps(); + mutate(steps); + + expect(validateE2eWorkflow(workflow)).toContain(error); +}); diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 9d5b2b57a92..acbc3659722 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -49,7 +49,10 @@ import { validateTrustedHermesSwapHelperSource, validateTrustedHermesSwapWorkflow, } from "./trusted-hermes-swap-workflow-boundary.mts"; -import { validateUploadE2eArtifactsWorkflowBoundary } from "./upload-e2e-artifacts-workflow-boundary.mts"; +import { + UPLOAD_E2E_ARTIFACTS_ACTION, + validateUploadE2eArtifactsWorkflowBoundary, +} from "./upload-e2e-artifacts-workflow-boundary.mts"; const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const DEFAULT_E2E_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e.yaml"); @@ -4091,6 +4094,42 @@ function validateRetiredSelectorCompatibilityJob(errors: string[], jobs: Workflo "retired-selector-compatibility job selector gate must match retired selector contract", ); } + + const steps = asSteps(job.steps); + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!checkout) { + errors.push("retired-selector-compatibility job must check out the candidate revision"); + } else { + requireFullShaAction(errors, checkout, "retired-selector-compatibility checkout"); + const checkoutWith = asRecord(checkout.with); + if ( + checkoutWith.repository !== "${{ inputs.checkout_repository || github.repository }}" || + checkoutWith.ref !== "${{ inputs.checkout_sha || github.sha }}" || + checkoutWith["persist-credentials"] !== false + ) { + errors.push("retired-selector-compatibility job must check out the candidate revision"); + } + } + + const verify = namedStep(steps, "Verify retired selector replacements"); + if ( + stringValue(verify?.run) !== "npx tsx tools/e2e/retired-selector-compatibility.mts" || + asRecord(verify?.env).JOBS !== "${{ inputs.jobs }}" + ) { + errors.push("retired-selector-compatibility job must invoke the replacement helper"); + } + + const upload = namedStep(steps, "Upload retired selector compatibility evidence"); + if ( + upload?.if !== "always()" || + upload?.uses !== UPLOAD_E2E_ARTIFACTS_ACTION || + !isDeepStrictEqual(asRecord(upload?.with), { + name: "e2e-retired-selector-compatibility", + path: "e2e-artifacts/live/retired-selector-compatibility/", + }) + ) { + errors.push("retired-selector-compatibility job must upload compatibility evidence"); + } } function validateTrustedE2eDispatchReceipt( From 8dd20936ed9b58f3d6bd029ffda1293f2309c1fc Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Mon, 27 Jul 2026 16:40:26 -0700 Subject: [PATCH 03/15] test(e2e): keep boundary fixtures linear Signed-off-by: Prekshi Vyas --- ...or-compatibility-workflow-boundary.test.ts | 40 +++++++++++++++---- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/test/e2e/support/retired-selector-compatibility-workflow-boundary.test.ts b/test/e2e/support/retired-selector-compatibility-workflow-boundary.test.ts index fd362845fab..a7ca917ebfe 100644 --- a/test/e2e/support/retired-selector-compatibility-workflow-boundary.test.ts +++ b/test/e2e/support/retired-selector-compatibility-workflow-boundary.test.ts @@ -8,6 +8,19 @@ import { readWorkflow } from "../../helpers/e2e-workflow-contract"; type WorkflowStep = Record; +function requiredStep( + steps: WorkflowStep[], + predicate: (step: WorkflowStep) => boolean, + description: string, +): WorkflowStep { + return ( + steps.find(predicate) ?? + (() => { + throw new Error(`${description} step is required`); + })() + ); +} + function compatibilitySteps(): { steps: WorkflowStep[]; workflow: ReturnType; @@ -15,8 +28,11 @@ function compatibilitySteps(): { const workflow = readWorkflow() as ReturnType & { jobs: Record; }; - const steps = workflow.jobs["retired-selector-compatibility"]?.steps; - if (!steps) throw new Error("retired-selector-compatibility steps are required"); + const steps = + workflow.jobs["retired-selector-compatibility"]?.steps ?? + (() => { + throw new Error("retired-selector-compatibility steps are required"); + })(); return { steps, workflow }; } @@ -24,28 +40,36 @@ const DRIFT_CASES = [ { name: "candidate checkout", mutate: (steps: WorkflowStep[]) => { - const index = steps.findIndex((step) => String(step.uses).startsWith("actions/checkout@")); - steps.splice(index, 1); + const checkout = requiredStep( + steps, + (step) => String(step.uses).startsWith("actions/checkout@"), + "candidate checkout", + ); + steps.splice(steps.indexOf(checkout), 1); }, error: "retired-selector-compatibility job must check out the candidate revision", }, { name: "replacement helper", mutate: (steps: WorkflowStep[]) => { - const step = steps.find( + const step = requiredStep( + steps, (candidate) => candidate.name === "Verify retired selector replacements", + "replacement helper", ); - if (step) step.run = "echo skipped"; + step.run = "echo skipped"; }, error: "retired-selector-compatibility job must invoke the replacement helper", }, { name: "compatibility artifact upload", mutate: (steps: WorkflowStep[]) => { - const step = steps.find( + const step = requiredStep( + steps, (candidate) => candidate.name === "Upload retired selector compatibility evidence", + "compatibility artifact upload", ); - if (step) step.uses = "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a"; + step.uses = "actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a"; }, error: "retired-selector-compatibility job must upload compatibility evidence", }, From a176245dc24638e384f41750e1c847bff282a045 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Mon, 27 Jul 2026 16:58:56 -0700 Subject: [PATCH 04/15] fix(e2e): preserve retired target selectors Signed-off-by: Prekshi Vyas --- .github/workflows/e2e.yaml | 3 +- test/e2e/README.md | 6 +- ...or-compatibility-workflow-boundary.test.ts | 40 ++++++++++--- .../retired-selector-compatibility.test.ts | 49 ++++++++++++++- test/e2e/support/workflow-plan.test.ts | 42 +++++++++++++ tools/e2e/retired-selector-compatibility.mts | 26 +++++--- tools/e2e/workflow-boundary.mts | 14 ++++- tools/e2e/workflow-plan.mts | 59 ++++++++++++++++--- 8 files changed, 206 insertions(+), 33 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index c72fed07d3d..7cde6a9d5f5 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -390,7 +390,7 @@ jobs: retired-selector-compatibility: needs: generate-matrix - if: ${{ inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',credential-migration,') || contains(format(',{0},', inputs.jobs), ',credential-sanitization,') || contains(format(',{0},', inputs.jobs), ',diagnostics,') || contains(format(',{0},', inputs.jobs), ',docs-validation,') || contains(format(',{0},', inputs.jobs), ',gateway-drift-preflight,') || contains(format(',{0},', inputs.jobs), ',gateway-health-honest,') || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths,') || contains(format(',{0},', inputs.jobs), ',openshell-version-pin,') || contains(format(',{0},', inputs.jobs), ',sandbox-rebuild,') || contains(format(',{0},', inputs.jobs), ',ubuntu-repo-cli-smoke,') || contains(format(',{0},', inputs.jobs), ',upgrade-stale-sandbox,')) }} + if: ${{ inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',credential-migration,') || contains(format(',{0},', inputs.jobs), ',credential-sanitization,') || contains(format(',{0},', inputs.jobs), ',diagnostics,') || contains(format(',{0},', inputs.jobs), ',docs-validation,') || contains(format(',{0},', inputs.jobs), ',gateway-drift-preflight,') || contains(format(',{0},', inputs.jobs), ',gateway-health-honest,') || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths,') || contains(format(',{0},', inputs.jobs), ',openshell-version-pin,') || contains(format(',{0},', inputs.jobs), ',sandbox-rebuild,') || contains(format(',{0},', inputs.jobs), ',ubuntu-repo-cli-smoke,') || contains(format(',{0},', inputs.jobs), ',upgrade-stale-sandbox,') || contains(format(',{0},', inputs.targets), ',sandbox-rebuild,') || contains(format(',{0},', inputs.targets), ',upgrade-stale-sandbox,')) }} runs-on: ubuntu-latest timeout-minutes: 15 env: @@ -408,6 +408,7 @@ jobs: - name: Verify retired selector replacements env: JOBS: ${{ inputs.jobs }} + TARGETS: ${{ inputs.targets }} run: npx tsx tools/e2e/retired-selector-compatibility.mts - name: Upload retired selector compatibility evidence diff --git a/test/e2e/README.md b/test/e2e/README.md index 19a1c79c722..1ca8249c9a0 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -84,9 +84,9 @@ the manually selected `mock`, `internal-nvidia`, or `public-nvidia` inference mode. PR-gate requests using the retired `sandbox-rebuild` and -`upgrade-stale-sandbox` job selectors run focused replacement tests through the -compatibility controller. `rebuild-openclaw` is the canonical live rebuild and -upgrade seam. +`upgrade-stale-sandbox` job or target selectors run focused replacement tests +through the compatibility controller. `rebuild-openclaw` is the canonical live +rebuild and upgrade seam. ## Larger-runner routing diff --git a/test/e2e/support/retired-selector-compatibility-workflow-boundary.test.ts b/test/e2e/support/retired-selector-compatibility-workflow-boundary.test.ts index a7ca917ebfe..17db1e41722 100644 --- a/test/e2e/support/retired-selector-compatibility-workflow-boundary.test.ts +++ b/test/e2e/support/retired-selector-compatibility-workflow-boundary.test.ts @@ -7,6 +7,9 @@ import { validateE2eWorkflow } from "../../../tools/e2e/workflow-boundary.mts"; import { readWorkflow } from "../../helpers/e2e-workflow-contract"; type WorkflowStep = Record; +type MutableWorkflow = ReturnType & { + jobs: Record; +}; function requiredStep( steps: WorkflowStep[], @@ -23,11 +26,9 @@ function requiredStep( function compatibilitySteps(): { steps: WorkflowStep[]; - workflow: ReturnType; + workflow: MutableWorkflow; } { - const workflow = readWorkflow() as ReturnType & { - jobs: Record; - }; + const workflow = readWorkflow() as MutableWorkflow; const steps = workflow.jobs["retired-selector-compatibility"]?.steps ?? (() => { @@ -37,9 +38,20 @@ function compatibilitySteps(): { } const DRIFT_CASES = [ + { + name: "target selector gate", + mutate: (_steps: WorkflowStep[], workflow: MutableWorkflow) => { + const job = workflow.jobs["retired-selector-compatibility"] as WorkflowStep; + job.if = String(job.if).replace( + " || contains(format(',{0},', inputs.targets), ',upgrade-stale-sandbox,')", + "", + ); + }, + error: "retired-selector-compatibility job selector gate must match retired selector contract", + }, { name: "candidate checkout", - mutate: (steps: WorkflowStep[]) => { + mutate: (steps: WorkflowStep[], _workflow: MutableWorkflow) => { const checkout = requiredStep( steps, (step) => String(step.uses).startsWith("actions/checkout@"), @@ -51,7 +63,7 @@ const DRIFT_CASES = [ }, { name: "replacement helper", - mutate: (steps: WorkflowStep[]) => { + mutate: (steps: WorkflowStep[], _workflow: MutableWorkflow) => { const step = requiredStep( steps, (candidate) => candidate.name === "Verify retired selector replacements", @@ -61,9 +73,21 @@ const DRIFT_CASES = [ }, error: "retired-selector-compatibility job must invoke the replacement helper", }, + { + name: "target selector forwarding", + mutate: (steps: WorkflowStep[], _workflow: MutableWorkflow) => { + const step = requiredStep( + steps, + (candidate) => candidate.name === "Verify retired selector replacements", + "replacement helper", + ); + delete (step.env as Record).TARGETS; + }, + error: "retired-selector-compatibility job must forward target selectors", + }, { name: "compatibility artifact upload", - mutate: (steps: WorkflowStep[]) => { + mutate: (steps: WorkflowStep[], _workflow: MutableWorkflow) => { const step = requiredStep( steps, (candidate) => candidate.name === "Upload retired selector compatibility evidence", @@ -80,7 +104,7 @@ it.each(DRIFT_CASES)("rejects retired-selector compatibility drift in $name (#76 error, }) => { const { steps, workflow } = compatibilitySteps(); - mutate(steps); + mutate(steps, workflow); expect(validateE2eWorkflow(workflow)).toContain(error); }); diff --git a/test/e2e/support/retired-selector-compatibility.test.ts b/test/e2e/support/retired-selector-compatibility.test.ts index 28c3172e35d..ce6af3f24b8 100644 --- a/test/e2e/support/retired-selector-compatibility.test.ts +++ b/test/e2e/support/retired-selector-compatibility.test.ts @@ -47,12 +47,17 @@ function workspace(): { artifactRoot: string; output: string; root: string } { }; } -function environment(target: ReturnType, jobs: string): NodeJS.ProcessEnv { +function environment( + target: ReturnType, + jobs: string, + targets = "", +): NodeJS.ProcessEnv { return { E2E_ARTIFACT_DIR: target.artifactRoot, GITHUB_OUTPUT: target.output, GITHUB_WORKSPACE: target.root, JOBS: jobs, + TARGETS: targets, NEMOCLAW_E2E_CORRELATION_ID: CORRELATION_ID, NEMOCLAW_E2E_EXPECTED_SHA: EXPECTED_SHA, NEMOCLAW_E2E_PLAN_HASH: PLAN_HASH, @@ -75,6 +80,14 @@ describe("retired E2E selector compatibility", () => { jobs: "cloud-onboard,gateway-health-honest", }), ).toEqual([]); + expect( + selectedRetiredControllerJobs({ + allowedJobs: ["cloud-onboard"], + expectedSha: EXPECTED_SHA, + jobs: "sandbox-rebuild", + targets: "diagnostics,sandbox-rebuild,upgrade-stale-sandbox", + }), + ).toEqual(["sandbox-rebuild", "upgrade-stale-sandbox"]); }); it("runs each ordinary replacement project once and emits bound signals (#7616)", () => { @@ -135,6 +148,40 @@ describe("retired E2E selector compatibility", () => { } }); + it("runs and reports both rebuild replacements selected through targets (#7615)", () => { + const target = workspace(); + const commands: string[] = []; + const suppliedEnvironment = environment(target, "", "sandbox-rebuild,upgrade-stale-sandbox"); + try { + const selected = runRetiredSelectorCompatibility(suppliedEnvironment, { + allowedJobs: ["cloud-onboard"], + repositoryRoot: target.root, + resolveHead: () => EXPECTED_SHA, + runCommand: (command, args) => { + commands.push([command, ...args].join(" ")); + }, + }); + + expect(selected).toEqual(["sandbox-rebuild", "upgrade-stale-sandbox"]); + expect(commands).toEqual([ + "npx vitest run --project cli src/lib/actions/sandbox/rebuild-finalization.test.ts src/lib/actions/sandbox/rebuild-flow-helpers.test.ts src/lib/sandbox/version.test.ts", + "npx vitest run --project integration test/cli/list-share-live-inference.test.ts test/rebuild-stale-recovery.test.ts", + ]); + for (const id of selected) { + const signalPath = path.join(target.artifactRoot, id, "risk-signal.json"); + expect(JSON.parse(fs.readFileSync(signalPath, "utf8"))).toMatchObject({ + jobId: id, + expectedSha: EXPECTED_SHA, + testedSha: EXPECTED_SHA, + passed: 1, + failed: 0, + }); + } + } finally { + fs.rmSync(target.root, { force: true, recursive: true }); + } + }); + it("fails before emitting passing evidence when a replacement command fails (#7616)", () => { const target = workspace(); try { diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts index 5c83dfb8ac7..aad54ea9d0e 100644 --- a/test/e2e/support/workflow-plan.test.ts +++ b/test/e2e/support/workflow-plan.test.ts @@ -207,6 +207,48 @@ describe("E2E workflow plan", () => { } }); + it("emits an empty live plan for P2 selectors handled by target compatibility (#7615)", () => { + const directory = mkdtempSync(path.join(tmpdir(), "nemoclaw-workflow-plan-cli-")); + const output = path.join(directory, "github-output"); + const summary = path.join(directory, "summary.md"); + const plan = { + matrix: [], + testMatrix: [], + hermesSelected: false, + explicitOnlyJobs: readFreeStandingJobsInventory().explicitOnlyJobs, + }; + try { + const result = spawnSync(TSX, [PLANNER_CLI, "--ci-output"], { + cwd: REPO_ROOT, + encoding: "utf8", + env: { + ...process.env, + GITHUB_OUTPUT: output, + GITHUB_STEP_SUMMARY: summary, + INFERENCE_MODE: "mock", + JOBS: "", + TARGETS: "sandbox-rebuild,upgrade-stale-sandbox", + NEMOCLAW_E2E_EXPECTED_SHA: "a".repeat(40), + }, + timeout: 30_000, + }); + + expect(result.status, result.stderr).toBe(0); + expect(readFileSync(output, "utf8")).toBe( + [ + "matrix=[]", + "test_matrix=[]", + "hermes_selected=false", + `explicit_only_jobs=${plan.explicitOnlyJobs.join(",")}`, + "", + ].join("\n"), + ); + expect(readFileSync(summary, "utf8")).toBe(renderE2eWorkflowPlanSummary(plan)); + } finally { + rmSync(directory, { force: true, recursive: true }); + } + }); + it("rejects the retired bootstrap job outside a PR controller checkout", () => { const directory = mkdtempSync(path.join(tmpdir(), "nemoclaw-workflow-plan-cli-")); try { diff --git a/tools/e2e/retired-selector-compatibility.mts b/tools/e2e/retired-selector-compatibility.mts index 6ecdd8d32ce..7dc24210c77 100644 --- a/tools/e2e/retired-selector-compatibility.mts +++ b/tools/e2e/retired-selector-compatibility.mts @@ -10,6 +10,7 @@ import { createPrivateRegularFile } from "./private-file.mts"; import * as importedRiskSignal from "./risk-signal.ts"; import { RETIRED_CONTROLLER_SELECTOR_IDS, + RETIRED_CONTROLLER_TARGET_SELECTOR_IDS, readFreeStandingJobsInventory, } from "./workflow-boundary.mts"; @@ -37,6 +38,7 @@ type Replacement = { }; const RETIRED_SELECTOR_ID_SET = new Set(RETIRED_CONTROLLER_SELECTOR_IDS); +const RETIRED_TARGET_SELECTOR_ID_SET = new Set(RETIRED_CONTROLLER_TARGET_SELECTOR_IDS); const REPLACEMENTS: Readonly> = { "credential-migration": { legacyFile: "test/e2e/live/credential-migration.test.ts", @@ -144,20 +146,25 @@ export function selectedRetiredControllerJobs(options: { allowedJobs: readonly string[]; expectedSha?: string; jobs?: string; + targets?: string; }): RetiredControllerSelectorId[] { - if (!SHA_PATTERN.test(options.expectedSha ?? "") || !options.jobs) return []; - if (!SELECTOR_LIST_PATTERN.test(options.jobs)) { - throw new Error("retired selector compatibility requires safe comma-separated job IDs"); + if (!SHA_PATTERN.test(options.expectedSha ?? "")) return []; + for (const selectors of [options.jobs, options.targets]) { + if (selectors && !SELECTOR_LIST_PATTERN.test(selectors)) { + throw new Error("retired selector compatibility requires safe comma-separated selector IDs"); + } } const allowedJobs = new Set(options.allowedJobs); + const requestedJobs = options.jobs?.split(",") ?? []; + const requestedTargets = (options.targets?.split(",") ?? []).filter((target) => + RETIRED_TARGET_SELECTOR_ID_SET.has(target), + ); return [ ...new Set( - options.jobs - .split(",") - .filter( - (job): job is RetiredControllerSelectorId => - RETIRED_SELECTOR_ID_SET.has(job) && !allowedJobs.has(job), - ), + [...requestedJobs, ...requestedTargets].filter( + (selector): selector is RetiredControllerSelectorId => + RETIRED_SELECTOR_ID_SET.has(selector) && !allowedJobs.has(selector), + ), ), ].sort(); } @@ -258,6 +265,7 @@ export function runRetiredSelectorCompatibility( allowedJobs, expectedSha: environment.NEMOCLAW_E2E_EXPECTED_SHA, jobs: environment.JOBS, + targets: environment.TARGETS, }); const output = environment.GITHUB_OUTPUT; if (!output) throw new Error("GITHUB_OUTPUT is required"); diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index acbc3659722..1c71bfbdee3 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -137,6 +137,10 @@ export const RETIRED_CONTROLLER_SELECTOR_IDS = [ "ubuntu-repo-cli-smoke", "upgrade-stale-sandbox", ] as const; +export const RETIRED_CONTROLLER_TARGET_SELECTOR_IDS = [ + "sandbox-rebuild", + "upgrade-stale-sandbox", +] as const; const LIVE_TEST_FILE_PATTERN = /test\/e2e\/live\/(?:[A-Za-z0-9._-]+\/)*[A-Za-z0-9._-]+\.test\.ts/g; const FREE_STANDING_JOB_MARKER = "E2E_JOB"; const FREE_STANDING_TARGET_MARKER = "E2E_TARGET_ID"; @@ -4085,10 +4089,13 @@ function validateRetiredSelectorCompatibilityJob(errors: string[], jobs: Workflo errors.push("workflow missing retired-selector-compatibility job"); return; } - const selectorGate = RETIRED_CONTROLLER_SELECTOR_IDS.map( + const jobSelectorGate = RETIRED_CONTROLLER_SELECTOR_IDS.map( (id) => `contains(format(',{0},', inputs.jobs), ',${id},')`, ).join(" || "); - const expectedIf = `\${{ inputs.checkout_sha != '' && (${selectorGate}) }}`; + const targetSelectorGate = RETIRED_CONTROLLER_TARGET_SELECTOR_IDS.map( + (id) => `contains(format(',{0},', inputs.targets), ',${id},')`, + ).join(" || "); + const expectedIf = `\${{ inputs.checkout_sha != '' && (${jobSelectorGate} || ${targetSelectorGate}) }}`; if (job.if !== expectedIf) { errors.push( "retired-selector-compatibility job selector gate must match retired selector contract", @@ -4118,6 +4125,9 @@ function validateRetiredSelectorCompatibilityJob(errors: string[], jobs: Workflo ) { errors.push("retired-selector-compatibility job must invoke the replacement helper"); } + if (asRecord(verify?.env).TARGETS !== "${{ inputs.targets }}") { + errors.push("retired-selector-compatibility job must forward target selectors"); + } const upload = namedStep(steps, "Upload retired selector compatibility evidence"); if ( diff --git a/tools/e2e/workflow-plan.mts b/tools/e2e/workflow-plan.mts index 01136af71ee..f2a296ffbc1 100644 --- a/tools/e2e/workflow-plan.mts +++ b/tools/e2e/workflow-plan.mts @@ -30,6 +30,11 @@ type WorkflowPlanCliOptions = WorkflowPlanSelectors & { ciOutput: boolean; }; +type TrustedControllerSelectorMap = { + retiredTargetSelected: boolean; + selectors: WorkflowPlanSelectors; +}; + const SAFE_SELECTOR_LIST_PATTERN = /^[A-Za-z0-9_-]+(?:,[A-Za-z0-9_-]+)*$/; const HERMES_JOB_ID = "hermes-e2e"; const LEGACY_BOOTSTRAP_INSTALL_JOB_ID = "launchable-smoke"; @@ -135,8 +140,10 @@ function selectTestRows( function mapTrustedControllerJobs( selectors: WorkflowPlanSelectors, environment: NodeJS.ProcessEnv, -): WorkflowPlanSelectors { - if (!COMMIT_SHA_PATTERN.test(environment.NEMOCLAW_E2E_EXPECTED_SHA ?? "")) return selectors; +): TrustedControllerSelectorMap { + if (!COMMIT_SHA_PATTERN.test(environment.NEMOCLAW_E2E_EXPECTED_SHA ?? "")) { + return { retiredTargetSelected: false, selectors }; + } const inventory = readFreeStandingJobsInventory(); const jobs = selectorIds(selectors.jobs, "jobs").map((job) => @@ -145,6 +152,7 @@ function mapTrustedControllerJobs( ? BOOTSTRAP_INSTALL_JOB_ID : job, ); + const targets = selectorIds(selectors.targets, "targets"); const retiredJobs = new Set( selectedRetiredControllerJobs({ allowedJobs: inventory.allowedJobs, @@ -152,7 +160,15 @@ function mapTrustedControllerJobs( jobs: jobs.join(","), }), ); + const retiredTargets = new Set( + selectedRetiredControllerJobs({ + allowedJobs: inventory.allowedJobs, + expectedSha: environment.NEMOCLAW_E2E_EXPECTED_SHA, + targets: targets.join(","), + }), + ); const compatibleJobs = jobs.filter((job) => !retiredJobs.has(job)); + const compatibleTargets = targets.filter((target) => !retiredTargets.has(target)); if (jobs.length > 0 && compatibleJobs.length === 0 && !selectors.targets) { throw new Error("retired selector compatibility requires another controller-selected job"); } @@ -161,8 +177,21 @@ function mapTrustedControllerJobs( // workflow becomes the controller. Keep the raw IDs for evidence, but plan // only jobs that still execute in the candidate. return { - ...selectors, - jobs: compatibleJobs.join(","), + retiredTargetSelected: retiredTargets.size > 0, + selectors: { + ...selectors, + jobs: compatibleJobs.join(","), + targets: compatibleTargets.join(","), + }, + }; +} + +function emptyE2eWorkflowPlan(): E2eWorkflowPlan { + return { + matrix: [], + testMatrix: [], + hermesSelected: false, + explicitOnlyJobs: readFreeStandingJobsInventory().explicitOnlyJobs, }; } @@ -221,12 +250,15 @@ export function validateE2eWorkflowPlan(plan: unknown): E2eWorkflowPlan { return plan as E2eWorkflowPlan; } -function expectedHermesSelection(selectors: WorkflowPlanSelectors): boolean { +function expectedHermesSelection( + selectors: WorkflowPlanSelectors, + retiredTargetSelected: boolean, +): boolean { const selected = [ ...selectorIds(selectors.jobs, "jobs"), ...selectorIds(selectors.targets, "targets"), ]; - return selected.length === 0 || selected.includes(HERMES_JOB_ID); + return (selected.length === 0 && !retiredTargetSelected) || selected.includes(HERMES_JOB_ID); } export function renderE2eWorkflowPlanSummary(plan: E2eWorkflowPlan): string { @@ -253,9 +285,18 @@ export function writeE2eWorkflowPlanCiOutput( if (!INFERENCE_MODES.has(inferenceMode)) { throw new Error(`Invalid inference_mode: ${inferenceMode}`); } - const plannerSelectors = mapTrustedControllerJobs(selectors, environment); - const plan = validateE2eWorkflowPlan(buildE2eWorkflowPlan(plannerSelectors)); - if (plan.hermesSelected !== expectedHermesSelection(plannerSelectors)) { + const controllerMap = mapTrustedControllerJobs(selectors, environment); + const plannerSelectors = controllerMap.selectors; + const hasPlannerSelectors = Boolean(plannerSelectors.jobs || plannerSelectors.targets); + const plan = validateE2eWorkflowPlan( + controllerMap.retiredTargetSelected && !hasPlannerSelectors + ? emptyE2eWorkflowPlan() + : buildE2eWorkflowPlan(plannerSelectors), + ); + if ( + plan.hermesSelected !== + expectedHermesSelection(plannerSelectors, controllerMap.retiredTargetSelected) + ) { throw new Error("E2E planner changed the trusted Hermes selection"); } const output = environment.GITHUB_OUTPUT; From e8b41235a00fc31b1ac13912e5828fbd9ffa0372 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 30 Jul 2026 11:18:36 -0700 Subject: [PATCH 05/15] perf(e2e): focus retained coverage Signed-off-by: Prekshi Vyas --- .github/workflows/e2e.yaml | 129 ++++++++++----- scripts/scorecard/summarize-jobs.mts | 10 +- test/e2e/README.md | 65 +++++++- ...openclaw-plugin-runtime-exdev-lifecycle.ts | 67 ++++++++ .../openclaw-plugin-runtime-exdev.test.ts | 112 ++++--------- ...e2e-report-to-pr-workflow-boundary.test.ts | 93 ++++++++++- test/e2e/support/e2e-scorecard.test.ts | 40 +++++ test/e2e/support/e2e-workflow.test.ts | 26 +++ .../openclaw-plugin-runtime-exdev-env.test.ts | 61 +++++++ ...-gateway-upgrade-workflow-boundary.test.ts | 55 +++--- test/pr-e2e-gate-signal-shards.test.ts | 12 +- test/pr-risk-plan.test.ts | 104 ++++++++++-- tools/advisors/risk-plan.mts | 11 +- ...hell-gateway-upgrade-workflow-boundary.mts | 156 ++++++++++++------ tools/e2e/report-e2e-results.mts | 19 ++- ...upload-e2e-artifacts-workflow-boundary.mts | 6 + tools/e2e/workflow-boundary.mts | 23 ++- 17 files changed, 770 insertions(+), 219 deletions(-) create mode 100644 test/e2e/live/openclaw-plugin-runtime-exdev-lifecycle.ts diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 7cde6a9d5f5..100aa5ed1c4 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -6,7 +6,8 @@ run-name: "${{ inputs.checkout_sha != '' && format('E2E PR #{0} ({1})', inputs.p on: schedule: - - cron: "0 0 * * *" + - cron: "0 0 * * 1-6" + - cron: "0 0 * * 0" workflow_dispatch: inputs: targets: @@ -15,7 +16,7 @@ on: default: "" type: string jobs: - description: "Optional comma-separated E2E test IDs. Empty runs default-enabled tests only when targets is also empty; explicit-only tests openshell-gateway-auth-contract, mcp-bridge-dev, hermes-gpu-startup, sandbox-rlimits-connect, jetson-nvmap-gpu, and staging-brev-launchable are skipped unless selected." + description: "Optional comma-separated E2E test IDs. Empty runs default-enabled tests only when targets is also empty; explicit-only tests openshell-gateway-auth-contract, openshell-gateway-upgrade-compatibility, mcp-bridge-dev, hermes-gpu-startup, sandbox-rlimits-connect, jetson-nvmap-gpu, and staging-brev-launchable are skipped unless selected." required: false default: "" type: string @@ -4129,45 +4130,11 @@ jobs: fail-fast: false matrix: include: - - id: v0.0.36-x86_64 - runner: ubuntu-latest - shard: v0-0-36-x86-64 - nemoclaw_ref: v0.0.36 - nemoclaw_commit: "3351fbdd4eb7d9b80ec471545083956327da2b10" - installer_sha256: "0c42400a0d3867739f1d75d612e069967be4506e169974bbbebf14b7af39144f" - sandbox_base_image_ref: "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:104151ffadc2ff0b6c815e3c95c2783ced61aee0d0f83fc327cc02be9b7e14e6" - openshell_version: 0.0.36 - openclaw_version: 2026.4.24 - - id: v0.0.55-x86_64 - runner: ubuntu-latest - shard: v0-0-55-x86-64 - nemoclaw_ref: v0.0.55 - nemoclaw_commit: "95d483fe2b6569d68e59493c60f19df09a068e8f" - installer_sha256: "ff8cf448e4d17b00421545a1f333262b615b1b0aa236d0cc5aeaf4e2cae2d897" - sandbox_base_image_ref: "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:10433a8cd2f2b809dd0fdf983514679e04c0f8aa1ff5bbff675029046033b108" - openshell_version: 0.0.44 - openclaw_version: 2026.5.22 - - id: v0.0.55-aarch64 - runner: ubuntu-24.04-arm - shard: v0-0-55-aarch64 - nemoclaw_ref: v0.0.55 - nemoclaw_commit: "95d483fe2b6569d68e59493c60f19df09a068e8f" - installer_sha256: "ff8cf448e4d17b00421545a1f333262b615b1b0aa236d0cc5aeaf4e2cae2d897" - sandbox_base_image_ref: "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:10433a8cd2f2b809dd0fdf983514679e04c0f8aa1ff5bbff675029046033b108" - openshell_version: 0.0.44 - openclaw_version: 2026.5.22 - - id: v0.0.74-x86_64 - runner: ubuntu-latest - shard: v0-0-74-x86-64 - nemoclaw_ref: v0.0.74 - nemoclaw_commit: "3a05b54e8ec3e1d5550ec5c728de54af872bffe3" - installer_sha256: "a0cd3feca488d247e53d59d7d8246d2b86e75e95acb5e7d78504b3c0c60fd7db" - sandbox_base_image_ref: "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:104151ffadc2ff0b6c815e3c95c2783ced61aee0d0f83fc327cc02be9b7e14e6" - openshell_version: 0.0.72 - openclaw_version: 2026.5.27 - id: v0.0.89-x86_64 runner: ubuntu-latest shard: v0-0-89-x86-64 + tier: nightly + boundary: current OpenClaw state migration nemoclaw_ref: v0.0.89 nemoclaw_commit: "1143aa5cce77f3bad1b3b5588bd7fddbe438237e" installer_sha256: "00f24959e5ca68104fe91221c0a015dab6a4154618497fa36b969b661f418cc2" @@ -4195,7 +4162,7 @@ jobs: NEMOCLAW_CURRENT_OPENCLAW_VERSION: ${{ matrix.current_openclaw_version }} NEMOCLAW_OPENCLAW_STATE_UPGRADE_PROOF: ${{ matrix.openclaw_state_upgrade }} OPENSHELL_GATEWAY: "nemoclaw" - steps: + steps: &openshell-gateway-upgrade-steps - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ inputs.checkout_repository || github.repository }} @@ -4208,11 +4175,8 @@ jobs: uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 - name: Run OpenShell gateway upgrade live Vitest test - # Keep the original v0.0.36 fixture on x86_64 and validate the exact - # v0.0.55/OpenShell 0.0.44 regression shape on x86_64 and arm64, plus - # the immediate v0.0.74/OpenShell 0.0.72 predecessor to this bump. The - # v0.0.89 row proves OpenClaw 2026.6.10 state survives the 2026.7.1 - # migration without crossing the OpenShell secret boundary. + # Each matrix row keeps one migration boundary. The workflow tier + # controls when the row runs; the live assertions remain identical. env: GITHUB_TOKEN: ${{ github.token }} run: | @@ -4231,6 +4195,82 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh + openshell-gateway-upgrade-compatibility: + name: openshell-gateway-upgrade-compatibility / ${{ matrix.id }} + needs: generate-matrix + if: ${{ (github.event_name == 'schedule' && github.event.schedule == '0 0 * * 0') || contains(format(',{0},', inputs.jobs), ',openshell-gateway-upgrade-compatibility,') || contains(format(',{0},', inputs.targets), ',openshell-gateway-upgrade-compatibility,') }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 70 + strategy: + fail-fast: false + matrix: + include: + - id: v0.0.36-x86_64 + runner: ubuntu-latest + shard: v0-0-36-x86-64 + tier: weekly + boundary: oldest retained registry migration + nemoclaw_ref: v0.0.36 + nemoclaw_commit: "3351fbdd4eb7d9b80ec471545083956327da2b10" + installer_sha256: "0c42400a0d3867739f1d75d612e069967be4506e169974bbbebf14b7af39144f" + sandbox_base_image_ref: "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:104151ffadc2ff0b6c815e3c95c2783ced61aee0d0f83fc327cc02be9b7e14e6" + openshell_version: 0.0.36 + openclaw_version: 2026.4.24 + - id: v0.0.55-x86_64 + runner: ubuntu-latest + shard: v0-0-55-x86-64 + tier: weekly + boundary: x86_64 OpenShell 0.0.44 regression + nemoclaw_ref: v0.0.55 + nemoclaw_commit: "95d483fe2b6569d68e59493c60f19df09a068e8f" + installer_sha256: "ff8cf448e4d17b00421545a1f333262b615b1b0aa236d0cc5aeaf4e2cae2d897" + sandbox_base_image_ref: "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:10433a8cd2f2b809dd0fdf983514679e04c0f8aa1ff5bbff675029046033b108" + openshell_version: 0.0.44 + openclaw_version: 2026.5.22 + - id: v0.0.55-aarch64 + runner: ubuntu-24.04-arm + shard: v0-0-55-aarch64 + tier: weekly + boundary: arm64 OpenShell 0.0.44 regression + nemoclaw_ref: v0.0.55 + nemoclaw_commit: "95d483fe2b6569d68e59493c60f19df09a068e8f" + installer_sha256: "ff8cf448e4d17b00421545a1f333262b615b1b0aa236d0cc5aeaf4e2cae2d897" + sandbox_base_image_ref: "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:10433a8cd2f2b809dd0fdf983514679e04c0f8aa1ff5bbff675029046033b108" + openshell_version: 0.0.44 + openclaw_version: 2026.5.22 + - id: v0.0.74-x86_64 + runner: ubuntu-latest + shard: v0-0-74-x86-64 + tier: weekly + boundary: immediate predecessor registry migration + nemoclaw_ref: v0.0.74 + nemoclaw_commit: "3a05b54e8ec3e1d5550ec5c728de54af872bffe3" + installer_sha256: "a0cd3feca488d247e53d59d7d8246d2b86e75e95acb5e7d78504b3c0c60fd7db" + sandbox_base_image_ref: "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:104151ffadc2ff0b6c815e3c95c2783ced61aee0d0f83fc327cc02be9b7e14e6" + openshell_version: 0.0.72 + openclaw_version: 2026.5.27 + env: + E2E_JOB: "1" + E2E_DEFAULT_ENABLED: "0" + E2E_TARGET_ID: "openshell-gateway-upgrade-compatibility" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openshell-gateway-upgrade-compatibility + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_E2E_SHARD: ${{ matrix.shard }} + NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_GATEWAY_UPGRADE_SURVIVOR_NAME: "e2e-gateway-upgrade-survivor" + NEMOCLAW_OLD_NEMOCLAW_REF: ${{ matrix.nemoclaw_ref }} + NEMOCLAW_OLD_NEMOCLAW_COMMIT: ${{ matrix.nemoclaw_commit }} + NEMOCLAW_OLD_INSTALLER_SHA256: ${{ matrix.installer_sha256 }} + NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF: ${{ matrix.sandbox_base_image_ref }} + NEMOCLAW_OLD_OPENSHELL_VERSION: ${{ matrix.openshell_version }} + NEMOCLAW_OLD_OPENCLAW_VERSION: ${{ matrix.openclaw_version }} + NEMOCLAW_CURRENT_OPENCLAW_VERSION: ${{ matrix.current_openclaw_version }} + NEMOCLAW_OPENCLAW_STATE_UPGRADE_PROOF: ${{ matrix.openclaw_state_upgrade }} + OPENSHELL_GATEWAY: "nemoclaw" + steps: *openshell-gateway-upgrade-steps + messaging-providers: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',messaging-providers,') || contains(format(',{0},', inputs.targets), ',messaging-providers,') }} @@ -5620,6 +5660,7 @@ jobs: overlayfs-autofix, state-backup-restore, openshell-gateway-upgrade, + openshell-gateway-upgrade-compatibility, token-rotation, messaging-compatible-endpoint, messaging-providers, diff --git a/scripts/scorecard/summarize-jobs.mts b/scripts/scorecard/summarize-jobs.mts index 7e45868a30d..3b7bafcadbf 100644 --- a/scripts/scorecard/summarize-jobs.mts +++ b/scripts/scorecard/summarize-jobs.mts @@ -184,7 +184,10 @@ function summarizeJobs(input: SummarizeJobsInput): JobSummary { if (input.apiJobs !== null) { const eligibleJobs = input.apiJobs.filter((job) => { const name = job.name.replace(/ \/ [^/]+$/u, ""); - return !metaJobs.has(name) && (!explicitOnly.has(name) || selected.has(name)); + return ( + !metaJobs.has(name) && + (!explicitOnly.has(name) || selected.has(name) || classifyApiJob(job) !== "skipped") + ); }); const jobs = normalizeApiJobs(eligibleJobs); const classified = jobs.map((job) => ({ job, result: classifyApiJob(job) })); @@ -202,7 +205,10 @@ function summarizeJobs(input: SummarizeJobsInput): JobSummary { const entries = Object.entries(input.needs) .filter(([name]) => !metaJobs.has(name)) - .filter(([name]) => !explicitOnly.has(name) || selected.has(name)) + .filter( + ([name, value]) => + !explicitOnly.has(name) || selected.has(name) || classifyNeed(value) !== "skipped", + ) .sort(([left], [right]) => left.localeCompare(right)); const classified = entries.map(([name, value]) => ({ name, result: classifyNeed(value) })); const counts = countResults(classified.map(({ result }) => result)); diff --git a/test/e2e/README.md b/test/e2e/README.md index 1ca8249c9a0..261d5da8638 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -88,6 +88,25 @@ PR-gate requests using the retired `sandbox-rebuild` and through the compatibility controller. `rebuild-openclaw` is the canonical live rebuild and upgrade seam. +### Current OpenClaw plugin EXDEV lifecycle + +The `openclaw-plugin-runtime-exdev` job keeps one current-version lifecycle: + +1. Onboard the custom weather plugin as v1. +2. Restart the gateway and verify v1. +3. Recreate the sandbox with the plugin changed to v2. +4. Run the cross-device runtime-dependency replacement probe. + +The recreation remains the replacement boundary. It verifies the v2 plugin +with runtime inspection, `tools.catalog`, and `tools.invoke`, and it preserves +the workspace marker. The job also keeps the test-only tmpfs mount, unchanged +stock policy-source bytes, and the distinct-device and source-side `EXDEV` +checks. The duplicate v3 rebuild is removed from this job. The +`rebuild-openclaw` job remains the canonical live rebuild coverage. + +The runtime target for `openclaw-plugin-runtime-exdev` is 16–17 minutes. +Scheduled-run timing for the reduced lifecycle has not yet been measured. + ## Larger-runner routing The larger-runner experiment is inactive while the configuration variable @@ -215,6 +234,31 @@ graph as the live targets: `post_to_slack=true`, which uses the preview Slack route. Branch-dispatched runs never receive Slack webhook secrets. +### Gateway migration tiers + +The OpenShell gateway migration matrix separates the current state-migration +boundary from retained historical and architecture boundaries: + +| Job | NemoClaw row | Runner | Boundary | Scheduled cadence | +| --- | --- | --- | --- | --- | +| `openshell-gateway-upgrade` | `v0.0.89-x86_64` | `ubuntu-latest` | current OpenClaw state migration | Daily | +| `openshell-gateway-upgrade-compatibility` | `v0.0.36-x86_64` | `ubuntu-latest` | oldest retained registry migration | Sunday | +| `openshell-gateway-upgrade-compatibility` | `v0.0.55-x86_64` | `ubuntu-latest` | x86_64 OpenShell 0.0.44 regression | Sunday | +| `openshell-gateway-upgrade-compatibility` | `v0.0.55-aarch64` | `ubuntu-24.04-arm` | arm64 OpenShell 0.0.44 regression | Sunday | +| `openshell-gateway-upgrade-compatibility` | `v0.0.74-x86_64` | `ubuntu-latest` | immediate predecessor registry migration | Sunday | + +The workflow schedules the nightly path Monday through Saturday and schedules +the complete tier set on Sunday. The current row runs on both schedules because +`openshell-gateway-upgrade` is default-enabled. +`openshell-gateway-upgrade-compatibility` is explicit-only outside the Sunday +schedule. Select that job directly through the `jobs` or `targets` input for a +focused PR run or another required qualification. + +Nightly reports list the compatibility job as skipped when it is not selected; +they do not classify that tier as failed. The split removes four shards from +each Monday-through-Saturday scheduled run. Runner-minute measurements for the +new cadence are pending. + A manual run with `include_staging_brev_launchable=true` and empty `jobs` and `targets` selectors is a full dispatch. Each full dispatch uses `github.run_id` in its workflow concurrency identity, so another full dispatch cannot supersede @@ -536,9 +580,17 @@ and advisor concurrency groups include that eligibility, so an ignored metadata-edit run cannot cancel an eligible run for the same PR. The trusted controller reads all changed files after eligible PR CI completes and builds the deterministic risk plan. -Runtime families and changes to workflow-wired live tests select -canonical selectors from the trusted `e2e.yaml` inventory independently of -advisor output. Ordinary internal changes execute those focused selections. +Runtime families and changes to workflow-wired live tests or their owning +helpers select canonical jobs from the trusted `e2e.yaml` inventory +independently of advisor output. A workflow-wired live test or owning helper +selects one to three focused E2E journeys. A gateway-migration live test or +owning helper selects both `openshell-gateway-upgrade` and +`openshell-gateway-upgrade-compatibility`. + +Changes only under `test/e2e/support/` select no credentialed live E2E job. +The `e2e-support` Vitest project runs those support tests in PR CI. A new or +renamed live test that does not match the trusted workflow inventory keeps the +conservative control-plane floor until its canonical job mapping is added. Gate initialization, CI coordination, and protected approval share one non-cancelling FIFO concurrency group for the exact repository, PR number, PR SHA, and base SHA. `queue: max` keeps pending jobs for @@ -558,9 +610,10 @@ GitHub consequently returns no head-repository object. Shared sandbox-boundary changes have a floor of `full-e2e`, `hermes-e2e`, and `security-posture`. E2E control-plane changes select `cloud-onboard`, `cloud-inference`, and `security-posture`. The `e2e-control-plane` -family is a conservative path boundary that includes non-documentation files -under `tools/e2e/` and `test/e2e/`, plus the E2E and PR-CI workflows, risk -policy, dependency and test configuration, and preparation and upload actions. +family remains the conservative boundary for shared E2E tools, workflow and +security files, unknown live test paths, risk policy, dependency and test +configuration, and preparation and upload actions. These cross-cutting changes +keep the broad three-job floor. Repository-root `Dockerfile` changes additionally select `full-e2e` alongside the platform-install `cloud-onboard` floor so OpenClaw final-image changes run through cold onboarding and a real first turn. diff --git a/test/e2e/live/openclaw-plugin-runtime-exdev-lifecycle.ts b/test/e2e/live/openclaw-plugin-runtime-exdev-lifecycle.ts new file mode 100644 index 00000000000..534a4e39b96 --- /dev/null +++ b/test/e2e/live/openclaw-plugin-runtime-exdev-lifecycle.ts @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const CURRENT_LIFECYCLE_PHASES = [ + "confirm Docker CLI and clear the current plugin sandbox", + "clone and prepare the current plugin fixture", + "install current OpenShell and onboard plugin v1", + "restart the gateway and confirm plugin v1", + "recreate the sandbox with plugin v2", + "prove cross-device runtime dependency replacement", +] as const; + +export type WeatherFixtureVersion = "v1" | "v2"; + +type LifecycleCommand = { + command: "node"; + args: string[]; +}; + +export function currentLifecycleCommands(input: { + cliEntrypoint: string; + dockerfilePath: string; + sandboxName: string; +}): { + onboard: LifecycleCommand; + recreate: LifecycleCommand; + restart: LifecycleCommand; +} { + return { + onboard: { + command: "node", + args: [ + input.cliEntrypoint, + "onboard", + "--fresh", + "--non-interactive", + "--yes-i-accept-third-party-software", + "--agent", + "openclaw", + "--from", + input.dockerfilePath, + ], + }, + recreate: { + command: "node", + args: [ + input.cliEntrypoint, + "onboard", + "--fresh", + "--recreate-sandbox", + "--non-interactive", + "--yes", + "--yes-i-accept-third-party-software", + "--name", + input.sandboxName, + "--agent", + "openclaw", + "--from", + input.dockerfilePath, + ], + }, + restart: { + command: "node", + args: [input.cliEntrypoint, input.sandboxName, "gateway", "restart"], + }, + }; +} diff --git a/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts b/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts index 9e37facc121..9a120507d18 100644 --- a/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts +++ b/test/e2e/live/openclaw-plugin-runtime-exdev.test.ts @@ -35,6 +35,11 @@ import { RELEASE_BASELINE_TEST_SELECTOR, RELEASE_SANDBOX_BASE_IMAGE_REF, } from "./openclaw-plugin-runtime-exdev-env.ts"; +import { + currentLifecycleCommands, + CURRENT_LIFECYCLE_PHASES, + type WeatherFixtureVersion, +} from "./openclaw-plugin-runtime-exdev-lifecycle.ts"; import { createOpenShellDriverConfigTestWrapper, type OpenShellComponents, @@ -44,7 +49,7 @@ import { } from "./openshell-driver-config-test-wrapper.ts"; // Keep this contract as a focused live test: build a deterministic custom plugin -// on top of the complete managed runtime, prove it survives restart/rebuild, then +// on top of the complete managed runtime, prove it survives restart/recreation, then // run the in-sandbox Node replacement probe that guards #3513/#3127's EXDEV // cross-device runtime-deps failure mode. No registry or ledger is required. @@ -86,7 +91,6 @@ const CURRENT_BUILDER_IMAGE_REF = const TOOL_DISCLOSURE_ENV_REFERENCE = "${NEMOCLAW_TOOL_DISCLOSURE}"; const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-openclaw-plugin-exdev"; const ONBOARD_TIMEOUT_MS = 25 * 60_000; -const REBUILD_TIMEOUT_MS = 20 * 60_000; const PROBE_TIMEOUT_MS = 60_000; const EXDEV_TMPFS_MOUNT = "/tmp/nemoclaw-exdev-tmpfs"; const EXDEV_TMPFS_SOURCE = `${EXDEV_TMPFS_MOUNT}/source`; @@ -121,8 +125,6 @@ const EXDEV_PATTERNS = [ /EXDEV: cross-device link not permitted/i, /cross-device link not permitted/i, ]; -type WeatherFixtureVersion = "v1" | "v2" | "v3"; - const GATEWAY_CATALOG_CALL_SOURCE = String.raw` import { Buffer } from "node:buffer"; import { accessSync, constants, realpathSync } from "node:fs"; @@ -1143,18 +1145,10 @@ test("the release-baseline custom plugin loads with its exact NemoClaw and OpenS }); }); -test("the current-lifecycle custom plugin survives restart, recreation, and rebuild without EXDEV failures (#6108)", { - timeout: ONBOARD_TIMEOUT_MS * 2 + REBUILD_TIMEOUT_MS + 15 * 60_000, +test("the current-lifecycle custom plugin survives restart and recreation without EXDEV failures (#6108)", { + timeout: ONBOARD_TIMEOUT_MS * 2 + 15 * 60_000, meta: { - e2ePhases: [ - "confirm Docker CLI and clear the current plugin sandbox", - "clone and prepare the current plugin fixture", - "install current OpenShell and onboard plugin v1", - "restart the gateway and confirm plugin v1", - "recreate the sandbox with plugin v2", - "rebuild the sandbox with plugin v3", - "prove cross-device runtime dependency replacement", - ], + e2ePhases: [...CURRENT_LIFECYCLE_PHASES], }, }, async ({ artifacts, cleanup, host, progress, sandbox, skip }) => { await artifacts.target.declare({ @@ -1165,10 +1159,10 @@ test("the current-lifecycle custom plugin survives restart, recreation, and rebu "the current CLI uses OpenShell 0.0.85 for current lifecycle coverage", "the current CLI selects and validates its compatible sandbox base image", "gateway log, runtime inspection, tools.catalog, and tools.invoke prove weather/get_weather", - "custom-plugin v1 survives restart, recreation installs v2, and rebuild installs v3", - "workspace state survives both onboarding recreation and rebuild", + "custom-plugin v1 survives restart and recreation installs v2", + "workspace state survives onboarding recreation", `test-only driver config mounts tmpfs at ${EXDEV_TMPFS_MOUNT} without changing production policies`, - "stock OpenClaw policy source bytes remain unchanged through onboard and rebuild", + "stock OpenClaw policy source bytes remain unchanged through onboard and recreation", `sandbox proves ${EXDEV_TMPFS_SOURCE} and plugin-runtime-deps are distinct devices`, `legacy source-side staging fails with EXDEV across the same ${EXDEV_TMPFS_SOURCE} to plugin-runtime-deps boundary`, "OpenClaw-style target-side plugin runtime-deps replacement completes without EXDEV", @@ -1263,20 +1257,15 @@ test("the current-lifecycle custom plugin survives restart, recreation, and rebu "current OpenShell wrapper and components must pass onboard coherence preflight", ).toBe(true); const sandboxEnv = withOpenShellWrapperEnv(deploymentEnv, openshellWrapper, pinnedOpenshell); + const lifecycleCommands = currentLifecycleCommands({ + cliEntrypoint: CLI_ENTRYPOINT, + dockerfilePath: customPluginContext.dockerfilePath, + sandboxName: SANDBOX_NAME, + }); const onboard = await host.command( - "node", - [ - CLI_ENTRYPOINT, - "onboard", - "--fresh", - "--non-interactive", - "--yes-i-accept-third-party-software", - "--agent", - "openclaw", - "--from", - customPluginContext.dockerfilePath, - ], + lifecycleCommands.onboard.command, + lifecycleCommands.onboard.args, { artifactName: "openclaw-plugin-exdev-onboard", env: sandboxEnv, @@ -1293,11 +1282,15 @@ test("the current-lifecycle custom plugin survives restart, recreation, and rebu const weatherAfterOnboard = await assertWeatherPluginRuntime(sandbox, "after-onboard", "v1"); progress.phase("restart the gateway and confirm plugin v1"); - const restart = await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "gateway", "restart"], { - artifactName: "openclaw-weather-plugin-gateway-restart", - env: sandboxEnv, - timeoutMs: 180_000, - }); + const restart = await host.command( + lifecycleCommands.restart.command, + lifecycleCommands.restart.args, + { + artifactName: "openclaw-weather-plugin-gateway-restart", + env: sandboxEnv, + timeoutMs: 180_000, + }, + ); expect(restart.exitCode, resultText(restart)).toBe(0); const weatherAfterRestart = await assertWeatherPluginRuntime(sandbox, "after-restart", "v1"); expect(weatherAfterRestart.imageMarker).toBe(weatherAfterOnboard.imageMarker); @@ -1305,28 +1298,14 @@ test("the current-lifecycle custom plugin survives restart, recreation, and rebu const workspaceMarker = `plugin-lifecycle-${randomUUID()}`; await writeWorkspaceMarker(sandbox, workspaceMarker); - // Change an actual build-context input so rebuild must produce a distinct - // plugin artifact. Onboarding recreation must preserve the fresh v2 + // Change an actual build-context input so recreation must produce a distinct + // plugin artifact. Recreation must preserve the fresh v2 // extension instead of replacing it with the backed-up v1 directory. progress.phase("recreate the sandbox with plugin v2"); writeCustomPluginVersion(customPluginContext.versionSourcePath, "v2"); const recreate = await host.command( - "node", - [ - CLI_ENTRYPOINT, - "onboard", - "--fresh", - "--recreate-sandbox", - "--non-interactive", - "--yes", - "--yes-i-accept-third-party-software", - "--name", - SANDBOX_NAME, - "--agent", - "openclaw", - "--from", - customPluginContext.dockerfilePath, - ], + lifecycleCommands.recreate.command, + lifecycleCommands.recreate.args, { artifactName: "openclaw-weather-plugin-recreate", env: sandboxEnv, @@ -1340,22 +1319,6 @@ test("the current-lifecycle custom plugin survives restart, recreation, and rebu expect(weatherAfterRecreate.imageMarker).not.toBe(weatherAfterOnboard.imageMarker); await assertWorkspaceMarker(sandbox, "after-recreate", workspaceMarker); - // A subsequent rebuild exercises the same semantic recreated-sandbox - // restore boundary with another fresh image artifact. - progress.phase("rebuild the sandbox with plugin v3"); - writeCustomPluginVersion(customPluginContext.versionSourcePath, "v3"); - const rebuild = await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "rebuild", "--yes"], { - artifactName: "openclaw-weather-plugin-rebuild", - env: sandboxEnv, - timeoutMs: REBUILD_TIMEOUT_MS, - }); - expect(rebuild.exitCode, resultText(rebuild)).toBe(0); - const tmpfsMountedAfterRebuild = await assertExdevTmpfsMounted(sandbox, "after-rebuild"); - assertPolicySourcesUnchanged(policySourceSnapshot, "rebuild"); - const weatherAfterRebuild = await assertWeatherPluginRuntime(sandbox, "after-rebuild", "v3"); - expect(weatherAfterRebuild.imageMarker).not.toBe(weatherAfterRecreate.imageMarker); - await assertWorkspaceMarker(sandbox, "after-rebuild", workspaceMarker); - progress.phase("prove cross-device runtime dependency replacement"); const df = await sandbox.execShell( SANDBOX_NAME, @@ -1392,7 +1355,6 @@ test("the current-lifecycle custom plugin survives restart, recreation, and rebu onboardExitCode: onboard.exitCode, restartExitCode: restart.exitCode, recreateExitCode: recreate.exitCode, - rebuildExitCode: rebuild.exitCode, filesystemProbeExitCode: df.exitCode, runtimeDepsProbeExitCode: probe.exitCode, testOnlyTmpfsSource: EXDEV_TMPFS_SOURCE, @@ -1409,10 +1371,6 @@ test("the current-lifecycle custom plugin survives restart, recreation, and rebu weatherAfterRecreate.inspectLoaded && weatherAfterRecreate.catalogToolIds.includes("get_weather") && weatherAfterRecreate.toolInvoked, - weatherAfterRebuild: - weatherAfterRebuild.inspectLoaded && - weatherAfterRebuild.catalogToolIds.includes("get_weather") && - weatherAfterRebuild.toolInvoked, v1MarkerStableThroughRestart: weatherAfterOnboard.imageMarker === weatherAfterRestart.imageMarker && weatherAfterOnboard.fixtureVersion === "v1" && @@ -1420,9 +1378,6 @@ test("the current-lifecycle custom plugin survives restart, recreation, and rebu recreatedV2ReplacedV1: weatherAfterRecreate.imageMarker !== weatherAfterOnboard.imageMarker && weatherAfterRecreate.fixtureVersion === "v2", - rebuiltV3ReplacedV2: - weatherAfterRebuild.imageMarker !== weatherAfterRecreate.imageMarker && - weatherAfterRebuild.fixtureVersion === "v3", distinctDevices: /source_device=\d+ target_device=\d+/.test(probeText), sourceSideExdevSelfCheck: probeText.includes( "source-side staging failure self-check completed", @@ -1430,8 +1385,7 @@ test("the current-lifecycle custom plugin survives restart, recreation, and rebu noExdevSignature: !EXDEV_PATTERNS.some((pattern) => pattern.test(probeText)), successMarker: probeText.includes("runtime deps replacement completed"), workspaceStatePreserved: true, - testOnlyTmpfsMounted: - tmpfsMountedAfterOnboard && tmpfsMountedAfterRecreate && tmpfsMountedAfterRebuild, + testOnlyTmpfsMounted: tmpfsMountedAfterOnboard && tmpfsMountedAfterRecreate, stockPolicySourcesUnchanged: true, }, }); diff --git a/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts b/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts index a52440b09e3..4e207098bc6 100644 --- a/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts @@ -178,6 +178,7 @@ async function executeReport(options: { testMatrix?: CredentialFreeTestMatrixRow[]; jobs?: string; needs?: ReportNeeds; + explicitOnlyJobs?: string; paginateError?: Error; }): Promise<{ body: string; @@ -188,6 +189,7 @@ async function executeReport(options: { apiJobs = [], testMatrix = DEFAULT_TEST_MATRIX, jobs = testMatrix.map(({ id }) => id).join(","), + explicitOnlyJobs = "", needs = { "generate-matrix": { result: "success" }, "shared-e2e": { result: "failure" }, @@ -204,7 +206,7 @@ async function executeReport(options: { const github = reportGithub({ createComment, paginate }); const core = { info: vi.fn(), setFailed, warning }; const env = { - EXPLICIT_ONLY_JOBS: "", + EXPLICIT_ONLY_JOBS: explicitOnlyJobs, TEST_MATRIX: JSON.stringify(testMatrix), JOB_PR_NUMBER: "42", JOB_TARGETS: "", @@ -579,6 +581,7 @@ it("lists explicit-only jobs skipped by default dispatch with their selection hi const report = renderE2eReport({ needs: { "generate-matrix": { result: "success" }, + "mcp-bridge-dev": { result: "skipped" }, }, env: { EXPLICIT_ONLY_JOBS: "mcp-bridge-dev", @@ -597,6 +600,61 @@ it("lists explicit-only jobs skipped by default dispatch with their selection hi ); }); +it("reports the weekly gateway migration tier as skipped on a nightly run (#7920)", async () => { + const { body, setFailed } = await executeReport({ + apiJobs: [ + { + conclusion: "success", + name: "OpenShell gateway upgrade (v0.0.89-x86_64)", + status: "completed", + }, + ], + explicitOnlyJobs: "openshell-gateway-upgrade-compatibility", + jobs: "", + needs: { + "generate-matrix": { result: "success" }, + "openshell-gateway-upgrade": { result: "success" }, + "openshell-gateway-upgrade-compatibility": { result: "skipped" }, + }, + testMatrix: [], + }); + + expect(setFailed).not.toHaveBeenCalled(); + expect(body).toContain("| openshell-gateway-upgrade | ✅ success | — |"); + expect(body).toContain("| openshell-gateway-upgrade-compatibility | ⏭️ skipped | — |"); + expect(body).toContain( + "`openshell-gateway-upgrade-compatibility` (nightly and default manual dispatches exclude weekly gateway compatibility migrations unless selected", + ); + expect(body).not.toContain("| openshell-gateway-upgrade-compatibility | ❌ failure |"); +}); + +it("does not report the weekly gateway migration tier as skipped after a Sunday run (#7920)", async () => { + const { body, setFailed } = await executeReport({ + apiJobs: [ + { + conclusion: "success", + name: "openshell-gateway-upgrade-compatibility / v0.0.36-x86_64", + status: "completed", + }, + ], + explicitOnlyJobs: "openshell-gateway-upgrade-compatibility", + jobs: "", + needs: { + "generate-matrix": { result: "success" }, + "openshell-gateway-upgrade": { result: "success" }, + "openshell-gateway-upgrade-compatibility": { result: "success" }, + }, + testMatrix: [], + }); + + expect(setFailed).not.toHaveBeenCalled(); + expect(body).toContain("| openshell-gateway-upgrade-compatibility | ✅ success | — |"); + expect(body).toContain( + "**Requested test IDs:** _(default — workflow policy selected the enabled tests for this run)_", + ); + expect(body).not.toContain("Explicit-only jobs skipped"); +}); + it("reports matrix children by test ID without fabricating a missing child result", async () => { const { body, setFailed, warning } = await executeReport({ apiJobs: [ @@ -756,6 +814,39 @@ it("reports one total wall clock span from valid matrix E2E jobs", async () => { expect(body).not.toContain("OpenShell gateway upgrade (v0.2.0)"); }); +it("aggregates the weekly gateway matrix under its tier job (#7920)", async () => { + const { body, setFailed } = await executeReport({ + apiJobs: [ + { + completed_at: "2026-07-15T00:05:00Z", + conclusion: "success", + name: "openshell-gateway-upgrade-compatibility / v0.0.36-x86_64", + started_at: "2026-07-15T00:00:00Z", + status: "completed", + }, + { + completed_at: "2026-07-15T00:11:00Z", + conclusion: "success", + name: "openshell-gateway-upgrade-compatibility / v0.0.55-aarch64", + started_at: "2026-07-15T00:02:00Z", + status: "completed", + }, + ], + explicitOnlyJobs: "openshell-gateway-upgrade-compatibility", + jobs: "openshell-gateway-upgrade-compatibility", + needs: { + "generate-matrix": { result: "success" }, + "openshell-gateway-upgrade-compatibility": { result: "success" }, + }, + testMatrix: [], + }); + + expect(setFailed).not.toHaveBeenCalled(); + expect(body).toContain("| openshell-gateway-upgrade-compatibility | ✅ success | 11m 0s |"); + expect(body).not.toContain("Explicit-only jobs skipped"); + expect(body).not.toContain("openshell-gateway-upgrade-compatibility / v0.0.36-x86_64"); +}); + it("reports one total wall clock span when matrix job names start with their job ID", async () => { const { body, setFailed } = await executeReport({ apiJobs: [ diff --git a/test/e2e/support/e2e-scorecard.test.ts b/test/e2e/support/e2e-scorecard.test.ts index 7c6e52eeebd..8f1a99ba712 100644 --- a/test/e2e/support/e2e-scorecard.test.ts +++ b/test/e2e/support/e2e-scorecard.test.ts @@ -546,6 +546,46 @@ describe("E2E scorecard", () => { }); }); + it("counts a scheduled explicit-only tier when it ran and excludes it when skipped (#7920)", () => { + const input = { + explicitOnlyJobNames: ["openshell-gateway-upgrade-compatibility"], + explicitlySelected: [], + metaJobNames: ["generate-matrix", "scorecard"], + }; + + expect( + scorecardJobs.summarizeJobs({ + ...input, + apiJobs: [ + { + conclusion: "success", + name: "openshell-gateway-upgrade-compatibility / v0.0.36-x86_64", + status: "completed", + }, + ], + needs: {}, + }), + ).toMatchObject({ ran: 1, skipped: 0, success: 1, total: 1 }); + expect( + scorecardJobs.summarizeJobs({ + ...input, + apiJobs: null, + needs: { + "openshell-gateway-upgrade-compatibility": { result: "success" }, + }, + }), + ).toMatchObject({ ran: 1, skipped: 0, success: 1, total: 1 }); + expect( + scorecardJobs.summarizeJobs({ + ...input, + apiJobs: null, + needs: { + "openshell-gateway-upgrade-compatibility": { result: "skipped" }, + }, + }), + ).toMatchObject({ ran: 0, skipped: 0, success: 0, total: 0 }); + }); + it("sanitizes raw traces into a timing-only artifact", () => { const directory = mkdtempSync(join(tmpdir(), "nemoclaw-trace-sanitize-")); const source = join(directory, "raw"); diff --git a/test/e2e/support/e2e-workflow.test.ts b/test/e2e/support/e2e-workflow.test.ts index 31d22166db3..956f2f61730 100644 --- a/test/e2e/support/e2e-workflow.test.ts +++ b/test/e2e/support/e2e-workflow.test.ts @@ -703,6 +703,32 @@ describe("e2e workflow boundary", () => { matchedFiles: ["test/e2e/live/token-rotation.test.ts"], }, ]); + expect( + focusedE2eJobsForChangedFiles( + ["test/e2e/live/openclaw-plugin-runtime-exdev-lifecycle.ts"], + inventory, + ), + ).toEqual([ + { + id: "openclaw-plugin-runtime-exdev", + matchedFiles: ["test/e2e/live/openclaw-plugin-runtime-exdev-lifecycle.ts"], + }, + ]); + expect( + focusedE2eJobsForChangedFiles( + ["test/e2e/live/openshell-gateway-upgrade-helpers.ts"], + inventory, + ), + ).toEqual([ + { + id: "openshell-gateway-upgrade", + matchedFiles: ["test/e2e/live/openshell-gateway-upgrade-helpers.ts"], + }, + { + id: "openshell-gateway-upgrade-compatibility", + matchedFiles: ["test/e2e/live/openshell-gateway-upgrade-helpers.ts"], + }, + ]); }); it("rejects malformed free-standing workflow metadata before matrix generation", { diff --git a/test/e2e/support/openclaw-plugin-runtime-exdev-env.test.ts b/test/e2e/support/openclaw-plugin-runtime-exdev-env.test.ts index c18f2a3b1d8..e091303bd10 100644 --- a/test/e2e/support/openclaw-plugin-runtime-exdev-env.test.ts +++ b/test/e2e/support/openclaw-plugin-runtime-exdev-env.test.ts @@ -9,6 +9,10 @@ import { RELEASE_BASELINE_TEST_SELECTOR, RELEASE_SANDBOX_BASE_IMAGE_REF, } from "../live/openclaw-plugin-runtime-exdev-env.ts"; +import { + currentLifecycleCommands, + CURRENT_LIFECYCLE_PHASES, +} from "../live/openclaw-plugin-runtime-exdev-lifecycle.ts"; describe("OpenClaw plugin runtime EXDEV base image selection", () => { it("pins the release baseline to its matching sandbox base image", () => { @@ -23,3 +27,60 @@ describe("OpenClaw plugin runtime EXDEV base image selection", () => { ); }); }); + +describe("OpenClaw plugin runtime EXDEV current lifecycle", () => { + it("maps the retained lifecycle to restart and recreation without a duplicate rebuild (#7917)", () => { + expect( + currentLifecycleCommands({ + cliEntrypoint: "/repo/bin/nemoclaw.js", + dockerfilePath: "/fixture/Dockerfile", + sandboxName: "e2e-openclaw-plugin-exdev", + }), + ).toEqual({ + onboard: { + command: "node", + args: [ + "/repo/bin/nemoclaw.js", + "onboard", + "--fresh", + "--non-interactive", + "--yes-i-accept-third-party-software", + "--agent", + "openclaw", + "--from", + "/fixture/Dockerfile", + ], + }, + recreate: { + command: "node", + args: [ + "/repo/bin/nemoclaw.js", + "onboard", + "--fresh", + "--recreate-sandbox", + "--non-interactive", + "--yes", + "--yes-i-accept-third-party-software", + "--name", + "e2e-openclaw-plugin-exdev", + "--agent", + "openclaw", + "--from", + "/fixture/Dockerfile", + ], + }, + restart: { + command: "node", + args: ["/repo/bin/nemoclaw.js", "e2e-openclaw-plugin-exdev", "gateway", "restart"], + }, + }); + expect(CURRENT_LIFECYCLE_PHASES).toEqual([ + "confirm Docker CLI and clear the current plugin sandbox", + "clone and prepare the current plugin fixture", + "install current OpenShell and onboard plugin v1", + "restart the gateway and confirm plugin v1", + "recreate the sandbox with plugin v2", + "prove cross-device runtime dependency replacement", + ]); + }); +}); diff --git a/test/e2e/support/openshell-gateway-upgrade-workflow-boundary.test.ts b/test/e2e/support/openshell-gateway-upgrade-workflow-boundary.test.ts index e0fb0bc2d0b..874cffdecaa 100644 --- a/test/e2e/support/openshell-gateway-upgrade-workflow-boundary.test.ts +++ b/test/e2e/support/openshell-gateway-upgrade-workflow-boundary.test.ts @@ -24,41 +24,56 @@ import { } from "../live/openshell-gateway-upgrade-helpers.ts"; describe("OpenShell gateway upgrade workflow boundary", () => { - it("pins architecture and immediate-predecessor fixtures to the canonical live test (#6114)", () => { + it("pins the nightly and weekly gateway migration tiers to the canonical live test (#7920)", () => { const workflow = readOpenShellGatewayUpgradeWorkflow(); expect(validateOpenShellGatewayUpgradeWorkflow(workflow)).toEqual([]); expect(validateE2eWorkflowBoundary()).toEqual([]); - const job = (workflow.jobs as Record>)[ - "openshell-gateway-upgrade" - ]; - job["runs-on"] = "ubuntu-latest"; - const strategy = job.strategy as Record>; - const fixtures = strategy.matrix.include as Array>; - fixtures.find((fixture) => fixture.id === "v0.0.55-x86_64")!.sandbox_base_image_ref = - "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:104151ffadc2ff0b6c815e3c95c2783ced61aee0d0f83fc327cc02be9b7e14e6"; - fixtures.find((fixture) => fixture.id === "v0.0.55-aarch64")!.runner = "ubuntu-latest"; - fixtures.find((fixture) => fixture.id === "v0.0.74-x86_64")!.openclaw_version = "latest"; - fixtures.find((fixture) => fixture.id === "v0.0.89-x86_64")!.openclaw_state_upgrade = "0"; - const env = job.env as Record; - env.NEMOCLAW_E2E_SHARD = "default"; - env.NEMOCLAW_CURRENT_OPENCLAW_VERSION = "latest"; - env.NEMOCLAW_OPENCLAW_STATE_UPGRADE_PROOF = "0"; - const run = (job.steps as Array>).find( + const jobs = workflow.jobs as Record>; + (workflow.on as Record).schedule = [{ cron: "0 0 * * *" }]; + const nightly = jobs["openshell-gateway-upgrade"]; + const compatibility = jobs["openshell-gateway-upgrade-compatibility"]; + nightly.if = "always()"; + compatibility.if = "always()"; + compatibility.name = "unrelated matrix name"; + nightly["runs-on"] = "ubuntu-latest"; + const nightlyFixtures = (nightly.strategy as Record>).matrix + .include as Array>; + nightlyFixtures[0]!.tier = "weekly"; + const nightlyEnv = nightly.env as Record; + nightlyEnv.E2E_DEFAULT_ENABLED = "0"; + nightlyEnv.E2E_TARGET_ID = "wrong-target"; + nightlyEnv.NEMOCLAW_E2E_SHARD = "default"; + nightlyEnv.NEMOCLAW_CURRENT_OPENCLAW_VERSION = "latest"; + nightlyEnv.NEMOCLAW_OPENCLAW_STATE_UPGRADE_PROOF = "0"; + const compatibilityFixtures = ( + compatibility.strategy as Record> + ).matrix.include as Array>; + compatibilityFixtures.find((fixture) => fixture.id === "v0.0.55-aarch64")!.runner = + "ubuntu-latest"; + const compatibilityEnv = compatibility.env as Record; + delete compatibilityEnv.E2E_DEFAULT_ENABLED; + const run = (nightly.steps as Array>).find( (step) => step.name === "Run OpenShell gateway upgrade live Vitest test", )!; run.run = "npx vitest run --project e2e-live unrelated.test.ts"; expect(validateE2eWorkflow(workflow)).toEqual( expect.arrayContaining([ + "E2E schedule must separate six nightly runs from the weekly compatibility run", + "openshell-gateway-upgrade must retain its execution-tier selector", "openshell-gateway-upgrade must run on ${{ matrix.runner }}", - "openshell-gateway-upgrade v0.0.55 matrix must pin x86_64 and arm64 upgrade fixtures", - "openshell-gateway-upgrade matrix must pin the immediate v0.0.74 x86_64 upgrade fixture", - "openshell-gateway-upgrade matrix must pin the v0.0.89 OpenClaw state-upgrade fixture", + "openshell-gateway-upgrade matrix must pin its tiered gateway upgrade fixtures", + "openshell-gateway-upgrade must publish its own target identity", + "openshell-gateway-upgrade must remain default-enabled", "openshell-gateway-upgrade must publish one risk-signal shard per legacy fixture", "openshell-gateway-upgrade must bind the current OpenClaw version from its fixture", "openshell-gateway-upgrade must bind the OpenClaw state-upgrade proof flag from its fixture", "openshell-gateway-upgrade step 'Run OpenShell gateway upgrade live Vitest test' must run: npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/openshell-gateway-upgrade.test.ts", + "openshell-gateway-upgrade-compatibility must retain its execution-tier selector", + "openshell-gateway-upgrade-compatibility must keep one scorecard identity across its matrix", + "openshell-gateway-upgrade-compatibility matrix must pin its tiered gateway upgrade fixtures", + "openshell-gateway-upgrade-compatibility must remain explicit-only outside the weekly schedule", ]), ); }); diff --git a/test/pr-e2e-gate-signal-shards.test.ts b/test/pr-e2e-gate-signal-shards.test.ts index 34e8d2338ac..c40d449e466 100644 --- a/test/pr-e2e-gate-signal-shards.test.ts +++ b/test/pr-e2e-gate-signal-shards.test.ts @@ -35,13 +35,19 @@ describe("PR E2E signal shard policy", () => { "hermes-inference-switch": ["anthropic"], "openclaw-inference-switch": ["anthropic"], }); - expect(expectedSignalShards(["openshell-gateway-upgrade"], undefined, [DCODE_TARGET])).toEqual({ - "openshell-gateway-upgrade": [ + expect( + expectedSignalShards( + ["openshell-gateway-upgrade", "openshell-gateway-upgrade-compatibility"], + undefined, + [DCODE_TARGET], + ), + ).toEqual({ + "openshell-gateway-upgrade": ["v0-0-89-x86-64"], + "openshell-gateway-upgrade-compatibility": [ "v0-0-36-x86-64", "v0-0-55-x86-64", "v0-0-55-aarch64", "v0-0-74-x86-64", - "v0-0-89-x86-64", ], [DCODE_TARGET]: ["default"], }); diff --git a/test/pr-risk-plan.test.ts b/test/pr-risk-plan.test.ts index 7cfe464ee94..5f1cb2b8487 100644 --- a/test/pr-risk-plan.test.ts +++ b/test/pr-risk-plan.test.ts @@ -28,7 +28,7 @@ describe("deterministic PR risk plan", () => { const second = plan("src/lib/onboard.ts", "src/lib/state/registry.ts"); expect(first).toEqual(second); - expect(first.version).toBe(8); + expect(first.version).toBe(9); expect(first.headSha).toBe(HEAD_SHA); expect(first.planHash).toMatch(/^[a-f0-9]{64}$/u); expect(first.changedFiles).toEqual(["src/lib/onboard.ts", "src/lib/state/registry.ts"]); @@ -43,22 +43,18 @@ describe("deterministic PR risk plan", () => { expect(result.requiredTargets).toEqual([]); }); - it("keeps every live test behind the control-plane exception and preserves the cloud floor (#6446)", () => { - const canonical = plan("test/e2e/live/cloud-onboard.test.ts"); - const ordinaryLiveTest = plan("test/e2e/live/full.test.ts"); + it("keeps an unmapped live test behind the control-plane exception and cloud floor (#6446)", () => { + const result = plan("test/e2e/live/full.test.ts"); - expect(canonical.families.map((family) => family.id)).toContain("platform-install"); - expect(canonical.families.map((family) => family.id)).toContain("e2e-control-plane"); - expect(riskPlanRequiredJobIds(canonical)).toContain("cloud-onboard"); - expect(ordinaryLiveTest.families.map((family) => family.id)).toEqual(["e2e-control-plane"]); - expect(riskPlanRequiredJobIds(ordinaryLiveTest)).toEqual([ + expect(result.families.map((family) => family.id)).toEqual(["e2e-control-plane"]); + expect(riskPlanRequiredJobIds(result)).toEqual([ "cloud-inference", "cloud-onboard", "security-posture", ]); }); - it("hashes trusted focused E2E selections into their canonical jobs", () => { + it("maps a workflow-wired live test only to its canonical job (#7921)", () => { const changedFiles = ["test/e2e/live/token-rotation.test.ts"]; const focusedE2eJobs = focusedE2eJobsForChangedFiles(changedFiles); const result = buildRiskPlan({ headSha: HEAD_SHA, changedFiles, focusedE2eJobs }); @@ -84,9 +80,97 @@ describe("deterministic PR risk plan", () => { matchedFiles: ["test/e2e/live/token-rotation.test.ts"], }), ); + expect(riskPlanRequiredJobIds(result)).toEqual(["token-rotation"]); + expect(result.families.map((family) => family.id)).toEqual(["focused-e2e"]); expect(result.planHash).not.toBe(withoutFocusedSelection.planHash); }); + it("leaves E2E support-only changes in the fast e2e-support project (#7921)", () => { + const changedFiles = ["test/e2e/support/workflow-plan.test.ts"]; + const focusedE2eJobs = focusedE2eJobsForChangedFiles(changedFiles); + const result = buildRiskPlan({ headSha: HEAD_SHA, changedFiles, focusedE2eJobs }); + + expect(focusedE2eJobs).toEqual([]); + expect(result.tier).toBe(0); + expect(result.families).toEqual([]); + expect(result.requiredJobs).toEqual([]); + expect(requiresCredentialedE2eAuthorization(result)).toBe(false); + }); + + it("maps a shared gateway live test to both retained migration tiers (#7921)", () => { + const changedFiles = ["test/e2e/live/openshell-gateway-upgrade.test.ts"]; + const focusedE2eJobs = focusedE2eJobsForChangedFiles(changedFiles); + const result = buildRiskPlan({ headSha: HEAD_SHA, changedFiles, focusedE2eJobs }); + + expect(focusedE2eJobs).toEqual([ + { + id: "openshell-gateway-upgrade", + matchedFiles: changedFiles, + }, + { + id: "openshell-gateway-upgrade-compatibility", + matchedFiles: changedFiles, + }, + ]); + expect(riskPlanRequiredJobIds(result)).toEqual([ + "openshell-gateway-upgrade", + "openshell-gateway-upgrade-compatibility", + ]); + }); + + it("keeps an unknown live test behind the broad control-plane floor (#7921)", () => { + const changedFiles = ["test/e2e/live/new-retained-journey.test.ts"]; + const focusedE2eJobs = focusedE2eJobsForChangedFiles(changedFiles); + const result = buildRiskPlan({ headSha: HEAD_SHA, changedFiles, focusedE2eJobs }); + + expect(focusedE2eJobs).toEqual([]); + expect(riskPlanRequiredJobIds(result)).toEqual([ + "cloud-inference", + "cloud-onboard", + "security-posture", + ]); + expect(result.families.map((family) => family.id)).toEqual(["e2e-control-plane"]); + }); + + it("keeps a renamed live test broad until the new path has an owning job (#7921)", () => { + const changedFiles = [ + "test/e2e/live/token-rotation.test.ts", + "test/e2e/live/token-rotation-renamed.test.ts", + ]; + const focusedE2eJobs = focusedE2eJobsForChangedFiles(changedFiles); + const result = buildRiskPlan({ headSha: HEAD_SHA, changedFiles, focusedE2eJobs }); + + expect(focusedE2eJobs).toEqual([ + { + id: "token-rotation", + matchedFiles: ["test/e2e/live/token-rotation.test.ts"], + }, + ]); + expect(riskPlanRequiredJobIds(result)).toEqual([ + "cloud-inference", + "cloud-onboard", + "security-posture", + "token-rotation", + ]); + expect( + result.families.find((family) => family.id === "e2e-control-plane")?.matchedFiles, + ).toEqual(["test/e2e/live/token-rotation-renamed.test.ts"]); + }); + + it("keeps a shared E2E workflow change behind the broad control-plane floor (#7921)", () => { + const result = plan(".github/workflows/e2e.yaml"); + + expect(riskPlanRequiredJobIds(result)).toEqual([ + "cloud-inference", + "cloud-onboard", + "security-posture", + ]); + expect(result.families.map((family) => family.id)).toEqual([ + "platform-install", + "e2e-control-plane", + ]); + }); + it("runs snapshot commands for restored-gateway pairing runtime changes (#7431)", () => { const runtimeFiles = [ "src/lib/actions/sandbox/restore-gateway-pairing.ts", diff --git a/tools/advisors/risk-plan.mts b/tools/advisors/risk-plan.mts index 6fc6a6d22a7..b4bf6576664 100644 --- a/tools/advisors/risk-plan.mts +++ b/tools/advisors/risk-plan.mts @@ -3,7 +3,7 @@ import { createHash } from "node:crypto"; -export const RISK_PLAN_VERSION = 8 as const; +export const RISK_PLAN_VERSION = 9 as const; export const PR_E2E_TYPED_TARGET_IDS = ["ubuntu-repo-cloud-langchain-deepagents-code"] as const; @@ -108,6 +108,7 @@ const RISK_RELEVANT_TEST_FILES = new Set([ "test/e2e/live/cloud-onboard.test.ts", "test/e2e/risk-signal-reporter.ts", ]); +const E2E_SUPPORT_FILE = /^test\/e2e\/support\//; const FOCUSED_E2E_SUMMARY = "Changed runtime surfaces and workflow-wired E2E tests must execute through their trusted canonical jobs or typed targets."; const FOCUSED_E2E_INVARIANTS = [ @@ -351,6 +352,7 @@ function normalizeFocusedE2eJobs( function isRuntimeRelevant(file: string): boolean { if (RISK_RELEVANT_TEST_FILES.has(file)) return true; + if (E2E_SUPPORT_FILE.test(file)) return false; if (file.startsWith("tools/e2e/") || file.startsWith("test/e2e/")) { return !/\.(?:md|mdx)$/u.test(file); } @@ -374,8 +376,12 @@ export function buildRiskPlan(options: { }): RiskPlan { const changedFiles = stableUnique(options.changedFiles); const runtimeFiles = changedFiles.filter(isRuntimeRelevant); + const focusedE2eJobs = normalizeFocusedE2eJobs(options.focusedE2eJobs ?? [], changedFiles); + const focusedLiveFiles = new Set(focusedE2eJobs.flatMap((selection) => selection.matchedFiles)); const staticFamilies: RiskPlanFamily[] = RISK_RULES.flatMap((rule) => { - const matchedFiles = runtimeFiles.filter(rule.matches); + const matchedFiles = runtimeFiles.filter( + (file) => rule.matches(file) && !(rule.id === "e2e-control-plane" && focusedLiveFiles.has(file)), + ); if (matchedFiles.length === 0) return []; return [ { @@ -389,7 +395,6 @@ export function buildRiskPlan(options: { }, ]; }); - const focusedE2eJobs = normalizeFocusedE2eJobs(options.focusedE2eJobs ?? [], changedFiles); const focusedE2eTargets = normalizeFocusedE2eJobs( focusedPrE2eTargetsForChangedFiles(changedFiles), changedFiles, diff --git a/tools/e2e/openshell-gateway-upgrade-workflow-boundary.mts b/tools/e2e/openshell-gateway-upgrade-workflow-boundary.mts index bf5fc49f93d..0190dec0e2f 100644 --- a/tools/e2e/openshell-gateway-upgrade-workflow-boundary.mts +++ b/tools/e2e/openshell-gateway-upgrade-workflow-boundary.mts @@ -9,19 +9,45 @@ import YAML from "yaml"; const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e.yaml"); -const JOB_NAME = "openshell-gateway-upgrade"; +const NIGHTLY_JOB_NAME = "openshell-gateway-upgrade"; +const COMPATIBILITY_JOB_NAME = "openshell-gateway-upgrade-compatibility"; +const NIGHTLY_CRON = "0 0 * * 1-6"; +const WEEKLY_CRON = "0 0 * * 0"; const RUN_STEP_NAME = "Run OpenShell gateway upgrade live Vitest test"; const RUN_COMMAND = "npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/openshell-gateway-upgrade.test.ts"; +const COMPATIBILITY_JOB_DISPLAY_NAME = + "openshell-gateway-upgrade-compatibility / ${{ matrix.id }}"; +const NIGHTLY_CONDITION = + "${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',openshell-gateway-upgrade,') || contains(format(',{0},', inputs.targets), ',openshell-gateway-upgrade,') }}"; +const COMPATIBILITY_CONDITION = + "${{ (github.event_name == 'schedule' && github.event.schedule == '0 0 * * 0') || contains(format(',{0},', inputs.jobs), ',openshell-gateway-upgrade-compatibility,') || contains(format(',{0},', inputs.targets), ',openshell-gateway-upgrade-compatibility,') }}"; type WorkflowRecord = Record; type WorkflowStep = WorkflowRecord & { name?: string; run?: string }; -const EXPECTED_V055_FIXTURES: WorkflowRecord[] = [ +const V036_FIXTURE: WorkflowRecord = { + id: "v0.0.36-x86_64", + runner: "ubuntu-latest", + shard: "v0-0-36-x86-64", + tier: "weekly", + boundary: "oldest retained registry migration", + nemoclaw_ref: "v0.0.36", + nemoclaw_commit: "3351fbdd4eb7d9b80ec471545083956327da2b10", + installer_sha256: "0c42400a0d3867739f1d75d612e069967be4506e169974bbbebf14b7af39144f", + sandbox_base_image_ref: + "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:104151ffadc2ff0b6c815e3c95c2783ced61aee0d0f83fc327cc02be9b7e14e6", + openshell_version: "0.0.36", + openclaw_version: "2026.4.24", +}; + +const V055_FIXTURES: WorkflowRecord[] = [ { id: "v0.0.55-x86_64", runner: "ubuntu-latest", shard: "v0-0-55-x86-64", + tier: "weekly", + boundary: "x86_64 OpenShell 0.0.44 regression", nemoclaw_ref: "v0.0.55", nemoclaw_commit: "95d483fe2b6569d68e59493c60f19df09a068e8f", installer_sha256: "ff8cf448e4d17b00421545a1f333262b615b1b0aa236d0cc5aeaf4e2cae2d897", @@ -34,6 +60,8 @@ const EXPECTED_V055_FIXTURES: WorkflowRecord[] = [ id: "v0.0.55-aarch64", runner: "ubuntu-24.04-arm", shard: "v0-0-55-aarch64", + tier: "weekly", + boundary: "arm64 OpenShell 0.0.44 regression", nemoclaw_ref: "v0.0.55", nemoclaw_commit: "95d483fe2b6569d68e59493c60f19df09a068e8f", installer_sha256: "ff8cf448e4d17b00421545a1f333262b615b1b0aa236d0cc5aeaf4e2cae2d897", @@ -44,10 +72,12 @@ const EXPECTED_V055_FIXTURES: WorkflowRecord[] = [ }, ]; -const EXPECTED_V074_FIXTURE: WorkflowRecord = { +const V074_FIXTURE: WorkflowRecord = { id: "v0.0.74-x86_64", runner: "ubuntu-latest", shard: "v0-0-74-x86-64", + tier: "weekly", + boundary: "immediate predecessor registry migration", nemoclaw_ref: "v0.0.74", nemoclaw_commit: "3a05b54e8ec3e1d5550ec5c728de54af872bffe3", installer_sha256: "a0cd3feca488d247e53d59d7d8246d2b86e75e95acb5e7d78504b3c0c60fd7db", @@ -57,10 +87,12 @@ const EXPECTED_V074_FIXTURE: WorkflowRecord = { openclaw_version: "2026.5.27", }; -const EXPECTED_V089_FIXTURE: WorkflowRecord = { +const V089_FIXTURE: WorkflowRecord = { id: "v0.0.89-x86_64", runner: "ubuntu-latest", shard: "v0-0-89-x86-64", + tier: "nightly", + boundary: "current OpenClaw state migration", nemoclaw_ref: "v0.0.89", nemoclaw_commit: "1143aa5cce77f3bad1b3b5588bd7fddbe438237e", installer_sha256: "00f24959e5ca68104fe91221c0a015dab6a4154618497fa36b969b661f418cc2", @@ -82,69 +114,99 @@ function jobSteps(job: WorkflowRecord): WorkflowStep[] { return Array.isArray(job.steps) ? (job.steps as WorkflowStep[]) : []; } -function v055Fixtures(job: WorkflowRecord): WorkflowRecord[] { +function matrixFixtures(job: WorkflowRecord): WorkflowRecord[] { const include = record(record(job.strategy).matrix).include; - return Array.isArray(include) - ? include.map(record).filter((fixture) => fixture.nemoclaw_ref === "v0.0.55") - : []; + return Array.isArray(include) ? include.map(record) : []; } -function v074Fixture(job: WorkflowRecord): WorkflowRecord { - const include = record(record(job.strategy).matrix).include; - return Array.isArray(include) - ? (include.map(record).find((fixture) => fixture.nemoclaw_ref === "v0.0.74") ?? {}) - : {}; -} - -function v089Fixture(job: WorkflowRecord): WorkflowRecord { - const include = record(record(job.strategy).matrix).include; - return Array.isArray(include) - ? (include.map(record).find((fixture) => fixture.nemoclaw_ref === "v0.0.89") ?? {}) - : {}; -} - -function requireRunContains(errors: string[], step: WorkflowStep, fragment: string): void { +function requireRunContains( + errors: string[], + jobName: string, + step: WorkflowStep, + fragment: string, +): void { if (!step.run?.includes(fragment)) { - errors.push(`${JOB_NAME} step '${RUN_STEP_NAME}' must run: ${fragment}`); + errors.push(`${jobName} step '${RUN_STEP_NAME}' must run: ${fragment}`); } } -export function readOpenShellGatewayUpgradeWorkflow( - workflowPath = DEFAULT_WORKFLOW_PATH, -): WorkflowRecord { - return YAML.parse(readFileSync(workflowPath, "utf8")) as WorkflowRecord; -} - -export function validateOpenShellGatewayUpgradeWorkflow(workflow: WorkflowRecord): string[] { - const errors: string[] = []; - const job = record(record(workflow.jobs)[JOB_NAME]); - - if (job["runs-on"] !== "${{ matrix.runner }}") { - errors.push(`${JOB_NAME} must run on \${{ matrix.runner }}`); +function validateJob( + errors: string[], + jobName: string, + job: WorkflowRecord, + fixtures: WorkflowRecord[], + condition: string, + explicitOnly: boolean, +): void { + if (job.if !== condition) { + errors.push(`${jobName} must retain its execution-tier selector`); } - if (!isDeepStrictEqual(v055Fixtures(job), EXPECTED_V055_FIXTURES)) { - errors.push(`${JOB_NAME} v0.0.55 matrix must pin x86_64 and arm64 upgrade fixtures`); + if (jobName === COMPATIBILITY_JOB_NAME && job.name !== COMPATIBILITY_JOB_DISPLAY_NAME) { + errors.push(`${jobName} must keep one scorecard identity across its matrix`); } - if (!isDeepStrictEqual(v074Fixture(job), EXPECTED_V074_FIXTURE)) { - errors.push(`${JOB_NAME} matrix must pin the immediate v0.0.74 x86_64 upgrade fixture`); + if (job["runs-on"] !== "${{ matrix.runner }}") { + errors.push(`${jobName} must run on \${{ matrix.runner }}`); } - if (!isDeepStrictEqual(v089Fixture(job), EXPECTED_V089_FIXTURE)) { - errors.push(`${JOB_NAME} matrix must pin the v0.0.89 OpenClaw state-upgrade fixture`); + if (!isDeepStrictEqual(matrixFixtures(job), fixtures)) { + errors.push(`${jobName} matrix must pin its tiered gateway upgrade fixtures`); } const env = record(job.env); + if (env.E2E_TARGET_ID !== jobName) { + errors.push(`${jobName} must publish its own target identity`); + } + if (explicitOnly ? env.E2E_DEFAULT_ENABLED !== "0" : Object.hasOwn(env, "E2E_DEFAULT_ENABLED")) { + errors.push( + explicitOnly + ? `${jobName} must remain explicit-only outside the weekly schedule` + : `${jobName} must remain default-enabled`, + ); + } if (env.NEMOCLAW_E2E_SHARD !== "${{ matrix.shard }}") { - errors.push(`${JOB_NAME} must publish one risk-signal shard per legacy fixture`); + errors.push(`${jobName} must publish one risk-signal shard per legacy fixture`); } if (env.NEMOCLAW_CURRENT_OPENCLAW_VERSION !== "${{ matrix.current_openclaw_version }}") { - errors.push(`${JOB_NAME} must bind the current OpenClaw version from its fixture`); + errors.push(`${jobName} must bind the current OpenClaw version from its fixture`); } if (env.NEMOCLAW_OPENCLAW_STATE_UPGRADE_PROOF !== "${{ matrix.openclaw_state_upgrade }}") { - errors.push(`${JOB_NAME} must bind the OpenClaw state-upgrade proof flag from its fixture`); + errors.push(`${jobName} must bind the OpenClaw state-upgrade proof flag from its fixture`); } - const run = jobSteps(job).find((step) => step.name === RUN_STEP_NAME) ?? {}; - requireRunContains(errors, run, RUN_COMMAND); + requireRunContains(errors, jobName, run, RUN_COMMAND); +} + +export function readOpenShellGatewayUpgradeWorkflow( + workflowPath = DEFAULT_WORKFLOW_PATH, +): WorkflowRecord { + return YAML.parse(readFileSync(workflowPath, "utf8")) as WorkflowRecord; +} + +export function validateOpenShellGatewayUpgradeWorkflow(workflow: WorkflowRecord): string[] { + const errors: string[] = []; + const jobs = record(workflow.jobs); + const triggers = record(workflow.on ?? workflow[true as unknown as string]); + const schedules = triggers.schedule; + if ( + !isDeepStrictEqual(schedules, [{ cron: NIGHTLY_CRON }, { cron: WEEKLY_CRON }]) + ) { + errors.push("E2E schedule must separate six nightly runs from the weekly compatibility run"); + } + validateJob( + errors, + NIGHTLY_JOB_NAME, + record(jobs[NIGHTLY_JOB_NAME]), + [V089_FIXTURE], + NIGHTLY_CONDITION, + false, + ); + validateJob( + errors, + COMPATIBILITY_JOB_NAME, + record(jobs[COMPATIBILITY_JOB_NAME]), + [V036_FIXTURE, ...V055_FIXTURES, V074_FIXTURE], + COMPATIBILITY_CONDITION, + true, + ); return errors; } diff --git a/tools/e2e/report-e2e-results.mts b/tools/e2e/report-e2e-results.mts index ba1ab62f001..4c8e6806339 100644 --- a/tools/e2e/report-e2e-results.mts +++ b/tools/e2e/report-e2e-results.mts @@ -188,6 +188,12 @@ export function renderE2eReport(input: { reason: "default dispatch excludes the resource-heavy OpenShell auth-contract probe unless selected", }, + "openshell-gateway-upgrade-compatibility": { + job: "openshell-gateway-upgrade-compatibility", + target: "openshell-gateway-upgrade-compatibility", + reason: + "nightly and default manual dispatches exclude weekly gateway compatibility migrations unless selected", + }, "mcp-bridge-dev": { job: "mcp-bridge-dev", target: "mcp-bridge-dev", @@ -205,9 +211,14 @@ export function renderE2eReport(input: { reason: "default dispatch excludes the destructive rlimit fork/connect probe unless selected", }, }; + const explicitlySelected = new Set( + [...requestedTestIdsCsv.split(","), ...requestedTargets.split(",")].filter(Boolean), + ); const explicitOnlySkippedJobs = (env.EXPLICIT_ONLY_JOBS || "") .split(",") .filter(Boolean) + .filter((job) => !explicitlySelected.has(job)) + .filter((job) => input.needs[job]?.result === "skipped") .map( (job) => explicitOnlyReasons[job] ?? { @@ -294,7 +305,9 @@ export function renderE2eReport(input: { for (const job of apiJobs) { const jobName = job.name || ""; const match = /^Shared E2E \(([A-Za-z0-9_-]+)\)$/.exec(jobName); - const aggregateJobName = aggregateJobNames.find((name) => jobName.startsWith(`${name} (`)); + const aggregateJobName = aggregateJobNames.find( + (name) => jobName.startsWith(`${name} (`) || jobName.startsWith(`${name} / `), + ); const reportEntryName = match?.[1] ?? aggregateJobName ?? @@ -428,14 +441,14 @@ export function renderE2eReport(input: { ? "**Requested test IDs:** _(selector rejected by workflow validation)_" : requestedTestIdsCsv ? `**Requested test IDs:** \`${requestedTestIdsCsv}\`` - : "**Requested test IDs:** _(default — all default-enabled tests; explicit-only tests `openshell-gateway-auth-contract`, `mcp-bridge-dev`, `hermes-gpu-startup`, `sandbox-rlimits-connect`, and `jetson-nvmap-gpu` are skipped unless selected)_", + : "**Requested test IDs:** _(default — workflow policy selected the enabled tests for this run)_", `**Summary:** ${passed.length} passed, ${failed.length} failed, ${cancelled.length} cancelled, ${skipped.length} skipped, ${unknown.length} unknown`, "", "| Test | Result | Total wall clock time |", "|-----|--------|-----------------------|", ...rows, ]; - if (!selectiveDispatch) { + if (!selectiveDispatch && explicitOnlySkippedJobs.length > 0) { const skippedJobHints = explicitOnlySkippedJobs .map( ({ job, target, reason }) => diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index ff8e0c9f331..1b470f9cc0a 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -196,6 +196,12 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ name: "e2e-openshell-gateway-upgrade-${{ matrix.id }}", }, ], + [ + "openshell-gateway-upgrade-compatibility", + { + name: "e2e-openshell-gateway-upgrade-${{ matrix.id }}", + }, + ], [ "openshell-gateway-auth-contract", { diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 1c71bfbdee3..6da51c612b9 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -156,6 +156,7 @@ const COMMON_SECRET_ENV_NAMES = [ const FREE_STANDING_SELECTOR_SPECIAL_CASES = new Set([ "hermes-e2e", "hermes-gpu-startup", + "openshell-gateway-upgrade-compatibility", "staging-brev-launchable", ]); const ADAPTER_MANAGED_INFERENCE_JOBS = new Set(["hermes-e2e"]); @@ -577,6 +578,20 @@ const RESTORED_GATEWAY_PAIRING_RUNTIME_FILES = new Set([ "src/lib/actions/sandbox/restore-gateway-pairing.ts", "src/lib/adapters/openshell/restore-gateway-pairing.ts", ]); +const LIVE_E2E_OWNING_FILE_JOBS = new Map([ + [ + "test/e2e/live/openclaw-plugin-runtime-exdev-lifecycle.ts", + ["openclaw-plugin-runtime-exdev"], + ], + [ + "test/e2e/live/openshell-gateway-upgrade-helpers.ts", + ["openshell-gateway-upgrade", "openshell-gateway-upgrade-compatibility"], + ], + [ + "test/e2e/live/openshell-gateway-upgrade-old-installer.ts", + ["openshell-gateway-upgrade", "openshell-gateway-upgrade-compatibility"], + ], +]); export function focusedE2eJobsForChangedFiles( changedFiles: readonly string[], @@ -587,6 +602,9 @@ export function focusedE2eJobsForChangedFiles( for (const job of inventory.liveTestToJobs.get(file) ?? []) { addMapValue(matchedFilesByJob, job, file); } + for (const job of LIVE_E2E_OWNING_FILE_JOBS.get(file) ?? []) { + if (inventory.allowedJobs.includes(job)) addMapValue(matchedFilesByJob, job, file); + } if (RESTORED_GATEWAY_PAIRING_RUNTIME_FILES.has(file)) { addMapValue(matchedFilesByJob, "snapshot-commands", file); } @@ -994,7 +1012,10 @@ function requireScheduledRun(errors: string[], triggers: WorkflowRecord): void { const cronEntries = schedule .map((entry) => asRecord(entry).cron) .filter((cron): cron is string => typeof cron === "string"); - if (!cronEntries.includes("0 0 * * *")) { + const runsDaily = + cronEntries.includes("0 0 * * *") || + (cronEntries.includes("0 0 * * 1-6") && cronEntries.includes("0 0 * * 0")); + if (!runsDaily) { errors.push("workflow schedule must run daily at 00:00 UTC"); } } From fbe230e4fbb2e0ee8375fcd850bb8a38ad3f71a0 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 30 Jul 2026 11:40:52 -0700 Subject: [PATCH 06/15] fix(e2e): retain retired selector compatibility Signed-off-by: Prekshi Vyas --- test/e2e/support/workflow-plan.test.ts | 31 ++++++++++++++++++++------ tools/e2e/workflow-plan.mts | 17 ++++++-------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts index aad54ea9d0e..7e93e990dee 100644 --- a/test/e2e/support/workflow-plan.test.ts +++ b/test/e2e/support/workflow-plan.test.ts @@ -180,28 +180,45 @@ describe("E2E workflow plan", () => { } }); - it("rejects a controller plan made only of retired selectors (#7616)", () => { + it.each( + RETIRED_CONTROLLER_SELECTOR_IDS, + )("emits an empty live plan for retired controller job %s (#7616)", (job) => { const directory = mkdtempSync(path.join(tmpdir(), "nemoclaw-workflow-plan-cli-")); + const output = path.join(directory, "github-output"); + const summary = path.join(directory, "summary.md"); + const plan = { + matrix: [], + testMatrix: [], + hermesSelected: false, + explicitOnlyJobs: readFreeStandingJobsInventory().explicitOnlyJobs, + }; try { const result = spawnSync(TSX, [PLANNER_CLI, "--ci-output"], { cwd: REPO_ROOT, encoding: "utf8", env: { ...process.env, - GITHUB_OUTPUT: path.join(directory, "github-output"), - GITHUB_STEP_SUMMARY: path.join(directory, "summary.md"), + GITHUB_OUTPUT: output, + GITHUB_STEP_SUMMARY: summary, INFERENCE_MODE: "mock", - JOBS: RETIRED_CONTROLLER_SELECTOR_IDS.join(","), + JOBS: job, TARGETS: "", NEMOCLAW_E2E_EXPECTED_SHA: "a".repeat(40), }, timeout: 30_000, }); - expect(result.status).toBe(1); - expect(result.stderr).toContain( - "::error::retired selector compatibility requires another controller-selected job", + expect(result.status, result.stderr).toBe(0); + expect(readFileSync(output, "utf8")).toBe( + [ + "matrix=[]", + "test_matrix=[]", + "hermes_selected=false", + `explicit_only_jobs=${plan.explicitOnlyJobs.join(",")}`, + "", + ].join("\n"), ); + expect(readFileSync(summary, "utf8")).toBe(renderE2eWorkflowPlanSummary(plan)); } finally { rmSync(directory, { force: true, recursive: true }); } diff --git a/tools/e2e/workflow-plan.mts b/tools/e2e/workflow-plan.mts index f2a296ffbc1..f3356f82f6f 100644 --- a/tools/e2e/workflow-plan.mts +++ b/tools/e2e/workflow-plan.mts @@ -31,7 +31,7 @@ type WorkflowPlanCliOptions = WorkflowPlanSelectors & { }; type TrustedControllerSelectorMap = { - retiredTargetSelected: boolean; + retiredSelectorSelected: boolean; selectors: WorkflowPlanSelectors; }; @@ -142,7 +142,7 @@ function mapTrustedControllerJobs( environment: NodeJS.ProcessEnv, ): TrustedControllerSelectorMap { if (!COMMIT_SHA_PATTERN.test(environment.NEMOCLAW_E2E_EXPECTED_SHA ?? "")) { - return { retiredTargetSelected: false, selectors }; + return { retiredSelectorSelected: false, selectors }; } const inventory = readFreeStandingJobsInventory(); @@ -169,15 +169,12 @@ function mapTrustedControllerJobs( ); const compatibleJobs = jobs.filter((job) => !retiredJobs.has(job)); const compatibleTargets = targets.filter((target) => !retiredTargets.has(target)); - if (jobs.length > 0 && compatibleJobs.length === 0 && !selectors.targets) { - throw new Error("retired selector compatibility requires another controller-selected job"); - } // Trusted main can select a renamed or newly retired job until the candidate // workflow becomes the controller. Keep the raw IDs for evidence, but plan // only jobs that still execute in the candidate. return { - retiredTargetSelected: retiredTargets.size > 0, + retiredSelectorSelected: retiredJobs.size > 0 || retiredTargets.size > 0, selectors: { ...selectors, jobs: compatibleJobs.join(","), @@ -252,13 +249,13 @@ export function validateE2eWorkflowPlan(plan: unknown): E2eWorkflowPlan { function expectedHermesSelection( selectors: WorkflowPlanSelectors, - retiredTargetSelected: boolean, + retiredSelectorSelected: boolean, ): boolean { const selected = [ ...selectorIds(selectors.jobs, "jobs"), ...selectorIds(selectors.targets, "targets"), ]; - return (selected.length === 0 && !retiredTargetSelected) || selected.includes(HERMES_JOB_ID); + return (selected.length === 0 && !retiredSelectorSelected) || selected.includes(HERMES_JOB_ID); } export function renderE2eWorkflowPlanSummary(plan: E2eWorkflowPlan): string { @@ -289,13 +286,13 @@ export function writeE2eWorkflowPlanCiOutput( const plannerSelectors = controllerMap.selectors; const hasPlannerSelectors = Boolean(plannerSelectors.jobs || plannerSelectors.targets); const plan = validateE2eWorkflowPlan( - controllerMap.retiredTargetSelected && !hasPlannerSelectors + controllerMap.retiredSelectorSelected && !hasPlannerSelectors ? emptyE2eWorkflowPlan() : buildE2eWorkflowPlan(plannerSelectors), ); if ( plan.hermesSelected !== - expectedHermesSelection(plannerSelectors, controllerMap.retiredTargetSelected) + expectedHermesSelection(plannerSelectors, controllerMap.retiredSelectorSelected) ) { throw new Error("E2E planner changed the trusted Hermes selection"); } From 5e00ea716c802381ded0b8f647d3a22db7822045 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 30 Jul 2026 12:21:38 -0700 Subject: [PATCH 07/15] test(e2e): align gateway release evidence Signed-off-by: Prekshi Vyas --- test/release-e2e-evidence.test.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/release-e2e-evidence.test.ts b/test/release-e2e-evidence.test.ts index 6900363b5bd..b37806f2cde 100644 --- a/test/release-e2e-evidence.test.ts +++ b/test/release-e2e-evidence.test.ts @@ -94,13 +94,14 @@ describe("release E2E evidence", () => { targets: "", }); const parallelExplicitJobs = plan.dispatches.parallelExplicit.jobs.split(","); - expect(parallelExplicitJobs).toHaveLength(4); + expect(parallelExplicitJobs).toHaveLength(5); expect(new Set(parallelExplicitJobs)).toEqual( new Set([ "openshell-gateway-auth-contract", "mcp-bridge-dev", "hermes-gpu-startup", "sandbox-rlimits-connect", + "openshell-gateway-upgrade-compatibility", ]), ); expect(plan.dispatches.conditional).toEqual([ @@ -116,7 +117,10 @@ describe("release E2E evidence", () => { expect(ids.filter((id) => id.startsWith("mcp-bridge-dev["))).toHaveLength(3); expect(ids.filter((id) => id.startsWith("hermes-gpu-startup["))).toHaveLength(3); - expect(ids.filter((id) => id.startsWith("openshell-gateway-upgrade["))).toHaveLength(5); + expect(ids.filter((id) => id.startsWith("openshell-gateway-upgrade["))).toHaveLength(1); + expect( + ids.filter((id) => id.startsWith("openshell-gateway-upgrade-compatibility[")), + ).toHaveLength(4); expect(ids).toContain("live[id=ubuntu-repo-cloud-openclaw]"); expect(ids).toContain("shared-e2e[id=vllm-docker-storage]"); expect(new Set(ids).size).toBe(ids.length); From 62bdeb9de53801be59bc89943d20f266704e950a Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 30 Jul 2026 12:41:22 -0700 Subject: [PATCH 08/15] test(e2e): retain batch upgrade coverage Signed-off-by: Prekshi Vyas --- .../upgrade-sandboxes-recovery.test.ts | 72 +++++++++++++++++++ .../retired-selector-compatibility.test.ts | 5 +- tools/e2e/retired-selector-compatibility.mts | 5 +- 3 files changed, 79 insertions(+), 3 deletions(-) diff --git a/src/lib/actions/upgrade-sandboxes-recovery.test.ts b/src/lib/actions/upgrade-sandboxes-recovery.test.ts index e0b32e6c0d7..446ec619784 100644 --- a/src/lib/actions/upgrade-sandboxes-recovery.test.ts +++ b/src/lib/actions/upgrade-sandboxes-recovery.test.ts @@ -7,7 +7,9 @@ import * as coreVersion from "../core/version"; import * as sandboxList from "../openshell-sandbox-list"; import * as sandboxVersion from "../sandbox/version"; import * as registry from "../state/registry"; +import type { SandboxRegistry } from "../state/registry/types"; import * as sandboxState from "../state/sandbox"; +import { commitRebuildRoutePreflight } from "./sandbox/rebuild-preflight-guards"; import { upgradeSandboxes, upgradeSandboxesDependencies } from "./upgrade-sandboxes"; type UpgradeSandboxes = typeof upgradeSandboxes; @@ -306,6 +308,76 @@ describe("upgrade-sandboxes prepared backup recovery (#6114)", () => { ); }); + it("migrates missing credential identities for two sandboxes on a shared route during upgrade-sandboxes --auto (#7615, #7798)", async () => { + const names = ["alpha", "beta"]; + const credentialEnv = "NVIDIA_INFERENCE_API_KEY"; + let persisted: SandboxRegistry = { + defaultSandbox: names[0], + sandboxes: Object.fromEntries( + names.map((name) => [ + name, + { + name, + agent: "openclaw", + agentVersion: "2026.5.27", + credentialEnv: null, + gatewayName: "nemoclaw", + gatewayPort: 8080, + model: "nvidia/llama-3.3-nemotron-super-49b-v1.5", + nemoclawVersion: "0.0.71", + provider: "nvidia-prod", + }, + ]), + ), + }; + const harness = createRecoveryHarness(names, { + liveOutput: names.map((name) => `${name} Ready`).join("\n"), + staleNames: names, + }); + vi.mocked(registry.listSandboxes).mockImplementation(() => { + const snapshot = structuredClone(persisted); + return { + defaultSandbox: snapshot.defaultSandbox, + sandboxes: Object.values(snapshot.sandboxes), + }; + }); + const routeResults: ReturnType[] = []; + harness.rebuildSpy.mockImplementation(async (sandboxName: string) => { + routeResults.push( + commitRebuildRoutePreflight( + { + sandboxName, + gatewayName: "nemoclaw", + targetUpdate: { + credentialEnv, + model: persisted.sandboxes[sandboxName]?.model, + provider: persisted.sandboxes[sandboxName]?.provider, + }, + }, + { + withLock: (fn: () => T): T => fn(), + load: () => structuredClone(persisted), + save: (next) => { + persisted = structuredClone(next); + }, + }, + ), + ); + }); + + await expect(harness.upgradeSandboxes(["--auto"])).resolves.toBeUndefined(); + + expect(harness.rebuildSpy.mock.calls.map((call) => call[0])).toEqual(names); + expect(routeResults).toMatchObject([ + { ok: true, receipt: { sandboxName: "alpha", migratedSandboxNames: ["beta"] } }, + { ok: true, receipt: { sandboxName: "beta", migratedSandboxNames: [] } }, + ]); + expect(names.map((name) => persisted.sandboxes[name]?.credentialEnv)).toEqual([ + credentialEnv, + credentialEnv, + ]); + }); + it("fails closed for a probed v0.0.55 custom image with matching backup agent version", async () => { const probedAgentVersion = "2026.5.27"; const harness = createRecoveryHarness(["custom-box"], { diff --git a/test/e2e/support/retired-selector-compatibility.test.ts b/test/e2e/support/retired-selector-compatibility.test.ts index ce6af3f24b8..25cf9443444 100644 --- a/test/e2e/support/retired-selector-compatibility.test.ts +++ b/test/e2e/support/retired-selector-compatibility.test.ts @@ -19,6 +19,7 @@ const CORRELATION_ID = "123e4567-e89b-42d3-a456-426614174000"; const REPLACEMENT_FILES = [ "src/lib/actions/sandbox/rebuild-finalization.test.ts", "src/lib/actions/sandbox/rebuild-flow-helpers.test.ts", + "src/lib/actions/upgrade-sandboxes-recovery.test.ts", "src/lib/sandbox/version.test.ts", "src/lib/security/credential-filter-secret-patterns.test.ts", "test/cli/list-share-live-inference.test.ts", @@ -110,7 +111,7 @@ describe("retired E2E selector compatibility", () => { expect(selected).toEqual([...RETIRED_CONTROLLER_SELECTOR_IDS].sort()); expect(commands).toEqual([ - "npx vitest run --project cli src/lib/actions/sandbox/rebuild-finalization.test.ts src/lib/actions/sandbox/rebuild-flow-helpers.test.ts src/lib/sandbox/version.test.ts src/lib/security/credential-filter-secret-patterns.test.ts", + "npx vitest run --project cli src/lib/actions/sandbox/rebuild-finalization.test.ts src/lib/actions/sandbox/rebuild-flow-helpers.test.ts src/lib/actions/upgrade-sandboxes-recovery.test.ts src/lib/sandbox/version.test.ts src/lib/security/credential-filter-secret-patterns.test.ts", "npx vitest run --project integration test/cli/list-share-live-inference.test.ts test/credential-migration-reconciliation.test.ts test/credentials.test.ts test/gateway-drift-preflight.test.ts test/gateway-health-honest.test.ts test/rebuild-stale-recovery.test.ts", "npx vitest run --project installer-integration test/install-openshell-version-pin.test.ts", "npx vitest run --project package-contract test/package-contract/cli/debug-cli-command.test.ts test/package-contract/cli/public-cli-contracts.test.ts test/package-contract/onboard/invalid-nvidia-key.test.ts", @@ -164,7 +165,7 @@ describe("retired E2E selector compatibility", () => { expect(selected).toEqual(["sandbox-rebuild", "upgrade-stale-sandbox"]); expect(commands).toEqual([ - "npx vitest run --project cli src/lib/actions/sandbox/rebuild-finalization.test.ts src/lib/actions/sandbox/rebuild-flow-helpers.test.ts src/lib/sandbox/version.test.ts", + "npx vitest run --project cli src/lib/actions/sandbox/rebuild-finalization.test.ts src/lib/actions/sandbox/rebuild-flow-helpers.test.ts src/lib/actions/upgrade-sandboxes-recovery.test.ts src/lib/sandbox/version.test.ts", "npx vitest run --project integration test/cli/list-share-live-inference.test.ts test/rebuild-stale-recovery.test.ts", ]); for (const id of selected) { diff --git a/tools/e2e/retired-selector-compatibility.mts b/tools/e2e/retired-selector-compatibility.mts index 7dc24210c77..c4137b8f2aa 100644 --- a/tools/e2e/retired-selector-compatibility.mts +++ b/tools/e2e/retired-selector-compatibility.mts @@ -131,7 +131,10 @@ const REPLACEMENTS: Readonly> = legacyFile: "test/e2e/live/upgrade-stale-sandbox.test.ts", tests: [ { - files: ["src/lib/sandbox/version.test.ts"], + files: [ + "src/lib/actions/upgrade-sandboxes-recovery.test.ts", + "src/lib/sandbox/version.test.ts", + ], project: "cli", }, { From c193e476604083ad8e4671b9af79fc01f482f14b Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 30 Jul 2026 12:54:53 -0700 Subject: [PATCH 09/15] test(e2e): separate batch upgrade boundaries Signed-off-by: Prekshi Vyas --- .../sandbox/rebuild-route-preflight.test.ts | 29 ++++++++ .../upgrade-sandboxes-recovery.test.ts | 71 +++---------------- .../retired-selector-compatibility.test.ts | 5 +- tools/e2e/retired-selector-compatibility.mts | 1 + 4 files changed, 42 insertions(+), 64 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-route-preflight.test.ts b/src/lib/actions/sandbox/rebuild-route-preflight.test.ts index 1226c19da28..15779b69117 100644 --- a/src/lib/actions/sandbox/rebuild-route-preflight.test.ts +++ b/src/lib/actions/sandbox/rebuild-route-preflight.test.ts @@ -124,6 +124,35 @@ describe("commitRebuildRoutePreflight", () => { expect(state.save).toHaveBeenCalledOnce(); }); + it("migrates two missing credential identities across sequential shared-route rebuilds (#7615, #7798)", () => { + const credentialEnv = "NVIDIA_INFERENCE_API_KEY"; + const alpha = sandbox("alpha", "nvidia-prod"); + const beta = sandbox("beta", "nvidia-prod"); + const state = transactionDependencies(registry(alpha, beta)); + + const results = [alpha, beta].map((entry) => + commitRebuildRoutePreflight( + { + sandboxName: entry.name, + gatewayName: "nemoclaw", + targetUpdate: targetUpdate({ ...entry, credentialEnv }), + }, + state.dependencies, + ), + ); + + expect(results).toMatchObject([ + { ok: true, receipt: { sandboxName: "alpha", migratedSandboxNames: ["beta"] } }, + { ok: true, receipt: { sandboxName: "beta", migratedSandboxNames: [] } }, + ]); + expect( + [alpha.name, beta.name].map( + (sandboxName) => state.persisted().sandboxes[sandboxName]?.credentialEnv, + ), + ).toEqual([credentialEnv, credentialEnv]); + expect(state.save).toHaveBeenCalledTimes(2); + }); + it.each( LOCAL_INFERENCE_PROVIDERS, )("keeps credential-free local provider %s compatible (#7798)", (provider) => { diff --git a/src/lib/actions/upgrade-sandboxes-recovery.test.ts b/src/lib/actions/upgrade-sandboxes-recovery.test.ts index 446ec619784..16d734ebb6b 100644 --- a/src/lib/actions/upgrade-sandboxes-recovery.test.ts +++ b/src/lib/actions/upgrade-sandboxes-recovery.test.ts @@ -7,9 +7,7 @@ import * as coreVersion from "../core/version"; import * as sandboxList from "../openshell-sandbox-list"; import * as sandboxVersion from "../sandbox/version"; import * as registry from "../state/registry"; -import type { SandboxRegistry } from "../state/registry/types"; import * as sandboxState from "../state/sandbox"; -import { commitRebuildRoutePreflight } from "./sandbox/rebuild-preflight-guards"; import { upgradeSandboxes, upgradeSandboxesDependencies } from "./upgrade-sandboxes"; type UpgradeSandboxes = typeof upgradeSandboxes; @@ -308,74 +306,23 @@ describe("upgrade-sandboxes prepared backup recovery (#6114)", () => { ); }); - it("migrates missing credential identities for two sandboxes on a shared route during upgrade-sandboxes --auto (#7615, #7798)", async () => { + it("forwards two stale shared-route sandboxes through upgrade-sandboxes --auto (#7615, #7798)", async () => { const names = ["alpha", "beta"]; - const credentialEnv = "NVIDIA_INFERENCE_API_KEY"; - let persisted: SandboxRegistry = { - defaultSandbox: names[0], - sandboxes: Object.fromEntries( - names.map((name) => [ - name, - { - name, - agent: "openclaw", - agentVersion: "2026.5.27", - credentialEnv: null, - gatewayName: "nemoclaw", - gatewayPort: 8080, - model: "nvidia/llama-3.3-nemotron-super-49b-v1.5", - nemoclawVersion: "0.0.71", - provider: "nvidia-prod", - }, - ]), - ), - }; const harness = createRecoveryHarness(names, { liveOutput: names.map((name) => `${name} Ready`).join("\n"), staleNames: names, }); - vi.mocked(registry.listSandboxes).mockImplementation(() => { - const snapshot = structuredClone(persisted); - return { - defaultSandbox: snapshot.defaultSandbox, - sandboxes: Object.values(snapshot.sandboxes), - }; - }); - const routeResults: ReturnType[] = []; - harness.rebuildSpy.mockImplementation(async (sandboxName: string) => { - routeResults.push( - commitRebuildRoutePreflight( - { - sandboxName, - gatewayName: "nemoclaw", - targetUpdate: { - credentialEnv, - model: persisted.sandboxes[sandboxName]?.model, - provider: persisted.sandboxes[sandboxName]?.provider, - }, - }, - { - withLock: (fn: () => T): T => fn(), - load: () => structuredClone(persisted), - save: (next) => { - persisted = structuredClone(next); - }, - }, - ), - ); - }); await expect(harness.upgradeSandboxes(["--auto"])).resolves.toBeUndefined(); - expect(harness.rebuildSpy.mock.calls.map((call) => call[0])).toEqual(names); - expect(routeResults).toMatchObject([ - { ok: true, receipt: { sandboxName: "alpha", migratedSandboxNames: ["beta"] } }, - { ok: true, receipt: { sandboxName: "beta", migratedSandboxNames: [] } }, - ]); - expect(names.map((name) => persisted.sandboxes[name]?.credentialEnv)).toEqual([ - credentialEnv, - credentialEnv, - ]); + expect(harness.rebuildSpy).toHaveBeenNthCalledWith(1, "alpha", ["--yes"], { + recoveryManifest: undefined, + throwOnError: true, + }); + expect(harness.rebuildSpy).toHaveBeenNthCalledWith(2, "beta", ["--yes"], { + recoveryManifest: undefined, + throwOnError: true, + }); }); it("fails closed for a probed v0.0.55 custom image with matching backup agent version", async () => { diff --git a/test/e2e/support/retired-selector-compatibility.test.ts b/test/e2e/support/retired-selector-compatibility.test.ts index 25cf9443444..6260ddadc11 100644 --- a/test/e2e/support/retired-selector-compatibility.test.ts +++ b/test/e2e/support/retired-selector-compatibility.test.ts @@ -19,6 +19,7 @@ const CORRELATION_ID = "123e4567-e89b-42d3-a456-426614174000"; const REPLACEMENT_FILES = [ "src/lib/actions/sandbox/rebuild-finalization.test.ts", "src/lib/actions/sandbox/rebuild-flow-helpers.test.ts", + "src/lib/actions/sandbox/rebuild-route-preflight.test.ts", "src/lib/actions/upgrade-sandboxes-recovery.test.ts", "src/lib/sandbox/version.test.ts", "src/lib/security/credential-filter-secret-patterns.test.ts", @@ -111,7 +112,7 @@ describe("retired E2E selector compatibility", () => { expect(selected).toEqual([...RETIRED_CONTROLLER_SELECTOR_IDS].sort()); expect(commands).toEqual([ - "npx vitest run --project cli src/lib/actions/sandbox/rebuild-finalization.test.ts src/lib/actions/sandbox/rebuild-flow-helpers.test.ts src/lib/actions/upgrade-sandboxes-recovery.test.ts src/lib/sandbox/version.test.ts src/lib/security/credential-filter-secret-patterns.test.ts", + "npx vitest run --project cli src/lib/actions/sandbox/rebuild-finalization.test.ts src/lib/actions/sandbox/rebuild-flow-helpers.test.ts src/lib/actions/sandbox/rebuild-route-preflight.test.ts src/lib/actions/upgrade-sandboxes-recovery.test.ts src/lib/sandbox/version.test.ts src/lib/security/credential-filter-secret-patterns.test.ts", "npx vitest run --project integration test/cli/list-share-live-inference.test.ts test/credential-migration-reconciliation.test.ts test/credentials.test.ts test/gateway-drift-preflight.test.ts test/gateway-health-honest.test.ts test/rebuild-stale-recovery.test.ts", "npx vitest run --project installer-integration test/install-openshell-version-pin.test.ts", "npx vitest run --project package-contract test/package-contract/cli/debug-cli-command.test.ts test/package-contract/cli/public-cli-contracts.test.ts test/package-contract/onboard/invalid-nvidia-key.test.ts", @@ -165,7 +166,7 @@ describe("retired E2E selector compatibility", () => { expect(selected).toEqual(["sandbox-rebuild", "upgrade-stale-sandbox"]); expect(commands).toEqual([ - "npx vitest run --project cli src/lib/actions/sandbox/rebuild-finalization.test.ts src/lib/actions/sandbox/rebuild-flow-helpers.test.ts src/lib/actions/upgrade-sandboxes-recovery.test.ts src/lib/sandbox/version.test.ts", + "npx vitest run --project cli src/lib/actions/sandbox/rebuild-finalization.test.ts src/lib/actions/sandbox/rebuild-flow-helpers.test.ts src/lib/actions/sandbox/rebuild-route-preflight.test.ts src/lib/actions/upgrade-sandboxes-recovery.test.ts src/lib/sandbox/version.test.ts", "npx vitest run --project integration test/cli/list-share-live-inference.test.ts test/rebuild-stale-recovery.test.ts", ]); for (const id of selected) { diff --git a/tools/e2e/retired-selector-compatibility.mts b/tools/e2e/retired-selector-compatibility.mts index c4137b8f2aa..df0e803facf 100644 --- a/tools/e2e/retired-selector-compatibility.mts +++ b/tools/e2e/retired-selector-compatibility.mts @@ -132,6 +132,7 @@ const REPLACEMENTS: Readonly> = tests: [ { files: [ + "src/lib/actions/sandbox/rebuild-route-preflight.test.ts", "src/lib/actions/upgrade-sandboxes-recovery.test.ts", "src/lib/sandbox/version.test.ts", ], From 4f640912163dc11f9ed48b038e6ac265db855831 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 30 Jul 2026 13:34:37 -0700 Subject: [PATCH 10/15] fix(e2e): keep gateway migration on the existing lane Signed-off-by: Apurv Kumaria --- .github/workflows/e2e.yaml | 129 +++++---------- scripts/scorecard/summarize-jobs.mts | 10 +- test/e2e/README.md | 28 +--- ...e2e-report-to-pr-workflow-boundary.test.ts | 93 +---------- test/e2e/support/e2e-scorecard.test.ts | 40 ----- test/e2e/support/e2e-workflow.test.ts | 4 - ...-gateway-upgrade-workflow-boundary.test.ts | 55 +++--- test/pr-e2e-gate-signal-shards.test.ts | 12 +- test/pr-risk-plan.test.ts | 11 +- test/release-e2e-evidence.test.ts | 8 +- ...hell-gateway-upgrade-workflow-boundary.mts | 156 ++++++------------ tools/e2e/report-e2e-results.mts | 19 +-- ...upload-e2e-artifacts-workflow-boundary.mts | 6 - tools/e2e/workflow-boundary.mts | 16 +- 14 files changed, 128 insertions(+), 459 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 0a5ae6dc7af..2fd66dde71a 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -6,8 +6,7 @@ run-name: "${{ inputs.checkout_sha != '' && format('E2E PR #{0} ({1})', inputs.p on: schedule: - - cron: "0 0 * * 1-6" - - cron: "0 0 * * 0" + - cron: "0 0 * * *" workflow_dispatch: inputs: targets: @@ -16,7 +15,7 @@ on: default: "" type: string jobs: - description: "Optional comma-separated E2E test IDs. Empty runs default-enabled tests only when targets is also empty; explicit-only tests openshell-gateway-auth-contract, openshell-gateway-upgrade-compatibility, mcp-bridge-dev, hermes-gpu-startup, sandbox-rlimits-connect, jetson-nvmap-gpu, and staging-brev-launchable are skipped unless selected." + description: "Optional comma-separated E2E test IDs. Empty runs default-enabled tests only when targets is also empty; explicit-only tests openshell-gateway-auth-contract, mcp-bridge-dev, hermes-gpu-startup, sandbox-rlimits-connect, jetson-nvmap-gpu, and staging-brev-launchable are skipped unless selected." required: false default: "" type: string @@ -4211,11 +4210,45 @@ jobs: fail-fast: false matrix: include: + - id: v0.0.36-x86_64 + runner: ubuntu-latest + shard: v0-0-36-x86-64 + nemoclaw_ref: v0.0.36 + nemoclaw_commit: "3351fbdd4eb7d9b80ec471545083956327da2b10" + installer_sha256: "0c42400a0d3867739f1d75d612e069967be4506e169974bbbebf14b7af39144f" + sandbox_base_image_ref: "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:104151ffadc2ff0b6c815e3c95c2783ced61aee0d0f83fc327cc02be9b7e14e6" + openshell_version: 0.0.36 + openclaw_version: 2026.4.24 + - id: v0.0.55-x86_64 + runner: ubuntu-latest + shard: v0-0-55-x86-64 + nemoclaw_ref: v0.0.55 + nemoclaw_commit: "95d483fe2b6569d68e59493c60f19df09a068e8f" + installer_sha256: "ff8cf448e4d17b00421545a1f333262b615b1b0aa236d0cc5aeaf4e2cae2d897" + sandbox_base_image_ref: "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:10433a8cd2f2b809dd0fdf983514679e04c0f8aa1ff5bbff675029046033b108" + openshell_version: 0.0.44 + openclaw_version: 2026.5.22 + - id: v0.0.55-aarch64 + runner: ubuntu-24.04-arm + shard: v0-0-55-aarch64 + nemoclaw_ref: v0.0.55 + nemoclaw_commit: "95d483fe2b6569d68e59493c60f19df09a068e8f" + installer_sha256: "ff8cf448e4d17b00421545a1f333262b615b1b0aa236d0cc5aeaf4e2cae2d897" + sandbox_base_image_ref: "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:10433a8cd2f2b809dd0fdf983514679e04c0f8aa1ff5bbff675029046033b108" + openshell_version: 0.0.44 + openclaw_version: 2026.5.22 + - id: v0.0.74-x86_64 + runner: ubuntu-latest + shard: v0-0-74-x86-64 + nemoclaw_ref: v0.0.74 + nemoclaw_commit: "3a05b54e8ec3e1d5550ec5c728de54af872bffe3" + installer_sha256: "a0cd3feca488d247e53d59d7d8246d2b86e75e95acb5e7d78504b3c0c60fd7db" + sandbox_base_image_ref: "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:104151ffadc2ff0b6c815e3c95c2783ced61aee0d0f83fc327cc02be9b7e14e6" + openshell_version: 0.0.72 + openclaw_version: 2026.5.27 - id: v0.0.89-x86_64 runner: ubuntu-latest shard: v0-0-89-x86-64 - tier: nightly - boundary: current OpenClaw state migration nemoclaw_ref: v0.0.89 nemoclaw_commit: "1143aa5cce77f3bad1b3b5588bd7fddbe438237e" installer_sha256: "00f24959e5ca68104fe91221c0a015dab6a4154618497fa36b969b661f418cc2" @@ -4243,7 +4276,7 @@ jobs: NEMOCLAW_CURRENT_OPENCLAW_VERSION: ${{ matrix.current_openclaw_version }} NEMOCLAW_OPENCLAW_STATE_UPGRADE_PROOF: ${{ matrix.openclaw_state_upgrade }} OPENSHELL_GATEWAY: "nemoclaw" - steps: &openshell-gateway-upgrade-steps + steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ inputs.checkout_repository || github.repository }} @@ -4256,8 +4289,11 @@ jobs: uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 - name: Run OpenShell gateway upgrade live Vitest test - # Each matrix row keeps one migration boundary. The workflow tier - # controls when the row runs; the live assertions remain identical. + # Keep the original v0.0.36 fixture on x86_64 and validate the exact + # v0.0.55/OpenShell 0.0.44 regression shape on x86_64 and arm64, plus + # the immediate v0.0.74/OpenShell 0.0.72 predecessor to this bump. The + # v0.0.89 row proves OpenClaw 2026.6.10 state survives the 2026.7.1 + # migration without crossing the OpenShell secret boundary. env: GITHUB_TOKEN: ${{ github.token }} run: | @@ -4276,82 +4312,6 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh - openshell-gateway-upgrade-compatibility: - name: openshell-gateway-upgrade-compatibility / ${{ matrix.id }} - needs: generate-matrix - if: ${{ (github.event_name == 'schedule' && github.event.schedule == '0 0 * * 0') || contains(format(',{0},', inputs.jobs), ',openshell-gateway-upgrade-compatibility,') || contains(format(',{0},', inputs.targets), ',openshell-gateway-upgrade-compatibility,') }} - runs-on: ${{ matrix.runner }} - timeout-minutes: 70 - strategy: - fail-fast: false - matrix: - include: - - id: v0.0.36-x86_64 - runner: ubuntu-latest - shard: v0-0-36-x86-64 - tier: weekly - boundary: oldest retained registry migration - nemoclaw_ref: v0.0.36 - nemoclaw_commit: "3351fbdd4eb7d9b80ec471545083956327da2b10" - installer_sha256: "0c42400a0d3867739f1d75d612e069967be4506e169974bbbebf14b7af39144f" - sandbox_base_image_ref: "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:104151ffadc2ff0b6c815e3c95c2783ced61aee0d0f83fc327cc02be9b7e14e6" - openshell_version: 0.0.36 - openclaw_version: 2026.4.24 - - id: v0.0.55-x86_64 - runner: ubuntu-latest - shard: v0-0-55-x86-64 - tier: weekly - boundary: x86_64 OpenShell 0.0.44 regression - nemoclaw_ref: v0.0.55 - nemoclaw_commit: "95d483fe2b6569d68e59493c60f19df09a068e8f" - installer_sha256: "ff8cf448e4d17b00421545a1f333262b615b1b0aa236d0cc5aeaf4e2cae2d897" - sandbox_base_image_ref: "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:10433a8cd2f2b809dd0fdf983514679e04c0f8aa1ff5bbff675029046033b108" - openshell_version: 0.0.44 - openclaw_version: 2026.5.22 - - id: v0.0.55-aarch64 - runner: ubuntu-24.04-arm - shard: v0-0-55-aarch64 - tier: weekly - boundary: arm64 OpenShell 0.0.44 regression - nemoclaw_ref: v0.0.55 - nemoclaw_commit: "95d483fe2b6569d68e59493c60f19df09a068e8f" - installer_sha256: "ff8cf448e4d17b00421545a1f333262b615b1b0aa236d0cc5aeaf4e2cae2d897" - sandbox_base_image_ref: "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:10433a8cd2f2b809dd0fdf983514679e04c0f8aa1ff5bbff675029046033b108" - openshell_version: 0.0.44 - openclaw_version: 2026.5.22 - - id: v0.0.74-x86_64 - runner: ubuntu-latest - shard: v0-0-74-x86-64 - tier: weekly - boundary: immediate predecessor registry migration - nemoclaw_ref: v0.0.74 - nemoclaw_commit: "3a05b54e8ec3e1d5550ec5c728de54af872bffe3" - installer_sha256: "a0cd3feca488d247e53d59d7d8246d2b86e75e95acb5e7d78504b3c0c60fd7db" - sandbox_base_image_ref: "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:104151ffadc2ff0b6c815e3c95c2783ced61aee0d0f83fc327cc02be9b7e14e6" - openshell_version: 0.0.72 - openclaw_version: 2026.5.27 - env: - E2E_JOB: "1" - E2E_DEFAULT_ENABLED: "0" - E2E_TARGET_ID: "openshell-gateway-upgrade-compatibility" - E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openshell-gateway-upgrade-compatibility - NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js - NEMOCLAW_E2E_SHARD: ${{ matrix.shard }} - NEMOCLAW_RUN_LIVE_E2E: "1" - NEMOCLAW_NON_INTERACTIVE: "1" - NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" - NEMOCLAW_GATEWAY_UPGRADE_SURVIVOR_NAME: "e2e-gateway-upgrade-survivor" - NEMOCLAW_OLD_NEMOCLAW_REF: ${{ matrix.nemoclaw_ref }} - NEMOCLAW_OLD_NEMOCLAW_COMMIT: ${{ matrix.nemoclaw_commit }} - NEMOCLAW_OLD_INSTALLER_SHA256: ${{ matrix.installer_sha256 }} - NEMOCLAW_OLD_SANDBOX_BASE_IMAGE_REF: ${{ matrix.sandbox_base_image_ref }} - NEMOCLAW_OLD_OPENSHELL_VERSION: ${{ matrix.openshell_version }} - NEMOCLAW_OLD_OPENCLAW_VERSION: ${{ matrix.openclaw_version }} - NEMOCLAW_CURRENT_OPENCLAW_VERSION: ${{ matrix.current_openclaw_version }} - NEMOCLAW_OPENCLAW_STATE_UPGRADE_PROOF: ${{ matrix.openclaw_state_upgrade }} - OPENSHELL_GATEWAY: "nemoclaw" - steps: *openshell-gateway-upgrade-steps - messaging-providers: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',messaging-providers,') || contains(format(',{0},', inputs.targets), ',messaging-providers,') }} @@ -5741,7 +5701,6 @@ jobs: overlayfs-autofix, state-backup-restore, openshell-gateway-upgrade, - openshell-gateway-upgrade-compatibility, token-rotation, messaging-compatible-endpoint, messaging-providers, diff --git a/scripts/scorecard/summarize-jobs.mts b/scripts/scorecard/summarize-jobs.mts index 3b7bafcadbf..7e45868a30d 100644 --- a/scripts/scorecard/summarize-jobs.mts +++ b/scripts/scorecard/summarize-jobs.mts @@ -184,10 +184,7 @@ function summarizeJobs(input: SummarizeJobsInput): JobSummary { if (input.apiJobs !== null) { const eligibleJobs = input.apiJobs.filter((job) => { const name = job.name.replace(/ \/ [^/]+$/u, ""); - return ( - !metaJobs.has(name) && - (!explicitOnly.has(name) || selected.has(name) || classifyApiJob(job) !== "skipped") - ); + return !metaJobs.has(name) && (!explicitOnly.has(name) || selected.has(name)); }); const jobs = normalizeApiJobs(eligibleJobs); const classified = jobs.map((job) => ({ job, result: classifyApiJob(job) })); @@ -205,10 +202,7 @@ function summarizeJobs(input: SummarizeJobsInput): JobSummary { const entries = Object.entries(input.needs) .filter(([name]) => !metaJobs.has(name)) - .filter( - ([name, value]) => - !explicitOnly.has(name) || selected.has(name) || classifyNeed(value) !== "skipped", - ) + .filter(([name]) => !explicitOnly.has(name) || selected.has(name)) .sort(([left], [right]) => left.localeCompare(right)); const classified = entries.map(([name, value]) => ({ name, result: classifyNeed(value) })); const counts = countResults(classified.map(({ result }) => result)); diff --git a/test/e2e/README.md b/test/e2e/README.md index 9064d016b27..dfd5ad0ab8d 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -254,31 +254,6 @@ graph as the live targets: `post_to_slack=true`, which uses the preview Slack route. Branch-dispatched runs never receive Slack webhook secrets. -### Gateway migration tiers - -The OpenShell gateway migration matrix separates the current state-migration -boundary from retained historical and architecture boundaries: - -| Job | NemoClaw row | Runner | Boundary | Scheduled cadence | -| --- | --- | --- | --- | --- | -| `openshell-gateway-upgrade` | `v0.0.89-x86_64` | `ubuntu-latest` | current OpenClaw state migration | Daily | -| `openshell-gateway-upgrade-compatibility` | `v0.0.36-x86_64` | `ubuntu-latest` | oldest retained registry migration | Sunday | -| `openshell-gateway-upgrade-compatibility` | `v0.0.55-x86_64` | `ubuntu-latest` | x86_64 OpenShell 0.0.44 regression | Sunday | -| `openshell-gateway-upgrade-compatibility` | `v0.0.55-aarch64` | `ubuntu-24.04-arm` | arm64 OpenShell 0.0.44 regression | Sunday | -| `openshell-gateway-upgrade-compatibility` | `v0.0.74-x86_64` | `ubuntu-latest` | immediate predecessor registry migration | Sunday | - -The workflow schedules the nightly path Monday through Saturday and schedules -the complete tier set on Sunday. The current row runs on both schedules because -`openshell-gateway-upgrade` is default-enabled. -`openshell-gateway-upgrade-compatibility` is explicit-only outside the Sunday -schedule. Select that job directly through the `jobs` or `targets` input for a -focused PR run or another required qualification. - -Nightly reports list the compatibility job as skipped when it is not selected; -they do not classify that tier as failed. The split removes four shards from -each Monday-through-Saturday scheduled run. Runner-minute measurements for the -new cadence are pending. - A manual run with `jobs=staging-brev-launchable` runs only the exact staging Launchable E2E job. Scheduled runs do not select this job. @@ -612,8 +587,7 @@ Runtime families and changes to workflow-wired live tests or their owning helpers select canonical jobs from the trusted `e2e.yaml` inventory independently of advisor output. A workflow-wired live test or owning helper selects one to three focused E2E journeys. A gateway-migration live test or -owning helper selects both `openshell-gateway-upgrade` and -`openshell-gateway-upgrade-compatibility`. +owning helper selects `openshell-gateway-upgrade`. Changes only under `test/e2e/support/` select no credentialed live E2E job. The `e2e-support` Vitest project runs those support tests in PR CI. A new or diff --git a/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts b/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts index 27746ccc572..882cd5d2347 100644 --- a/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts +++ b/test/e2e/support/e2e-report-to-pr-workflow-boundary.test.ts @@ -179,7 +179,6 @@ async function executeReport(options: { testMatrix?: CredentialFreeTestMatrixRow[]; jobs?: string; needs?: ReportNeeds; - explicitOnlyJobs?: string; paginateError?: Error; }): Promise<{ body: string; @@ -190,7 +189,6 @@ async function executeReport(options: { apiJobs = [], testMatrix = DEFAULT_TEST_MATRIX, jobs = testMatrix.map(({ id }) => id).join(","), - explicitOnlyJobs = "", needs = { "generate-matrix": { result: "success" }, "shared-e2e": { result: "failure" }, @@ -207,7 +205,7 @@ async function executeReport(options: { const github = reportGithub({ createComment, paginate }); const core = { info: vi.fn(), setFailed, warning }; const env = { - EXPLICIT_ONLY_JOBS: explicitOnlyJobs, + EXPLICIT_ONLY_JOBS: "", TEST_MATRIX: JSON.stringify(testMatrix), JOB_PR_NUMBER: "42", JOB_TARGETS: "", @@ -582,7 +580,6 @@ it("lists explicit-only jobs skipped by default dispatch with their selection hi const report = renderE2eReport({ needs: { "generate-matrix": { result: "success" }, - "mcp-bridge-dev": { result: "skipped" }, }, env: { EXPLICIT_ONLY_JOBS: "mcp-bridge-dev", @@ -601,61 +598,6 @@ it("lists explicit-only jobs skipped by default dispatch with their selection hi ); }); -it("reports the weekly gateway migration tier as skipped on a nightly run (#7920)", async () => { - const { body, setFailed } = await executeReport({ - apiJobs: [ - { - conclusion: "success", - name: "OpenShell gateway upgrade (v0.0.89-x86_64)", - status: "completed", - }, - ], - explicitOnlyJobs: "openshell-gateway-upgrade-compatibility", - jobs: "", - needs: { - "generate-matrix": { result: "success" }, - "openshell-gateway-upgrade": { result: "success" }, - "openshell-gateway-upgrade-compatibility": { result: "skipped" }, - }, - testMatrix: [], - }); - - expect(setFailed).not.toHaveBeenCalled(); - expect(body).toContain("| openshell-gateway-upgrade | ✅ success | — |"); - expect(body).toContain("| openshell-gateway-upgrade-compatibility | ⏭️ skipped | — |"); - expect(body).toContain( - "`openshell-gateway-upgrade-compatibility` (nightly and default manual dispatches exclude weekly gateway compatibility migrations unless selected", - ); - expect(body).not.toContain("| openshell-gateway-upgrade-compatibility | ❌ failure |"); -}); - -it("does not report the weekly gateway migration tier as skipped after a Sunday run (#7920)", async () => { - const { body, setFailed } = await executeReport({ - apiJobs: [ - { - conclusion: "success", - name: "openshell-gateway-upgrade-compatibility / v0.0.36-x86_64", - status: "completed", - }, - ], - explicitOnlyJobs: "openshell-gateway-upgrade-compatibility", - jobs: "", - needs: { - "generate-matrix": { result: "success" }, - "openshell-gateway-upgrade": { result: "success" }, - "openshell-gateway-upgrade-compatibility": { result: "success" }, - }, - testMatrix: [], - }); - - expect(setFailed).not.toHaveBeenCalled(); - expect(body).toContain("| openshell-gateway-upgrade-compatibility | ✅ success | — |"); - expect(body).toContain( - "**Requested test IDs:** _(default — workflow policy selected the enabled tests for this run)_", - ); - expect(body).not.toContain("Explicit-only jobs skipped"); -}); - it("reports matrix children by test ID without fabricating a missing child result", async () => { const { body, setFailed, warning } = await executeReport({ apiJobs: [ @@ -815,39 +757,6 @@ it("reports one total wall clock span from valid matrix E2E jobs", async () => { expect(body).not.toContain("OpenShell gateway upgrade (v0.2.0)"); }); -it("aggregates the weekly gateway matrix under its tier job (#7920)", async () => { - const { body, setFailed } = await executeReport({ - apiJobs: [ - { - completed_at: "2026-07-15T00:05:00Z", - conclusion: "success", - name: "openshell-gateway-upgrade-compatibility / v0.0.36-x86_64", - started_at: "2026-07-15T00:00:00Z", - status: "completed", - }, - { - completed_at: "2026-07-15T00:11:00Z", - conclusion: "success", - name: "openshell-gateway-upgrade-compatibility / v0.0.55-aarch64", - started_at: "2026-07-15T00:02:00Z", - status: "completed", - }, - ], - explicitOnlyJobs: "openshell-gateway-upgrade-compatibility", - jobs: "openshell-gateway-upgrade-compatibility", - needs: { - "generate-matrix": { result: "success" }, - "openshell-gateway-upgrade-compatibility": { result: "success" }, - }, - testMatrix: [], - }); - - expect(setFailed).not.toHaveBeenCalled(); - expect(body).toContain("| openshell-gateway-upgrade-compatibility | ✅ success | 11m 0s |"); - expect(body).not.toContain("Explicit-only jobs skipped"); - expect(body).not.toContain("openshell-gateway-upgrade-compatibility / v0.0.36-x86_64"); -}); - it("reports one total wall clock span when matrix job names start with their job ID", async () => { const { body, setFailed } = await executeReport({ apiJobs: [ diff --git a/test/e2e/support/e2e-scorecard.test.ts b/test/e2e/support/e2e-scorecard.test.ts index 8f1a99ba712..7c6e52eeebd 100644 --- a/test/e2e/support/e2e-scorecard.test.ts +++ b/test/e2e/support/e2e-scorecard.test.ts @@ -546,46 +546,6 @@ describe("E2E scorecard", () => { }); }); - it("counts a scheduled explicit-only tier when it ran and excludes it when skipped (#7920)", () => { - const input = { - explicitOnlyJobNames: ["openshell-gateway-upgrade-compatibility"], - explicitlySelected: [], - metaJobNames: ["generate-matrix", "scorecard"], - }; - - expect( - scorecardJobs.summarizeJobs({ - ...input, - apiJobs: [ - { - conclusion: "success", - name: "openshell-gateway-upgrade-compatibility / v0.0.36-x86_64", - status: "completed", - }, - ], - needs: {}, - }), - ).toMatchObject({ ran: 1, skipped: 0, success: 1, total: 1 }); - expect( - scorecardJobs.summarizeJobs({ - ...input, - apiJobs: null, - needs: { - "openshell-gateway-upgrade-compatibility": { result: "success" }, - }, - }), - ).toMatchObject({ ran: 1, skipped: 0, success: 1, total: 1 }); - expect( - scorecardJobs.summarizeJobs({ - ...input, - apiJobs: null, - needs: { - "openshell-gateway-upgrade-compatibility": { result: "skipped" }, - }, - }), - ).toMatchObject({ ran: 0, skipped: 0, success: 0, total: 0 }); - }); - it("sanitizes raw traces into a timing-only artifact", () => { const directory = mkdtempSync(join(tmpdir(), "nemoclaw-trace-sanitize-")); const source = join(directory, "raw"); diff --git a/test/e2e/support/e2e-workflow.test.ts b/test/e2e/support/e2e-workflow.test.ts index 7f787935bfe..894ed0edbd9 100644 --- a/test/e2e/support/e2e-workflow.test.ts +++ b/test/e2e/support/e2e-workflow.test.ts @@ -769,10 +769,6 @@ describe("e2e workflow boundary", () => { id: "openshell-gateway-upgrade", matchedFiles: ["test/e2e/live/openshell-gateway-upgrade-helpers.ts"], }, - { - id: "openshell-gateway-upgrade-compatibility", - matchedFiles: ["test/e2e/live/openshell-gateway-upgrade-helpers.ts"], - }, ]); }); diff --git a/test/e2e/support/openshell-gateway-upgrade-workflow-boundary.test.ts b/test/e2e/support/openshell-gateway-upgrade-workflow-boundary.test.ts index 874cffdecaa..e0fb0bc2d0b 100644 --- a/test/e2e/support/openshell-gateway-upgrade-workflow-boundary.test.ts +++ b/test/e2e/support/openshell-gateway-upgrade-workflow-boundary.test.ts @@ -24,56 +24,41 @@ import { } from "../live/openshell-gateway-upgrade-helpers.ts"; describe("OpenShell gateway upgrade workflow boundary", () => { - it("pins the nightly and weekly gateway migration tiers to the canonical live test (#7920)", () => { + it("pins architecture and immediate-predecessor fixtures to the canonical live test (#6114)", () => { const workflow = readOpenShellGatewayUpgradeWorkflow(); expect(validateOpenShellGatewayUpgradeWorkflow(workflow)).toEqual([]); expect(validateE2eWorkflowBoundary()).toEqual([]); - const jobs = workflow.jobs as Record>; - (workflow.on as Record).schedule = [{ cron: "0 0 * * *" }]; - const nightly = jobs["openshell-gateway-upgrade"]; - const compatibility = jobs["openshell-gateway-upgrade-compatibility"]; - nightly.if = "always()"; - compatibility.if = "always()"; - compatibility.name = "unrelated matrix name"; - nightly["runs-on"] = "ubuntu-latest"; - const nightlyFixtures = (nightly.strategy as Record>).matrix - .include as Array>; - nightlyFixtures[0]!.tier = "weekly"; - const nightlyEnv = nightly.env as Record; - nightlyEnv.E2E_DEFAULT_ENABLED = "0"; - nightlyEnv.E2E_TARGET_ID = "wrong-target"; - nightlyEnv.NEMOCLAW_E2E_SHARD = "default"; - nightlyEnv.NEMOCLAW_CURRENT_OPENCLAW_VERSION = "latest"; - nightlyEnv.NEMOCLAW_OPENCLAW_STATE_UPGRADE_PROOF = "0"; - const compatibilityFixtures = ( - compatibility.strategy as Record> - ).matrix.include as Array>; - compatibilityFixtures.find((fixture) => fixture.id === "v0.0.55-aarch64")!.runner = - "ubuntu-latest"; - const compatibilityEnv = compatibility.env as Record; - delete compatibilityEnv.E2E_DEFAULT_ENABLED; - const run = (nightly.steps as Array>).find( + const job = (workflow.jobs as Record>)[ + "openshell-gateway-upgrade" + ]; + job["runs-on"] = "ubuntu-latest"; + const strategy = job.strategy as Record>; + const fixtures = strategy.matrix.include as Array>; + fixtures.find((fixture) => fixture.id === "v0.0.55-x86_64")!.sandbox_base_image_ref = + "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:104151ffadc2ff0b6c815e3c95c2783ced61aee0d0f83fc327cc02be9b7e14e6"; + fixtures.find((fixture) => fixture.id === "v0.0.55-aarch64")!.runner = "ubuntu-latest"; + fixtures.find((fixture) => fixture.id === "v0.0.74-x86_64")!.openclaw_version = "latest"; + fixtures.find((fixture) => fixture.id === "v0.0.89-x86_64")!.openclaw_state_upgrade = "0"; + const env = job.env as Record; + env.NEMOCLAW_E2E_SHARD = "default"; + env.NEMOCLAW_CURRENT_OPENCLAW_VERSION = "latest"; + env.NEMOCLAW_OPENCLAW_STATE_UPGRADE_PROOF = "0"; + const run = (job.steps as Array>).find( (step) => step.name === "Run OpenShell gateway upgrade live Vitest test", )!; run.run = "npx vitest run --project e2e-live unrelated.test.ts"; expect(validateE2eWorkflow(workflow)).toEqual( expect.arrayContaining([ - "E2E schedule must separate six nightly runs from the weekly compatibility run", - "openshell-gateway-upgrade must retain its execution-tier selector", "openshell-gateway-upgrade must run on ${{ matrix.runner }}", - "openshell-gateway-upgrade matrix must pin its tiered gateway upgrade fixtures", - "openshell-gateway-upgrade must publish its own target identity", - "openshell-gateway-upgrade must remain default-enabled", + "openshell-gateway-upgrade v0.0.55 matrix must pin x86_64 and arm64 upgrade fixtures", + "openshell-gateway-upgrade matrix must pin the immediate v0.0.74 x86_64 upgrade fixture", + "openshell-gateway-upgrade matrix must pin the v0.0.89 OpenClaw state-upgrade fixture", "openshell-gateway-upgrade must publish one risk-signal shard per legacy fixture", "openshell-gateway-upgrade must bind the current OpenClaw version from its fixture", "openshell-gateway-upgrade must bind the OpenClaw state-upgrade proof flag from its fixture", "openshell-gateway-upgrade step 'Run OpenShell gateway upgrade live Vitest test' must run: npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/openshell-gateway-upgrade.test.ts", - "openshell-gateway-upgrade-compatibility must retain its execution-tier selector", - "openshell-gateway-upgrade-compatibility must keep one scorecard identity across its matrix", - "openshell-gateway-upgrade-compatibility matrix must pin its tiered gateway upgrade fixtures", - "openshell-gateway-upgrade-compatibility must remain explicit-only outside the weekly schedule", ]), ); }); diff --git a/test/pr-e2e-gate-signal-shards.test.ts b/test/pr-e2e-gate-signal-shards.test.ts index c40d449e466..34e8d2338ac 100644 --- a/test/pr-e2e-gate-signal-shards.test.ts +++ b/test/pr-e2e-gate-signal-shards.test.ts @@ -35,19 +35,13 @@ describe("PR E2E signal shard policy", () => { "hermes-inference-switch": ["anthropic"], "openclaw-inference-switch": ["anthropic"], }); - expect( - expectedSignalShards( - ["openshell-gateway-upgrade", "openshell-gateway-upgrade-compatibility"], - undefined, - [DCODE_TARGET], - ), - ).toEqual({ - "openshell-gateway-upgrade": ["v0-0-89-x86-64"], - "openshell-gateway-upgrade-compatibility": [ + expect(expectedSignalShards(["openshell-gateway-upgrade"], undefined, [DCODE_TARGET])).toEqual({ + "openshell-gateway-upgrade": [ "v0-0-36-x86-64", "v0-0-55-x86-64", "v0-0-55-aarch64", "v0-0-74-x86-64", + "v0-0-89-x86-64", ], [DCODE_TARGET]: ["default"], }); diff --git a/test/pr-risk-plan.test.ts b/test/pr-risk-plan.test.ts index 4ceabaac852..bc92c7d0592 100644 --- a/test/pr-risk-plan.test.ts +++ b/test/pr-risk-plan.test.ts @@ -95,7 +95,7 @@ describe("deterministic PR risk plan", () => { expect(result.requiredJobs).toEqual([]); }); - it("maps a shared gateway live test to both retained migration tiers (#7921)", () => { + it("maps a shared gateway live test to the retained migration job (#7921)", () => { const changedFiles = ["test/e2e/live/openshell-gateway-upgrade.test.ts"]; const focusedE2eJobs = focusedE2eJobsForChangedFiles(changedFiles); const result = buildRiskPlan({ headSha: HEAD_SHA, changedFiles, focusedE2eJobs }); @@ -105,15 +105,8 @@ describe("deterministic PR risk plan", () => { id: "openshell-gateway-upgrade", matchedFiles: changedFiles, }, - { - id: "openshell-gateway-upgrade-compatibility", - matchedFiles: changedFiles, - }, - ]); - expect(riskPlanRequiredJobIds(result)).toEqual([ - "openshell-gateway-upgrade", - "openshell-gateway-upgrade-compatibility", ]); + expect(riskPlanRequiredJobIds(result)).toEqual(["openshell-gateway-upgrade"]); }); it("keeps an unknown live test behind the broad control-plane floor (#7921)", () => { diff --git a/test/release-e2e-evidence.test.ts b/test/release-e2e-evidence.test.ts index b37806f2cde..6900363b5bd 100644 --- a/test/release-e2e-evidence.test.ts +++ b/test/release-e2e-evidence.test.ts @@ -94,14 +94,13 @@ describe("release E2E evidence", () => { targets: "", }); const parallelExplicitJobs = plan.dispatches.parallelExplicit.jobs.split(","); - expect(parallelExplicitJobs).toHaveLength(5); + expect(parallelExplicitJobs).toHaveLength(4); expect(new Set(parallelExplicitJobs)).toEqual( new Set([ "openshell-gateway-auth-contract", "mcp-bridge-dev", "hermes-gpu-startup", "sandbox-rlimits-connect", - "openshell-gateway-upgrade-compatibility", ]), ); expect(plan.dispatches.conditional).toEqual([ @@ -117,10 +116,7 @@ describe("release E2E evidence", () => { expect(ids.filter((id) => id.startsWith("mcp-bridge-dev["))).toHaveLength(3); expect(ids.filter((id) => id.startsWith("hermes-gpu-startup["))).toHaveLength(3); - expect(ids.filter((id) => id.startsWith("openshell-gateway-upgrade["))).toHaveLength(1); - expect( - ids.filter((id) => id.startsWith("openshell-gateway-upgrade-compatibility[")), - ).toHaveLength(4); + expect(ids.filter((id) => id.startsWith("openshell-gateway-upgrade["))).toHaveLength(5); expect(ids).toContain("live[id=ubuntu-repo-cloud-openclaw]"); expect(ids).toContain("shared-e2e[id=vllm-docker-storage]"); expect(new Set(ids).size).toBe(ids.length); diff --git a/tools/e2e/openshell-gateway-upgrade-workflow-boundary.mts b/tools/e2e/openshell-gateway-upgrade-workflow-boundary.mts index 0190dec0e2f..bf5fc49f93d 100644 --- a/tools/e2e/openshell-gateway-upgrade-workflow-boundary.mts +++ b/tools/e2e/openshell-gateway-upgrade-workflow-boundary.mts @@ -9,45 +9,19 @@ import YAML from "yaml"; const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e.yaml"); -const NIGHTLY_JOB_NAME = "openshell-gateway-upgrade"; -const COMPATIBILITY_JOB_NAME = "openshell-gateway-upgrade-compatibility"; -const NIGHTLY_CRON = "0 0 * * 1-6"; -const WEEKLY_CRON = "0 0 * * 0"; +const JOB_NAME = "openshell-gateway-upgrade"; const RUN_STEP_NAME = "Run OpenShell gateway upgrade live Vitest test"; const RUN_COMMAND = "npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/openshell-gateway-upgrade.test.ts"; -const COMPATIBILITY_JOB_DISPLAY_NAME = - "openshell-gateway-upgrade-compatibility / ${{ matrix.id }}"; -const NIGHTLY_CONDITION = - "${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',openshell-gateway-upgrade,') || contains(format(',{0},', inputs.targets), ',openshell-gateway-upgrade,') }}"; -const COMPATIBILITY_CONDITION = - "${{ (github.event_name == 'schedule' && github.event.schedule == '0 0 * * 0') || contains(format(',{0},', inputs.jobs), ',openshell-gateway-upgrade-compatibility,') || contains(format(',{0},', inputs.targets), ',openshell-gateway-upgrade-compatibility,') }}"; type WorkflowRecord = Record; type WorkflowStep = WorkflowRecord & { name?: string; run?: string }; -const V036_FIXTURE: WorkflowRecord = { - id: "v0.0.36-x86_64", - runner: "ubuntu-latest", - shard: "v0-0-36-x86-64", - tier: "weekly", - boundary: "oldest retained registry migration", - nemoclaw_ref: "v0.0.36", - nemoclaw_commit: "3351fbdd4eb7d9b80ec471545083956327da2b10", - installer_sha256: "0c42400a0d3867739f1d75d612e069967be4506e169974bbbebf14b7af39144f", - sandbox_base_image_ref: - "ghcr.io/nvidia/nemoclaw/sandbox-base@sha256:104151ffadc2ff0b6c815e3c95c2783ced61aee0d0f83fc327cc02be9b7e14e6", - openshell_version: "0.0.36", - openclaw_version: "2026.4.24", -}; - -const V055_FIXTURES: WorkflowRecord[] = [ +const EXPECTED_V055_FIXTURES: WorkflowRecord[] = [ { id: "v0.0.55-x86_64", runner: "ubuntu-latest", shard: "v0-0-55-x86-64", - tier: "weekly", - boundary: "x86_64 OpenShell 0.0.44 regression", nemoclaw_ref: "v0.0.55", nemoclaw_commit: "95d483fe2b6569d68e59493c60f19df09a068e8f", installer_sha256: "ff8cf448e4d17b00421545a1f333262b615b1b0aa236d0cc5aeaf4e2cae2d897", @@ -60,8 +34,6 @@ const V055_FIXTURES: WorkflowRecord[] = [ id: "v0.0.55-aarch64", runner: "ubuntu-24.04-arm", shard: "v0-0-55-aarch64", - tier: "weekly", - boundary: "arm64 OpenShell 0.0.44 regression", nemoclaw_ref: "v0.0.55", nemoclaw_commit: "95d483fe2b6569d68e59493c60f19df09a068e8f", installer_sha256: "ff8cf448e4d17b00421545a1f333262b615b1b0aa236d0cc5aeaf4e2cae2d897", @@ -72,12 +44,10 @@ const V055_FIXTURES: WorkflowRecord[] = [ }, ]; -const V074_FIXTURE: WorkflowRecord = { +const EXPECTED_V074_FIXTURE: WorkflowRecord = { id: "v0.0.74-x86_64", runner: "ubuntu-latest", shard: "v0-0-74-x86-64", - tier: "weekly", - boundary: "immediate predecessor registry migration", nemoclaw_ref: "v0.0.74", nemoclaw_commit: "3a05b54e8ec3e1d5550ec5c728de54af872bffe3", installer_sha256: "a0cd3feca488d247e53d59d7d8246d2b86e75e95acb5e7d78504b3c0c60fd7db", @@ -87,12 +57,10 @@ const V074_FIXTURE: WorkflowRecord = { openclaw_version: "2026.5.27", }; -const V089_FIXTURE: WorkflowRecord = { +const EXPECTED_V089_FIXTURE: WorkflowRecord = { id: "v0.0.89-x86_64", runner: "ubuntu-latest", shard: "v0-0-89-x86-64", - tier: "nightly", - boundary: "current OpenClaw state migration", nemoclaw_ref: "v0.0.89", nemoclaw_commit: "1143aa5cce77f3bad1b3b5588bd7fddbe438237e", installer_sha256: "00f24959e5ca68104fe91221c0a015dab6a4154618497fa36b969b661f418cc2", @@ -114,99 +82,69 @@ function jobSteps(job: WorkflowRecord): WorkflowStep[] { return Array.isArray(job.steps) ? (job.steps as WorkflowStep[]) : []; } -function matrixFixtures(job: WorkflowRecord): WorkflowRecord[] { +function v055Fixtures(job: WorkflowRecord): WorkflowRecord[] { const include = record(record(job.strategy).matrix).include; - return Array.isArray(include) ? include.map(record) : []; + return Array.isArray(include) + ? include.map(record).filter((fixture) => fixture.nemoclaw_ref === "v0.0.55") + : []; } -function requireRunContains( - errors: string[], - jobName: string, - step: WorkflowStep, - fragment: string, -): void { +function v074Fixture(job: WorkflowRecord): WorkflowRecord { + const include = record(record(job.strategy).matrix).include; + return Array.isArray(include) + ? (include.map(record).find((fixture) => fixture.nemoclaw_ref === "v0.0.74") ?? {}) + : {}; +} + +function v089Fixture(job: WorkflowRecord): WorkflowRecord { + const include = record(record(job.strategy).matrix).include; + return Array.isArray(include) + ? (include.map(record).find((fixture) => fixture.nemoclaw_ref === "v0.0.89") ?? {}) + : {}; +} + +function requireRunContains(errors: string[], step: WorkflowStep, fragment: string): void { if (!step.run?.includes(fragment)) { - errors.push(`${jobName} step '${RUN_STEP_NAME}' must run: ${fragment}`); + errors.push(`${JOB_NAME} step '${RUN_STEP_NAME}' must run: ${fragment}`); } } -function validateJob( - errors: string[], - jobName: string, - job: WorkflowRecord, - fixtures: WorkflowRecord[], - condition: string, - explicitOnly: boolean, -): void { - if (job.if !== condition) { - errors.push(`${jobName} must retain its execution-tier selector`); - } - if (jobName === COMPATIBILITY_JOB_NAME && job.name !== COMPATIBILITY_JOB_DISPLAY_NAME) { - errors.push(`${jobName} must keep one scorecard identity across its matrix`); - } +export function readOpenShellGatewayUpgradeWorkflow( + workflowPath = DEFAULT_WORKFLOW_PATH, +): WorkflowRecord { + return YAML.parse(readFileSync(workflowPath, "utf8")) as WorkflowRecord; +} + +export function validateOpenShellGatewayUpgradeWorkflow(workflow: WorkflowRecord): string[] { + const errors: string[] = []; + const job = record(record(workflow.jobs)[JOB_NAME]); + if (job["runs-on"] !== "${{ matrix.runner }}") { - errors.push(`${jobName} must run on \${{ matrix.runner }}`); + errors.push(`${JOB_NAME} must run on \${{ matrix.runner }}`); } - if (!isDeepStrictEqual(matrixFixtures(job), fixtures)) { - errors.push(`${jobName} matrix must pin its tiered gateway upgrade fixtures`); + if (!isDeepStrictEqual(v055Fixtures(job), EXPECTED_V055_FIXTURES)) { + errors.push(`${JOB_NAME} v0.0.55 matrix must pin x86_64 and arm64 upgrade fixtures`); } - const env = record(job.env); - if (env.E2E_TARGET_ID !== jobName) { - errors.push(`${jobName} must publish its own target identity`); + if (!isDeepStrictEqual(v074Fixture(job), EXPECTED_V074_FIXTURE)) { + errors.push(`${JOB_NAME} matrix must pin the immediate v0.0.74 x86_64 upgrade fixture`); } - if (explicitOnly ? env.E2E_DEFAULT_ENABLED !== "0" : Object.hasOwn(env, "E2E_DEFAULT_ENABLED")) { - errors.push( - explicitOnly - ? `${jobName} must remain explicit-only outside the weekly schedule` - : `${jobName} must remain default-enabled`, - ); + if (!isDeepStrictEqual(v089Fixture(job), EXPECTED_V089_FIXTURE)) { + errors.push(`${JOB_NAME} matrix must pin the v0.0.89 OpenClaw state-upgrade fixture`); } + const env = record(job.env); if (env.NEMOCLAW_E2E_SHARD !== "${{ matrix.shard }}") { - errors.push(`${jobName} must publish one risk-signal shard per legacy fixture`); + errors.push(`${JOB_NAME} must publish one risk-signal shard per legacy fixture`); } if (env.NEMOCLAW_CURRENT_OPENCLAW_VERSION !== "${{ matrix.current_openclaw_version }}") { - errors.push(`${jobName} must bind the current OpenClaw version from its fixture`); + errors.push(`${JOB_NAME} must bind the current OpenClaw version from its fixture`); } if (env.NEMOCLAW_OPENCLAW_STATE_UPGRADE_PROOF !== "${{ matrix.openclaw_state_upgrade }}") { - errors.push(`${jobName} must bind the OpenClaw state-upgrade proof flag from its fixture`); + errors.push(`${JOB_NAME} must bind the OpenClaw state-upgrade proof flag from its fixture`); } - const run = jobSteps(job).find((step) => step.name === RUN_STEP_NAME) ?? {}; - requireRunContains(errors, jobName, run, RUN_COMMAND); -} -export function readOpenShellGatewayUpgradeWorkflow( - workflowPath = DEFAULT_WORKFLOW_PATH, -): WorkflowRecord { - return YAML.parse(readFileSync(workflowPath, "utf8")) as WorkflowRecord; -} - -export function validateOpenShellGatewayUpgradeWorkflow(workflow: WorkflowRecord): string[] { - const errors: string[] = []; - const jobs = record(workflow.jobs); - const triggers = record(workflow.on ?? workflow[true as unknown as string]); - const schedules = triggers.schedule; - if ( - !isDeepStrictEqual(schedules, [{ cron: NIGHTLY_CRON }, { cron: WEEKLY_CRON }]) - ) { - errors.push("E2E schedule must separate six nightly runs from the weekly compatibility run"); - } + const run = jobSteps(job).find((step) => step.name === RUN_STEP_NAME) ?? {}; + requireRunContains(errors, run, RUN_COMMAND); - validateJob( - errors, - NIGHTLY_JOB_NAME, - record(jobs[NIGHTLY_JOB_NAME]), - [V089_FIXTURE], - NIGHTLY_CONDITION, - false, - ); - validateJob( - errors, - COMPATIBILITY_JOB_NAME, - record(jobs[COMPATIBILITY_JOB_NAME]), - [V036_FIXTURE, ...V055_FIXTURES, V074_FIXTURE], - COMPATIBILITY_CONDITION, - true, - ); return errors; } diff --git a/tools/e2e/report-e2e-results.mts b/tools/e2e/report-e2e-results.mts index 4c8e6806339..ba1ab62f001 100644 --- a/tools/e2e/report-e2e-results.mts +++ b/tools/e2e/report-e2e-results.mts @@ -188,12 +188,6 @@ export function renderE2eReport(input: { reason: "default dispatch excludes the resource-heavy OpenShell auth-contract probe unless selected", }, - "openshell-gateway-upgrade-compatibility": { - job: "openshell-gateway-upgrade-compatibility", - target: "openshell-gateway-upgrade-compatibility", - reason: - "nightly and default manual dispatches exclude weekly gateway compatibility migrations unless selected", - }, "mcp-bridge-dev": { job: "mcp-bridge-dev", target: "mcp-bridge-dev", @@ -211,14 +205,9 @@ export function renderE2eReport(input: { reason: "default dispatch excludes the destructive rlimit fork/connect probe unless selected", }, }; - const explicitlySelected = new Set( - [...requestedTestIdsCsv.split(","), ...requestedTargets.split(",")].filter(Boolean), - ); const explicitOnlySkippedJobs = (env.EXPLICIT_ONLY_JOBS || "") .split(",") .filter(Boolean) - .filter((job) => !explicitlySelected.has(job)) - .filter((job) => input.needs[job]?.result === "skipped") .map( (job) => explicitOnlyReasons[job] ?? { @@ -305,9 +294,7 @@ export function renderE2eReport(input: { for (const job of apiJobs) { const jobName = job.name || ""; const match = /^Shared E2E \(([A-Za-z0-9_-]+)\)$/.exec(jobName); - const aggregateJobName = aggregateJobNames.find( - (name) => jobName.startsWith(`${name} (`) || jobName.startsWith(`${name} / `), - ); + const aggregateJobName = aggregateJobNames.find((name) => jobName.startsWith(`${name} (`)); const reportEntryName = match?.[1] ?? aggregateJobName ?? @@ -441,14 +428,14 @@ export function renderE2eReport(input: { ? "**Requested test IDs:** _(selector rejected by workflow validation)_" : requestedTestIdsCsv ? `**Requested test IDs:** \`${requestedTestIdsCsv}\`` - : "**Requested test IDs:** _(default — workflow policy selected the enabled tests for this run)_", + : "**Requested test IDs:** _(default — all default-enabled tests; explicit-only tests `openshell-gateway-auth-contract`, `mcp-bridge-dev`, `hermes-gpu-startup`, `sandbox-rlimits-connect`, and `jetson-nvmap-gpu` are skipped unless selected)_", `**Summary:** ${passed.length} passed, ${failed.length} failed, ${cancelled.length} cancelled, ${skipped.length} skipped, ${unknown.length} unknown`, "", "| Test | Result | Total wall clock time |", "|-----|--------|-----------------------|", ...rows, ]; - if (!selectiveDispatch && explicitOnlySkippedJobs.length > 0) { + if (!selectiveDispatch) { const skippedJobHints = explicitOnlySkippedJobs .map( ({ job, target, reason }) => diff --git a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts index 4a92ca4237a..66933d2e1cd 100644 --- a/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts +++ b/tools/e2e/upload-e2e-artifacts-workflow-boundary.mts @@ -196,12 +196,6 @@ const EXPLICIT_UPLOAD_CONTRACTS = new Map([ name: "e2e-openshell-gateway-upgrade-${{ matrix.id }}", }, ], - [ - "openshell-gateway-upgrade-compatibility", - { - name: "e2e-openshell-gateway-upgrade-${{ matrix.id }}", - }, - ], [ "openshell-gateway-auth-contract", { diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 3851d7f1dee..0d0c90d4a12 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -155,7 +155,6 @@ const COMMON_SECRET_ENV_NAMES = [ const FREE_STANDING_SELECTOR_SPECIAL_CASES = new Set([ "hermes-e2e", "hermes-gpu-startup", - "openshell-gateway-upgrade-compatibility", "staging-brev-launchable", ]); const ADAPTER_MANAGED_INFERENCE_JOBS = new Set(["hermes-e2e"]); @@ -579,14 +578,8 @@ const RESTORED_GATEWAY_PAIRING_RUNTIME_FILES = new Set([ ]); const LIVE_E2E_OWNING_FILE_JOBS = new Map([ ["test/e2e/live/openclaw-plugin-runtime-exdev-lifecycle.ts", ["openclaw-plugin-runtime-exdev"]], - [ - "test/e2e/live/openshell-gateway-upgrade-helpers.ts", - ["openshell-gateway-upgrade", "openshell-gateway-upgrade-compatibility"], - ], - [ - "test/e2e/live/openshell-gateway-upgrade-old-installer.ts", - ["openshell-gateway-upgrade", "openshell-gateway-upgrade-compatibility"], - ], + ["test/e2e/live/openshell-gateway-upgrade-helpers.ts", ["openshell-gateway-upgrade"]], + ["test/e2e/live/openshell-gateway-upgrade-old-installer.ts", ["openshell-gateway-upgrade"]], ]); export function focusedE2eJobsForChangedFiles( @@ -1005,10 +998,7 @@ function requireScheduledRun(errors: string[], triggers: WorkflowRecord): void { const cronEntries = schedule .map((entry) => asRecord(entry).cron) .filter((cron): cron is string => typeof cron === "string"); - const runsDaily = - cronEntries.includes("0 0 * * *") || - (cronEntries.includes("0 0 * * 1-6") && cronEntries.includes("0 0 * * 0")); - if (!runsDaily) { + if (!cronEntries.includes("0 0 * * *")) { errors.push("workflow schedule must run daily at 00:00 UTC"); } } From d8cd881bed877d7f72d18005905a3c6d9d4b269f Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 30 Jul 2026 13:42:35 -0700 Subject: [PATCH 11/15] fix(e2e): reject hosted inference step overrides Signed-off-by: Apurv Kumaria --- .../inference-switch-workflow-boundary.test.ts | 15 +++++++++++++++ tools/e2e/inference-switch-workflow-boundary.mts | 5 +++++ 2 files changed, 20 insertions(+) diff --git a/test/e2e/support/inference-switch-workflow-boundary.test.ts b/test/e2e/support/inference-switch-workflow-boundary.test.ts index b27328c51ed..6095b8ee339 100644 --- a/test/e2e/support/inference-switch-workflow-boundary.test.ts +++ b/test/e2e/support/inference-switch-workflow-boundary.test.ts @@ -106,6 +106,21 @@ describe("inference switch workflow boundary", () => { ); }); + it("rejects a step-scoped hosted inference override in a local switch job", () => { + const hostedInference = readInferenceSwitchWorkflow(); + const runStep = hostedInference.jobs["openclaw-inference-switch"].steps!.find( + (step) => step.name === "Run OpenClaw inference switch live test", + )!; + runStep.env = { + ...runStep.env, + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1", + }; + + expect(validateInferenceSwitchWorkflow(hostedInference)).toContain( + "openclaw-inference-switch must not define NEMOCLAW_E2E_USE_HOSTED_INFERENCE at step scope for its Anthropic-compatible mode", + ); + }); + it("accepts shared guarded Docker authentication without mode-specific auth scripts", () => { const workflow = readInferenceSwitchWorkflow(); const steps = workflow.jobs["openclaw-inference-switch"].steps!; diff --git a/tools/e2e/inference-switch-workflow-boundary.mts b/tools/e2e/inference-switch-workflow-boundary.mts index 3ca24e262fa..364b049bad5 100644 --- a/tools/e2e/inference-switch-workflow-boundary.mts +++ b/tools/e2e/inference-switch-workflow-boundary.mts @@ -103,6 +103,11 @@ function validateJob(errors: string[], spec: JobSpec, job: WorkflowJob): void { errors.push(`${spec.job} must not enable hosted inference for its Anthropic-compatible mode`); } for (const step of job.steps ?? []) { + if (step.env?.NEMOCLAW_E2E_USE_HOSTED_INFERENCE !== undefined) { + errors.push( + `${spec.job} must not define NEMOCLAW_E2E_USE_HOSTED_INFERENCE at step scope for its Anthropic-compatible mode`, + ); + } if (step.env?.NVIDIA_INFERENCE_API_KEY !== undefined) { errors.push( `${spec.job} must not expose NVIDIA_INFERENCE_API_KEY in its Anthropic-compatible mode`, From f37fb1da1b48a24580454f7bfbecb37144f88b49 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 30 Jul 2026 14:12:09 -0700 Subject: [PATCH 12/15] fix(e2e): preserve selector transition jobs --- .github/workflows/e2e.yaml | 160 +++++++++ test/e2e/README.md | 12 +- test/e2e/live/sandbox-rebuild.test.ts | 263 ++++++++++++++ .../e2e/live/upgrade-stale-sandbox-helpers.ts | 327 ++++++++++++++++++ test/e2e/live/upgrade-stale-sandbox.test.ts | 223 ++++++++++++ test/e2e/mock-parity.json | 14 + ...or-compatibility-workflow-boundary.test.ts | 36 ++ test/e2e/support/workflow-plan.test.ts | 11 +- tools/e2e/retired-selector-compatibility.mts | 4 +- tools/e2e/workflow-boundary.mts | 243 +++++++++++++ 10 files changed, 1285 insertions(+), 8 deletions(-) create mode 100644 test/e2e/live/sandbox-rebuild.test.ts create mode 100644 test/e2e/live/upgrade-stale-sandbox-helpers.ts create mode 100644 test/e2e/live/upgrade-stale-sandbox.test.ts diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 2fd66dde71a..292ef10bb1e 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -3246,6 +3246,85 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh + sandbox-rebuild: + needs: generate-matrix + if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',sandbox-rebuild,') || contains(format(',{0},', inputs.targets), ',sandbox-rebuild,') }} + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + E2E_JOB: "1" + E2E_TARGET_ID: "sandbox-rebuild" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/sandbox-rebuild + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + OPENSHELL_GATEWAY: nemoclaw + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ inputs.checkout_repository || github.repository }} + ref: ${{ inputs.checkout_sha || github.sha }} + persist-credentials: false + + - *dockerhub-auth + + - id: selector_compatibility + name: Classify sandbox-rebuild candidate selector + env: + LEGACY_TEST_FILE: test/e2e/live/sandbox-rebuild.test.ts + shell: bash + run: | + set -euo pipefail + if [[ -f "$LEGACY_TEST_FILE" ]]; then + retired=false + else + retired=true + fi + printf 'retired=%s\n' "$retired" >> "$GITHUB_OUTPUT" + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 + + - name: Install OpenShell + if: ${{ steps.selector_compatibility.outputs.retired != 'true' }} + env: + NEMOCLAW_NON_INTERACTIVE: "1" + run: | + set -euo pipefail + env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh + + - name: Run sandbox rebuild live test + if: ${{ steps.selector_compatibility.outputs.retired != 'true' }} + env: + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + if command -v openshell >/dev/null 2>&1; then + OPENSHELL_BIN="$(command -v openshell)" + elif [ -x "$HOME/.local/bin/openshell" ]; then + OPENSHELL_BIN="$HOME/.local/bin/openshell" + else + echo "::error::OpenShell CLI not found after install" + ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true + exit 1 + fi + export OPENSHELL_BIN + "$OPENSHELL_BIN" --version + npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/sandbox-rebuild.test.ts + + - name: Upload sandbox rebuild artifacts + if: always() + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + + - name: Clean up Docker auth + if: always() + shell: bash + run: bash .github/scripts/docker-auth-cleanup.sh + + sandbox-rlimits-connect: needs: generate-matrix if: ${{ contains(format(',{0},', inputs.jobs), ',sandbox-rlimits-connect,') || contains(format(',{0},', inputs.targets), ',sandbox-rlimits-connect,') }} @@ -3418,6 +3497,85 @@ jobs: shell: bash run: bash .github/scripts/docker-auth-cleanup.sh + upgrade-stale-sandbox: + needs: generate-matrix + if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',upgrade-stale-sandbox,') || contains(format(',{0},', inputs.targets), ',upgrade-stale-sandbox,') }} + runs-on: ubuntu-latest + timeout-minutes: 85 + env: + E2E_JOB: "1" + E2E_TARGET_ID: "upgrade-stale-sandbox" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/upgrade-stale-sandbox + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_LIVE_E2E: "1" + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-upgrade-stale" + OPENSHELL_GATEWAY: "nemoclaw" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ inputs.checkout_repository || github.repository }} + ref: ${{ inputs.checkout_sha || github.sha }} + persist-credentials: false + + - *dockerhub-auth + + - id: selector_compatibility + name: Classify upgrade-stale-sandbox candidate selector + env: + LEGACY_TEST_FILE: test/e2e/live/upgrade-stale-sandbox.test.ts + shell: bash + run: | + set -euo pipefail + if [[ -f "$LEGACY_TEST_FILE" ]]; then + retired=false + else + retired=true + fi + printf 'retired=%s\n' "$retired" >> "$GITHUB_OUTPUT" + + - name: Prepare E2E workspace + uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75 + + - name: Install OpenShell CLI + if: ${{ steps.selector_compatibility.outputs.retired != 'true' }} + run: | + set -euo pipefail + env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN \ + bash scripts/install-openshell.sh + + - name: Run upgrade stale sandbox live Vitest test + if: ${{ steps.selector_compatibility.outputs.retired != 'true' }} + env: + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + if command -v openshell >/dev/null 2>&1; then + OPENSHELL_BIN="$(command -v openshell)" + elif [ -x "$HOME/.local/bin/openshell" ]; then + OPENSHELL_BIN="$HOME/.local/bin/openshell" + else + echo "::error::OpenShell CLI not found after install" + ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true + exit 1 + fi + export OPENSHELL_BIN + "$OPENSHELL_BIN" --version + npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/upgrade-stale-sandbox.test.ts + + - name: Upload upgrade stale sandbox artifacts + if: always() + uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57 + + - name: Clean up Docker auth + if: always() + shell: bash + run: bash .github/scripts/docker-auth-cleanup.sh + + double-onboard: needs: generate-matrix if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',double-onboard,') || contains(format(',{0},', inputs.targets), ',double-onboard,') }} @@ -5697,9 +5855,11 @@ jobs: rebuild-openclaw, rebuild-hermes, rebuild-hermes-stale-base, + sandbox-rebuild, sandbox-rlimits-connect, overlayfs-autofix, state-backup-restore, + upgrade-stale-sandbox, openshell-gateway-upgrade, token-rotation, messaging-compatible-endpoint, diff --git a/test/e2e/README.md b/test/e2e/README.md index 283891df53c..11c57030315 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -103,10 +103,14 @@ The retired `hermes-dashboard` selector remains a compatibility alias for the manually selected `mock`, `internal-nvidia`, or `public-nvidia` inference mode. -PR-gate requests using the retired `sandbox-rebuild` and -`upgrade-stale-sandbox` job or target selectors run focused replacement tests -through the compatibility controller. `rebuild-openclaw` is the canonical live -rebuild and upgrade seam. +## Retired selector transition + +When a candidate no longer contains `test/e2e/live/sandbox-rebuild.test.ts` or +`test/e2e/live/upgrade-stale-sandbox.test.ts`, PR gate requests for the +corresponding job or target selector run focused replacement tests through the +compatibility controller. While the file remains, the selector runs its +dedicated live E2E test. `rebuild-openclaw` is the canonical live rebuild and +upgrade target. ## Larger-runner routing diff --git a/test/e2e/live/sandbox-rebuild.test.ts b/test/e2e/live/sandbox-rebuild.test.ts new file mode 100644 index 00000000000..dc32f56a2db --- /dev/null +++ b/test/e2e/live/sandbox-rebuild.test.ts @@ -0,0 +1,263 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { resultText } from "../fixtures/clients/command.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { + latestRebuildBackupDir, + listCredentialLeakPaths, + patchRegistrySandboxEntry, + readRegistrySandboxEntry, + restoreRegistryAndSession, + snapshotRegistryAndSession, +} from "../fixtures/phases/state-validation.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +// This dependent migration reuses the rebuild/state helper shape seeded by the +// OpenClaw rebuild anchor while keeping the contract focused: onboard a real +// sandbox, mark workspace state, force stale registry metadata, run the real +// `nemoclaw rebuild --yes`, then verify state preservation, registry +// refresh, and backup credential hygiene. + +const MARKER_FILE = "/sandbox/.openclaw/workspace/rebuild-marker.txt"; +const STALE_AGENT_VERSION = "0.0.1"; +const TEST_SANDBOX_PREFIX = "e2e-sandbox-rebuild"; +const SANDBOX_NAME = + process.env.NEMOCLAW_SANDBOX_NAME ?? + [TEST_SANDBOX_PREFIX, process.env.GITHUB_RUN_ID, process.env.GITHUB_RUN_ATTEMPT, process.pid] + .filter(Boolean) + .join("-"); +const TEST_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_TIMEOUT_SECONDS ?? 1_200) * 1_000; +const STATUS_TIMEOUT_MS = 60_000; +const ONBOARD_TIMEOUT_MS = TEST_TIMEOUT_MS; +const REBUILD_TIMEOUT_MS = TEST_TIMEOUT_MS; +const MARKER_CONTENT = `REBUILD_E2E_${Date.now()}`; + +function sandboxRebuildEnv(apiKey: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + ...extra, + NVIDIA_INFERENCE_API_KEY: apiKey, + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + }; +} + +function assertTestOwnedSandboxName(): void { + if (!SANDBOX_NAME.startsWith(TEST_SANDBOX_PREFIX)) { + throw new Error( + `sandbox-rebuild live test is destructive and only accepts sandbox names with prefix ${TEST_SANDBOX_PREFIX}; got ${SANDBOX_NAME}`, + ); + } +} + +async function bestEffortPreclean(run: () => Promise): Promise { + try { + await run(); + } catch { + // Cleanup remains best-effort so earlier lifecycle failures stay visible. + } +} + +// biome-ignore format: preserve legacy live-test body formatting so phase-only changes stay reviewable. +test( + "sandbox-rebuild: rebuild preserves marker state and refreshes registry metadata", + { + timeout: TEST_TIMEOUT_MS * 3, + meta: { + e2ePhases: [ + "confirm Docker and environment readiness", + "onboard the OpenClaw rebuild sandbox", + "write state and mark the registry stale", + "rebuild the stale sandbox", + "confirm marker preservation and registry refresh", + "scan the rebuild backup for credential leaks", + ], + }, + }, + async ({ + artifacts, + cleanup, + environment, + host, + lifecycle, + onboard, + progress, + sandbox, + secrets, + skip, + stateValidation, + }) => { + assertTestOwnedSandboxName(); + const apiKey = secrets.required("NVIDIA_INFERENCE_API_KEY"); + const dockerInfo = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + if (dockerInfo.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error( + `Docker is required for sandbox-rebuild live coverage: ${resultText(dockerInfo)}`, + ); + } + skip("Docker is required for sandbox-rebuild live coverage"); + } + + const ready = await environment.assertReady({ + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-openclaw", + }); + + await artifacts.writeJson("contract.json", { + sandboxName: SANDBOX_NAME, + markerFile: MARKER_FILE, + staleAgentVersion: STALE_AGENT_VERSION, + preservedBoundaries: [ + "real nemoclaw onboard with Docker/OpenShell", + "openshell sandbox exec marker write/read", + "local registry stale agentVersion mutation", + "real nemoclaw rebuild --yes", + "backup credential leak scan under ~/.nemoclaw/rebuild-backups", + ], + }); + + const stateSnapshot = snapshotRegistryAndSession(); + const backupRoot = path.join( + process.env.HOME ?? os.homedir(), + ".nemoclaw", + "rebuild-backups", + SANDBOX_NAME, + ); + cleanup.trackDisposable(`restore NemoClaw state files for ${SANDBOX_NAME}`, () => { + restoreRegistryAndSession(stateSnapshot); + fs.rmSync(backupRoot, { recursive: true, force: true }); + }); + if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX !== "1") { + cleanup.trackDisposable(`delete OpenShell sandbox ${SANDBOX_NAME}`, () => + sandbox.cleanupSandbox(SANDBOX_NAME, { + artifactName: "cleanup-openshell-sandbox-delete", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }), + ); + cleanup.trackSandbox(host, SANDBOX_NAME, { + artifactName: "cleanup-nemoclaw-destroy", + env: buildAvailabilityProbeEnv(), + timeoutMs: 15 * 60_000, + }); + } + + await bestEffortPreclean(() => + onboard.destroySandbox(SANDBOX_NAME, "pre-cleanup-nemoclaw-destroy"), + ); + await bestEffortPreclean(() => + sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "pre-cleanup-openshell-sandbox-delete", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }), + ); + + progress.phase("onboard the OpenClaw rebuild sandbox"); + const instance = await onboard.from(ready, { + sandboxName: SANDBOX_NAME, + timeoutMs: ONBOARD_TIMEOUT_MS, + }); + + const status = await host.nemoclaw([SANDBOX_NAME, "status"], { + artifactName: "phase-2-status-version-detection", + env: sandboxRebuildEnv(apiKey), + redactionValues: [apiKey], + timeoutMs: STATUS_TIMEOUT_MS, + }); + await artifacts.writeText("phase-2-status-output.txt", resultText(status)); + if (/Agent:.*v?\d+\.\d+/i.test(resultText(status))) { + await artifacts.writeJson("phase-2-status-version-summary.json", { versionVisible: true }); + } else { + await artifacts.writeJson("phase-2-status-version-summary.json", { + versionVisible: false, + note: "Legacy shell accepted first-run status output without cached version.", + }); + } + + progress.phase("write state and mark the registry stale"); + await stateValidationWriteMarker(); + + patchRegistrySandboxEntry(SANDBOX_NAME, { agentVersion: STALE_AGENT_VERSION }); + await artifacts.writeJson("phase-4-stale-registry-summary.json", { + sandboxName: SANDBOX_NAME, + agentVersion: readRegistrySandboxEntry(SANDBOX_NAME).agentVersion, + }); + + const staleStatus = await host.nemoclaw([SANDBOX_NAME, "status"], { + artifactName: "phase-4-status-stale-warning", + env: sandboxRebuildEnv(apiKey), + redactionValues: [apiKey], + timeoutMs: STATUS_TIMEOUT_MS, + }); + expect(staleStatus.exitCode, resultText(staleStatus)).toBe(0); + expect(resultText(staleStatus)).toMatch(/rebuild/i); + + progress.phase("rebuild the stale sandbox"); + await lifecycle.rebuildSandbox(instance, { + artifactName: "phase-5-nemoclaw-rebuild", + env: sandboxRebuildEnv(apiKey), + redactionValues: [apiKey], + timeoutMs: REBUILD_TIMEOUT_MS, + }); + await lifecycle.assertSandboxReadyAfterRebuild(instance, { + artifactNamePrefix: "phase-5-sandbox-ready-after-rebuild", + env: buildAvailabilityProbeEnv(), + attempts: 12, + delayMs: 5_000, + }); + + progress.phase("confirm marker preservation and registry refresh"); + await stateValidation.expectMarkerFileContent(instance, MARKER_FILE, MARKER_CONTENT, { + artifactName: "phase-6-read-marker-after-rebuild", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + + const updatedVersion = stateValidation.expectRegistryAgentVersionUpdated( + SANDBOX_NAME, + STALE_AGENT_VERSION, + ); + await artifacts.writeJson("phase-7-registry-version-summary.json", { + sandboxName: SANDBOX_NAME, + staleVersion: STALE_AGENT_VERSION, + updatedVersion, + }); + + progress.phase("scan the rebuild backup for credential leaks"); + const backupDir = latestRebuildBackupDir(SANDBOX_NAME); + const leaks = listCredentialLeakPaths(backupDir, { extraSecrets: [apiKey] }); + await artifacts.writeJson("phase-8-backup-credential-scan.json", { + backupDir: backupDir ?? null, + leaks, + note: backupDir ? undefined : "No backup directory found; former shell skipped this check.", + }); + expect(leaks, "backup files must not contain credential-shaped values").toEqual([]); + + async function stateValidationWriteMarker(): Promise { + await stateValidation.writeMarkerFile(instance, MARKER_FILE, MARKER_CONTENT, { + artifactName: "phase-3-write-marker", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + await stateValidation.expectMarkerFileContent(instance, MARKER_FILE, MARKER_CONTENT, { + artifactName: "phase-3-read-marker-before-rebuild", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + } + }, +); diff --git a/test/e2e/live/upgrade-stale-sandbox-helpers.ts b/test/e2e/live/upgrade-stale-sandbox-helpers.ts new file mode 100644 index 00000000000..588ba3bc8ce --- /dev/null +++ b/test/e2e/live/upgrade-stale-sandbox-helpers.ts @@ -0,0 +1,327 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { findAvailableDashboardPort } from "../../../src/lib/onboard/dashboard-port.ts"; +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { CleanupRegistry } from "../fixtures/cleanup.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { assertExitZero, resultText } from "../fixtures/clients/index.ts"; +import { type SandboxClient, validateSandboxName } from "../fixtures/clients/sandbox.ts"; +import { expect } from "../fixtures/e2e-test.ts"; +import { + readJsonFileOrFallback, + restoreFile, + snapshotFile, + writeJsonFile, +} from "../fixtures/file-state.ts"; +import { REPO_ROOT } from "../fixtures/paths.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts"; +import { createOldBaseBuildContext } from "./rebuild-openclaw-old-base-context.ts"; + +export { REPO_ROOT }; + +const TEST_SANDBOX_PREFIX = "e2e-upgrade-stale"; +export const SANDBOX_NAME = + process.env.NEMOCLAW_SANDBOX_NAME ?? + [TEST_SANDBOX_PREFIX, process.env.GITHUB_RUN_ID, process.env.GITHUB_RUN_ATTEMPT, process.pid] + .filter(Boolean) + .join("-"); +export const SIBLING_SANDBOX_NAME = `${SANDBOX_NAME}-peer`; +export const SANDBOX_NAMES = [SANDBOX_NAME, SIBLING_SANDBOX_NAME] as const; +validateSandboxName(SANDBOX_NAME); +validateSandboxName(SIBLING_SANDBOX_NAME); +assertSafeSandboxName(); +export const OLD_OPENCLAW_VERSION = "2026.3.11"; +export const OLD_BASE_TAG = `nemoclaw-old-base:${SANDBOX_NAME.toLowerCase().replace(/[^a-z0-9_.-]+/g, "-")}`; +const REGISTRY_FILE = path.join(os.homedir(), ".nemoclaw", "sandboxes.json"); +const SESSION_FILE = path.join(os.homedir(), ".nemoclaw", "onboard-session.json"); +const INSTALL_ATTEMPTS = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true" ? 3 : 1; + +function assertSafeSandboxName(): void { + for (const sandboxName of SANDBOX_NAMES) { + if (!sandboxName.startsWith(TEST_SANDBOX_PREFIX)) { + throw new Error( + `upgrade-stale-sandbox live test is destructive and only accepts sandbox names with prefix ${TEST_SANDBOX_PREFIX}; got ${sandboxName}`, + ); + } + } +} + +export function commandEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + ...extra, + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_REBUILD_VERBOSE: "1", + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", + }; +} + +async function bestEffortPreclean(run: () => Promise): Promise { + try { + await run(); + } catch { + // Cleanup must not mask the primary assertion failure. + } +} + +export function allocateSiblingDashboardPort(forwardListOutput: string | null): number { + const registry = readJsonFileOrFallback<{ + sandboxes?: Record>; + }>(REGISTRY_FILE, {}); + const primaryDashboardPort = registry.sandboxes?.[SANDBOX_NAME]?.dashboardPort; + expect( + typeof primaryDashboardPort === "number" && + Number.isInteger(primaryDashboardPort) && + primaryDashboardPort > 0 && + primaryDashboardPort <= 65535, + "initial onboard must persist the dashboard port used by authoritative rebuild", + ).toBe(true); + const occupied = new Map([[String(primaryDashboardPort), SANDBOX_NAME]]); + return findAvailableDashboardPort( + SIBLING_SANDBOX_NAME, + primaryDashboardPort === 18790 ? 18791 : 18790, + forwardListOutput, + undefined, + occupied, + ); +} + +export function writeStaleRegistryEntries(siblingDashboardPort: number): void { + const session = readJsonFileOrFallback>(SESSION_FILE, {}); + const envProvider = + process.env.NEMOCLAW_PROVIDER === "custom" + ? "compatible-endpoint" + : process.env.NEMOCLAW_PROVIDER; + const provider = + typeof session.provider === "string" && session.provider + ? session.provider + : envProvider || "compatible-endpoint"; + const model = + (typeof session.model === "string" && session.model) || + process.env.NEMOCLAW_MODEL || + process.env.NEMOCLAW_COMPAT_MODEL || + "nvidia/nvidia/nemotron-3-ultra"; + const registry = readJsonFileOrFallback<{ + sandboxes?: Record>; + defaultSandbox?: string; + }>(REGISTRY_FILE, {}); + const currentEntry = registry.sandboxes?.[SANDBOX_NAME] ?? {}; + const dashboardPort = currentEntry.dashboardPort; + expect( + typeof dashboardPort === "number" && + Number.isInteger(dashboardPort) && + dashboardPort > 0 && + dashboardPort <= 65535, + "initial onboard must persist the dashboard port used by authoritative rebuild", + ).toBe(true); + const endpointUrl = + (typeof currentEntry.endpointUrl === "string" && currentEntry.endpointUrl) || + (typeof session.endpointUrl === "string" && session.endpointUrl) || + null; + const preferredInferenceApi = + (typeof currentEntry.preferredInferenceApi === "string" && + currentEntry.preferredInferenceApi) || + (typeof session.preferredInferenceApi === "string" && session.preferredInferenceApi) || + null; + if (provider === "compatible-endpoint" || provider === "compatible-anthropic-endpoint") { + expect(endpointUrl, "custom stale route must retain its durable endpoint").toBeTruthy(); + expect( + preferredInferenceApi, + "custom stale route must retain its durable inference API family", + ).toBeTruthy(); + } + registry.sandboxes = registry.sandboxes ?? {}; + for (const [sandboxName, assignedDashboardPort] of [ + [SANDBOX_NAME, dashboardPort], + [SIBLING_SANDBOX_NAME, siblingDashboardPort], + ] as const) { + registry.sandboxes[sandboxName] = { + name: sandboxName, + createdAt: new Date().toISOString(), + model, + provider, + endpointUrl, + preferredInferenceApi, + gpuEnabled: false, + policies: [], + policyTier: null, + fromDockerfile: null, + dashboardPort: assignedDashboardPort, + gatewayName: "nemoclaw", + openshellVersion: "0.0.71", + nemoclawVersion: "0.0.71", + agent: null, + agentVersion: OLD_OPENCLAW_VERSION, + // Deliberately omit credentialEnv on both legacy rows. Rebuild must + // migrate the shared provider identity before deleting either sandbox. + }; + } + registry.defaultSandbox = SANDBOX_NAME; + writeJsonFile(REGISTRY_FILE, registry); + writeJsonFile(SESSION_FILE, { ...session, sandboxName: SANDBOX_NAME, status: "complete" }); +} + +export function assertDockerAvailable( + result: ShellProbeResult, + skip: (note?: string) => never, +): void { + result.exitCode === 0 || process.env.GITHUB_ACTIONS === "true" + ? undefined + : skip(`Docker is required for stale sandbox upgrade E2E: ${resultText(result)}`); + result.exitCode === 0 || + process.env.GITHUB_ACTIONS !== "true" || + (() => { + throw new Error(`Docker is required for stale sandbox upgrade E2E: ${resultText(result)}`); + })(); +} + +export function registerStateRestore(cleanup: Pick): void { + const registrySnapshot = snapshotFile(REGISTRY_FILE); + const sessionSnapshot = snapshotFile(SESSION_FILE); + cleanup.trackDisposable(`restore NemoClaw state files for ${SANDBOX_NAME}`, () => { + restoreFile(REGISTRY_FILE, registrySnapshot); + restoreFile(SESSION_FILE, sessionSnapshot); + }); +} + +export async function precleanStaleSandbox( + host: HostCliClient, + sandbox: SandboxClient, +): Promise { + for (const sandboxName of SANDBOX_NAMES) { + await bestEffortPreclean(() => + host.nemoclaw([sandboxName, "destroy", "--yes"], { + artifactName: `cleanup-nemoclaw-destroy-${sandboxName}`, + env: commandEnv(), + timeoutMs: 120_000, + }), + ); + await bestEffortPreclean(() => + sandbox.openshell(["sandbox", "delete", sandboxName], { + artifactName: `cleanup-openshell-delete-${sandboxName}`, + env: commandEnv(), + timeoutMs: 60_000, + }), + ); + } +} + +export async function cleanupOldImage(host: HostCliClient): Promise { + const result = await host.command("docker", ["image", "rm", "-f", OLD_BASE_TAG], { + artifactName: "cleanup-docker-image-upgrade-stale", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + if (result.exitCode === 0 || /No such image|image[^\n]*not found/i.test(resultText(result))) + return; + assertExitZero(result, `cleanup Docker image ${OLD_BASE_TAG}`); +} + +export async function installCurrentNemoclaw( + host: HostCliClient, + hosted: { apiKey: string; env: NodeJS.ProcessEnv }, +): Promise { + let install: ShellProbeResult | undefined; + for (let attempt = 1; attempt <= INSTALL_ATTEMPTS; attempt += 1) { + install = await host.command("bash", ["install.sh", "--non-interactive"], { + artifactName: + attempt === 1 + ? "phase-1-install-current-nemoclaw" + : `phase-1-install-current-nemoclaw-attempt-${attempt}`, + cwd: REPO_ROOT, + env: commandEnv(hosted.env), + redactionValues: [hosted.apiKey], + timeoutMs: 20 * 60_000, + }); + const retry = + install.exitCode !== 0 && + isTransientProviderValidationFailure(install) && + attempt < INSTALL_ATTEMPTS; + install.exitCode === 0 && (attempt = INSTALL_ATTEMPTS + 1); + retry && (await new Promise((resolve) => setTimeout(resolve, 10_000 * attempt))); + !retry && install.exitCode !== 0 && (attempt = INSTALL_ATTEMPTS + 1); + } + if (!install) throw new Error("install command did not run"); + return install; +} + +export function assertDeleteInstalledSandboxAllowed(result: ShellProbeResult): void { + result.exitCode === 0 || expect(result.exitCode, resultText(result)).toBe(1); + result.exitCode === 0 || + expect(resultText(result)).toMatch(/not found|does not exist|no sandbox/i); +} + +export async function buildOldOpenClawBase(host: HostCliClient): Promise { + const oldBaseBuildContext = createOldBaseBuildContext(); + try { + return await host.command( + "docker", + [ + "build", + "--build-arg", + `OPENCLAW_VERSION=${OLD_OPENCLAW_VERSION}`, + "--build-arg", + "NEMOCLAW_E2E_FIXTURE_LEGACY_OPENCLAW=1", + "-f", + path.join(REPO_ROOT, "Dockerfile.base"), + "-t", + OLD_BASE_TAG, + oldBaseBuildContext, + ], + { + artifactName: "phase-2-build-old-openclaw-base", + env: buildAvailabilityProbeEnv(), + timeoutMs: 20 * 60_000, + }, + ); + } finally { + fs.rmSync(oldBaseBuildContext, { recursive: true, force: true }); + } +} + +export function createFixtureDockerfile(cleanup: Pick): string { + const fixtureDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-old-openclaw-")); + cleanup.trackDisposable("remove stale sandbox fixture Dockerfile", () => + fs.rmSync(fixtureDir, { recursive: true, force: true }), + ); + const fixtureDockerfile = path.join(fixtureDir, "Dockerfile"); + fs.writeFileSync( + fixtureDockerfile, + [ + `FROM ${OLD_BASE_TAG}`, + "USER sandbox", + "WORKDIR /sandbox", + "RUN mkdir -p /sandbox/.openclaw/workspace /sandbox/.openclaw && echo '{}' > /sandbox/.openclaw/openclaw.json", + 'CMD ["/bin/bash"]', + "", + ].join("\n"), + ); + return fixtureDockerfile; +} + +export async function waitSandboxReady( + host: HostCliClient, + sandboxName: string, + artifactName: string, +): Promise { + return await host.command( + "bash", + [ + "-lc", + `for _i in $(seq 1 30); do openshell sandbox list 2>/dev/null | grep -q '${sandboxName}.*Ready' && exit 0; sleep 5; done; openshell sandbox list >&2; exit 1`, + ], + { artifactName, env: commandEnv(), timeoutMs: 180_000 }, + ); +} + +export function registeredStaleSandboxJson(): string { + return fs.readFileSync(REGISTRY_FILE, "utf8"); +} diff --git a/test/e2e/live/upgrade-stale-sandbox.test.ts b/test/e2e/live/upgrade-stale-sandbox.test.ts new file mode 100644 index 00000000000..d328556e9f6 --- /dev/null +++ b/test/e2e/live/upgrade-stale-sandbox.test.ts @@ -0,0 +1,223 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Preserves the #1904 contract with real Docker/OpenShell/NemoClaw + * boundaries: onboard current NemoClaw, create an old OpenClaw sandbox from a + * real image, register two legacy sandboxes on one gateway, prove + * upgrade-sandboxes detects both, rebuild them as one batch, and prove the + * stale version and missing shared-route credential metadata are gone. + */ + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { resultText } from "../fixtures/clients/index.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; +import { + allocateSiblingDashboardPort, + assertDeleteInstalledSandboxAllowed, + assertDockerAvailable, + buildOldOpenClawBase, + cleanupOldImage, + commandEnv, + createFixtureDockerfile, + installCurrentNemoclaw, + OLD_OPENCLAW_VERSION, + precleanStaleSandbox, + registeredStaleSandboxJson, + registerStateRestore, + SANDBOX_NAME, + SANDBOX_NAMES, + SIBLING_SANDBOX_NAME, + waitSandboxReady, + writeStaleRegistryEntries, +} from "./upgrade-stale-sandbox-helpers.ts"; + +const LIVE_TIMEOUT_MS = 75 * 60_000; + +test("upgrade-sandboxes rebuilds two legacy sandboxes on one shared route (#1904, #7798)", { + timeout: LIVE_TIMEOUT_MS, + meta: { + e2ePhases: [ + "confirm Docker and install current NemoClaw", + "construct two old OpenClaw sandboxes on one gateway", + "register stale shared-route sandbox metadata", + "detect both stale sandboxes", + "rebuild both to the current OpenClaw runtime", + "confirm the upgrade check is clean", + ], + }, +}, async ({ artifacts, cleanup, host, progress, sandbox, secrets, skip }) => { + const hosted = requireHostedInferenceConfig(secrets); + + await artifacts.target.declare({ + id: "upgrade-stale-sandbox", + boundary: "install.sh + Docker old base image + OpenShell sandbox create + NemoClaw rebuild", + sandboxName: SANDBOX_NAME, + sandboxNames: [...SANDBOX_NAMES], + oldOpenClawVersion: OLD_OPENCLAW_VERSION, + contracts: [ + "current NemoClaw install/onboard succeeds before stale fixture creation", + "an old OpenClaw base image can be created with the legacy version", + "two legacy sandboxes with missing credentialEnv share one complete inference route", + "both sandboxes are reported stale by upgrade-sandboxes --check", + "upgrade-sandboxes --auto upgrades both without orphaning the first sandbox", + "both registry rows carry the canonical shared credential identity after rebuild", + "upgrade-sandboxes --check reports up-to-date after rebuild", + ], + }); + + const dockerInfo = await host.command("docker", ["info"], { + artifactName: "phase-0-docker-info", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + assertDockerAvailable(dockerInfo, skip); + + registerStateRestore(cleanup); + cleanup.trackDisposable("remove stale OpenClaw test image", () => cleanupOldImage(host)); + for (const sandboxName of SANDBOX_NAMES) { + cleanup.trackDisposable(`delete OpenShell sandbox ${sandboxName}`, () => + sandbox.cleanupSandbox(sandboxName, { + artifactName: `cleanup-openshell-delete-${sandboxName}`, + env: commandEnv(), + timeoutMs: 60_000, + }), + ); + cleanup.trackSandbox(host, sandboxName, { + artifactName: `cleanup-nemoclaw-destroy-${sandboxName}`, + env: commandEnv(), + timeoutMs: 120_000, + }); + } + await precleanStaleSandbox(host, sandbox); + + const install = await installCurrentNemoclaw(host, hosted); + expect(install.exitCode, resultText(install)).toBe(0); + + progress.phase("construct two old OpenClaw sandboxes on one gateway"); + const deleteInstalledSandbox = await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "phase-2-delete-installed-sandbox", + env: commandEnv(), + timeoutMs: 120_000, + }); + assertDeleteInstalledSandboxAllowed(deleteInstalledSandbox); + const forwardList = await sandbox.openshell(["forward", "list"], { + artifactName: "phase-2-forward-list-before-stale-fixture", + env: commandEnv(), + timeoutMs: 60_000, + }); + expect(forwardList.exitCode, resultText(forwardList)).toBe(0); + const siblingDashboardPort = allocateSiblingDashboardPort(resultText(forwardList)); + + const buildOldBase = await buildOldOpenClawBase(host); + expect(buildOldBase.exitCode, resultText(buildOldBase)).toBe(0); + + const fixtureDockerfile = createFixtureDockerfile(cleanup); + for (const sandboxName of SANDBOX_NAMES) { + const createOldSandbox = await sandbox.openshell( + [ + "sandbox", + "create", + "--name", + sandboxName, + "--from", + fixtureDockerfile, + "--gateway", + "nemoclaw", + "--no-tty", + "--", + "true", + ], + { + artifactName: `phase-3-create-old-openclaw-${sandboxName}`, + env: commandEnv(), + timeoutMs: 15 * 60_000, + }, + ); + expect(createOldSandbox.exitCode, resultText(createOldSandbox)).toBe(0); + + const waitReady = await waitSandboxReady( + host, + sandboxName, + `phase-3-wait-old-${sandboxName}-ready`, + ); + expect(waitReady.exitCode, resultText(waitReady)).toBe(0); + + const oldVersion = await sandbox.exec(sandboxName, ["openclaw", "--version"], { + artifactName: `phase-3-old-openclaw-version-${sandboxName}`, + env: commandEnv(), + timeoutMs: 60_000, + }); + expect(oldVersion.exitCode, resultText(oldVersion)).toBe(0); + expect(resultText(oldVersion)).toContain(OLD_OPENCLAW_VERSION); + } + + progress.phase("register stale shared-route sandbox metadata"); + writeStaleRegistryEntries(siblingDashboardPort); + const staleRegistryJson = registeredStaleSandboxJson(); + await artifacts.writeText("registered-stale-sandboxes.json", staleRegistryJson); + const staleRegistry = JSON.parse(staleRegistryJson) as { + sandboxes: Record>; + }; + for (const sandboxName of SANDBOX_NAMES) { + expect(staleRegistry.sandboxes[sandboxName]).toBeDefined(); + expect(Object.hasOwn(staleRegistry.sandboxes[sandboxName]!, "credentialEnv")).toBe(false); + } + + progress.phase("detect both stale sandboxes"); + const staleCheck = await host.nemoclaw(["upgrade-sandboxes", "--check"], { + artifactName: "phase-5-upgrade-sandboxes-check-stale", + env: commandEnv(hosted.env), + redactionValues: [hosted.apiKey], + timeoutMs: 120_000, + }); + expect(staleCheck.exitCode, resultText(staleCheck)).toBe(0); + expect(resultText(staleCheck)).toMatch(/stale|need upgrading/i); + expect(resultText(staleCheck)).not.toMatch(/up to date/i); + expect(resultText(staleCheck)).toContain(SANDBOX_NAME); + expect(resultText(staleCheck)).toContain(SIBLING_SANDBOX_NAME); + + progress.phase("rebuild both to the current OpenClaw runtime"); + const rebuild = await host.nemoclaw(["upgrade-sandboxes", "--auto"], { + artifactName: "phase-6-upgrade-both-stale-sandboxes", + env: commandEnv(hosted.env), + redactionValues: [hosted.apiKey], + timeoutMs: 45 * 60_000, + }); + expect(rebuild.exitCode, resultText(rebuild)).toBe(0); + expect(resultText(rebuild)).toMatch(/2 sandbox\(es\) rebuilt/i); + + for (const sandboxName of SANDBOX_NAMES) { + const waitRebuiltReady = await waitSandboxReady( + host, + sandboxName, + `phase-6-wait-rebuilt-${sandboxName}-ready`, + ); + expect(waitRebuiltReady.exitCode, resultText(waitRebuiltReady)).toBe(0); + + const newVersion = await sandbox.exec(sandboxName, ["openclaw", "--version"], { + artifactName: `phase-6-new-openclaw-version-${sandboxName}`, + env: commandEnv(), + timeoutMs: 60_000, + }); + expect(newVersion.exitCode, resultText(newVersion)).toBe(0); + expect(resultText(newVersion)).not.toContain(OLD_OPENCLAW_VERSION); + } + const rebuiltRegistry = JSON.parse(registeredStaleSandboxJson()) as { + sandboxes: Record>; + }; + for (const sandboxName of SANDBOX_NAMES) { + expect(rebuiltRegistry.sandboxes[sandboxName]?.credentialEnv).toBe(hosted.credentialEnv); + } + + progress.phase("confirm the upgrade check is clean"); + const cleanCheck = await host.nemoclaw(["upgrade-sandboxes", "--check"], { + artifactName: "phase-7-upgrade-sandboxes-check-clean", + env: commandEnv(hosted.env), + redactionValues: [hosted.apiKey], + timeoutMs: 120_000, + }); + expect(cleanCheck.exitCode, resultText(cleanCheck)).toBe(0); + expect(resultText(cleanCheck)).toMatch(/up to date/i); +}); diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index 7911db86cf1..256810f497c 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -412,6 +412,13 @@ "test/e2e/support/e2e-clients.test.ts" ] }, + { + "live": "test/e2e/live/sandbox-rebuild.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, { "live": "test/e2e/live/sandbox-survival.test.ts", "fast": [ @@ -468,6 +475,13 @@ "test/e2e/support/e2e-clients.test.ts" ] }, + { + "live": "test/e2e/live/upgrade-stale-sandbox.test.ts", + "fast": [ + "test/e2e/support/e2e-cleanup-resources.test.ts", + "test/e2e/support/e2e-clients.test.ts" + ] + }, { "live": "test/e2e/live/state-dir-guard-metadata.test.ts", "liveOnlyReason": "Runs the installed guard from both production images against a real bind-mounted Linux filesystem; mocks cannot reproduce kernel xattr copying, POSIX ACL mask synchronization, inode replacement, or allocated-byte behavior." diff --git a/test/e2e/support/retired-selector-compatibility-workflow-boundary.test.ts b/test/e2e/support/retired-selector-compatibility-workflow-boundary.test.ts index 17db1e41722..45d72fcee11 100644 --- a/test/e2e/support/retired-selector-compatibility-workflow-boundary.test.ts +++ b/test/e2e/support/retired-selector-compatibility-workflow-boundary.test.ts @@ -37,6 +37,15 @@ function compatibilitySteps(): { return { steps, workflow }; } +function jobSteps(workflow: MutableWorkflow, jobName: string): WorkflowStep[] { + return ( + workflow.jobs[jobName]?.steps ?? + (() => { + throw new Error(`${jobName} steps are required`); + })() + ); +} + const DRIFT_CASES = [ { name: "target selector gate", @@ -97,6 +106,33 @@ const DRIFT_CASES = [ }, error: "retired-selector-compatibility job must upload compatibility evidence", }, + { + name: "sandbox rebuild candidate classifier", + mutate: (_steps: WorkflowStep[], workflow: MutableWorkflow) => { + const steps = jobSteps(workflow, "sandbox-rebuild"); + const classifier = requiredStep( + steps, + (candidate) => candidate.name === "Classify sandbox-rebuild candidate selector", + "sandbox rebuild candidate classifier", + ); + steps.splice(steps.indexOf(classifier), 1); + }, + error: "sandbox-rebuild job must classify candidate selector retirement", + }, + { + name: "upgrade stale sandbox candidate guard", + mutate: (_steps: WorkflowStep[], workflow: MutableWorkflow) => { + const steps = jobSteps(workflow, "upgrade-stale-sandbox"); + const run = requiredStep( + steps, + (candidate) => candidate.name === "Run upgrade stale sandbox live Vitest test", + "upgrade stale sandbox live test", + ); + run.if = "always()"; + }, + error: + "upgrade-stale-sandbox step 'Run upgrade stale sandbox live Vitest test' must skip a retired candidate selector", + }, ] as const; it.each(DRIFT_CASES)("rejects retired-selector compatibility drift in $name (#7615)", ({ diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts index aad54ea9d0e..d6360813f4d 100644 --- a/test/e2e/support/workflow-plan.test.ts +++ b/test/e2e/support/workflow-plan.test.ts @@ -29,6 +29,11 @@ function firstId(rows: readonly T[], label: string): s return rows[0]!.id; } +function retiredControllerSelectorIds(): string[] { + const allowedJobs = new Set(readFreeStandingJobsInventory().allowedJobs); + return RETIRED_CONTROLLER_SELECTOR_IDS.filter((id) => !allowedJobs.has(id)); +} + describe("E2E workflow plan", () => { it("defaults to every supported registry target and tagged credential-free test", () => { const plan = buildE2eWorkflowPlan(); @@ -157,7 +162,7 @@ describe("E2E workflow plan", () => { GITHUB_OUTPUT: output, GITHUB_STEP_SUMMARY: summary, INFERENCE_MODE: "mock", - JOBS: [activeJobs, ...RETIRED_CONTROLLER_SELECTOR_IDS].join(","), + JOBS: [activeJobs, ...retiredControllerSelectorIds()].join(","), TARGETS: "", NEMOCLAW_E2E_EXPECTED_SHA: "a".repeat(40), }, @@ -191,7 +196,7 @@ describe("E2E workflow plan", () => { GITHUB_OUTPUT: path.join(directory, "github-output"), GITHUB_STEP_SUMMARY: path.join(directory, "summary.md"), INFERENCE_MODE: "mock", - JOBS: RETIRED_CONTROLLER_SELECTOR_IDS.join(","), + JOBS: retiredControllerSelectorIds().join(","), TARGETS: "", NEMOCLAW_E2E_EXPECTED_SHA: "a".repeat(40), }, @@ -207,7 +212,7 @@ describe("E2E workflow plan", () => { } }); - it("emits an empty live plan for P2 selectors handled by target compatibility (#7615)", () => { + it("emits an empty matrix for free-standing rebuild selectors during transition (#7615)", () => { const directory = mkdtempSync(path.join(tmpdir(), "nemoclaw-workflow-plan-cli-")); const output = path.join(directory, "github-output"); const summary = path.join(directory, "summary.md"); diff --git a/tools/e2e/retired-selector-compatibility.mts b/tools/e2e/retired-selector-compatibility.mts index df0e803facf..6df89560e9d 100644 --- a/tools/e2e/retired-selector-compatibility.mts +++ b/tools/e2e/retired-selector-compatibility.mts @@ -155,7 +155,9 @@ export function selectedRetiredControllerJobs(options: { if (!SHA_PATTERN.test(options.expectedSha ?? "")) return []; for (const selectors of [options.jobs, options.targets]) { if (selectors && !SELECTOR_LIST_PATTERN.test(selectors)) { - throw new Error("retired selector compatibility requires safe comma-separated selector IDs"); + throw new Error( + "retired selector compatibility requires comma-separated selector IDs containing only letters, numbers, underscores, and hyphens", + ); } } const allowedJobs = new Set(options.allowedJobs); diff --git a/tools/e2e/workflow-boundary.mts b/tools/e2e/workflow-boundary.mts index 00eb6e5a5bd..280a62b2900 100644 --- a/tools/e2e/workflow-boundary.mts +++ b/tools/e2e/workflow-boundary.mts @@ -1059,6 +1059,61 @@ function validateFreeStandingJobSelector( } } +const CANDIDATE_SELECTOR_ACTIVE_IF = + "${{ steps.selector_compatibility.outputs.retired != 'true' }}"; +const CANDIDATE_SELECTOR_CLASSIFIER = [ + "set -euo pipefail", + 'if [[ -f "$LEGACY_TEST_FILE" ]]; then', + " retired=false", + "else", + " retired=true", + "fi", + `printf 'retired=%s\\n' "$retired" >> "$GITHUB_OUTPUT"`, +].join("\n"); + +function validateCandidateSelectorCompatibility( + errors: string[], + options: { + installStep: string; + jobName: string; + legacyTestFile: string; + runStep: string; + steps: readonly WorkflowStep[]; + }, +): void { + const classifierName = `Classify ${options.jobName} candidate selector`; + const classifier = namedStep(options.steps, classifierName); + if ( + stringValue(classifier?.id) !== "selector_compatibility" || + asRecord(classifier?.env).LEGACY_TEST_FILE !== options.legacyTestFile || + stringValue(classifier?.run).trim() !== CANDIDATE_SELECTOR_CLASSIFIER + ) { + errors.push(`${options.jobName} job must classify candidate selector retirement`); + } + + const dockerAuthIndex = options.steps.findIndex( + (step) => step.name === "Authenticate to Docker Hub", + ); + const classifierIndex = options.steps.indexOf(classifier ?? {}); + const prepareIndex = options.steps.findIndex((step) => step.name === "Prepare E2E workspace"); + if ( + dockerAuthIndex < 0 || + classifierIndex <= dockerAuthIndex || + prepareIndex < 0 || + classifierIndex >= prepareIndex + ) { + errors.push( + `${options.jobName} candidate selector classifier must follow Docker Hub authentication`, + ); + } + + for (const stepName of [options.installStep, options.runStep]) { + if (namedStep(options.steps, stepName)?.if !== CANDIDATE_SELECTOR_ACTIVE_IF) { + errors.push(`${options.jobName} step '${stepName}' must skip a retired candidate selector`); + } + } +} + function validateGatewayGuardRecoveryJob(errors: string[], jobs: WorkflowRecord): void { const job = asRecord(jobs["gateway-guard-recovery"]); if (Object.keys(job).length === 0) return; @@ -1982,6 +2037,94 @@ function validateRebuildHermesJob( } } +function validateSandboxRebuildJob(errors: string[], jobs: WorkflowRecord): void { + const jobName = "sandbox-rebuild"; + const targetName = "sandbox-rebuild"; + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) { + errors.push("workflow missing sandbox-rebuild job"); + return; + } + + if (job["runs-on"] !== "ubuntu-latest") { + errors.push("sandbox-rebuild job must run on ubuntu-latest"); + } + validateFreeStandingJobSelector(errors, jobs, jobName, targetName); + if (job["timeout-minutes"] !== 90) { + errors.push("sandbox-rebuild job must keep the legacy 90 minute timeout"); + } + const jobEnv = asRecord(job.env); + if (jobEnv.NEMOCLAW_RUN_LIVE_E2E !== "1") { + errors.push("sandbox-rebuild job must set NEMOCLAW_RUN_LIVE_E2E=1"); + } + if (jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/live/sandbox-rebuild") { + errors.push( + "sandbox-rebuild job must write artifacts under e2e-artifacts/live/sandbox-rebuild", + ); + } + if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") { + errors.push("sandbox-rebuild job must point NEMOCLAW_CLI_BIN at the repo CLI"); + } + if (jobEnv.OPENSHELL_GATEWAY !== "nemoclaw") { + errors.push("sandbox-rebuild job must force OPENSHELL_GATEWAY=nemoclaw"); + } + for (const secret of [ + "NVIDIA_INFERENCE_API_KEY", + "DOCKERHUB_USERNAME", + "DOCKERHUB_TOKEN", + "GITHUB_TOKEN", + ]) { + requireEnvDoesNotExposeSecret(errors, "sandbox-rebuild job", jobEnv, secret); + } + + const steps = asSteps(job.steps); + validateCandidateSelectorCompatibility(errors, { + installStep: "Install OpenShell", + jobName, + legacyTestFile: "test/e2e/live/sandbox-rebuild.test.ts", + runStep: "Run sandbox rebuild live test", + steps, + }); + requireNoDispatchInputInterpolation(errors, steps); + for (const step of steps) { + const stepName = `sandbox-rebuild step '${step.name ?? step.uses ?? ""}'`; + const stepEnv = asRecord(step.env); + if (step.name !== "Run sandbox rebuild live test") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_INFERENCE_API_KEY"); + } + if (step.name !== "Authenticate to Docker Hub") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); + requireNoDockerHubAuthInRun(errors, stepName, stringValue(step.run)); + } + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "GITHUB_TOKEN"); + } + + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!checkout) errors.push("sandbox-rebuild job missing checkout step"); + requireFullShaAction(errors, checkout, "sandbox-rebuild checkout"); + if (asRecord(checkout?.with)["persist-credentials"] !== false) { + errors.push("sandbox-rebuild checkout step must set persist-credentials=false"); + } + + const installOpenShell = requireJobStep(errors, jobName, steps, "Install OpenShell"); + requireRunContains(errors, installOpenShell, "bash scripts/install-openshell.sh"); + requireRunContains(errors, installOpenShell, "env -u DOCKER_CONFIG"); + requireRunContains(errors, installOpenShell, "-u DOCKERHUB_USERNAME"); + requireRunContains(errors, installOpenShell, "-u DOCKERHUB_TOKEN"); + requireRunContains(errors, installOpenShell, "-u NVIDIA_INFERENCE_API_KEY"); + requireRunContains(errors, installOpenShell, "-u GITHUB_TOKEN"); + + const runVitest = requireJobStep(errors, jobName, steps, "Run sandbox rebuild live test"); + const runVitestEnv = asRecord(runVitest?.env); + if (runVitestEnv.NVIDIA_INFERENCE_API_KEY !== "${{ secrets.NVIDIA_INFERENCE_API_KEY }}") { + errors.push("sandbox-rebuild step must receive NVIDIA_INFERENCE_API_KEY from secrets"); + } + requireRunContains(errors, runVitest, "OPENSHELL_BIN"); + requireRunContains(errors, runVitest, "tools/e2e/live-vitest-invocation.mts run --test-path"); + requireRunContains(errors, runVitest, "test/e2e/live/sandbox-rebuild.test.ts"); +} + function validateStateBackupRestoreJob(errors: string[], jobs: WorkflowRecord): void { const jobName = "state-backup-restore"; const targetName = "state-backup-restore"; @@ -2074,6 +2217,104 @@ function validateStateBackupRestoreJob(errors: string[], jobs: WorkflowRecord): requireRunContains(errors, runVitest, "test/e2e/live/state-backup-restore.test.ts"); } +function validateUpgradeStaleSandboxJob(errors: string[], jobs: WorkflowRecord): void { + const jobName = "upgrade-stale-sandbox"; + const targetName = "upgrade-stale-sandbox"; + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) { + errors.push("workflow missing upgrade-stale-sandbox job"); + return; + } + + if (job["runs-on"] !== "ubuntu-latest") { + errors.push("upgrade-stale-sandbox job must run on ubuntu-latest"); + } + validateFreeStandingJobSelector(errors, jobs, jobName, targetName); + if (job["timeout-minutes"] !== 85) { + errors.push("upgrade-stale-sandbox job must keep the two-sandbox 85 minute timeout"); + } + + const jobEnv = asRecord(job.env); + if (jobEnv.NEMOCLAW_RUN_LIVE_E2E !== "1") { + errors.push("upgrade-stale-sandbox job must set NEMOCLAW_RUN_LIVE_E2E=1"); + } + if ( + jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/live/upgrade-stale-sandbox" + ) { + errors.push( + "upgrade-stale-sandbox job must write artifacts under e2e-artifacts/live/upgrade-stale-sandbox", + ); + } + if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") { + errors.push("upgrade-stale-sandbox job must point NEMOCLAW_CLI_BIN at the repo CLI"); + } + if (jobEnv.OPENSHELL_GATEWAY !== "nemoclaw") { + errors.push("upgrade-stale-sandbox job must force OPENSHELL_GATEWAY=nemoclaw"); + } + if (jobEnv.NEMOCLAW_SANDBOX_NAME !== "e2e-upgrade-stale") { + errors.push("upgrade-stale-sandbox job must set NEMOCLAW_SANDBOX_NAME=e2e-upgrade-stale"); + } + if ("DOCKER_CONFIG" in jobEnv) { + errors.push("upgrade-stale-sandbox job must not set DOCKER_CONFIG at job level"); + } + for (const secret of [...COMMON_SECRET_ENV_NAMES]) { + requireEnvDoesNotExposeSecret(errors, "upgrade-stale-sandbox job", jobEnv, secret); + } + + const steps = asSteps(job.steps); + validateCandidateSelectorCompatibility(errors, { + installStep: "Install OpenShell CLI", + jobName, + legacyTestFile: "test/e2e/live/upgrade-stale-sandbox.test.ts", + runStep: "Run upgrade stale sandbox live Vitest test", + steps, + }); + requireNoDispatchInputInterpolation(errors, steps); + for (const step of steps) { + const stepName = `upgrade-stale-sandbox step '${step.name ?? step.uses ?? ""}'`; + const stepEnv = asRecord(step.env); + if (step.name !== "Run upgrade stale sandbox live Vitest test") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_INFERENCE_API_KEY"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_API_KEY"); + } + if (step.name !== "Authenticate to Docker Hub") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); + requireNoDockerHubAuthInRun(errors, stepName, stringValue(step.run)); + } + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "GITHUB_TOKEN"); + } + + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!checkout) errors.push("upgrade-stale-sandbox job missing checkout step"); + requireFullShaAction(errors, checkout, "upgrade-stale-sandbox checkout"); + if (asRecord(checkout?.with)["persist-credentials"] !== false) { + errors.push("upgrade-stale-sandbox checkout step must set persist-credentials=false"); + } + + const installOpenShell = requireJobStep(errors, jobName, steps, "Install OpenShell CLI"); + requireRunContains(errors, installOpenShell, "bash scripts/install-openshell.sh"); + requireRunContains(errors, installOpenShell, "env -u DOCKER_CONFIG"); + requireRunContains(errors, installOpenShell, "-u DOCKERHUB_USERNAME"); + requireRunContains(errors, installOpenShell, "-u DOCKERHUB_TOKEN"); + requireRunContains(errors, installOpenShell, "-u NVIDIA_INFERENCE_API_KEY"); + requireRunContains(errors, installOpenShell, "-u GITHUB_TOKEN"); + + const runVitest = requireJobStep( + errors, + jobName, + steps, + "Run upgrade stale sandbox live Vitest test", + ); + const runVitestEnv = asRecord(runVitest?.env); + if (runVitestEnv.NVIDIA_INFERENCE_API_KEY !== "${{ secrets.NVIDIA_INFERENCE_API_KEY }}") { + errors.push("upgrade-stale-sandbox step must receive NVIDIA_INFERENCE_API_KEY from secrets"); + } + requireRunContains(errors, runVitest, "OPENSHELL_BIN"); + requireRunContains(errors, runVitest, "tools/e2e/live-vitest-invocation.mts run --test-path"); + requireRunContains(errors, runVitest, "test/e2e/live/upgrade-stale-sandbox.test.ts"); +} + function validateTokenRotationJob(errors: string[], jobs: WorkflowRecord): void { const jobName = "token-rotation"; const job = asRecord(jobs[jobName]); @@ -4821,7 +5062,9 @@ export function validateE2eWorkflow(workflowValue: unknown): string[] { validateRebuildOpenClawJob(errors, jobs); validateRebuildHermesJob(errors, jobs, { staleBase: false }); validateRebuildHermesJob(errors, jobs, { staleBase: true }); + validateSandboxRebuildJob(errors, jobs); validateStateBackupRestoreJob(errors, jobs); + validateUpgradeStaleSandboxJob(errors, jobs); validateTokenRotationJob(errors, jobs); validateMessagingCompatibleEndpointJob(errors, jobs); validateFreeStandingJobSelector(errors, jobs, "gateway-guard-recovery", "gateway-guard-recovery"); From 7cc422d9de1f1cd1895470da2f7f19c6f36f25d0 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 30 Jul 2026 14:28:55 -0700 Subject: [PATCH 13/15] test(e2e): bound batch rebuild calls --- src/lib/actions/upgrade-sandboxes-recovery.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/actions/upgrade-sandboxes-recovery.test.ts b/src/lib/actions/upgrade-sandboxes-recovery.test.ts index 16d734ebb6b..715538f53f9 100644 --- a/src/lib/actions/upgrade-sandboxes-recovery.test.ts +++ b/src/lib/actions/upgrade-sandboxes-recovery.test.ts @@ -323,6 +323,7 @@ describe("upgrade-sandboxes prepared backup recovery (#6114)", () => { recoveryManifest: undefined, throwOnError: true, }); + expect(harness.rebuildSpy).toHaveBeenCalledTimes(2); }); it("fails closed for a probed v0.0.55 custom image with matching backup agent version", async () => { From f933abffeae2a465f7d4d703c9ca031603a2fa6b Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Thu, 30 Jul 2026 14:57:14 -0700 Subject: [PATCH 14/15] test(e2e): require retired selector cases --- test/e2e/support/workflow-plan.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/e2e/support/workflow-plan.test.ts b/test/e2e/support/workflow-plan.test.ts index 58d17ece4bb..fba5f7f1ba8 100644 --- a/test/e2e/support/workflow-plan.test.ts +++ b/test/e2e/support/workflow-plan.test.ts @@ -31,7 +31,9 @@ function firstId(rows: readonly T[], label: string): s function retiredControllerSelectorIds(): string[] { const allowedJobs = new Set(readFreeStandingJobsInventory().allowedJobs); - return RETIRED_CONTROLLER_SELECTOR_IDS.filter((id) => !allowedJobs.has(id)); + const retiredIds = RETIRED_CONTROLLER_SELECTOR_IDS.filter((id) => !allowedJobs.has(id)); + expect(retiredIds).toEqual([...RETIRED_CONTROLLER_SELECTOR_IDS]); + return retiredIds; } describe("E2E workflow plan", () => { From 3747a7817eec99688755d4d384c3bd36d8703071 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Thu, 30 Jul 2026 15:56:06 -0700 Subject: [PATCH 15/15] fix(e2e): bridge retired rebuild selectors Signed-off-by: Apurv Kumaria --- test/e2e/README.md | 4 +- .../retired-rebuild-selector-compatibility.ts | 40 +++++++++++++++++ test/e2e/live/sandbox-rebuild.test.ts | 13 ++++++ test/e2e/live/upgrade-stale-sandbox.test.ts | 13 ++++++ .../support/e2e-semantic-phase-check.test.ts | 16 +++++++ ...red-rebuild-selector-compatibility.test.ts | 45 +++++++++++++++++++ .../retired-selector-compatibility.test.ts | 37 +++++++++++++++ tools/e2e/check-semantic-phases.mts | 2 + tools/e2e/retired-selector-compatibility.mts | 19 +++++++- 9 files changed, 186 insertions(+), 3 deletions(-) create mode 100644 test/e2e/live/retired-rebuild-selector-compatibility.ts create mode 100644 test/e2e/live/sandbox-rebuild.test.ts create mode 100644 test/e2e/live/upgrade-stale-sandbox.test.ts create mode 100644 test/e2e/support/retired-rebuild-selector-compatibility.test.ts diff --git a/test/e2e/README.md b/test/e2e/README.md index cdf2234e1b1..da98d5a48af 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -108,7 +108,9 @@ mode. PR gate requests using the retired `sandbox-rebuild` and `upgrade-stale-sandbox` job or target selectors run focused replacement tests through the compatibility controller. `rebuild-openclaw` is the canonical live -rebuild and upgrade target. +rebuild and upgrade target. Thin entrypoints at the two former test paths +delegate to that canonical target only while trusted `main` can still execute +the pre-retirement workflow graph; they do not restore either retired lane. ## Current OpenClaw plugin EXDEV lifecycle diff --git a/test/e2e/live/retired-rebuild-selector-compatibility.ts b/test/e2e/live/retired-rebuild-selector-compatibility.ts new file mode 100644 index 00000000000..7eaa1e74c48 --- /dev/null +++ b/test/e2e/live/retired-rebuild-selector-compatibility.ts @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +export const RETIRED_REBUILD_SELECTOR_IDS = ["sandbox-rebuild", "upgrade-stale-sandbox"] as const; + +export type RetiredRebuildSelectorId = (typeof RETIRED_REBUILD_SELECTOR_IDS)[number]; + +function numericIdentity(value: string | undefined, fallback: string, label: string): string { + const resolved = value?.trim() || fallback; + if (!/^[0-9]+$/u.test(resolved)) { + throw new Error(`${label} must contain only decimal digits`); + } + return resolved; +} + +export function retiredRebuildCompatibilitySandboxName( + selector: RetiredRebuildSelectorId, + environment: NodeJS.ProcessEnv = process.env, +): string { + const runId = numericIdentity(environment.GITHUB_RUN_ID, String(process.pid), "GITHUB_RUN_ID"); + const runAttempt = numericIdentity(environment.GITHUB_RUN_ATTEMPT, "1", "GITHUB_RUN_ATTEMPT"); + const name = `e2e-rebuild-openclaw-${selector}-${runId}-${runAttempt}`; + if (name.length > 63) { + throw new Error(`retired rebuild compatibility sandbox name exceeds 63 characters: ${name}`); + } + return name; +} + +export function prepareRetiredRebuildSelectorCompatibility( + selector: RetiredRebuildSelectorId, + environment: NodeJS.ProcessEnv = process.env, +): void { + const targetId = environment.E2E_TARGET_ID ?? environment.GITHUB_JOB; + if (targetId !== selector) { + throw new Error( + `retired rebuild compatibility entrypoint ${selector} requires matching E2E_TARGET_ID or GITHUB_JOB; got ${JSON.stringify(targetId ?? "")}`, + ); + } + environment.NEMOCLAW_SANDBOX_NAME = retiredRebuildCompatibilitySandboxName(selector, environment); +} diff --git a/test/e2e/live/sandbox-rebuild.test.ts b/test/e2e/live/sandbox-rebuild.test.ts new file mode 100644 index 00000000000..15c645a47b3 --- /dev/null +++ b/test/e2e/live/sandbox-rebuild.test.ts @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// @retired-selector-compatibility-entrypoint sandbox-rebuild +// Trusted main can still invoke this path while the candidate removes its +// legacy job graph. Exercise the retained canonical rebuild test during that +// rollout window instead of restoring the retired coverage. +import { prepareRetiredRebuildSelectorCompatibility } from "./retired-rebuild-selector-compatibility.ts"; + +if (process.env.NEMOCLAW_E2E_PHASE_COLLECTION !== "1") { + prepareRetiredRebuildSelectorCompatibility("sandbox-rebuild"); + await import("./rebuild-openclaw.test.ts"); +} diff --git a/test/e2e/live/upgrade-stale-sandbox.test.ts b/test/e2e/live/upgrade-stale-sandbox.test.ts new file mode 100644 index 00000000000..4bfb4fca85f --- /dev/null +++ b/test/e2e/live/upgrade-stale-sandbox.test.ts @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// @retired-selector-compatibility-entrypoint upgrade-stale-sandbox +// Trusted main can still invoke this path while the candidate removes its +// legacy job graph. Exercise the retained canonical rebuild test during that +// rollout window instead of restoring the retired coverage. +import { prepareRetiredRebuildSelectorCompatibility } from "./retired-rebuild-selector-compatibility.ts"; + +if (process.env.NEMOCLAW_E2E_PHASE_COLLECTION !== "1") { + prepareRetiredRebuildSelectorCompatibility("upgrade-stale-sandbox"); + await import("./rebuild-openclaw.test.ts"); +} diff --git a/test/e2e/support/e2e-semantic-phase-check.test.ts b/test/e2e/support/e2e-semantic-phase-check.test.ts index 4e3b2066041..01123c070df 100644 --- a/test/e2e/support/e2e-semantic-phase-check.test.ts +++ b/test/e2e/support/e2e-semantic-phase-check.test.ts @@ -218,6 +218,22 @@ describe("semantic E2E phase checker", () => { ); }); + test.each([ + "sandbox-rebuild", + "upgrade-stale-sandbox", + ])("accepts the %s rollout entrypoint as an exact rebuild forwarder (#7615)", (selector) => { + const relativeModuleId = `test/e2e/live/${selector}.test.ts`; + expect( + validateCollectedSemanticPhaseModule({ + relativeModuleId, + project: "e2e-live", + errors: [], + tests: [], + source: scanLiveSourceGraph(path.join(REPO_ROOT, relativeModuleId)), + }), + ).toEqual([]); + }); + test("rejects each direct child-process boundary that can hide a heartbeat", () => { const source = scanLiveSourceGraph(CHILD_PROCESS_SOURCE_FIXTURE); const failures = validateCollectedSemanticPhaseModule({ diff --git a/test/e2e/support/retired-rebuild-selector-compatibility.test.ts b/test/e2e/support/retired-rebuild-selector-compatibility.test.ts new file mode 100644 index 00000000000..427202e04a2 --- /dev/null +++ b/test/e2e/support/retired-rebuild-selector-compatibility.test.ts @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { + prepareRetiredRebuildSelectorCompatibility, + retiredRebuildCompatibilitySandboxName, +} from "../live/retired-rebuild-selector-compatibility.ts"; + +describe("retired rebuild selector compatibility", () => { + it.each([ + "sandbox-rebuild", + "upgrade-stale-sandbox", + ] as const)("prepares %s to use isolated canonical rebuild state (#7615)", (selector) => { + const environment = { + E2E_TARGET_ID: selector, + GITHUB_RUN_ATTEMPT: "2", + GITHUB_RUN_ID: "30587862523", + NEMOCLAW_SANDBOX_NAME: "legacy-name", + }; + + prepareRetiredRebuildSelectorCompatibility(selector, environment); + + expect(environment.NEMOCLAW_SANDBOX_NAME).toBe( + `e2e-rebuild-openclaw-${selector}-30587862523-2`, + ); + }); + + it("rejects a legacy path invoked for another job (#7615)", () => { + expect(() => + prepareRetiredRebuildSelectorCompatibility("sandbox-rebuild", { + E2E_TARGET_ID: "upgrade-stale-sandbox", + }), + ).toThrow("requires matching E2E_TARGET_ID or GITHUB_JOB"); + }); + + it("rejects untrusted run identity fragments (#7615)", () => { + expect(() => + retiredRebuildCompatibilitySandboxName("sandbox-rebuild", { + GITHUB_RUN_ID: "3058;echo", + }), + ).toThrow("GITHUB_RUN_ID must contain only decimal digits"); + }); +}); diff --git a/test/e2e/support/retired-selector-compatibility.test.ts b/test/e2e/support/retired-selector-compatibility.test.ts index 6260ddadc11..ce36af250f1 100644 --- a/test/e2e/support/retired-selector-compatibility.test.ts +++ b/test/e2e/support/retired-selector-compatibility.test.ts @@ -34,6 +34,10 @@ const REPLACEMENT_FILES = [ "test/install-openshell-version-pin.test.ts", "test/rebuild-stale-recovery.test.ts", ] as const; +const TRANSITION_ENTRYPOINTS = { + "sandbox-rebuild": "test/e2e/live/sandbox-rebuild.test.ts", + "upgrade-stale-sandbox": "test/e2e/live/upgrade-stale-sandbox.test.ts", +} as const; function workspace(): { artifactRoot: string; output: string; root: string } { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-retired-selector-")); @@ -42,6 +46,19 @@ function workspace(): { artifactRoot: string; output: string; root: string } { fs.mkdirSync(path.dirname(fullPath), { recursive: true }); fs.writeFileSync(fullPath, "export {};\n", "utf8"); } + for (const [id, file] of Object.entries(TRANSITION_ENTRYPOINTS)) { + const fullPath = path.join(root, file); + fs.mkdirSync(path.dirname(fullPath), { recursive: true }); + fs.writeFileSync( + fullPath, + [ + `// @retired-selector-compatibility-entrypoint ${id}`, + `prepareRetiredRebuildSelectorCompatibility("${id}");`, + "", + ].join("\n"), + "utf8", + ); + } return { artifactRoot: path.join(root, "artifacts"), output: path.join(root, "github-output"), @@ -226,4 +243,24 @@ describe("retired E2E selector compatibility", () => { fs.rmSync(target.root, { force: true, recursive: true }); } }); + + it("rejects a rebuild transition entrypoint that stops delegating (#7615)", () => { + const target = workspace(); + fs.writeFileSync( + path.join(target.root, TRANSITION_ENTRYPOINTS["sandbox-rebuild"]), + "export {};\n", + "utf8", + ); + try { + expect(() => + runRetiredSelectorCompatibility(environment(target, "sandbox-rebuild"), { + allowedJobs: ["cloud-onboard"], + repositoryRoot: target.root, + runCommand: () => undefined, + }), + ).toThrow("sandbox-rebuild transition entrypoint must remain a thin canonical delegate"); + } finally { + fs.rmSync(target.root, { force: true, recursive: true }); + } + }); }); diff --git a/tools/e2e/check-semantic-phases.mts b/tools/e2e/check-semantic-phases.mts index 58c7887d063..a815d67aa5d 100644 --- a/tools/e2e/check-semantic-phases.mts +++ b/tools/e2e/check-semantic-phases.mts @@ -97,6 +97,8 @@ const E2E_RUNTIME_OBSERVABILITY_FILES = [path.join(E2E_ROOT, "risk-signal-report const REGISTRY_TARGET_TEST = "test/e2e/live/registry-targets.test.ts"; const LIVE_TEST_FORWARDERS = new Map([ ["test/e2e/live/bootstrap-install-smoke.test.ts", "test/e2e/live/launchable-smoke.test.ts"], + ["test/e2e/live/sandbox-rebuild.test.ts", "test/e2e/live/rebuild-openclaw.test.ts"], + ["test/e2e/live/upgrade-stale-sandbox.test.ts", "test/e2e/live/rebuild-openclaw.test.ts"], ]); const LIVE_TEST_FIXTURE_SUFFIX = "/fixtures/e2e-test.ts"; const WORKFLOW_TEST_FIXTURE_SUFFIX = "/e2e/fixtures/workflow-e2e-test.ts"; diff --git a/tools/e2e/retired-selector-compatibility.mts b/tools/e2e/retired-selector-compatibility.mts index 6df89560e9d..82794df43a7 100644 --- a/tools/e2e/retired-selector-compatibility.mts +++ b/tools/e2e/retired-selector-compatibility.mts @@ -34,6 +34,7 @@ type ReplacementTest = { }; type Replacement = { legacyFile?: string; + transitionEntrypoint?: string; tests: readonly ReplacementTest[]; }; @@ -103,7 +104,7 @@ const REPLACEMENTS: Readonly> = ], }, "sandbox-rebuild": { - legacyFile: "test/e2e/live/sandbox-rebuild.test.ts", + transitionEntrypoint: "test/e2e/live/sandbox-rebuild.test.ts", tests: [ { files: [ @@ -128,7 +129,7 @@ const REPLACEMENTS: Readonly> = ], }, "upgrade-stale-sandbox": { - legacyFile: "test/e2e/live/upgrade-stale-sandbox.test.ts", + transitionEntrypoint: "test/e2e/live/upgrade-stale-sandbox.test.ts", tests: [ { files: [ @@ -243,6 +244,20 @@ function verifyReplacementBoundary( ) { throw new Error(`${id} compatibility requires its live E2E file to remain retired`); } + if (replacement.transitionEntrypoint) { + const entrypoint = path.join(repositoryRoot, replacement.transitionEntrypoint); + if (!fs.existsSync(entrypoint)) { + throw new Error( + `${id} transition entrypoint is missing: ${replacement.transitionEntrypoint}`, + ); + } + const source = fs.readFileSync(entrypoint, "utf8"); + const marker = `@retired-selector-compatibility-entrypoint ${id}`; + const registration = `prepareRetiredRebuildSelectorCompatibility("${id}")`; + if (!source.includes(marker) || !source.includes(registration)) { + throw new Error(`${id} transition entrypoint must remain a thin canonical delegate`); + } + } for (const test of replacement.tests) { for (const file of test.files) { const fullPath = path.join(repositoryRoot, file);