test(e2e): consolidate priority 2 live seams

.github/workflows/e2e.yaml

@@ -440,7 +440,7 @@ jobs:
 
   retired-selector-compatibility:
     needs: generate-matrix
-    if: ${{ inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',credential-migration,') || contains(format(',{0},', inputs.jobs), ',credential-sanitization,') || contains(format(',{0},', inputs.jobs), ',diagnostics,') || contains(format(',{0},', inputs.jobs), ',docs-validation,') || contains(format(',{0},', inputs.jobs), ',gateway-drift-preflight,') || contains(format(',{0},', inputs.jobs), ',gateway-health-honest,') || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths,') || contains(format(',{0},', inputs.jobs), ',openshell-version-pin,') || contains(format(',{0},', inputs.jobs), ',ubuntu-repo-cli-smoke,')) }}
+    if: ${{ inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',credential-migration,') || contains(format(',{0},', inputs.jobs), ',credential-sanitization,') || contains(format(',{0},', inputs.jobs), ',diagnostics,') || contains(format(',{0},', inputs.jobs), ',docs-validation,') || contains(format(',{0},', inputs.jobs), ',gateway-drift-preflight,') || contains(format(',{0},', inputs.jobs), ',gateway-health-honest,') || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths,') || contains(format(',{0},', inputs.jobs), ',openshell-version-pin,') || contains(format(',{0},', inputs.jobs), ',sandbox-rebuild,') || contains(format(',{0},', inputs.jobs), ',ubuntu-repo-cli-smoke,') || contains(format(',{0},', inputs.jobs), ',upgrade-stale-sandbox,') || contains(format(',{0},', inputs.targets), ',sandbox-rebuild,') || contains(format(',{0},', inputs.targets), ',upgrade-stale-sandbox,')) }}
     runs-on: ubuntu-latest
     timeout-minutes: 15
     env:
@@ -458,6 +458,7 @@ jobs:
       - name: Verify retired selector replacements
         env:
           JOBS: ${{ inputs.jobs }}
+          TARGETS: ${{ inputs.targets }}
         run: npx tsx tools/e2e/retired-selector-compatibility.mts
 
       - name: Upload retired selector compatibility evidence
@@ -1742,12 +1743,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - mode: hosted
-            sandbox_name: e2e-hermes-inference-switch
-            switch_provider: nvidia-prod
-            switch_model: nvidia/nemotron-3-super-120b-a12b
-            switch_inference_api: openai-completions
-            switch_mock_anthropic: "0"
           - mode: anthropic
             sandbox_name: e2e-hermes-anthropic-inference-switch
             switch_provider: compatible-anthropic-endpoint
@@ -1760,7 +1755,6 @@ jobs:
       E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-inference-switch/${{ matrix.mode }}
       NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
       NEMOCLAW_RUN_LIVE_E2E: "1"
-      NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
       NEMOCLAW_NON_INTERACTIVE: "1"
       NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
       NEMOCLAW_AGENT: "hermes"
@@ -1809,9 +1803,6 @@ jobs:
       - name: Install OpenShell CLI
         run: bash scripts/install-openshell.sh
       - name: Run Hermes inference switch live Vitest test
-        env:
-          NVIDIA_API_KEY: ${{ matrix.mode == 'hosted' && secrets.NVIDIA_API_KEY || '' }}
-          NVIDIA_INFERENCE_API_KEY: ${{ matrix.mode == 'hosted' && secrets.NVIDIA_INFERENCE_API_KEY || '' }}
         run: |
           set -euo pipefail
           export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
@@ -3255,68 +3246,6 @@ jobs:
         shell: bash
         run: bash .github/scripts/docker-auth-cleanup.sh
 
-  sandbox-rebuild:
-    needs: generate-matrix
-    if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',sandbox-rebuild,') || contains(format(',{0},', inputs.targets), ',sandbox-rebuild,') }}
-    runs-on: ubuntu-latest
-    timeout-minutes: 90
-    env:
-      E2E_JOB: "1"
-      E2E_TARGET_ID: "sandbox-rebuild"
-      E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/sandbox-rebuild
-      NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
-      NEMOCLAW_RUN_LIVE_E2E: "1"
-      NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
-      NEMOCLAW_NON_INTERACTIVE: "1"
-      NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
-      OPENSHELL_GATEWAY: nemoclaw
-    steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-        with:
-          repository: ${{ inputs.checkout_repository || github.repository }}
-          ref: ${{ inputs.checkout_sha || github.sha }}
-          persist-credentials: false
-
-      - *dockerhub-auth
-
-      - name: Prepare E2E workspace
-        uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
-
-      - name: Install OpenShell
-        env:
-          NEMOCLAW_NON_INTERACTIVE: "1"
-        run: |
-          set -euo pipefail
-          env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh
-
-      - name: Run sandbox rebuild live test
-        env:
-          NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
-        run: |
-          set -euo pipefail
-          export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
-          if command -v openshell >/dev/null 2>&1; then
-            OPENSHELL_BIN="$(command -v openshell)"
-          elif [ -x "$HOME/.local/bin/openshell" ]; then
-            OPENSHELL_BIN="$HOME/.local/bin/openshell"
-          else
-            echo "::error::OpenShell CLI not found after install"
-            ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true
-            exit 1
-          fi
-          export OPENSHELL_BIN
-          "$OPENSHELL_BIN" --version
-          npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/sandbox-rebuild.test.ts
-
-      - name: Upload sandbox rebuild artifacts
-        if: always()
-        uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
-
-      - name: Clean up Docker auth
-        if: always()
-        shell: bash
-        run: bash .github/scripts/docker-auth-cleanup.sh
-
   sandbox-rlimits-connect:
     needs: generate-matrix
     if: ${{ contains(format(',{0},', inputs.jobs), ',sandbox-rlimits-connect,') || contains(format(',{0},', inputs.targets), ',sandbox-rlimits-connect,') }}
@@ -3489,68 +3418,6 @@ jobs:
         shell: bash
         run: bash .github/scripts/docker-auth-cleanup.sh
 
-  upgrade-stale-sandbox:
-    needs: generate-matrix
-    if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',upgrade-stale-sandbox,') || contains(format(',{0},', inputs.targets), ',upgrade-stale-sandbox,') }}
-    runs-on: ubuntu-latest
-    timeout-minutes: 85
-    env:
-      E2E_JOB: "1"
-      E2E_TARGET_ID: "upgrade-stale-sandbox"
-      E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/upgrade-stale-sandbox
-      NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
-      NEMOCLAW_RUN_LIVE_E2E: "1"
-      NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
-      NEMOCLAW_NON_INTERACTIVE: "1"
-      NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
-      NEMOCLAW_SANDBOX_NAME: "e2e-upgrade-stale"
-      OPENSHELL_GATEWAY: "nemoclaw"
-    steps:
-      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-        with:
-          repository: ${{ inputs.checkout_repository || github.repository }}
-          ref: ${{ inputs.checkout_sha || github.sha }}
-          persist-credentials: false
-
-      - *dockerhub-auth
-
-      - name: Prepare E2E workspace
-        uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
-
-      - name: Install OpenShell CLI
-        run: |
-          set -euo pipefail
-          env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN \
-            bash scripts/install-openshell.sh
-
-      - name: Run upgrade stale sandbox live Vitest test
-        env:
-          NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
-        run: |
-          set -euo pipefail
-          export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
-          if command -v openshell >/dev/null 2>&1; then
-            OPENSHELL_BIN="$(command -v openshell)"
-          elif [ -x "$HOME/.local/bin/openshell" ]; then
-            OPENSHELL_BIN="$HOME/.local/bin/openshell"
-          else
-            echo "::error::OpenShell CLI not found after install"
-            ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true
-            exit 1
-          fi
-          export OPENSHELL_BIN
-          "$OPENSHELL_BIN" --version
-          npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/upgrade-stale-sandbox.test.ts
-
-      - name: Upload upgrade stale sandbox artifacts
-        if: always()
-        uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
-
-      - name: Clean up Docker auth
-        if: always()
-        shell: bash
-        run: bash .github/scripts/docker-auth-cleanup.sh
-
   double-onboard:
     needs: generate-matrix
     if: ${{ (github.event_name != 'workflow_dispatch' || (inputs.jobs == '' && inputs.targets == '')) || contains(format(',{0},', inputs.jobs), ',double-onboard,') || contains(format(',{0},', inputs.targets), ',double-onboard,') }}
@@ -5054,12 +4921,6 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - mode: hosted
-            sandbox_name: e2e-openclaw-inference-switch
-            switch_provider: nvidia-prod
-            switch_model: nvidia/nemotron-3-super-120b-a12b
-            switch_inference_api: openai-completions
-            switch_mock_anthropic: "0"
           - mode: anthropic
             sandbox_name: e2e-openclaw-anthropic-inference-switch
             switch_provider: compatible-anthropic-endpoint
@@ -5072,7 +4933,6 @@ jobs:
       E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-inference-switch/${{ matrix.mode }}
       NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
       NEMOCLAW_RUN_LIVE_E2E: "1"
-      NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
       NEMOCLAW_NON_INTERACTIVE: "1"
       NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
       NEMOCLAW_AGENT: "openclaw"
@@ -5096,13 +4956,9 @@ jobs:
         uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
 
       - name: Run OpenClaw inference switch live test
-        # Preserves
-        # the real install.sh/OpenShell/Docker/live-provider boundary while
+        # Preserves the real install.sh/OpenShell/Docker/managed-proxy boundary while
         # Vitest owns the route, config, registry, inference.local, and agent
         # assertions.
-        env:
-          NVIDIA_API_KEY: ${{ matrix.mode == 'hosted' && secrets.NVIDIA_API_KEY || '' }}
-          NVIDIA_INFERENCE_API_KEY: ${{ matrix.mode == 'hosted' && secrets.NVIDIA_INFERENCE_API_KEY || '' }}
         run: |
           set -euo pipefail
           npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/openclaw-inference-switch.test.ts
@@ -5128,7 +4984,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        agent: [openclaw, hermes]
+        agent: [openclaw]
     env:
       E2E_JOB: "1"
       E2E_TARGET_ID: "bedrock-runtime-compatible-anthropic"
@@ -5143,26 +4999,6 @@ jobs:
       NEMOCLAW_SANDBOX_NAME: e2e-bedrock-${{ matrix.agent }}
       OPENSHELL_GATEWAY: "nemoclaw"
     steps:
-      - id: trusted_hermes_swap
-        name: Provision trusted Hermes E2E swap
-        if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && matrix.agent == 'hermes' }}
-        shell: /bin/bash --noprofile --norc -e -o pipefail {0}
-        env:
-          BASH_ENV: /dev/null
-          CHECKOUT_SHA: ${{ inputs.checkout_sha }}
-          DISPATCH_SHA: ${{ github.sha }}
-          ENV: /dev/null
-          EVENT_NAME: ${{ github.event_name }}
-          EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }}
-          LC_ALL: C
-          REF: ${{ github.ref }}
-          REPOSITORY: ${{ github.repository }}
-          RUNNER_ARCH_KIND: ${{ runner.arch }}
-          RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }}
-          RUNNER_OS_KIND: ${{ runner.os }}
-          WORKFLOW_SHA: ${{ github.workflow_sha }}
-        run: *trusted-hermes-e2e-swap
-
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           repository: ${{ inputs.checkout_repository || github.repository }}
@@ -5174,28 +5010,14 @@ jobs:
       - name: Prepare E2E workspace
         uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
 
-      - name: Initialize runner comparison telemetry
-        if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && matrix.agent == 'hermes' }}
-        continue-on-error: true
-        shell: bash
-        run: npx tsx tools/e2e/runner-comparison.mts initialize
-
       - name: Run Bedrock Runtime compatible Anthropic live test
-        # Direct Vitest coverage for
-        # Preserves the
-        # fake Bedrock Runtime endpoint, /etc/hosts mapping, source CLI
-        # onboard, OpenShell adapter route, agent-specific runtime probes, and
-        # leak-scan contract for both OpenClaw and Hermes.
+        # Preserves the fake Bedrock Runtime endpoint, /etc/hosts mapping,
+        # source CLI onboard, OpenShell adapter route, OpenClaw runtime probe,
+        # and leak-scan contract.
         run: |
           set -euo pipefail
           npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/bedrock-runtime-compatible-anthropic.test.ts
 
-      - name: Finalize runner comparison telemetry
-        if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && matrix.agent == 'hermes' }}
-        continue-on-error: true
-        shell: bash
-        run: npx tsx tools/e2e/runner-comparison.mts finalize
-
       - name: Upload Bedrock Runtime compatible Anthropic artifacts
         if: always()
         uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
@@ -5875,11 +5697,9 @@ jobs:
         rebuild-openclaw,
         rebuild-hermes,
         rebuild-hermes-stale-base,
-        sandbox-rebuild,
         sandbox-rlimits-connect,
         overlayfs-autofix,
         state-backup-restore,
-        upgrade-stale-sandbox,
         openshell-gateway-upgrade,
         token-rotation,
         messaging-compatible-endpoint,

src/lib/actions/upgrade-sandboxes-recovery.test.ts

@@ -7,7 +7,9 @@ import * as coreVersion from "../core/version";
 import * as sandboxList from "../openshell-sandbox-list";
 import * as sandboxVersion from "../sandbox/version";
 import * as registry from "../state/registry";
+import type { SandboxRegistry } from "../state/registry/types";
 import * as sandboxState from "../state/sandbox";
+import { commitRebuildRoutePreflight } from "./sandbox/rebuild-preflight-guards";
 import { upgradeSandboxes, upgradeSandboxesDependencies } from "./upgrade-sandboxes";
 
 type UpgradeSandboxes = typeof upgradeSandboxes;
@@ -306,6 +308,76 @@ describe("upgrade-sandboxes prepared backup recovery (#6114)", () => {
     );
   });
 
+  it("migrates missing credential identities for two sandboxes on a shared route during upgrade-sandboxes --auto (#7615, #7798)", async () => {
+    const names = ["alpha", "beta"];
+    const credentialEnv = "NVIDIA_INFERENCE_API_KEY";
+    let persisted: SandboxRegistry = {
+      defaultSandbox: names[0],
+      sandboxes: Object.fromEntries(
+        names.map((name) => [
+          name,
+          {
+            name,
+            agent: "openclaw",
+            agentVersion: "2026.5.27",
+            credentialEnv: null,
+            gatewayName: "nemoclaw",
+            gatewayPort: 8080,
+            model: "nvidia/llama-3.3-nemotron-super-49b-v1.5",
+            nemoclawVersion: "0.0.71",
+            provider: "nvidia-prod",
+          },
+        ]),
+      ),
+    };
+    const harness = createRecoveryHarness(names, {
+      liveOutput: names.map((name) => `${name} Ready`).join("\n"),
+      staleNames: names,
+    });
+    vi.mocked(registry.listSandboxes).mockImplementation(() => {
+      const snapshot = structuredClone(persisted);
+      return {
+        defaultSandbox: snapshot.defaultSandbox,
+        sandboxes: Object.values(snapshot.sandboxes),
+      };
+    });
+    const routeResults: ReturnType<typeof commitRebuildRoutePreflight>[] = [];
+    harness.rebuildSpy.mockImplementation(async (sandboxName: string) => {
+      routeResults.push(
+        commitRebuildRoutePreflight(
+          {
+            sandboxName,
+            gatewayName: "nemoclaw",
+            targetUpdate: {
+              credentialEnv,
+              model: persisted.sandboxes[sandboxName]?.model,
+              provider: persisted.sandboxes[sandboxName]?.provider,
+            },
+          },
+          {
+            withLock: <T>(fn: () => T): T => fn(),
+            load: () => structuredClone(persisted),
+            save: (next) => {
+              persisted = structuredClone(next);
+            },
+          },
+        ),
+      );
+    });
+
+    await expect(harness.upgradeSandboxes(["--auto"])).resolves.toBeUndefined();
+
+    expect(harness.rebuildSpy.mock.calls.map((call) => call[0])).toEqual(names);
+    expect(routeResults).toMatchObject([
+      { ok: true, receipt: { sandboxName: "alpha", migratedSandboxNames: ["beta"] } },
+      { ok: true, receipt: { sandboxName: "beta", migratedSandboxNames: [] } },
+    ]);
+    expect(names.map((name) => persisted.sandboxes[name]?.credentialEnv)).toEqual([
+      credentialEnv,
+      credentialEnv,
+    ]);
+  });
+
   it("fails closed for a probed v0.0.55 custom image with matching backup agent version", async () => {
     const probedAgentVersion = "2026.5.27";
     const harness = createRecoveryHarness(["custom-box"], {