From 5ca8abbefda40d4d285d9ceeafc79398d44f5419 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 27 Jun 2026 23:25:29 -0700 Subject: [PATCH 01/16] test(e2e): wire sandbox operations into Vitest Signed-off-by: Carlos Villela --- .github/workflows/e2e-vitest-scenarios.yaml | 93 +++++++++++++++++++ ...ndbox-operations-workflow-boundary.test.ts | 31 +++++++ 2 files changed, 124 insertions(+) create mode 100644 test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 5cc5eb06d1c..4052ab9b9b4 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -4147,6 +4147,98 @@ jobs: docker logout docker.io || true rm -rf "${DOCKER_CONFIG}" + sandbox-operations-vitest: + needs: generate-matrix + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',sandbox-operations-vitest,') || contains(format(',{0},', inputs.scenarios), ',sandbox-operations,') }} + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "sandbox-operations" + DOCKER_CONFIG: ${{ github.workspace }}/.docker-config-sandbox-operations + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/sandbox-operations + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_POLICY_TIER: "open" + OPENSHELL_GATEWAY: "nemoclaw" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Authenticate to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + mkdir -p "${DOCKER_CONFIG}" + chmod 700 "${DOCKER_CONFIG}" + login_succeeded=0 + for attempt in 1 2 3; do + if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then + login_succeeded=1 + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." + sleep 5 + fi + done + if [[ "$login_succeeded" -ne 1 ]]; then + echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." + fi + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Install OpenShell CLI + run: bash scripts/install-openshell.sh + + - name: Run sandbox operations live test + env: + NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/sandbox-operations.test.ts \ + --silent=false --reporter=default + + - name: Upload sandbox operations artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-sandbox-operations + path: e2e-artifacts/vitest/sandbox-operations/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + + - name: Clean up Docker auth + if: always() + run: | + set -euo pipefail + docker logout docker.io || true + rm -rf "${DOCKER_CONFIG}" + sandbox-survival-vitest: needs: generate-matrix if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',sandbox-survival-vitest,') || contains(format(',{0},', inputs.scenarios), ',sandbox-survival,') }} @@ -5760,6 +5852,7 @@ jobs: issue-4462-scope-upgrade-approval-vitest, onboard-resume-vitest, model-router-provider-routed-inference-vitest, + sandbox-operations-vitest, sandbox-survival-vitest, diagnostics-vitest, snapshot-commands-vitest, diff --git a/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts b/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts new file mode 100644 index 00000000000..76ebd1f914d --- /dev/null +++ b/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { + evaluateE2eVitestWorkflowDispatchSelectors, + readFreeStandingJobsInventory, + validateE2eVitestScenariosWorkflowBoundary, +} from "../../../tools/e2e-scenarios/workflow-boundary.mts"; + +describe("sandbox operations workflow boundary", () => { + it("runs by default and through either selective dispatch input", () => { + const inventory = readFreeStandingJobsInventory(); + expect(validateE2eVitestScenariosWorkflowBoundary()).toEqual([]); + expect(inventory.scenarioToJob.get("sandbox-operations")).toBe("sandbox-operations-vitest"); + + for (const selector of [ + { scenarios: "sandbox-operations" }, + { jobs: "sandbox-operations-vitest" }, + ]) { + expect(evaluateE2eVitestWorkflowDispatchSelectors(selector)).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["sandbox-operations-vitest"], + }); + } + expect(evaluateE2eVitestWorkflowDispatchSelectors({}).selectedFreeStandingJobs).toContain( + "sandbox-operations-vitest", + ); + }); +}); From 9aec3353909002d598c754d6bc5f1337168ea518 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 27 Jun 2026 23:30:45 -0700 Subject: [PATCH 02/16] fix(e2e): configure sandbox operations inference Signed-off-by: Carlos Villela --- .../e2e-scenario/live/sandbox-operations.test.ts | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/test/e2e-scenario/live/sandbox-operations.test.ts b/test/e2e-scenario/live/sandbox-operations.test.ts index bbcf380d7af..a46525867de 100644 --- a/test/e2e-scenario/live/sandbox-operations.test.ts +++ b/test/e2e-scenario/live/sandbox-operations.test.ts @@ -18,6 +18,7 @@ import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { type SandboxClient, trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; +import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { ubuntuRepoDocker } from "../scenarios/matrix.ts"; @@ -100,6 +101,7 @@ async function onboardSandbox( cleanup: CleanupRegistry, sandboxName: string, artifactName: string, + providerEnv: NodeJS.ProcessEnv, extraEnv: NodeJS.ProcessEnv = {}, ): Promise { cleanup.add(`destroy sandbox ${sandboxName}`, () => cleanupSandbox(host, sandboxName)); @@ -109,14 +111,16 @@ async function onboardSandbox( artifactName, env: { ...buildAvailabilityProbeEnv(), + ...providerEnv, ...extraEnv, NEMOCLAW_AGENT: "openclaw", - NEMOCLAW_PROVIDER: "cloud", NEMOCLAW_SANDBOX_NAME: sandboxName, NEMOCLAW_RECREATE_SANDBOX: "1", - NVIDIA_INFERENCE_API_KEY: process.env.NVIDIA_INFERENCE_API_KEY ?? "", }, - redactionValues: [process.env.NVIDIA_INFERENCE_API_KEY ?? ""], + redactionValues: [ + providerEnv.NVIDIA_INFERENCE_API_KEY ?? "", + providerEnv.COMPATIBLE_API_KEY ?? "", + ], timeoutMs: 20 * 60_000, }, ); @@ -601,7 +605,7 @@ async function assertGatewayRecovery(host: HostCliClient, sandboxName: string): liveTest( "sandbox operations preserve list/status/logs/recovery/multi-sandbox contracts", async ({ artifacts, cleanup, environment, host, sandbox, secrets, skip }) => { - secrets.required("NVIDIA_INFERENCE_API_KEY"); + const hosted = requireHostedInferenceConfig(secrets); await artifacts.writeJson("scenario.json", { id: "sandbox-operations", @@ -641,7 +645,7 @@ liveTest( await bestEffortCleanupSandbox(host, SANDBOX_B); await bestEffortCleanupSandbox(host, SANDBOX_A); - await onboardSandbox(host, cleanup, SANDBOX_A, "onboard-sandbox-a"); + await onboardSandbox(host, cleanup, SANDBOX_A, "onboard-sandbox-a", hosted.env); await expectListed(host, SANDBOX_A, "tc-sbx-01-list-sandbox-a"); await assertAgentCanAnswer(host, SANDBOX_A); @@ -652,7 +656,7 @@ liveTest( await assertRegistryRebuild(host, SANDBOX_A); await assertProcessRecovery(host, sandbox, SANDBOX_A); - await onboardSandbox(host, cleanup, SANDBOX_B, "tc-sbx-10-onboard-sandbox-b", { + await onboardSandbox(host, cleanup, SANDBOX_B, "tc-sbx-10-onboard-sandbox-b", hosted.env, { CHAT_UI_URL: "http://127.0.0.1:18790", }); await assertMetadataForBothSandboxes(host, SANDBOX_A, SANDBOX_B); From 7a9a099f6f21c1c51f8a331b685ae5a2a52913b7 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 27 Jun 2026 23:37:24 -0700 Subject: [PATCH 03/16] fix(ci): isolate sandbox operations credentials Signed-off-by: Carlos Villela --- .github/workflows/e2e-vitest-scenarios.yaml | 43 +++-- ...ndbox-operations-workflow-boundary.test.ts | 31 ++++ .../sandbox-operations-workflow-boundary.mts | 150 ++++++++++++++++++ 3 files changed, 208 insertions(+), 16 deletions(-) create mode 100644 tools/e2e-scenarios/sandbox-operations-workflow-boundary.mts diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 4052ab9b9b4..d59f85e88b6 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -4151,11 +4151,12 @@ jobs: needs: generate-matrix if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',sandbox-operations-vitest,') || contains(format(',{0},', inputs.scenarios), ',sandbox-operations,') }} runs-on: ubuntu-latest + # The test onboards two sandboxes, exercises process and gateway recovery, + # and tears both down; its live-test budget is 45 minutes. timeout-minutes: 60 env: FREE_STANDING_VITEST_JOB: "1" FREE_STANDING_SCENARIO_ID: "sandbox-operations" - DOCKER_CONFIG: ${{ github.workspace }}/.docker-config-sandbox-operations E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/sandbox-operations NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js NEMOCLAW_RUN_E2E_SCENARIOS: "1" @@ -4169,6 +4170,31 @@ jobs: with: persist-credentials: false + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Install OpenShell CLI + run: | + env -u DOCKER_CONFIG \ + -u DOCKERHUB_USERNAME \ + -u DOCKERHUB_TOKEN \ + -u NVIDIA_API_KEY \ + -u NVIDIA_INFERENCE_API_KEY \ + -u GITHUB_TOKEN \ + bash scripts/install-openshell.sh + + - name: Configure isolated Docker auth directory + run: echo "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config-sandbox-operations" >> "$GITHUB_ENV" + - name: Authenticate to Docker Hub env: DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} @@ -4197,21 +4223,6 @@ jobs: echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." fi - - name: Set up Node - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 - with: - node-version: 22 - cache: npm - - - name: Install root dependencies - run: npm ci --ignore-scripts - - - name: Build CLI - run: npm run build:cli - - - name: Install OpenShell CLI - run: bash scripts/install-openshell.sh - - name: Run sandbox operations live test env: NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }} diff --git a/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts b/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts index 76ebd1f914d..1815fa42a02 100644 --- a/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts +++ b/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts @@ -2,6 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it } from "vitest"; +import { + readSandboxOperationsWorkflow, + validateSandboxOperationsWorkflow, + validateSandboxOperationsWorkflowBoundary, +} from "../../../tools/e2e-scenarios/sandbox-operations-workflow-boundary.mts"; import { evaluateE2eVitestWorkflowDispatchSelectors, readFreeStandingJobsInventory, @@ -12,6 +17,7 @@ describe("sandbox operations workflow boundary", () => { it("runs by default and through either selective dispatch input", () => { const inventory = readFreeStandingJobsInventory(); expect(validateE2eVitestScenariosWorkflowBoundary()).toEqual([]); + expect(validateSandboxOperationsWorkflowBoundary()).toEqual([]); expect(inventory.scenarioToJob.get("sandbox-operations")).toBe("sandbox-operations-vitest"); for (const selector of [ @@ -28,4 +34,29 @@ describe("sandbox operations workflow boundary", () => { "sandbox-operations-vitest", ); }); + + it("rejects workspace-scoped auth, unsanitized installs, and broad inference secrets", () => { + const workspaceAuth = readSandboxOperationsWorkflow(); + workspaceAuth.jobs["sandbox-operations-vitest"].env!.DOCKER_CONFIG = + "${{ github.workspace }}/docker"; + expect(validateSandboxOperationsWorkflow(workspaceAuth)).toContain( + "sandbox-operations-vitest must not configure Docker auth at job scope", + ); + + const unsanitizedInstall = readSandboxOperationsWorkflow(); + unsanitizedInstall.jobs["sandbox-operations-vitest"].steps!.find( + (step) => step.name === "Install OpenShell CLI", + )!.run = "bash scripts/install-openshell.sh"; + expect(validateSandboxOperationsWorkflow(unsanitizedInstall)).toContain( + "sandbox-operations-vitest step 'Install OpenShell CLI' must run: -u DOCKER_CONFIG", + ); + + const broadInferenceSecret = readSandboxOperationsWorkflow(); + broadInferenceSecret.jobs["sandbox-operations-vitest"].steps!.find( + (step) => step.name === "Build CLI", + )!.env = { NVIDIA_INFERENCE_API_KEY: "${{ secrets.NVIDIA_INFERENCE_API_KEY }}" }; + expect(validateSandboxOperationsWorkflow(broadInferenceSecret)).toContain( + "sandbox-operations-vitest exposes the inference key outside the live test step", + ); + }); }); diff --git a/tools/e2e-scenarios/sandbox-operations-workflow-boundary.mts b/tools/e2e-scenarios/sandbox-operations-workflow-boundary.mts new file mode 100644 index 00000000000..1a6b6385030 --- /dev/null +++ b/tools/e2e-scenarios/sandbox-operations-workflow-boundary.mts @@ -0,0 +1,150 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import YAML from "yaml"; + +const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e-vitest-scenarios.yaml"); +const JOB_NAME = "sandbox-operations-vitest"; +const FULL_SHA_ACTION = /^[^\s@]+@[0-9a-f]{40}$/u; + +type WorkflowStep = { + env?: Record; + if?: string; + name?: string; + run?: string; + uses?: string; + with?: Record; +}; + +type WorkflowJob = { + env?: Record; + steps?: WorkflowStep[]; +}; + +export type SandboxOperationsWorkflow = { + jobs: Record; +}; + +export function readSandboxOperationsWorkflow( + workflowPath = DEFAULT_WORKFLOW_PATH, +): SandboxOperationsWorkflow { + return YAML.parse(readFileSync(workflowPath, "utf8")) as SandboxOperationsWorkflow; +} + +function findStep(job: WorkflowJob, name: string): WorkflowStep { + return job.steps?.find((step) => step.name === name) ?? {}; +} + +function requireRunContains(errors: string[], step: WorkflowStep, fragment: string): void { + if (!step.run?.includes(fragment)) { + errors.push(`${JOB_NAME} step '${step.name ?? ""}' must run: ${fragment}`); + } +} + +function requireStepOrder( + errors: string[], + steps: WorkflowStep[], + beforeName: string, + afterName: string, +): void { + const before = steps.findIndex((step) => step.name === beforeName); + const after = steps.findIndex((step) => step.name === afterName); + if (before < 0 || after < 0 || before >= after) { + errors.push(`${JOB_NAME} step '${beforeName}' must precede '${afterName}'`); + } +} + +export function validateSandboxOperationsWorkflow(workflow: SandboxOperationsWorkflow): string[] { + const errors: string[] = []; + const job = workflow.jobs[JOB_NAME] ?? {}; + const jobEnv = job.env ?? {}; + const steps = job.steps ?? []; + + if (Object.hasOwn(jobEnv, "DOCKER_CONFIG")) { + errors.push(`${JOB_NAME} must not configure Docker auth at job scope`); + } + + const checkout = steps.find((step) => step.uses?.startsWith("actions/checkout@")) ?? {}; + if (!FULL_SHA_ACTION.test(checkout.uses ?? "")) { + errors.push(`${JOB_NAME} checkout must pin a full action SHA`); + } + if (checkout.with?.["persist-credentials"] !== false) { + errors.push(`${JOB_NAME} checkout must disable persisted credentials`); + } + for (const step of steps.filter((entry) => entry.uses)) { + if (!FULL_SHA_ACTION.test(step.uses ?? "")) { + errors.push(`${JOB_NAME} action '${step.name ?? step.uses}' must pin a full SHA`); + } + } + + const install = findStep(job, "Install OpenShell CLI"); + for (const variable of [ + "DOCKER_CONFIG", + "DOCKERHUB_USERNAME", + "DOCKERHUB_TOKEN", + "NVIDIA_API_KEY", + "NVIDIA_INFERENCE_API_KEY", + "GITHUB_TOKEN", + ]) { + requireRunContains(errors, install, `-u ${variable}`); + } + requireRunContains(errors, install, "bash scripts/install-openshell.sh"); + + const configure = findStep(job, "Configure isolated Docker auth directory"); + requireRunContains( + errors, + configure, + "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config-sandbox-operations", + ); + requireRunContains(errors, configure, '>> "$GITHUB_ENV"'); + + const authenticate = findStep(job, "Authenticate to Docker Hub"); + if (authenticate.env?.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") { + errors.push(`${JOB_NAME} Docker username must be scoped to the auth step`); + } + if (authenticate.env?.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") { + errors.push(`${JOB_NAME} Docker token must be scoped to the auth step`); + } + + requireStepOrder(errors, steps, "Install OpenShell CLI", configure.name ?? ""); + requireStepOrder(errors, steps, configure.name ?? "", authenticate.name ?? ""); + requireStepOrder(errors, steps, authenticate.name ?? "", "Run sandbox operations live test"); + + const run = findStep(job, "Run sandbox operations live test"); + if (run.env?.NVIDIA_INFERENCE_API_KEY !== "${{ secrets.NVIDIA_INFERENCE_API_KEY }}") { + errors.push(`${JOB_NAME} inference key must be scoped to the live test step`); + } + for (const step of steps.filter((entry) => entry !== run)) { + if (step.env?.NVIDIA_INFERENCE_API_KEY !== undefined) { + errors.push(`${JOB_NAME} exposes the inference key outside the live test step`); + } + } + requireRunContains(errors, run, "npx vitest run --project e2e-scenarios-live"); + requireRunContains(errors, run, "test/e2e-scenario/live/sandbox-operations.test.ts"); + + const upload = findStep(job, "Upload sandbox operations artifacts"); + if (upload.if !== "always()") errors.push(`${JOB_NAME} artifact upload must always run`); + if (upload.with?.path !== "e2e-artifacts/vitest/sandbox-operations/") { + errors.push(`${JOB_NAME} must upload sandbox operations artifacts`); + } + if (upload.with?.["include-hidden-files"] !== false) { + errors.push(`${JOB_NAME} artifact upload must exclude hidden files`); + } + + const cleanup = findStep(job, "Clean up Docker auth"); + if (cleanup.if !== "always()") errors.push(`${JOB_NAME} Docker auth cleanup must always run`); + requireRunContains(errors, cleanup, "docker logout docker.io"); + requireRunContains(errors, cleanup, 'rm -rf "${DOCKER_CONFIG}"'); + + return errors; +} + +export function validateSandboxOperationsWorkflowBoundary( + workflowPath = DEFAULT_WORKFLOW_PATH, +): string[] { + return validateSandboxOperationsWorkflow(readSandboxOperationsWorkflow(workflowPath)); +} From ac07ef9a36494833e2b7ff5195dbcfcd9cff842f Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 27 Jun 2026 23:43:38 -0700 Subject: [PATCH 04/16] test(e2e): compose sandbox operations boundaries Signed-off-by: Carlos Villela --- .../sandbox-operations-workflow-boundary.test.ts | 2 -- .../e2e-scenarios/sandbox-operations-workflow-boundary.mts | 7 ++++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts b/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts index 1815fa42a02..65072c4134f 100644 --- a/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts +++ b/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts @@ -10,13 +10,11 @@ import { import { evaluateE2eVitestWorkflowDispatchSelectors, readFreeStandingJobsInventory, - validateE2eVitestScenariosWorkflowBoundary, } from "../../../tools/e2e-scenarios/workflow-boundary.mts"; describe("sandbox operations workflow boundary", () => { it("runs by default and through either selective dispatch input", () => { const inventory = readFreeStandingJobsInventory(); - expect(validateE2eVitestScenariosWorkflowBoundary()).toEqual([]); expect(validateSandboxOperationsWorkflowBoundary()).toEqual([]); expect(inventory.scenarioToJob.get("sandbox-operations")).toBe("sandbox-operations-vitest"); diff --git a/tools/e2e-scenarios/sandbox-operations-workflow-boundary.mts b/tools/e2e-scenarios/sandbox-operations-workflow-boundary.mts index 1a6b6385030..e28fd4cd1ae 100644 --- a/tools/e2e-scenarios/sandbox-operations-workflow-boundary.mts +++ b/tools/e2e-scenarios/sandbox-operations-workflow-boundary.mts @@ -6,6 +6,8 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import YAML from "yaml"; +import { validateE2eVitestScenariosWorkflowBoundary } from "./workflow-boundary.mts"; + const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e-vitest-scenarios.yaml"); const JOB_NAME = "sandbox-operations-vitest"; @@ -146,5 +148,8 @@ export function validateSandboxOperationsWorkflow(workflow: SandboxOperationsWor export function validateSandboxOperationsWorkflowBoundary( workflowPath = DEFAULT_WORKFLOW_PATH, ): string[] { - return validateSandboxOperationsWorkflow(readSandboxOperationsWorkflow(workflowPath)); + return [ + ...validateE2eVitestScenariosWorkflowBoundary(workflowPath), + ...validateSandboxOperationsWorkflow(readSandboxOperationsWorkflow(workflowPath)), + ]; } From 238224a5dd0d23fe39d41daa4da5378109c5eedc Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 27 Jun 2026 23:53:08 -0700 Subject: [PATCH 05/16] fix(e2e): align agent transport assertion Signed-off-by: Carlos Villela --- .../live/sandbox-operations.test.ts | 25 +------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/test/e2e-scenario/live/sandbox-operations.test.ts b/test/e2e-scenario/live/sandbox-operations.test.ts index a46525867de..639b4c106df 100644 --- a/test/e2e-scenario/live/sandbox-operations.test.ts +++ b/test/e2e-scenario/live/sandbox-operations.test.ts @@ -36,10 +36,6 @@ function resultText(result: ProcessResult): string { return [result.stdout, result.stderr].filter(Boolean).join("\n"); } -function shellQuote(value: string): string { - return `'${value.replaceAll("'", "'\\''")}'`; -} - function outputContainsSandbox(result: ProcessResult, sandboxName: string): boolean { const escaped = sandboxName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); return new RegExp(`(^|\\s)${escaped}(\\s|$)`, "m").test(resultText(result)); @@ -299,25 +295,6 @@ async function assertAgentJsonTransportBoundaries( expect(invalidFlag.exitCode, resultText(invalidFlag)).not.toBeNull(); expect(invalidFlag.exitCode, resultText(invalidFlag)).not.toBe(0); - const stdinPrompt = "What is 6 multiplied by 7? Reply with only the integer, no extra words."; - const stdinSessionId = `e2e-sbx-02b-stdin-${Date.now()}-${process.pid}`; - const stdinScript = [ - "set -euo pipefail", - `printf '%s\\n' ${shellQuote(stdinPrompt)} | ${shellQuote(host.commandPath)} ${shellQuote( - sandboxName, - )} agent --agent main --json --session-id ${shellQuote(stdinSessionId)}`, - ].join("\n"); - const stdinResult = await host.command("bash", ["-lc", stdinScript], { - artifactName: "tc-sbx-02b-agent-json-stdin", - env: buildAvailabilityProbeEnv(), - timeoutMs: 120_000, - }); - expectExitZero(stdinResult, `printf prompt | nemoclaw ${sandboxName} agent --json`); - expectJsonStdout(stdinResult, "stdin agent --json"); - expect(parseOpenClawAgentText(stdinResult.stdout), resultText(stdinResult)).toMatch( - /(^|[^0-9])42([^0-9]|$)/, - ); - const provenanceMarker = `NEMOCLAW_PROVENANCE_E2E_${Date.now()}_${process.pid}`; const failureSessionId = `e2e-sbx-02b-failure-${Date.now()}-${process.pid}`; const failure = await host.nemoclaw( @@ -615,7 +592,7 @@ liveTest( contracts: [ "TC-SBX-01 list shows onboarded sandbox", "TC-SBX-02 nemoclaw agent --json answers through sandbox inference.local", - "TC-SBX-02b agent --json preserves stdin, nonzero status, and failed-tool provenance boundaries", + "TC-SBX-02b agent --json preserves nonzero status and failed-tool provenance boundaries", "TC-SBX-03 status renders Sandbox/Model/Provider/GPU fields", "TC-SBX-04 logs and logs --follow behave as streaming commands", "TC-SBX-05 destroy removes NemoClaw and OpenShell entries", From 19a920b0247a1d1bda1d077ba82de93abe725f84 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 27 Jun 2026 23:59:36 -0700 Subject: [PATCH 06/16] test(e2e): centralize sandbox operations guardrail Signed-off-by: Carlos Villela --- ...ndbox-operations-workflow-boundary.test.ts | 36 +++++++++++++++++-- .../sandbox-operations-workflow-boundary.mts | 17 +++------ tools/e2e-scenarios/workflow-boundary.mts | 3 ++ 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts b/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts index 65072c4134f..6db7c5f2f48 100644 --- a/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts +++ b/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts @@ -1,21 +1,38 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + import { describe, expect, it } from "vitest"; import { readSandboxOperationsWorkflow, validateSandboxOperationsWorkflow, - validateSandboxOperationsWorkflowBoundary, } from "../../../tools/e2e-scenarios/sandbox-operations-workflow-boundary.mts"; import { evaluateE2eVitestWorkflowDispatchSelectors, readFreeStandingJobsInventory, + validateE2eVitestScenariosWorkflowBoundary, } from "../../../tools/e2e-scenarios/workflow-boundary.mts"; +const WORKFLOW_PATH = join(process.cwd(), ".github", "workflows", "e2e-vitest-scenarios.yaml"); + +function validateCentralWorkflowMutation(mutate: (source: string) => string): string[] { + const directory = mkdtempSync(join(tmpdir(), "nemoclaw-sandbox-operations-boundary-")); + const workflowPath = join(directory, "workflow.yaml"); + try { + writeFileSync(workflowPath, mutate(readFileSync(WORKFLOW_PATH, "utf8"))); + return validateE2eVitestScenariosWorkflowBoundary(workflowPath); + } finally { + rmSync(directory, { force: true, recursive: true }); + } +} + describe("sandbox operations workflow boundary", () => { it("runs by default and through either selective dispatch input", () => { const inventory = readFreeStandingJobsInventory(); - expect(validateSandboxOperationsWorkflowBoundary()).toEqual([]); + expect(validateE2eVitestScenariosWorkflowBoundary()).toEqual([]); expect(inventory.scenarioToJob.get("sandbox-operations")).toBe("sandbox-operations-vitest"); for (const selector of [ @@ -34,6 +51,21 @@ describe("sandbox operations workflow boundary", () => { }); it("rejects workspace-scoped auth, unsanitized installs, and broad inference secrets", () => { + const jobMarker = [ + ' FREE_STANDING_VITEST_JOB: "1"', + ' FREE_STANDING_SCENARIO_ID: "sandbox-operations"', + "", + ].join("\n"); + expect( + validateCentralWorkflowMutation((source) => { + expect(source).toContain(jobMarker); + return source.replace( + jobMarker, + `${jobMarker} DOCKER_CONFIG: \${{ github.workspace }}/docker\n`, + ); + }), + ).toContain("sandbox-operations-vitest must not configure Docker auth at job scope"); + const workspaceAuth = readSandboxOperationsWorkflow(); workspaceAuth.jobs["sandbox-operations-vitest"].env!.DOCKER_CONFIG = "${{ github.workspace }}/docker"; diff --git a/tools/e2e-scenarios/sandbox-operations-workflow-boundary.mts b/tools/e2e-scenarios/sandbox-operations-workflow-boundary.mts index e28fd4cd1ae..efac5a31da1 100644 --- a/tools/e2e-scenarios/sandbox-operations-workflow-boundary.mts +++ b/tools/e2e-scenarios/sandbox-operations-workflow-boundary.mts @@ -6,8 +6,6 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import YAML from "yaml"; -import { validateE2eVitestScenariosWorkflowBoundary } from "./workflow-boundary.mts"; - const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e-vitest-scenarios.yaml"); const JOB_NAME = "sandbox-operations-vitest"; @@ -60,9 +58,11 @@ function requireStepOrder( } } -export function validateSandboxOperationsWorkflow(workflow: SandboxOperationsWorkflow): string[] { +export function validateSandboxOperationsWorkflow(workflow: { + jobs: Record; +}): string[] { const errors: string[] = []; - const job = workflow.jobs[JOB_NAME] ?? {}; + const job = (workflow.jobs[JOB_NAME] ?? {}) as WorkflowJob; const jobEnv = job.env ?? {}; const steps = job.steps ?? []; @@ -144,12 +144,3 @@ export function validateSandboxOperationsWorkflow(workflow: SandboxOperationsWor return errors; } - -export function validateSandboxOperationsWorkflowBoundary( - workflowPath = DEFAULT_WORKFLOW_PATH, -): string[] { - return [ - ...validateE2eVitestScenariosWorkflowBoundary(workflowPath), - ...validateSandboxOperationsWorkflow(readSandboxOperationsWorkflow(workflowPath)), - ]; -} diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 999c5e398c7..150b93865e7 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -6,6 +6,8 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import YAML from "yaml"; +import { validateSandboxOperationsWorkflow } from "./sandbox-operations-workflow-boundary.mts"; + const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const DEFAULT_VITEST_WORKFLOW_PATH = join( REPO_ROOT, @@ -7859,6 +7861,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( validateDiagnosticsVitestJob(errors, jobs); validateModelRouterProviderRoutedInferenceVitestJob(errors, jobs); validateSnapshotCommandsVitestJob(errors, jobs); + errors.push(...validateSandboxOperationsWorkflow({ jobs })); validateSparkInstallVitestJob(errors, jobs); validateGatewayDriftPreflightVitestJob(errors, jobs); From 0e41783d32ea6be1f441c2497fd5f3d1fe7a14ec Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 28 Jun 2026 00:19:19 -0700 Subject: [PATCH 07/16] test(e2e): remove unreliable provenance prompt Signed-off-by: Carlos Villela --- .../live/sandbox-operations.test.ts | 48 ++----------------- 1 file changed, 5 insertions(+), 43 deletions(-) diff --git a/test/e2e-scenario/live/sandbox-operations.test.ts b/test/e2e-scenario/live/sandbox-operations.test.ts index 639b4c106df..df2e13fff95 100644 --- a/test/e2e-scenario/live/sandbox-operations.test.ts +++ b/test/e2e-scenario/live/sandbox-operations.test.ts @@ -45,17 +45,6 @@ function expectExitZero(result: ProcessResult, label: string): void { expect(result.exitCode, `${label}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`).toBe(0); } -function expectJsonStdout(result: ProcessResult, label: string): void { - expect( - result.stdout.trim(), - `${label} produced empty stdout\nstderr:\n${result.stderr}`, - ).not.toBe(""); - expect( - () => JSON.parse(result.stdout), - `${label} stdout is not JSON:\n${result.stdout}`, - ).not.toThrow(); -} - async function cleanupSandbox(host: HostCliClient, sandboxName: string): Promise { const result = await host.nemoclaw([sandboxName, "destroy", "--yes"], { artifactName: `cleanup-destroy-${sandboxName}`, @@ -279,10 +268,7 @@ async function assertAgentCanAnswer(host: HostCliClient, sandboxName: string): P expect(containsInteger42Answer(reply), resultText(result)).toBe(true); } -async function assertAgentJsonTransportBoundaries( - host: HostCliClient, - sandboxName: string, -): Promise { +async function assertAgentJsonNonzeroExit(host: HostCliClient, sandboxName: string): Promise { const invalidFlag = await host.nemoclaw( [sandboxName, "agent", "--json", "--nemoclaw-e2e-invalid-openclaw-agent-flag"], { @@ -295,32 +281,8 @@ async function assertAgentJsonTransportBoundaries( expect(invalidFlag.exitCode, resultText(invalidFlag)).not.toBeNull(); expect(invalidFlag.exitCode, resultText(invalidFlag)).not.toBe(0); - const provenanceMarker = `NEMOCLAW_PROVENANCE_E2E_${Date.now()}_${process.pid}`; - const failureSessionId = `e2e-sbx-02b-failure-${Date.now()}-${process.pid}`; - const failure = await host.nemoclaw( - [ - sandboxName, - "agent", - "--agent", - "main", - "--json", - "--thinking", - "off", - "--session-id", - failureSessionId, - "-m", - `Use the exec tool to run exactly: /bin/sh -lc 'echo ${provenanceMarker}; exit 7'. Do not retry the command. Then reply with exactly: observed expected failure.`, - ], - { - artifactName: "tc-sbx-02b-agent-json-failed-tool-provenance", - env: buildAvailabilityProbeEnv(), - timeoutMs: 180_000, - }, - ); - expectExitZero(failure, `nemoclaw ${sandboxName} agent --json failed-tool provenance`); - expectJsonStdout(failure, "failed-tool agent --json"); - expect(failure.stderr, resultText(failure)).toContain("[openclaw provenance] failed tool result"); - expect(failure.stderr, resultText(failure)).toContain(provenanceMarker); + // Failed-tool provenance is covered by deterministic source/package tests. + // A live prompt cannot require upstream OpenClaw to emit failed tool-result metadata. } async function assertStatusFields(host: HostCliClient, sandboxName: string): Promise { @@ -592,7 +554,7 @@ liveTest( contracts: [ "TC-SBX-01 list shows onboarded sandbox", "TC-SBX-02 nemoclaw agent --json answers through sandbox inference.local", - "TC-SBX-02b agent --json preserves nonzero status and failed-tool provenance boundaries", + "TC-SBX-02b agent --json preserves nonzero transport status", "TC-SBX-03 status renders Sandbox/Model/Provider/GPU fields", "TC-SBX-04 logs and logs --follow behave as streaming commands", "TC-SBX-05 destroy removes NemoClaw and OpenShell entries", @@ -626,7 +588,7 @@ liveTest( await expectListed(host, SANDBOX_A, "tc-sbx-01-list-sandbox-a"); await assertAgentCanAnswer(host, SANDBOX_A); - await assertAgentJsonTransportBoundaries(host, SANDBOX_A); + await assertAgentJsonNonzeroExit(host, SANDBOX_A); await assertStatusFields(host, SANDBOX_A); await assertLogsStream(host, SANDBOX_A); await assertTmuxPtyFlow(sandbox, SANDBOX_A); From f3fc8f1e3e32644b0af95c95427cffd58037706d Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 28 Jun 2026 13:40:37 -0700 Subject: [PATCH 08/16] test(e2e): harden sandbox operations evidence --- .github/workflows/e2e-vitest-scenarios.yaml | 5 +- .../live/sandbox-operations.test.ts | 87 +++++++++-------- ...ndbox-operations-workflow-boundary.test.ts | 97 +++++++++++++++++++ .../sandbox-operations-workflow-boundary.mts | 38 ++++++++ 4 files changed, 183 insertions(+), 44 deletions(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 58d2ad72919..698e43a5f27 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -4154,8 +4154,9 @@ jobs: needs: generate-matrix if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',sandbox-operations-vitest,') || contains(format(',{0},', inputs.scenarios), ',sandbox-operations,') }} runs-on: ubuntu-latest - # The test onboards two sandboxes, exercises process and gateway recovery, - # and tears both down; its live-test budget is 45 minutes. + # The live test receives 45 minutes for two onboards plus process/gateway + # recovery. The remaining 15 minutes cover checkout, build, OpenShell setup, + # artifact upload, and unconditional credential/resource cleanup. timeout-minutes: 60 env: FREE_STANDING_VITEST_JOB: "1" diff --git a/test/e2e-scenario/live/sandbox-operations.test.ts b/test/e2e-scenario/live/sandbox-operations.test.ts index df2e13fff95..4a7ae318b02 100644 --- a/test/e2e-scenario/live/sandbox-operations.test.ts +++ b/test/e2e-scenario/live/sandbox-operations.test.ts @@ -18,7 +18,10 @@ import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; import type { HostCliClient } from "../fixtures/clients/host.ts"; import { type SandboxClient, trustedSandboxShellScript } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; -import { requireHostedInferenceConfig } from "../fixtures/hosted-inference.ts"; +import { + type HostedInferenceConfig, + requireHostedInferenceConfig, +} from "../fixtures/hosted-inference.ts"; import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; import { ubuntuRepoDocker } from "../scenarios/matrix.ts"; @@ -57,28 +60,21 @@ async function cleanupSandbox(host: HostCliClient, sandboxName: string): Promise expectExitZero(result, `cleanup destroy sandbox ${sandboxName}`); } -async function bestEffortCleanupSandbox(host: HostCliClient, sandboxName: string): Promise { - try { - await cleanupSandbox(host, sandboxName); - } catch { - // Best-effort pre-cleanup mirrors the legacy script's stale sandbox removal. - } -} - async function destroyGateway(host: HostCliClient, artifactName = "cleanup-gateway-destroy") { - await host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { + const result = await host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { artifactName, env: buildAvailabilityProbeEnv(), timeoutMs: 5 * 60_000, }); -} - -async function bestEffortDestroyGateway(host: HostCliClient): Promise { - try { - await destroyGateway(host); - } catch { - // Mirrors legacy teardown: gateway cleanup must not mask the test failure. + if (result.exitCode === 0) return; + if ( + /gateway[^\n]*(?:does not exist|not found)|No (?:active )?gateway|No gateway metadata found/i.test( + resultText(result), + ) + ) { + return; } + expectExitZero(result, "cleanup destroy shared NemoClaw gateway"); } async function onboardSandbox( @@ -86,7 +82,7 @@ async function onboardSandbox( cleanup: CleanupRegistry, sandboxName: string, artifactName: string, - providerEnv: NodeJS.ProcessEnv, + hosted: HostedInferenceConfig, extraEnv: NodeJS.ProcessEnv = {}, ): Promise { cleanup.add(`destroy sandbox ${sandboxName}`, () => cleanupSandbox(host, sandboxName)); @@ -96,16 +92,15 @@ async function onboardSandbox( artifactName, env: { ...buildAvailabilityProbeEnv(), - ...providerEnv, + // The shared hosted configuration intentionally wins over availability + // defaults; extraEnv remains the per-sandbox override boundary. + ...hosted.env, ...extraEnv, NEMOCLAW_AGENT: "openclaw", NEMOCLAW_SANDBOX_NAME: sandboxName, NEMOCLAW_RECREATE_SANDBOX: "1", }, - redactionValues: [ - providerEnv.NVIDIA_INFERENCE_API_KEY ?? "", - providerEnv.COMPATIBLE_API_KEY ?? "", - ], + redactionValues: [hosted.apiKey], timeoutMs: 20 * 60_000, }, ); @@ -281,8 +276,11 @@ async function assertAgentJsonNonzeroExit(host: HostCliClient, sandboxName: stri expect(invalidFlag.exitCode, resultText(invalidFlag)).not.toBeNull(); expect(invalidFlag.exitCode, resultText(invalidFlag)).not.toBe(0); - // Failed-tool provenance is covered by deterministic source/package tests. - // A live prompt cannot require upstream OpenClaw to emit failed tool-result metadata. + // The v0.0.69 legacy job did not exercise piped stdin. That experimental + // migration-only assertion was retired instead of expanding the parity lane. + // Failed-tool provenance remains covered deterministically by + // test/openclaw-agent-json.test.ts; a live prompt cannot require upstream + // OpenClaw to emit failed tool-result metadata. } async function assertStatusFields(host: HostCliClient, sandboxName: string): Promise { @@ -479,7 +477,16 @@ async function assertDestroyRemovesSandbox( expect(outputContainsSandbox(openshellList, sandboxName), resultText(openshellList)).toBe(false); } -async function assertGatewayRecovery(host: HostCliClient, sandboxName: string): Promise { +type GatewayRecoveryOutcome = + | "recovered" + | "skipped-gateway-absent" + | "skipped-docker-restarted-before-probe" + | "skipped-docker-did-not-restart"; + +async function assertGatewayRecovery( + host: HostCliClient, + sandboxName: string, +): Promise { const running = await host.command( "docker", ["ps", "-q", "--filter", `name=${GATEWAY_CONTAINER}`], @@ -490,9 +497,7 @@ async function assertGatewayRecovery(host: HostCliClient, sandboxName: string): }, ); if (!running.stdout.trim()) { - // Preserve the legacy script's soft-skip when the shared gateway is already - // absent before the destructive recovery probe can exercise it. - return; + return "skipped-gateway-absent"; } await host.command("docker", ["kill", GATEWAY_CONTAINER], { @@ -512,9 +517,7 @@ async function assertGatewayRecovery(host: HostCliClient, sandboxName: string): }, ); if (afterKill.stdout.trim() === "true") { - // Preserve the legacy script's soft-skip when Docker restarts the gateway - // before the recovery path can observe a stopped container. - return; + return "skipped-docker-restarted-before-probe"; } const status = await host.nemoclaw([sandboxName, "status"], { @@ -522,7 +525,7 @@ async function assertGatewayRecovery(host: HostCliClient, sandboxName: string): env: buildAvailabilityProbeEnv(), timeoutMs: 10 * 60_000, }); - if (status.exitCode === 0) return; + if (status.exitCode === 0) return "recovered"; const afterStatus = await host.command( "docker", @@ -534,11 +537,10 @@ async function assertGatewayRecovery(host: HostCliClient, sandboxName: string): }, ); if (afterStatus.stdout.trim() !== "true") { - // Same legacy soft-skip: Docker did not restart the gateway container on - // this runner, so there is no recovery signal to assert. - return; + return "skipped-docker-did-not-restart"; } expectExitZero(status, `nemoclaw ${sandboxName} status after gateway kill`); + return "recovered"; } liveTest( @@ -580,11 +582,11 @@ liveTest( } await environment.assertReady(ENVIRONMENT); - cleanup.add("destroy shared NemoClaw gateway", () => bestEffortDestroyGateway(host)); - await bestEffortCleanupSandbox(host, SANDBOX_B); - await bestEffortCleanupSandbox(host, SANDBOX_A); + cleanup.add("destroy shared NemoClaw gateway", () => destroyGateway(host)); + await cleanupSandbox(host, SANDBOX_B); + await cleanupSandbox(host, SANDBOX_A); - await onboardSandbox(host, cleanup, SANDBOX_A, "onboard-sandbox-a", hosted.env); + await onboardSandbox(host, cleanup, SANDBOX_A, "onboard-sandbox-a", hosted); await expectListed(host, SANDBOX_A, "tc-sbx-01-list-sandbox-a"); await assertAgentCanAnswer(host, SANDBOX_A); @@ -595,7 +597,7 @@ liveTest( await assertRegistryRebuild(host, SANDBOX_A); await assertProcessRecovery(host, sandbox, SANDBOX_A); - await onboardSandbox(host, cleanup, SANDBOX_B, "tc-sbx-10-onboard-sandbox-b", hosted.env, { + await onboardSandbox(host, cleanup, SANDBOX_B, "tc-sbx-10-onboard-sandbox-b", hosted, { CHAT_UI_URL: "http://127.0.0.1:18790", }); await assertMetadataForBothSandboxes(host, SANDBOX_A, SANDBOX_B); @@ -603,11 +605,12 @@ liveTest( await assertNetworkIsolation(sandbox, SANDBOX_B, SANDBOX_A, "tc-sbx-11-b-cannot-reach-a"); await assertDestroyRemovesSandbox(host, sandbox, SANDBOX_B); - await assertGatewayRecovery(host, SANDBOX_A); + const gatewayRecovery = await assertGatewayRecovery(host, SANDBOX_A); await artifacts.writeJson("scenario-result.json", { id: "sandbox-operations", status: "passed", + gatewayRecovery, legacySource: "test/e2e/test-sandbox-operations.sh", }); }, diff --git a/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts b/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts index 6db7c5f2f48..760dfad16a4 100644 --- a/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts +++ b/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts @@ -29,6 +29,19 @@ function validateCentralWorkflowMutation(mutate: (source: string) => string): st } } +function mutateSandboxOperationsJob(source: string, mutate: (jobSource: string) => string): string { + const startMarker = " sandbox-operations-vitest:\n"; + const endMarker = " sandbox-survival-vitest:\n"; + const start = source.indexOf(startMarker); + const end = source.indexOf(endMarker, start + startMarker.length); + expect(start).toBeGreaterThanOrEqual(0); + expect(end).toBeGreaterThan(start); + const jobSource = source.slice(start, end); + const mutated = mutate(jobSource); + expect(mutated).not.toBe(jobSource); + return `${source.slice(0, start)}${mutated}${source.slice(end)}`; +} + describe("sandbox operations workflow boundary", () => { it("runs by default and through either selective dispatch input", () => { const inventory = readFreeStandingJobsInventory(); @@ -89,4 +102,88 @@ describe("sandbox operations workflow boundary", () => { "sandbox-operations-vitest exposes the inference key outside the live test step", ); }); + + it.each([ + { + label: "Docker credentials at job scope", + mutate: (source: string) => + mutateSandboxOperationsJob(source, (jobSource) => + jobSource.replace( + ' FREE_STANDING_SCENARIO_ID: "sandbox-operations"', + [ + ' FREE_STANDING_SCENARIO_ID: "sandbox-operations"', + " DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}", + ].join("\n"), + ), + ), + expected: "sandbox-operations-vitest must not expose DOCKERHUB_TOKEN at job scope", + }, + { + label: "Docker credentials on another step", + mutate: (source: string) => + mutateSandboxOperationsJob(source, (jobSource) => + jobSource.replace( + " - name: Build CLI\n run: npm run build:cli", + [ + " - name: Build CLI", + " env:", + " DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}", + " run: npm run build:cli", + ].join("\n"), + ), + ), + expected: + "sandbox-operations-vitest exposes DOCKERHUB_USERNAME outside the Docker authentication step", + }, + { + label: "step-scoped Docker config", + mutate: (source: string) => + mutateSandboxOperationsJob(source, (jobSource) => + jobSource.replace( + " - name: Build CLI\n run: npm run build:cli", + [ + " - name: Build CLI", + " env:", + ' DOCKER_CONFIG: "${{ runner.temp }}/docker"', + " run: npm run build:cli", + ].join("\n"), + ), + ), + expected: "sandbox-operations-vitest must not expose DOCKER_CONFIG through step 'Build CLI'", + }, + { + label: "persistent environment write outside the configure step", + mutate: (source: string) => + mutateSandboxOperationsJob(source, (jobSource) => + jobSource.replace( + " - name: Build CLI\n run: npm run build:cli", + [ + " - name: Build CLI", + " run: |", + " npm run build:cli", + ' echo "DOCKER_CONFIG=${{ github.workspace }}/docker" >> "$GITHUB_ENV"', + ].join("\n"), + ), + ), + expected: "sandbox-operations-vitest step 'Build CLI' must not write persistent environment", + }, + { + label: "workspace override in the configure step", + mutate: (source: string) => + mutateSandboxOperationsJob(source, (jobSource) => + jobSource.replace( + ' run: echo "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config-sandbox-operations" >> "$GITHUB_ENV"', + [ + " run: |", + ' echo "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config-sandbox-operations" >> "$GITHUB_ENV"', + ' echo "DOCKER_CONFIG=${{ github.workspace }}/docker" >> "$GITHUB_ENV"', + ].join("\n"), + ), + ), + expected: + "sandbox-operations-vitest Docker auth directory must not use the checkout workspace", + }, + ])("rejects $label", ({ expected, mutate }) => { + expect(validateCentralWorkflowMutation(mutate)).toContain(expected); + }); }); diff --git a/tools/e2e-scenarios/sandbox-operations-workflow-boundary.mts b/tools/e2e-scenarios/sandbox-operations-workflow-boundary.mts index efac5a31da1..5712cd2bd03 100644 --- a/tools/e2e-scenarios/sandbox-operations-workflow-boundary.mts +++ b/tools/e2e-scenarios/sandbox-operations-workflow-boundary.mts @@ -6,10 +6,17 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import YAML from "yaml"; +// Current-state security boundary for the default sandbox-operations job. +// It pins the isolated Docker-auth pattern that free-standing live jobs should +// reuse: trusted setup first, credentials only on the login step, target code +// only after login, and unconditional artifact/auth cleanup. const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); const DEFAULT_WORKFLOW_PATH = join(REPO_ROOT, ".github", "workflows", "e2e-vitest-scenarios.yaml"); const JOB_NAME = "sandbox-operations-vitest"; const FULL_SHA_ACTION = /^[^\s@]+@[0-9a-f]{40}$/u; +const GITHUB_ENV_REFERENCE = /\$\{?GITHUB_ENV\}?/u; +const WORKSPACE_REFERENCE = /github\.workspace|GITHUB_WORKSPACE/u; +const DOCKER_CREDENTIALS = ["DOCKERHUB_USERNAME", "DOCKERHUB_TOKEN"] as const; type WorkflowStep = { env?: Record; @@ -69,6 +76,11 @@ export function validateSandboxOperationsWorkflow(workflow: { if (Object.hasOwn(jobEnv, "DOCKER_CONFIG")) { errors.push(`${JOB_NAME} must not configure Docker auth at job scope`); } + for (const variable of DOCKER_CREDENTIALS) { + if (Object.hasOwn(jobEnv, variable) || JSON.stringify(jobEnv).includes(`secrets.${variable}`)) { + errors.push(`${JOB_NAME} must not expose ${variable} at job scope`); + } + } const checkout = steps.find((step) => step.uses?.startsWith("actions/checkout@")) ?? {}; if (!FULL_SHA_ACTION.test(checkout.uses ?? "")) { @@ -103,6 +115,22 @@ export function validateSandboxOperationsWorkflow(workflow: { "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config-sandbox-operations", ); requireRunContains(errors, configure, '>> "$GITHUB_ENV"'); + if (WORKSPACE_REFERENCE.test(configure.run ?? "")) { + errors.push(`${JOB_NAME} Docker auth directory must not use the checkout workspace`); + } + + for (const step of steps) { + if (step.env?.DOCKER_CONFIG !== undefined) { + errors.push( + `${JOB_NAME} must not expose DOCKER_CONFIG through step '${step.name ?? ""}'`, + ); + } + if (step !== configure && GITHUB_ENV_REFERENCE.test(step.run ?? "")) { + errors.push( + `${JOB_NAME} step '${step.name ?? ""}' must not write persistent environment`, + ); + } + } const authenticate = findStep(job, "Authenticate to Docker Hub"); if (authenticate.env?.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") { @@ -111,6 +139,16 @@ export function validateSandboxOperationsWorkflow(workflow: { if (authenticate.env?.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") { errors.push(`${JOB_NAME} Docker token must be scoped to the auth step`); } + for (const step of steps.filter((entry) => entry !== authenticate)) { + for (const variable of DOCKER_CREDENTIALS) { + if ( + step.env?.[variable] !== undefined || + JSON.stringify(step.env ?? {}).includes(`secrets.${variable}`) + ) { + errors.push(`${JOB_NAME} exposes ${variable} outside the Docker authentication step`); + } + } + } requireStepOrder(errors, steps, "Install OpenShell CLI", configure.name ?? ""); requireStepOrder(errors, steps, configure.name ?? "", authenticate.name ?? ""); From 066aeb2320467fda33faed08910a14c687e01cb3 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 28 Jun 2026 13:50:44 -0700 Subject: [PATCH 09/16] test(e2e): keep gateway cleanup assertion linear Signed-off-by: Carlos Villela --- test/e2e-scenario/live/sandbox-operations.test.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/test/e2e-scenario/live/sandbox-operations.test.ts b/test/e2e-scenario/live/sandbox-operations.test.ts index 4a7ae318b02..0caee77f528 100644 --- a/test/e2e-scenario/live/sandbox-operations.test.ts +++ b/test/e2e-scenario/live/sandbox-operations.test.ts @@ -66,15 +66,14 @@ async function destroyGateway(host: HostCliClient, artifactName = "cleanup-gatew env: buildAvailabilityProbeEnv(), timeoutMs: 5 * 60_000, }); - if (result.exitCode === 0) return; - if ( + const gatewayAlreadyAbsent = /gateway[^\n]*(?:does not exist|not found)|No (?:active )?gateway|No gateway metadata found/i.test( resultText(result), - ) - ) { - return; - } - expectExitZero(result, "cleanup destroy shared NemoClaw gateway"); + ); + expect( + result.exitCode === 0 || gatewayAlreadyAbsent, + `cleanup destroy shared NemoClaw gateway\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + ).toBe(true); } async function onboardSandbox( From 5f790e5f80c6e8ee17a7a3cbac52848cebb7be4e Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 28 Jun 2026 13:58:27 -0700 Subject: [PATCH 10/16] test(e2e): fail closed on gateway recovery Signed-off-by: Carlos Villela --- .../live/sandbox-operations.test.ts | 19 +++++-------------- ...ndbox-operations-workflow-boundary.test.ts | 7 ++++--- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/test/e2e-scenario/live/sandbox-operations.test.ts b/test/e2e-scenario/live/sandbox-operations.test.ts index 0caee77f528..ec6df3cead2 100644 --- a/test/e2e-scenario/live/sandbox-operations.test.ts +++ b/test/e2e-scenario/live/sandbox-operations.test.ts @@ -476,11 +476,7 @@ async function assertDestroyRemovesSandbox( expect(outputContainsSandbox(openshellList, sandboxName), resultText(openshellList)).toBe(false); } -type GatewayRecoveryOutcome = - | "recovered" - | "skipped-gateway-absent" - | "skipped-docker-restarted-before-probe" - | "skipped-docker-did-not-restart"; +type GatewayRecoveryOutcome = "recovered" | "skipped-gateway-absent"; async function assertGatewayRecovery( host: HostCliClient, @@ -499,11 +495,12 @@ async function assertGatewayRecovery( return "skipped-gateway-absent"; } - await host.command("docker", ["kill", GATEWAY_CONTAINER], { + const kill = await host.command("docker", ["kill", GATEWAY_CONTAINER], { artifactName: "tc-sbx-06-docker-kill-gateway", env: buildAvailabilityProbeEnv(), timeoutMs: 30_000, }); + expectExitZero(kill, "kill shared NemoClaw gateway container"); await new Promise((resolve) => setTimeout(resolve, 5_000)); const afterKill = await host.command( @@ -515,17 +512,13 @@ async function assertGatewayRecovery( timeoutMs: 15_000, }, ); - if (afterKill.stdout.trim() === "true") { - return "skipped-docker-restarted-before-probe"; - } + expect(afterKill.stdout.trim(), resultText(afterKill)).not.toBe("true"); const status = await host.nemoclaw([sandboxName, "status"], { artifactName: "tc-sbx-06-status-recovers-gateway", env: buildAvailabilityProbeEnv(), timeoutMs: 10 * 60_000, }); - if (status.exitCode === 0) return "recovered"; - const afterStatus = await host.command( "docker", ["inspect", "-f", "{{.State.Running}}", GATEWAY_CONTAINER], @@ -535,10 +528,8 @@ async function assertGatewayRecovery( timeoutMs: 15_000, }, ); - if (afterStatus.stdout.trim() !== "true") { - return "skipped-docker-did-not-restart"; - } expectExitZero(status, `nemoclaw ${sandboxName} status after gateway kill`); + expect(afterStatus.stdout.trim(), resultText(afterStatus)).toBe("true"); return "recovered"; } diff --git a/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts b/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts index 760dfad16a4..ab030d1658a 100644 --- a/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts +++ b/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts @@ -31,11 +31,12 @@ function validateCentralWorkflowMutation(mutate: (source: string) => string): st function mutateSandboxOperationsJob(source: string, mutate: (jobSource: string) => string): string { const startMarker = " sandbox-operations-vitest:\n"; - const endMarker = " sandbox-survival-vitest:\n"; const start = source.indexOf(startMarker); - const end = source.indexOf(endMarker, start + startMarker.length); expect(start).toBeGreaterThanOrEqual(0); - expect(end).toBeGreaterThan(start); + const rest = source.slice(start + startMarker.length); + const nextJob = /^ [A-Za-z0-9_-]+:\n/m.exec(rest); + const end = nextJob ? start + startMarker.length + nextJob.index : source.length; + expect(end).toBeGreaterThan(start + startMarker.length); const jobSource = source.slice(start, end); const mutated = mutate(jobSource); expect(mutated).not.toBe(jobSource); From a6aae71f4b7395501f2d59776d25ecbb9789049a Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 28 Jun 2026 14:03:56 -0700 Subject: [PATCH 11/16] test(e2e): support current gateway cleanup Signed-off-by: Carlos Villela --- .../live/sandbox-operations.test.ts | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/test/e2e-scenario/live/sandbox-operations.test.ts b/test/e2e-scenario/live/sandbox-operations.test.ts index ec6df3cead2..7dbd187addf 100644 --- a/test/e2e-scenario/live/sandbox-operations.test.ts +++ b/test/e2e-scenario/live/sandbox-operations.test.ts @@ -60,19 +60,28 @@ async function cleanupSandbox(host: HostCliClient, sandboxName: string): Promise expectExitZero(result, `cleanup destroy sandbox ${sandboxName}`); } -async function destroyGateway(host: HostCliClient, artifactName = "cleanup-gateway-destroy") { - const result = await host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { - artifactName, +async function cleanupGateway(host: HostCliClient, artifactName = "cleanup-gateway") { + const remove = await host.command("openshell", ["gateway", "remove", "nemoclaw"], { + artifactName: `${artifactName}-remove`, + env: buildAvailabilityProbeEnv(), + timeoutMs: 5 * 60_000, + }); + if (remove.exitCode === 0) return; + + // OpenShell builds before the registration-only gateway API exposed the + // equivalent cleanup operation as `gateway destroy`. + const destroy = await host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: `${artifactName}-legacy-destroy`, env: buildAvailabilityProbeEnv(), timeoutMs: 5 * 60_000, }); const gatewayAlreadyAbsent = /gateway[^\n]*(?:does not exist|not found)|No (?:active )?gateway|No gateway metadata found/i.test( - resultText(result), + `${resultText(remove)}\n${resultText(destroy)}`, ); expect( - result.exitCode === 0 || gatewayAlreadyAbsent, - `cleanup destroy shared NemoClaw gateway\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`, + destroy.exitCode === 0 || gatewayAlreadyAbsent, + `cleanup shared NemoClaw gateway registration\nremove:\n${resultText(remove)}\nlegacy destroy:\n${resultText(destroy)}`, ).toBe(true); } @@ -572,7 +581,7 @@ liveTest( } await environment.assertReady(ENVIRONMENT); - cleanup.add("destroy shared NemoClaw gateway", () => destroyGateway(host)); + cleanup.add("remove shared NemoClaw gateway registration", () => cleanupGateway(host)); await cleanupSandbox(host, SANDBOX_B); await cleanupSandbox(host, SANDBOX_A); From ff8df5a39ce6bc16b51d64acd538e25ae33c9a3a Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 28 Jun 2026 14:10:38 -0700 Subject: [PATCH 12/16] test(e2e): centralize sandbox cleanup semantics Signed-off-by: Carlos Villela --- .../live/sandbox-operations.test.ts | 18 ++--------- .../support-tests/e2e-clients.test.ts | 31 ++++++++++++++++--- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/test/e2e-scenario/live/sandbox-operations.test.ts b/test/e2e-scenario/live/sandbox-operations.test.ts index 7dbd187addf..88e453c13ab 100644 --- a/test/e2e-scenario/live/sandbox-operations.test.ts +++ b/test/e2e-scenario/live/sandbox-operations.test.ts @@ -48,18 +48,6 @@ function expectExitZero(result: ProcessResult, label: string): void { expect(result.exitCode, `${label}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`).toBe(0); } -async function cleanupSandbox(host: HostCliClient, sandboxName: string): Promise { - const result = await host.nemoclaw([sandboxName, "destroy", "--yes"], { - artifactName: `cleanup-destroy-${sandboxName}`, - env: buildAvailabilityProbeEnv(), - timeoutMs: 15 * 60_000, - }); - if (result.exitCode === 0) return; - const text = resultText(result); - if (/Sandbox '.+' does not exist|Run 'nemoclaw onboard' to create one/i.test(text)) return; - expectExitZero(result, `cleanup destroy sandbox ${sandboxName}`); -} - async function cleanupGateway(host: HostCliClient, artifactName = "cleanup-gateway") { const remove = await host.command("openshell", ["gateway", "remove", "nemoclaw"], { artifactName: `${artifactName}-remove`, @@ -93,7 +81,7 @@ async function onboardSandbox( hosted: HostedInferenceConfig, extraEnv: NodeJS.ProcessEnv = {}, ): Promise { - cleanup.add(`destroy sandbox ${sandboxName}`, () => cleanupSandbox(host, sandboxName)); + cleanup.add(`destroy sandbox ${sandboxName}`, () => host.cleanupSandbox(sandboxName)); const result = await host.nemoclaw( ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], { @@ -582,8 +570,8 @@ liveTest( await environment.assertReady(ENVIRONMENT); cleanup.add("remove shared NemoClaw gateway registration", () => cleanupGateway(host)); - await cleanupSandbox(host, SANDBOX_B); - await cleanupSandbox(host, SANDBOX_A); + await host.cleanupSandbox(SANDBOX_B); + await host.cleanupSandbox(SANDBOX_A); await onboardSandbox(host, cleanup, SANDBOX_A, "onboard-sandbox-a", hosted); diff --git a/test/e2e-scenario/support-tests/e2e-clients.test.ts b/test/e2e-scenario/support-tests/e2e-clients.test.ts index b8d1f170e25..b42c2e30d57 100644 --- a/test/e2e-scenario/support-tests/e2e-clients.test.ts +++ b/test/e2e-scenario/support-tests/e2e-clients.test.ts @@ -6,17 +6,17 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, expectTypeOf, it } from "vitest"; - -import { assertExitZero, type CommandRunner } from "../fixtures/clients/index.ts"; import { + assertExitZero, + type CommandRunner, GatewayClient, HostCliClient, ProviderClient, SandboxClient, StateClient, - trustedSandboxShellScript, - trustedProviderEndpoint, type TrustedSandboxShellScript, + trustedProviderEndpoint, + trustedSandboxShellScript, } from "../fixtures/clients/index.ts"; import type { ShellProbeResult, @@ -100,6 +100,29 @@ describe("E2E fixture clients", () => { ]); }); + it.each([ + "Error: sandbox assistant not found", + "no such sandbox: assistant", + ])("host client accepts canonical already-absent cleanup output: %s", async (stderr) => { + const runner = new FakeRunner(); + runner.exitCode = 1; + runner.stderr = stderr; + const host = new HostCliClient(runner, { cliPath: "nemoclaw" }); + + await expect(host.cleanupSandbox("assistant")).resolves.toBeUndefined(); + }); + + it("host client surfaces unexpected sandbox cleanup failures", async () => { + const runner = new FakeRunner(); + runner.exitCode = 1; + runner.stderr = "permission denied"; + const host = new HostCliClient(runner, { cliPath: "nemoclaw" }); + + await expect(host.cleanupSandbox("assistant")).rejects.toThrow( + "cleanup destroy sandbox assistant failed: permission denied", + ); + }); + it("host client propagates cwd, env, and timeout options", async () => { const runner = new FakeRunner(); const host = new HostCliClient(runner, { From 22fa85ed0e96916112440a97377dee3eafb776fb Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 28 Jun 2026 14:15:50 -0700 Subject: [PATCH 13/16] test(e2e): ratchet launcher and recovery evidence Signed-off-by: Carlos Villela --- .github/workflows/e2e-vitest-scenarios.yaml | 7 +++++++ .../live/sandbox-operations.test.ts | 14 +++++++++---- ...ndbox-operations-workflow-boundary.test.ts | 20 +++++++++++++++++++ .../sandbox-operations-workflow-boundary.mts | 9 +++++++++ 4 files changed, 46 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 698e43a5f27..6ef91b76501 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -4167,6 +4167,8 @@ jobs: NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1" NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + # Open permits the scenario's inference/log probes; TC-SBX-11 separately + # proves that sandbox-to-sandbox network isolation remains enforced. NEMOCLAW_POLICY_TIER: "open" OPENSHELL_GATEWAY: "nemoclaw" steps: @@ -4186,6 +4188,11 @@ jobs: - name: Build CLI run: npm run build:cli + - name: Verify CLI launcher + run: | + test -x "${NEMOCLAW_CLI_BIN}" + "${NEMOCLAW_CLI_BIN}" --version + - name: Install OpenShell CLI run: | env -u DOCKER_CONFIG \ diff --git a/test/e2e-scenario/live/sandbox-operations.test.ts b/test/e2e-scenario/live/sandbox-operations.test.ts index 88e453c13ab..80725820fb5 100644 --- a/test/e2e-scenario/live/sandbox-operations.test.ts +++ b/test/e2e-scenario/live/sandbox-operations.test.ts @@ -276,7 +276,9 @@ async function assertAgentJsonNonzeroExit(host: HostCliClient, sandboxName: stri // migration-only assertion was retired instead of expanding the parity lane. // Failed-tool provenance remains covered deterministically by // test/openclaw-agent-json.test.ts; a live prompt cannot require upstream - // OpenClaw to emit failed tool-result metadata. + // OpenClaw to emit failed tool-result metadata. Re-add live stdin coverage if + // the frozen parity source gains that contract or transport validation is + // explicitly added to this lane's scope. } async function assertStatusFields(host: HostCliClient, sandboxName: string): Promise { @@ -473,7 +475,10 @@ async function assertDestroyRemovesSandbox( expect(outputContainsSandbox(openshellList, sandboxName), resultText(openshellList)).toBe(false); } -type GatewayRecoveryOutcome = "recovered" | "skipped-gateway-absent"; +type GatewayRecoveryOutcome = + | "recovered-before-status" + | "recovered-by-status" + | "skipped-gateway-absent"; async function assertGatewayRecovery( host: HostCliClient, @@ -509,7 +514,8 @@ async function assertGatewayRecovery( timeoutMs: 15_000, }, ); - expect(afterKill.stdout.trim(), resultText(afterKill)).not.toBe("true"); + const recoveryOutcome = + afterKill.stdout.trim() === "true" ? "recovered-before-status" : "recovered-by-status"; const status = await host.nemoclaw([sandboxName, "status"], { artifactName: "tc-sbx-06-status-recovers-gateway", @@ -527,7 +533,7 @@ async function assertGatewayRecovery( ); expectExitZero(status, `nemoclaw ${sandboxName} status after gateway kill`); expect(afterStatus.stdout.trim(), resultText(afterStatus)).toBe("true"); - return "recovered"; + return recoveryOutcome; } liveTest( diff --git a/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts b/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts index ab030d1658a..a69f349e024 100644 --- a/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts +++ b/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts @@ -105,6 +105,26 @@ describe("sandbox operations workflow boundary", () => { }); it.each([ + { + label: "a non-launcher CLI path", + mutate: (source: string) => + mutateSandboxOperationsJob(source, (jobSource) => + jobSource.replace( + " NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js", + " NEMOCLAW_CLI_BIN: ${{ github.workspace }}/dist/nemoclaw.js", + ), + ), + expected: "sandbox-operations-vitest must use the stable bin/nemoclaw.js CLI launcher", + }, + { + label: "a missing CLI launcher preflight", + mutate: (source: string) => + mutateSandboxOperationsJob(source, (jobSource) => + jobSource.replace(' test -x "${NEMOCLAW_CLI_BIN}"', " true"), + ), + expected: + "sandbox-operations-vitest step 'Verify CLI launcher' must run: test -x \"${NEMOCLAW_CLI_BIN}\"", + }, { label: "Docker credentials at job scope", mutate: (source: string) => diff --git a/tools/e2e-scenarios/sandbox-operations-workflow-boundary.mts b/tools/e2e-scenarios/sandbox-operations-workflow-boundary.mts index 5712cd2bd03..dbda86c228f 100644 --- a/tools/e2e-scenarios/sandbox-operations-workflow-boundary.mts +++ b/tools/e2e-scenarios/sandbox-operations-workflow-boundary.mts @@ -81,6 +81,9 @@ export function validateSandboxOperationsWorkflow(workflow: { errors.push(`${JOB_NAME} must not expose ${variable} at job scope`); } } + if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") { + errors.push(`${JOB_NAME} must use the stable bin/nemoclaw.js CLI launcher`); + } const checkout = steps.find((step) => step.uses?.startsWith("actions/checkout@")) ?? {}; if (!FULL_SHA_ACTION.test(checkout.uses ?? "")) { @@ -108,6 +111,10 @@ export function validateSandboxOperationsWorkflow(workflow: { } requireRunContains(errors, install, "bash scripts/install-openshell.sh"); + const verifyLauncher = findStep(job, "Verify CLI launcher"); + requireRunContains(errors, verifyLauncher, 'test -x "${NEMOCLAW_CLI_BIN}"'); + requireRunContains(errors, verifyLauncher, '"${NEMOCLAW_CLI_BIN}" --version'); + const configure = findStep(job, "Configure isolated Docker auth directory"); requireRunContains( errors, @@ -150,6 +157,8 @@ export function validateSandboxOperationsWorkflow(workflow: { } } + requireStepOrder(errors, steps, "Build CLI", verifyLauncher.name ?? ""); + requireStepOrder(errors, steps, verifyLauncher.name ?? "", "Install OpenShell CLI"); requireStepOrder(errors, steps, "Install OpenShell CLI", configure.name ?? ""); requireStepOrder(errors, steps, configure.name ?? "", authenticate.name ?? ""); requireStepOrder(errors, steps, authenticate.name ?? "", "Run sandbox operations live test"); From ac1e912791e11d7d01826cd07e0a7167685f5f2d Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 28 Jun 2026 14:19:36 -0700 Subject: [PATCH 14/16] test(e2e): centralize gateway cleanup semantics Signed-off-by: Carlos Villela --- test/e2e-scenario/fixtures/clients/host.ts | 26 +++++++- .../live/sandbox-operations.test.ts | 32 ++-------- .../support-tests/e2e-clients.test.ts | 62 +++++++++++++++++-- 3 files changed, 89 insertions(+), 31 deletions(-) diff --git a/test/e2e-scenario/fixtures/clients/host.ts b/test/e2e-scenario/fixtures/clients/host.ts index 5fbb37b8876..a1e34f1bd10 100644 --- a/test/e2e-scenario/fixtures/clients/host.ts +++ b/test/e2e-scenario/fixtures/clients/host.ts @@ -7,9 +7,9 @@ import { trustedShellCommand } from "../shell-probe.ts"; import { artifactLabel, assertExitZero, + type CommandRunner, outputContainsSandbox, resultText, - type CommandRunner, } from "./command.ts"; export interface HostClientOptions { @@ -17,6 +17,9 @@ export interface HostClientOptions { cwd?: string; } +const GATEWAY_ALREADY_ABSENT = + /gateway[^\n]*(?:does not exist|not found)|No (?:active )?gateway|No gateway metadata found/i; + export class HostCliClient { private readonly runner: CommandRunner; private readonly cliPath: string; @@ -122,6 +125,27 @@ export class HostCliClient { assertExitZero(result, `cleanup destroy sandbox ${sandboxName}`); } + async cleanupGatewayRegistration( + gatewayName: string, + options: ShellProbeRunOptions = {}, + ): Promise { + const artifactName = options.artifactName ?? `cleanup-gateway-${artifactLabel(gatewayName)}`; + const remove = await this.command("openshell", ["gateway", "remove", gatewayName], { + ...options, + artifactName: `${artifactName}-remove`, + }); + if (remove.exitCode === 0 || GATEWAY_ALREADY_ABSENT.test(resultText(remove))) return; + + // Remove this fallback once the supported OpenShell floor no longer + // includes builds whose local-registration verb was `gateway destroy`. + const destroy = await this.command("openshell", ["gateway", "destroy", "-g", gatewayName], { + ...options, + artifactName: `${artifactName}-legacy-destroy`, + }); + if (destroy.exitCode === 0 || GATEWAY_ALREADY_ABSENT.test(resultText(destroy))) return; + assertExitZero(destroy, `cleanup gateway registration ${gatewayName}`); + } + async bestEffortCleanupSandbox( sandboxName: string, options: ShellProbeRunOptions = {}, diff --git a/test/e2e-scenario/live/sandbox-operations.test.ts b/test/e2e-scenario/live/sandbox-operations.test.ts index 80725820fb5..1e167ee27c2 100644 --- a/test/e2e-scenario/live/sandbox-operations.test.ts +++ b/test/e2e-scenario/live/sandbox-operations.test.ts @@ -48,31 +48,6 @@ function expectExitZero(result: ProcessResult, label: string): void { expect(result.exitCode, `${label}\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`).toBe(0); } -async function cleanupGateway(host: HostCliClient, artifactName = "cleanup-gateway") { - const remove = await host.command("openshell", ["gateway", "remove", "nemoclaw"], { - artifactName: `${artifactName}-remove`, - env: buildAvailabilityProbeEnv(), - timeoutMs: 5 * 60_000, - }); - if (remove.exitCode === 0) return; - - // OpenShell builds before the registration-only gateway API exposed the - // equivalent cleanup operation as `gateway destroy`. - const destroy = await host.command("openshell", ["gateway", "destroy", "-g", "nemoclaw"], { - artifactName: `${artifactName}-legacy-destroy`, - env: buildAvailabilityProbeEnv(), - timeoutMs: 5 * 60_000, - }); - const gatewayAlreadyAbsent = - /gateway[^\n]*(?:does not exist|not found)|No (?:active )?gateway|No gateway metadata found/i.test( - `${resultText(remove)}\n${resultText(destroy)}`, - ); - expect( - destroy.exitCode === 0 || gatewayAlreadyAbsent, - `cleanup shared NemoClaw gateway registration\nremove:\n${resultText(remove)}\nlegacy destroy:\n${resultText(destroy)}`, - ).toBe(true); -} - async function onboardSandbox( host: HostCliClient, cleanup: CleanupRegistry, @@ -575,7 +550,12 @@ liveTest( } await environment.assertReady(ENVIRONMENT); - cleanup.add("remove shared NemoClaw gateway registration", () => cleanupGateway(host)); + cleanup.add("remove shared NemoClaw gateway registration", () => + host.cleanupGatewayRegistration("nemoclaw", { + env: buildAvailabilityProbeEnv(), + timeoutMs: 5 * 60_000, + }), + ); await host.cleanupSandbox(SANDBOX_B); await host.cleanupSandbox(SANDBOX_A); diff --git a/test/e2e-scenario/support-tests/e2e-clients.test.ts b/test/e2e-scenario/support-tests/e2e-clients.test.ts index b42c2e30d57..226531837dc 100644 --- a/test/e2e-scenario/support-tests/e2e-clients.test.ts +++ b/test/e2e-scenario/support-tests/e2e-clients.test.ts @@ -30,13 +30,22 @@ interface RunnerCall { options?: ShellProbeRunOptions; } +type FakeRunnerResponse = Partial< + Pick +>; + class FakeRunner implements CommandRunner { readonly calls: RunnerCall[] = []; + readonly responses: FakeRunnerResponse[] = []; stdout = ""; stderr = ""; exitCode: number | null = 0; signal: NodeJS.Signals | null = null; + enqueue(response: FakeRunnerResponse): void { + this.responses.push(response); + } + async run( command: TrustedShellCommand, options?: ShellProbeRunOptions, @@ -46,13 +55,14 @@ class FakeRunner implements CommandRunner { args: [...command.args], options, }); + const response = this.responses.shift(); return { command: [command.command, ...command.args], - exitCode: this.exitCode, - signal: this.signal, + exitCode: response?.exitCode === undefined ? this.exitCode : response.exitCode, + signal: response?.signal === undefined ? this.signal : response.signal, timedOut: false, - stdout: this.stdout, - stderr: this.stderr, + stdout: response?.stdout ?? this.stdout, + stderr: response?.stderr ?? this.stderr, artifacts: { stdout: "/tmp/stdout.txt", stderr: "/tmp/stderr.txt", @@ -123,6 +133,50 @@ describe("E2E fixture clients", () => { ); }); + it("host client removes a current OpenShell gateway registration", async () => { + const runner = new FakeRunner(); + const host = new HostCliClient(runner, { cliPath: "nemoclaw" }); + + await host.cleanupGatewayRegistration("nemoclaw"); + + expect(runner.calls.map((call) => call.args)).toEqual([["gateway", "remove", "nemoclaw"]]); + }); + + it("host client falls back to the legacy gateway destroy verb", async () => { + const runner = new FakeRunner(); + runner.enqueue({ exitCode: 2, stderr: "unrecognized subcommand 'remove'" }); + runner.enqueue({ exitCode: 0 }); + const host = new HostCliClient(runner, { cliPath: "nemoclaw" }); + + await host.cleanupGatewayRegistration("nemoclaw"); + + expect(runner.calls.map((call) => call.args)).toEqual([ + ["gateway", "remove", "nemoclaw"], + ["gateway", "destroy", "-g", "nemoclaw"], + ]); + }); + + it("host client accepts an already-absent gateway without a legacy fallback", async () => { + const runner = new FakeRunner(); + runner.enqueue({ exitCode: 1, stderr: "No gateway metadata found" }); + const host = new HostCliClient(runner, { cliPath: "nemoclaw" }); + + await host.cleanupGatewayRegistration("nemoclaw"); + + expect(runner.calls.map((call) => call.args)).toEqual([["gateway", "remove", "nemoclaw"]]); + }); + + it("host client surfaces an unexpected legacy gateway cleanup failure", async () => { + const runner = new FakeRunner(); + runner.enqueue({ exitCode: 2, stderr: "unrecognized subcommand 'remove'" }); + runner.enqueue({ exitCode: 1, stderr: "permission denied" }); + const host = new HostCliClient(runner, { cliPath: "nemoclaw" }); + + await expect(host.cleanupGatewayRegistration("nemoclaw")).rejects.toThrow( + "cleanup gateway registration nemoclaw failed: permission denied", + ); + }); + it("host client propagates cwd, env, and timeout options", async () => { const runner = new FakeRunner(); const host = new HostCliClient(runner, { From 70dd42af080d210a97fa17606ac959844340500f Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 28 Jun 2026 14:33:01 -0700 Subject: [PATCH 15/16] test(e2e): close cleanup and dispatch guard branches Signed-off-by: Carlos Villela --- .../e2e-scenario/support-tests/e2e-clients.test.ts | 14 ++++++++++++++ .../sandbox-operations-workflow-boundary.test.ts | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/test/e2e-scenario/support-tests/e2e-clients.test.ts b/test/e2e-scenario/support-tests/e2e-clients.test.ts index 226531837dc..10fed91be99 100644 --- a/test/e2e-scenario/support-tests/e2e-clients.test.ts +++ b/test/e2e-scenario/support-tests/e2e-clients.test.ts @@ -166,6 +166,20 @@ describe("E2E fixture clients", () => { expect(runner.calls.map((call) => call.args)).toEqual([["gateway", "remove", "nemoclaw"]]); }); + it("host client accepts an already-absent legacy gateway registration", async () => { + const runner = new FakeRunner(); + runner.enqueue({ exitCode: 2, stderr: "unrecognized subcommand 'remove'" }); + runner.enqueue({ exitCode: 1, stderr: "No gateway metadata found" }); + const host = new HostCliClient(runner, { cliPath: "nemoclaw" }); + + await host.cleanupGatewayRegistration("nemoclaw"); + + expect(runner.calls.map((call) => call.args)).toEqual([ + ["gateway", "remove", "nemoclaw"], + ["gateway", "destroy", "-g", "nemoclaw"], + ]); + }); + it("host client surfaces an unexpected legacy gateway cleanup failure", async () => { const runner = new FakeRunner(); runner.enqueue({ exitCode: 2, stderr: "unrecognized subcommand 'remove'" }); diff --git a/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts b/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts index a69f349e024..cb3f2b685c7 100644 --- a/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts +++ b/test/e2e-scenario/support-tests/sandbox-operations-workflow-boundary.test.ts @@ -104,6 +104,20 @@ describe("sandbox operations workflow boundary", () => { ); }); + it("keeps secret-bearing live jobs on manual dispatch with read-only contents", () => { + expect( + validateCentralWorkflowMutation((source) => + source.replace("on:\n workflow_dispatch:", "on:\n pull_request:\n workflow_dispatch:"), + ), + ).toContain("workflow must not run on pull_request"); + + expect( + validateCentralWorkflowMutation((source) => + source.replace("permissions:\n contents: read", "permissions:\n contents: write"), + ), + ).toContain("workflow permissions.contents must be read"); + }); + it.each([ { label: "a non-launcher CLI path", From ca68bbdcb1e05ed4f97a3840b0f5a32bd54f13e6 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 28 Jun 2026 14:46:00 -0700 Subject: [PATCH 16/16] test(e2e): restrict legacy gateway cleanup fallback Signed-off-by: Carlos Villela --- test/e2e-scenario/fixtures/clients/host.ts | 5 +++++ test/e2e-scenario/support-tests/e2e-clients.test.ts | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/test/e2e-scenario/fixtures/clients/host.ts b/test/e2e-scenario/fixtures/clients/host.ts index a1e34f1bd10..6f09e8e26c6 100644 --- a/test/e2e-scenario/fixtures/clients/host.ts +++ b/test/e2e-scenario/fixtures/clients/host.ts @@ -19,6 +19,8 @@ export interface HostClientOptions { const GATEWAY_ALREADY_ABSENT = /gateway[^\n]*(?:does not exist|not found)|No (?:active )?gateway|No gateway metadata found/i; +const GATEWAY_REMOVE_UNSUPPORTED = + /unrecognized subcommand ['"]remove['"]|unknown command ['"]remove['"]/i; export class HostCliClient { private readonly runner: CommandRunner; @@ -135,6 +137,9 @@ export class HostCliClient { artifactName: `${artifactName}-remove`, }); if (remove.exitCode === 0 || GATEWAY_ALREADY_ABSENT.test(resultText(remove))) return; + if (!GATEWAY_REMOVE_UNSUPPORTED.test(resultText(remove))) { + assertExitZero(remove, `cleanup gateway registration ${gatewayName}`); + } // Remove this fallback once the supported OpenShell floor no longer // includes builds whose local-registration verb was `gateway destroy`. diff --git a/test/e2e-scenario/support-tests/e2e-clients.test.ts b/test/e2e-scenario/support-tests/e2e-clients.test.ts index 10fed91be99..9d004c04c81 100644 --- a/test/e2e-scenario/support-tests/e2e-clients.test.ts +++ b/test/e2e-scenario/support-tests/e2e-clients.test.ts @@ -180,6 +180,17 @@ describe("E2E fixture clients", () => { ]); }); + it("host client does not hide a current gateway remove failure behind the legacy verb", async () => { + const runner = new FakeRunner(); + runner.enqueue({ exitCode: 1, stderr: "permission denied" }); + const host = new HostCliClient(runner, { cliPath: "nemoclaw" }); + + await expect(host.cleanupGatewayRegistration("nemoclaw")).rejects.toThrow( + "cleanup gateway registration nemoclaw failed: permission denied", + ); + expect(runner.calls.map((call) => call.args)).toEqual([["gateway", "remove", "nemoclaw"]]); + }); + it("host client surfaces an unexpected legacy gateway cleanup failure", async () => { const runner = new FakeRunner(); runner.enqueue({ exitCode: 2, stderr: "unrecognized subcommand 'remove'" });