diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 4b5d9b2d3f0..70a62345675 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -2179,6 +2179,94 @@ jobs: docker logout docker.io || true rm -rf "${DOCKER_CONFIG}" + snapshot-commands-vitest: + needs: generate-matrix + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',snapshot-commands-vitest,') || contains(format(',{0},', inputs.scenarios), ',snapshot-commands,') }} + runs-on: ubuntu-latest + timeout-minutes: 40 + env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "snapshot-commands" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/snapshot-commands + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-snapshot" + OPENSHELL_GATEWAY: "nemoclaw" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Configure isolated Docker auth directory + run: echo "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config-snapshot-commands" >> "$GITHUB_ENV" + + - name: Authenticate to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + mkdir -p "${DOCKER_CONFIG}" + chmod 700 "${DOCKER_CONFIG}" + login_succeeded=0 + for attempt in 1 2 3; do + if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then + login_succeeded=1 + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." + sleep 5 + fi + done + if [[ "$login_succeeded" -ne 1 ]]; then + echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." + fi + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Run snapshot commands live test + # Migrated from test/e2e/test-snapshot-commands.sh. Preserves the + # original ubuntu-latest + Docker/OpenShell + NVIDIA_API_KEY runner + # contract by driving install.sh and the real snapshot CLI lifecycle. + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/snapshot-commands.test.ts \ + --silent=false --reporter=default + + - name: Upload snapshot commands artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-snapshot-commands + path: e2e-artifacts/vitest/snapshot-commands/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + + - name: Clean up Docker auth + if: always() + run: | + set -euo pipefail + docker logout docker.io || true + rm -rf "${DOCKER_CONFIG}" + sandbox-survival-vitest: needs: generate-matrix if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',sandbox-survival-vitest,') || contains(format(',{0},', inputs.scenarios), ',sandbox-survival,') }} @@ -2988,6 +3076,7 @@ jobs: double-onboard-vitest, model-router-provider-routed-inference-vitest, sandbox-survival-vitest, + snapshot-commands-vitest, gateway-drift-preflight-vitest, openclaw-tui-chat-correlation-vitest, gateway-guard-recovery, diff --git a/test/e2e-scenario/live/snapshot-commands.test.ts b/test/e2e-scenario/live/snapshot-commands.test.ts new file mode 100644 index 00000000000..7d8aa01d121 --- /dev/null +++ b/test/e2e-scenario/live/snapshot-commands.test.ts @@ -0,0 +1,398 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Live Vitest replacement for test/e2e/test-snapshot-commands.sh. + * + * Preserves the legacy real boundaries: install.sh non-interactive onboard, + * NemoClaw snapshot create/list/restore commands, OpenShell sandbox exec for + * workspace mutation/verification, host rebuild-backups inspection, artifact + * capture, cleanup, and secret redaction. + */ + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; +import { type SandboxClient, validateSandboxName } from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; +import { isTransientProviderValidationFailure } from "./network-policy-transient-provider.ts"; + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? "e2e-snapshot"; +validateSandboxName(SANDBOX_NAME); +const BACKUP_ROOT = path.join(os.homedir(), ".nemoclaw", "rebuild-backups"); +const BACKUP_DIR = path.resolve(BACKUP_ROOT, SANDBOX_NAME); +if (!BACKUP_DIR.startsWith(`${path.resolve(BACKUP_ROOT)}${path.sep}`)) { + throw new Error(`snapshot backup directory escaped rebuild-backups root: ${BACKUP_DIR}`); +} +const MARKER_FILE = "/sandbox/.openclaw/workspace/snapshot-marker.txt"; +const SECOND_MARKER = "/sandbox/.openclaw/workspace/snapshot-marker-2.txt"; +const LIVE_TIMEOUT_MS = 30 * 60_000; +const INSTALL_ATTEMPTS = process.env.CI === "true" || process.env.GITHUB_ACTIONS === "true" ? 3 : 1; +const CREDENTIAL_TOKEN_VALUE_PATTERN = /(?:nvapi-|sk-|Bearer )/; +const CREDENTIAL_ENV_ASSIGNMENT_PATTERN = + /(?:^|\n)\s*(?:export\s+)?(?:NVIDIA_API_KEY|NVIDIA_INFERENCE_API_KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|COMPATIBLE_API_KEY|NGC_API_KEY|AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY)\s*=/i; +const STRUCTURED_CREDENTIAL_KEY_PATTERN = + /["']?(?:apiKey|api_key|accessToken|access_token|secretKey|secret_key|bearerToken|bearer_token)["']?\s*[:=]\s*["'][^"']+["']/i; + +function resultText(result: Pick): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +function commandEnv(apiKey?: string): NodeJS.ProcessEnv { + const env: NodeJS.ProcessEnv = { + ...buildAvailabilityProbeEnv(), + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", + }; + if (apiKey) env.NVIDIA_API_KEY = apiKey; + return env; +} + +async function bestEffort(run: () => Promise): Promise { + try { + await run(); + } catch { + // Mirrors the legacy teardown: cleanup attempts should not hide the main failure. + } +} + +async function cleanupSnapshotSandbox( + host: HostCliClient, + sandbox: SandboxClient, + label: string, +): Promise { + await bestEffort(() => + host.command("nemoclaw", [SANDBOX_NAME, "destroy", "--yes"], { + artifactName: `${label}-nemoclaw-destroy`, + env: commandEnv(), + timeoutMs: 120_000, + }), + ); + await bestEffort(() => + sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: `${label}-openshell-sandbox-delete`, + env: commandEnv(), + timeoutMs: 60_000, + }), + ); + await bestEffort(() => + sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: `${label}-openshell-gateway-destroy`, + env: commandEnv(), + timeoutMs: 60_000, + }), + ); +} + +async function expectSandboxFileContent( + sandbox: SandboxClient, + filePath: string, + expected: string, + artifactName: string, +): Promise { + const result = await sandbox.exec(SANDBOX_NAME, ["cat", filePath], { + artifactName, + env: commandEnv(), + timeoutMs: 30_000, + }); + expect(result.exitCode, resultText(result)).toBe(0); + expect(result.stdout.trim()).toBe(expected); +} + +function firstSnapshotTimestamp(listOutput: string): string { + const match = listOutput.match(/\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d+Z/); + if (!match) + throw new Error(`Failed to parse snapshot timestamp from list output:\n${listOutput}`); + return match[0]; +} + +function scanSnapshotCredentialLeaks(root: string): string[] { + if (!fs.existsSync(root)) throw new Error(`Backup directory missing: ${root}`); + const ignored = new Set([ + "package-lock.json", + "npm-shrinkwrap.json", + "yarn.lock", + "pnpm-lock.yaml", + "pnpm-lock.yml", + ]); + const leaks: string[] = []; + const visit = (dir: string): void => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + visit(fullPath); + continue; + } + if (!entry.isFile()) continue; + if (ignored.has(entry.name)) continue; + if (!(entry.name === ".env" || entry.name.endsWith(".env") || entry.name.endsWith(".json"))) { + continue; + } + const body = fs.readFileSync(fullPath, "utf8"); + const tokenValueLeak = CREDENTIAL_TOKEN_VALUE_PATTERN.test(body); + const envAssignmentLeak = CREDENTIAL_ENV_ASSIGNMENT_PATTERN.test(body); + // openclaw.json may legitimately contain non-secret provider metadata + // such as credential env-var references. Still fail it on token-shaped + // values or concrete env assignments, but reserve generic structured-key + // checks for other env/json files where such keys indicate persisted + // credentials rather than configuration schema. + const structuredKeyLeak = + entry.name !== "openclaw.json" && STRUCTURED_CREDENTIAL_KEY_PATTERN.test(body); + if (tokenValueLeak || envAssignmentLeak || structuredKeyLeak) leaks.push(fullPath); + } + }; + visit(root); + return leaks; +} + +test.skipIf(!shouldRunLiveE2EScenarios())( + "snapshot commands preserve create/list/latest restore/targeted restore/no-leak lifecycle", + { timeout: LIVE_TIMEOUT_MS }, + async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { + const apiKey = secrets.required("NVIDIA_API_KEY"); + expect(apiKey.startsWith("nvapi-"), "NVIDIA_API_KEY must start with nvapi-").toBe(true); + + await artifacts.writeJson("scenario.json", { + id: "snapshot-commands", + runner: "vitest", + legacySource: "test/e2e/test-snapshot-commands.sh", + boundary: "install.sh + nemoclaw snapshot commands + openshell sandbox exec", + sandboxName: SANDBOX_NAME, + backupDir: BACKUP_DIR, + contracts: [ + "install.sh onboards a live OpenClaw sandbox", + "snapshot create reports Snapshot v created", + "snapshot list shows versioned snapshots and parseable timestamps", + "latest snapshot restore recovers latest workspace state", + "timestamp-targeted restore recovers the first snapshot state", + "snapshot directory excludes credential-bearing env/json files", + "snapshot help advertises create/list/restore", + ], + }); + + const dockerInfo = await host.command("docker", ["info"], { + artifactName: "phase-0-docker-info", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + if (dockerInfo.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error(`Docker is required for snapshot commands E2E: ${resultText(dockerInfo)}`); + } + skip(`Docker is required for snapshot commands E2E: ${resultText(dockerInfo)}`); + } + + cleanup.add(`destroy snapshot sandbox ${SANDBOX_NAME}`, () => + cleanupSnapshotSandbox(host, sandbox, "cleanup"), + ); + + await cleanupSnapshotSandbox(host, sandbox, "pre-cleanup"); + fs.rmSync(BACKUP_DIR, { recursive: true, force: true }); + + let install: ShellProbeResult | undefined; + for (let attempt = 1; attempt <= INSTALL_ATTEMPTS; attempt += 1) { + install = await host.command("bash", ["install.sh", "--non-interactive", "--fresh"], { + artifactName: + attempt === 1 + ? "phase-1-install-nemoclaw" + : `phase-1-install-nemoclaw-attempt-${attempt}`, + cwd: REPO_ROOT, + env: commandEnv(apiKey), + redactionValues: [apiKey], + timeoutMs: 20 * 60_000, + }); + if (install.exitCode === 0) break; + if (isTransientProviderValidationFailure(install) && attempt < INSTALL_ATTEMPTS) { + await new Promise((resolve) => setTimeout(resolve, 10_000 * attempt)); + continue; + } + if (isTransientProviderValidationFailure(install) && process.env.GITHUB_ACTIONS === "true") { + await artifacts.writeJson("transient-provider-validation.skip.json", { + reason: "transient NVIDIA Endpoints validation failure during install.sh onboard", + attempts: INSTALL_ATTEMPTS, + sourceBoundary: "external NVIDIA Endpoints provider availability", + removalCondition: + "remove once CI endpoint validation is stable for a release cycle or covered by a hermetic provider-validation fixture", + }); + skip( + `NVIDIA Endpoints validation hit a transient upstream/rate-limit failure after ${INSTALL_ATTEMPTS} attempts`, + ); + } + break; + } + expect(install?.exitCode, install ? resultText(install) : "install did not run").toBe(0); + + const cliProbe = await host.command( + "bash", + ["-lc", "command -v nemoclaw && command -v openshell"], + { + artifactName: "phase-1-cli-probe", + env: commandEnv(), + timeoutMs: 30_000, + }, + ); + expect(cliProbe.exitCode, resultText(cliProbe)).toBe(0); + expect(cliProbe.stdout).toContain("nemoclaw"); + expect(cliProbe.stdout).toContain("openshell"); + + const markerContent = `SNAPSHOT_E2E_${Date.now()}`; + const secondContent = `SNAPSHOT_E2E_SECOND_${Date.now()}`; + + const writeMarker = await sandbox.exec( + SANDBOX_NAME, + [ + "sh", + "-lc", + `mkdir -p /sandbox/.openclaw/workspace && printf '%s' '${markerContent}' > ${MARKER_FILE}`, + ], + { + artifactName: "phase-2-write-marker", + env: commandEnv(), + timeoutMs: 60_000, + }, + ); + expect(writeMarker.exitCode, resultText(writeMarker)).toBe(0); + await expectSandboxFileContent(sandbox, MARKER_FILE, markerContent, "phase-2-read-marker"); + + const firstCreate = await host.command("nemoclaw", [SANDBOX_NAME, "snapshot", "create"], { + artifactName: "phase-3-snapshot-create-first", + env: commandEnv(), + timeoutMs: 120_000, + }); + expect(firstCreate.exitCode, resultText(firstCreate)).toBe(0); + expect(resultText(firstCreate)).toMatch(/Snapshot v\d+.*created/); + expect(resultText(firstCreate)).toContain("rebuild-backups"); + + const list = await host.command("nemoclaw", [SANDBOX_NAME, "snapshot", "list"], { + artifactName: "phase-4-snapshot-list", + env: commandEnv(), + timeoutMs: 60_000, + }); + expect(list.exitCode, resultText(list)).toBe(0); + expect(resultText(list)).toContain("snapshot(s)"); + const timestamp = firstSnapshotTimestamp(resultText(list)); + await artifacts.writeJson("phase-4-first-snapshot.json", { timestamp }); + + const modify = await sandbox.exec( + SANDBOX_NAME, + ["sh", "-lc", `rm -f ${MARKER_FILE} && printf '%s' '${secondContent}' > ${SECOND_MARKER}`], + { + artifactName: "phase-5-modify-workspace", + env: commandEnv(), + timeoutMs: 60_000, + }, + ); + expect(modify.exitCode, resultText(modify)).toBe(0); + + const firstGone = await sandbox.exec(SANDBOX_NAME, ["sh", "-lc", `test ! -e ${MARKER_FILE}`], { + artifactName: "phase-5-first-marker-gone", + env: commandEnv(), + timeoutMs: 30_000, + }); + expect(firstGone.exitCode, resultText(firstGone)).toBe(0); + + const secondCreate = await host.command("nemoclaw", [SANDBOX_NAME, "snapshot", "create"], { + artifactName: "phase-5-snapshot-create-second", + env: commandEnv(), + timeoutMs: 120_000, + }); + expect(secondCreate.exitCode, resultText(secondCreate)).toBe(0); + expect(resultText(secondCreate)).toMatch(/Snapshot v\d+.*created/); + + const perturb = await sandbox.exec( + SANDBOX_NAME, + ["sh", "-lc", `rm -f ${SECOND_MARKER} && printf '%s' 'BROKEN' > ${MARKER_FILE}`], + { + artifactName: "phase-5-perturb-workspace", + env: commandEnv(), + timeoutMs: 60_000, + }, + ); + expect(perturb.exitCode, resultText(perturb)).toBe(0); + + const latestRestore = await host.command("nemoclaw", [SANDBOX_NAME, "snapshot", "restore"], { + artifactName: "phase-6-snapshot-restore-latest", + env: commandEnv(), + timeoutMs: 120_000, + }); + expect(latestRestore.exitCode, resultText(latestRestore)).toBe(0); + expect(resultText(latestRestore)).toContain("Restored"); + await expectSandboxFileContent( + sandbox, + SECOND_MARKER, + secondContent, + "phase-6-read-second-marker-after-latest-restore", + ); + const firstGoneAfterLatest = await sandbox.exec( + SANDBOX_NAME, + ["sh", "-lc", `test ! -e ${MARKER_FILE}`], + { + artifactName: "phase-6-first-marker-absent-after-latest-restore", + env: commandEnv(), + timeoutMs: 30_000, + }, + ); + expect(firstGoneAfterLatest.exitCode, resultText(firstGoneAfterLatest)).toBe(0); + + const targetedRestore = await host.command( + "nemoclaw", + [SANDBOX_NAME, "snapshot", "restore", timestamp], + { + artifactName: "phase-7-snapshot-restore-first-timestamp", + env: commandEnv(), + timeoutMs: 120_000, + }, + ); + expect(targetedRestore.exitCode, resultText(targetedRestore)).toBe(0); + expect(resultText(targetedRestore)).toContain("Restored"); + await expectSandboxFileContent( + sandbox, + MARKER_FILE, + markerContent, + "phase-7-read-first-marker-after-targeted-restore", + ); + const secondGone = await sandbox.exec( + SANDBOX_NAME, + ["sh", "-lc", `test ! -e ${SECOND_MARKER}`], + { + artifactName: "phase-7-second-marker-absent-after-targeted-restore", + env: commandEnv(), + timeoutMs: 30_000, + }, + ); + expect(secondGone.exitCode, resultText(secondGone)).toBe(0); + + const credentialLeaks = scanSnapshotCredentialLeaks(BACKUP_DIR); + await artifacts.writeJson("phase-8-credential-scan.json", { + backupDir: BACKUP_DIR, + leakedFiles: credentialLeaks, + }); + expect(credentialLeaks).toEqual([]); + + const help = await host.command("nemoclaw", [SANDBOX_NAME, "snapshot"], { + artifactName: "phase-9-snapshot-help", + env: commandEnv(), + timeoutMs: 30_000, + }); + expect(help.exitCode, resultText(help)).toBe(0); + expect(resultText(help)).toContain("snapshot create"); + expect(resultText(help)).toContain("snapshot list"); + expect(resultText(help)).toContain("snapshot restore"); + + await artifacts.writeJson("scenario-result.json", { + id: "snapshot-commands", + status: "passed", + firstSnapshotTimestamp: timestamp, + backupDir: BACKUP_DIR, + }); + }, +); diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index e13343d9986..50c8209fa40 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -865,6 +865,7 @@ jobs: "workflow missing hermes-e2e-vitest job", "workflow missing skill-agent-vitest job", "workflow missing model-router-provider-routed-inference-vitest job", + "workflow missing snapshot-commands-vitest job", "report-to-pr job must wait for live-scenarios", "report-to-pr step must pass jobs through JOBS env", "step 'Post Vitest scenario results to PR' run script must check selector validation before echoing selectors", @@ -897,6 +898,87 @@ jobs: } }); + it("requires snapshot commands workflow boundary coverage", () => { + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); + const workflowPath = path.join(tmp, "workflow.yaml"); + const workflow = fs.readFileSync( + path.join(process.cwd(), ".github/workflows/e2e-vitest-scenarios.yaml"), + "utf8", + ); + const parsedWorkflow = YAML.parse(workflow) as { + jobs: Record< + string, + { + env: Record; + steps: Array>; + "timeout-minutes"?: number; + } + >; + }; + const snapshotJob = parsedWorkflow.jobs["snapshot-commands-vitest"]; + snapshotJob["timeout-minutes"] = 30; + snapshotJob.env.DOCKER_CONFIG = "${{ github.workspace }}/.docker-config-shared"; + snapshotJob.env.NVIDIA_API_KEY = "${{ secrets.NVIDIA_API_KEY }}"; + for (const step of snapshotJob.steps) { + if (typeof step.uses === "string" && step.uses.startsWith("actions/checkout@")) { + step.with = { ...(step.with as Record), "persist-credentials": true }; + } + if (step.name === "Configure isolated Docker auth directory") { + step.run = + 'echo "DOCKER_CONFIG=${{ github.workspace }}/.docker-config-shared" >> "$GITHUB_ENV"'; + } + if (step.name === "Set up Node") { + step.env = { NVIDIA_API_KEY: "${{ secrets.NVIDIA_API_KEY }}" }; + } + if (step.name === "Install root dependencies") { + step.env = { + DOCKERHUB_USERNAME: "${{ secrets.DOCKERHUB_USERNAME }}", + DOCKERHUB_TOKEN: "${{ secrets.DOCKERHUB_TOKEN }}", + }; + step.run = "npm install"; + } + if (step.name === "Run snapshot commands live test") { + step.run = String(step.run).replace( + "test/e2e-scenario/live/snapshot-commands.test.ts", + "test/e2e-scenario/live/registry-scenarios.test.ts", + ); + } + if (step.name === "Upload snapshot commands artifacts") { + step.with = { + ...(step.with as Record), + path: "e2e-artifacts/vitest/", + "include-hidden-files": true, + }; + } + if (step.name === "Clean up Docker auth") { + step.run = String(step.run).replace('rm -rf "${DOCKER_CONFIG}"', 'echo "missing cleanup"'); + } + } + fs.writeFileSync(workflowPath, YAML.stringify(parsedWorkflow)); + + try { + expect(validateE2eVitestScenariosWorkflowBoundary(workflowPath)).toEqual( + expect.arrayContaining([ + "snapshot-commands-vitest job must keep a 40 minute timeout", + "snapshot-commands-vitest job must not set DOCKER_CONFIG at job level", + 'step \'Configure isolated Docker auth directory\' run script must include echo "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config-snapshot-commands" >> "$GITHUB_ENV"', + "snapshot-commands-vitest checkout step must set persist-credentials=false", + "snapshot-commands-vitest job env must not include NVIDIA_API_KEY", + "snapshot-commands-vitest step 'Set up Node' env must not include NVIDIA_API_KEY", + "snapshot-commands-vitest step 'Install root dependencies' env must not include DOCKERHUB_USERNAME", + "snapshot-commands-vitest step 'Install root dependencies' env must not include DOCKERHUB_TOKEN", + "snapshot-commands-vitest artifact upload must set include-hidden-files: false", + "artifact upload path must include e2e-artifacts/vitest/snapshot-commands/", + "step 'Clean up Docker auth' run script must include rm -rf \"${DOCKER_CONFIG}\"", + "step 'Install root dependencies' run script must include npm ci --ignore-scripts", + "step 'Run snapshot commands live test' run script must include test/e2e-scenario/live/snapshot-commands.test.ts", + ]), + ); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } + }); + it("applies boundary checks to newly marked free-standing jobs", () => { const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-vitest-workflow-")); const workflowPath = path.join(tmp, "workflow.yaml"); diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 1afd0be5d25..5fd96ea0d14 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -2457,6 +2457,162 @@ function validateHermesRootEntrypointSmokeVitestJob(errors: string[], jobs: Work } } +function validateSnapshotCommandsVitestJob(errors: string[], jobs: WorkflowRecord): void { + const jobName = "snapshot-commands-vitest"; + const scenarioName = "snapshot-commands"; + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) { + errors.push("workflow missing snapshot-commands-vitest job"); + return; + } + + if (job["runs-on"] !== "ubuntu-latest") { + errors.push("snapshot-commands-vitest job must run on ubuntu-latest"); + } + if (job["timeout-minutes"] !== 40) { + errors.push("snapshot-commands-vitest job must keep a 40 minute timeout"); + } + validateFreeStandingJobSelector(errors, jobs, jobName, scenarioName); + + const jobEnv = asRecord(job.env); + if ("DOCKER_CONFIG" in jobEnv) { + errors.push("snapshot-commands-vitest job must not set DOCKER_CONFIG at job level"); + } + if ( + jobEnv.E2E_ARTIFACT_DIR !== "${{ github.workspace }}/e2e-artifacts/vitest/snapshot-commands" + ) { + errors.push( + "snapshot-commands-vitest job must write artifacts under e2e-artifacts/vitest/snapshot-commands", + ); + } + if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { + errors.push("snapshot-commands-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); + } + if (jobEnv.NEMOCLAW_NON_INTERACTIVE !== "1") { + errors.push("snapshot-commands-vitest job must set NEMOCLAW_NON_INTERACTIVE=1"); + } + if (jobEnv.NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE !== "1") { + errors.push("snapshot-commands-vitest job must accept third-party software non-interactively"); + } + if (jobEnv.NEMOCLAW_SANDBOX_NAME !== "e2e-snapshot") { + errors.push("snapshot-commands-vitest job must use the stable e2e-snapshot sandbox name"); + } + if (jobEnv.OPENSHELL_GATEWAY !== "nemoclaw") { + errors.push("snapshot-commands-vitest job must force OPENSHELL_GATEWAY=nemoclaw"); + } + for (const secret of [ + "NVIDIA_API_KEY", + "DOCKERHUB_USERNAME", + "DOCKERHUB_TOKEN", + "GITHUB_TOKEN", + ]) { + requireEnvDoesNotExposeSecret(errors, "snapshot-commands-vitest job", jobEnv, secret); + } + + const steps = asSteps(job.steps); + requireNoDispatchInputInterpolation(errors, steps); + for (const step of steps) { + const stepName = `snapshot-commands-vitest step '${step.name ?? step.uses ?? ""}'`; + const stepEnv = asRecord(step.env); + if (step.name !== "Run snapshot commands live test") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_API_KEY"); + } + if (step.name !== "Authenticate to Docker Hub") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); + requireNoDockerHubAuthInRun(errors, stepName, stringValue(step.run)); + } + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "GITHUB_TOKEN"); + } + + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!checkout) { + errors.push("snapshot-commands-vitest job missing checkout step"); + } + requireFullShaAction(errors, checkout, "snapshot-commands-vitest checkout"); + if (asRecord(checkout?.with)["persist-credentials"] !== false) { + errors.push("snapshot-commands-vitest checkout step must set persist-credentials=false"); + } + + const configureDockerAuth = requireJobStep( + errors, + jobName, + steps, + "Configure isolated Docker auth directory", + ); + requireRunContains( + errors, + configureDockerAuth, + 'echo "DOCKER_CONFIG=${RUNNER_TEMP}/docker-config-snapshot-commands" >> "$GITHUB_ENV"', + ); + requireRunDoesNotContain(errors, configureDockerAuth, "${{ runner.temp }}"); + + const dockerLogin = requireJobStep(errors, jobName, steps, "Authenticate to Docker Hub"); + const dockerLoginEnv = asRecord(dockerLogin?.env); + if (dockerLoginEnv.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") { + errors.push( + "snapshot-commands-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets", + ); + } + if (dockerLoginEnv.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") { + errors.push( + "snapshot-commands-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets", + ); + } + requireRunContains(errors, dockerLogin, 'mkdir -p "${DOCKER_CONFIG}"'); + requireRunContains(errors, dockerLogin, 'chmod 700 "${DOCKER_CONFIG}"'); + requireRunContains(errors, dockerLogin, "docker login docker.io"); + requireRunContains(errors, dockerLogin, "--password-stdin"); + requireRunContains(errors, dockerLogin, "continuing with anonymous pulls"); + + const setupNode = namedStep(steps, "Set up Node"); + if (!setupNode) { + errors.push("snapshot-commands-vitest job missing step: Set up Node"); + } + requireFullShaAction(errors, setupNode, "snapshot-commands-vitest setup-node"); + + const installRootDependencies = requireJobStep( + errors, + jobName, + steps, + "Install root dependencies", + ); + requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); + + const runVitest = requireJobStep(errors, jobName, steps, "Run snapshot commands live test"); + const runVitestEnv = asRecord(runVitest?.env); + if (runVitestEnv.NVIDIA_API_KEY !== "${{ secrets.NVIDIA_API_KEY }}") { + errors.push("snapshot-commands-vitest Vitest step must receive NVIDIA_API_KEY from secrets"); + } + requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); + requireRunContains(errors, runVitest, "test/e2e-scenario/live/snapshot-commands.test.ts"); + + const upload = requireJobStep(errors, jobName, steps, "Upload snapshot commands artifacts"); + requireFullShaAction(errors, upload, "snapshot-commands-vitest upload-artifact"); + const uploadWith = asRecord(upload?.with); + if (uploadWith.name !== "e2e-vitest-scenarios-snapshot-commands") { + errors.push("snapshot-commands-vitest artifact upload name must be stable"); + } + const uploadPath = stringValue(uploadWith.path); + requireUploadPathContains(errors, uploadPath, "e2e-artifacts/vitest/snapshot-commands/"); + if (uploadWith["include-hidden-files"] !== false) { + errors.push("snapshot-commands-vitest artifact upload must set include-hidden-files: false"); + } + if (uploadWith["if-no-files-found"] !== "ignore") { + errors.push("snapshot-commands-vitest artifact upload must ignore missing fixture artifacts"); + } + if (uploadWith["retention-days"] !== 14) { + errors.push("snapshot-commands-vitest artifact upload retention-days must be 14"); + } + + const cleanup = requireJobStep(errors, jobName, steps, "Clean up Docker auth"); + if (cleanup?.if !== "always()") { + errors.push("snapshot-commands-vitest Docker auth cleanup must always run"); + } + requireRunContains(errors, cleanup, "docker logout docker.io"); + requireRunContains(errors, cleanup, 'rm -rf "${DOCKER_CONFIG}"'); +} + function validateModelRouterProviderRoutedInferenceVitestJob( errors: string[], jobs: WorkflowRecord, @@ -3512,6 +3668,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( "issue-4434-tui-unreachable-inference", ); validateModelRouterProviderRoutedInferenceVitestJob(errors, jobs); + validateSnapshotCommandsVitestJob(errors, jobs); validateFreeStandingJobSelector( errors, jobs,