diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 5e8700ad1e6..55615fb6d26 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -1298,6 +1298,102 @@ jobs: if-no-files-found: ignore retention-days: 14 + state-backup-restore-vitest: + needs: generate-matrix + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',state-backup-restore-vitest,') || contains(format(',{0},', inputs.scenarios), ',state-backup-restore,') }} + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + FREE_STANDING_VITEST_JOB: "1" + FREE_STANDING_SCENARIO_ID: "state-backup-restore" + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/state-backup-restore + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + NEMOCLAW_NON_INTERACTIVE: "1" + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: e2e-state-backup + OPENSHELL_GATEWAY: nemoclaw + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Authenticate to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + login_succeeded=0 + for attempt in 1 2 3; do + if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then + login_succeeded=1 + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." + sleep 5 + fi + done + if [[ "$login_succeeded" -ne 1 ]]; then + echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." + fi + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Install OpenShell + env: + NEMOCLAW_NON_INTERACTIVE: "1" + run: | + set -euo pipefail + env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh + + - name: Run state backup restore live test + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + run: | + set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + if command -v openshell >/dev/null 2>&1; then + OPENSHELL_BIN="$(command -v openshell)" + elif [ -x "$HOME/.local/bin/openshell" ]; then + OPENSHELL_BIN="$HOME/.local/bin/openshell" + else + echo "::error::OpenShell CLI not found after install" + ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true + exit 1 + fi + export OPENSHELL_BIN + "$OPENSHELL_BIN" --version + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/state-backup-restore.test.ts \ + --silent=false --reporter=default + + - name: Upload state backup restore artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-state-backup-restore + path: e2e-artifacts/vitest/state-backup-restore/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + double-onboard-vitest: needs: generate-matrix if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',double-onboard-vitest,') || contains(format(',{0},', inputs.scenarios), ',double-onboard,') }} @@ -2014,6 +2110,7 @@ jobs: shields-config-vitest, rebuild-openclaw-vitest, sandbox-rebuild-vitest, + state-backup-restore-vitest, token-rotation-vitest, messaging-providers-vitest, launchable-smoke-vitest, diff --git a/test/e2e-scenario/fixtures/phases/onboarding.ts b/test/e2e-scenario/fixtures/phases/onboarding.ts index 865d817587a..23000d53908 100644 --- a/test/e2e-scenario/fixtures/phases/onboarding.ts +++ b/test/e2e-scenario/fixtures/phases/onboarding.ts @@ -38,6 +38,7 @@ const MISSING_SANDBOX_DELETE_PATTERNS = [ /sandbox not found/i, /sandbox .* not found/i, /sandbox .* not present/i, + /sandbox .* does not exist/i, /sandbox does not exist/i, /no such sandbox/i, ]; diff --git a/test/e2e-scenario/live/state-backup-restore.test.ts b/test/e2e-scenario/live/state-backup-restore.test.ts new file mode 100644 index 00000000000..a7a7ec8f526 --- /dev/null +++ b/test/e2e-scenario/live/state-backup-restore.test.ts @@ -0,0 +1,431 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { resultText } from "../fixtures/clients/index.ts"; +import { sandboxAccessEnv, validateSandboxName } from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import type { NemoClawInstance } from "../fixtures/phases/onboarding.ts"; +import { + restoreRegistryAndSession, + snapshotRegistryAndSession, +} from "../fixtures/phases/state-validation.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +// Direct Vitest replacement coverage for test/e2e/test-state-backup-restore.sh. +// Keep the core boundary identical to the legacy shell lane: write durable +// workspace state in a real OpenClaw sandbox, run scripts/backup-workspace.sh +// backup, destroy and recreate the sandbox, run scripts/backup-workspace.sh +// restore, then verify the five top-level workspace files plus memory/ return. + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const WORKSPACE_PATH = "/sandbox/.openclaw/workspace"; +const WORKSPACE_FILES = ["SOUL.md", "USER.md", "IDENTITY.md", "AGENTS.md", "MEMORY.md"]; +const MEMORY_FILE = "memory/2026-04-20.md"; +const TEST_SANDBOX_PREFIX = "e2e-state-backup"; +const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? TEST_SANDBOX_PREFIX; +const TEST_TIMEOUT_MS = Number(process.env.NEMOCLAW_E2E_TIMEOUT_SECONDS ?? 3_600) * 1_000; +const ONBOARD_TIMEOUT_MS = 30 * 60_000; +const BACKUP_RESTORE_TIMEOUT_MS = 5 * 60_000; +const DESTROY_ATTEMPTS = 3; +const DESTROY_RETRY_DELAY_MS = 10_000; + +validateSandboxName(SANDBOX_NAME); + +type BackupExpectation = { + relativePath: string; + expected: string; +}; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function assertTestOwnedSandboxName(): void { + if (!SANDBOX_NAME.startsWith(TEST_SANDBOX_PREFIX)) { + throw new Error( + `state-backup-restore live test is destructive and only accepts sandbox names with prefix ${TEST_SANDBOX_PREFIX}; got ${SANDBOX_NAME}`, + ); + } +} + +function backupRoot(): string { + return path.join(process.env.HOME ?? os.homedir(), ".nemoclaw", "backups"); +} + +function listBackupDirs(root = backupRoot()): string[] { + if (!fs.existsSync(root)) return []; + return fs + .readdirSync(root, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => path.join(root, entry.name)) + .sort(); +} + +function latestBackupDir(candidates: readonly string[]): string | undefined { + return [...candidates] + .filter((candidate) => fs.existsSync(candidate)) + .sort((a, b) => fs.statSync(b).mtimeMs - fs.statSync(a).mtimeMs) + .at(0); +} + +function backupRestoreEnv(): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + OPENSHELL_GATEWAY: process.env.OPENSHELL_GATEWAY ?? "nemoclaw", + }; +} + +function commandFailed(result: ShellProbeResult): boolean { + return result.exitCode !== 0 || result.timedOut; +} + +function errorText(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function isNvidiaEndpointValidationUnavailable(text: string): boolean { + return ( + /NVIDIA Endpoints endpoint validation failed/i.test(text) && + (/Validation details were omitted/i.test(text) || + /HTTP 429|rate limit|quota|temporarily unavailable|timed out|timeout/i.test(text)) + ); +} + +async function bestEffort(run: () => Promise): Promise { + try { + await run(); + } catch { + // Cleanup remains best-effort so the primary E2E failure stays visible. + } +} + +function hostFileContains(filePath: string, expected: string): boolean { + return fs.existsSync(filePath) && fs.readFileSync(filePath, "utf8").includes(expected); +} + +async function destroySandboxUntilAbsent( + sandboxName: string, + destroy: (artifactName: string) => Promise, + list: (artifactName: string) => Promise, +): Promise { + let lastList = ""; + for (let attempt = 1; attempt <= DESTROY_ATTEMPTS; attempt += 1) { + await bestEffort(() => destroy(`phase-3-destroy-attempt-${attempt}`)); + const listResult = await list(`phase-3-list-after-destroy-${attempt}`); + lastList = resultText(listResult); + if (listResult.exitCode === 0 && !lastList.includes(sandboxName)) return; + if (attempt < DESTROY_ATTEMPTS) await sleep(DESTROY_RETRY_DELAY_MS); + } + throw new Error( + `TC-STATE-01: Destroy failed; sandbox ${sandboxName} still exists after ${DESTROY_ATTEMPTS} attempts:\n${lastList}`, + ); +} + +test.skipIf(!shouldRunLiveE2EScenarios())( + "state-backup-restore: backup-workspace.sh restores workspace files and memory directory", + { timeout: TEST_TIMEOUT_MS }, + async ({ + artifacts, + cleanup, + environment, + host, + onboard, + sandbox, + secrets, + skip, + stateValidation, + }) => { + assertTestOwnedSandboxName(); + const apiKey = secrets.required("NVIDIA_API_KEY"); + expect(apiKey.startsWith("nvapi-"), "NVIDIA_API_KEY must start with nvapi-").toBe(true); + expect(fs.existsSync(path.join(REPO_ROOT, "scripts", "backup-workspace.sh"))).toBe(true); + + const dockerInfo = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + if (dockerInfo.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error( + `Docker is required for state-backup-restore live coverage: ${resultText(dockerInfo)}`, + ); + } + skip("Docker is required for state-backup-restore live coverage"); + } + + await artifacts.writeJson("contract.json", { + legacySource: "test/e2e/test-state-backup-restore.sh", + sandboxName: SANDBOX_NAME, + workspacePath: WORKSPACE_PATH, + restoredFiles: WORKSPACE_FILES, + restoredDirectoryProbe: MEMORY_FILE, + preservedBoundaries: [ + "real nemoclaw onboard with Docker/OpenShell", + "openshell sandbox exec workspace marker writes and reads", + "real scripts/backup-workspace.sh backup host process", + "real nemoclaw destroy --yes", + "real scripts/backup-workspace.sh restore host process", + ], + }); + + const stateSnapshot = snapshotRegistryAndSession(); + let createdBackupDir: string | undefined; + cleanup.add(`restore NemoClaw state files for ${SANDBOX_NAME}`, () => { + restoreRegistryAndSession(stateSnapshot); + }); + cleanup.add("remove generated backup-workspace.sh backup", () => { + if (!createdBackupDir) return; + const root = backupRoot(); + const resolved = path.resolve(createdBackupDir); + if (resolved !== root && resolved.startsWith(`${path.resolve(root)}${path.sep}`)) { + fs.rmSync(resolved, { recursive: true, force: true }); + } + }); + cleanup.add(`destroy sandbox ${SANDBOX_NAME}`, async () => { + if (process.env.NEMOCLAW_E2E_KEEP_SANDBOX === "1") return; + await bestEffort(() => onboard.destroySandbox(SANDBOX_NAME, "cleanup-nemoclaw-destroy")); + await bestEffort(() => + sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "cleanup-openshell-sandbox-delete", + env: sandboxAccessEnv(), + timeoutMs: 60_000, + }), + ); + }); + cleanup.add("stop NemoClaw gateway", async () => { + await bestEffort(() => + host.nemoclaw(["stop"], { + artifactName: "cleanup-nemoclaw-stop", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }), + ); + }); + + await bestEffort(() => onboard.destroySandbox(SANDBOX_NAME, "pre-cleanup-nemoclaw-destroy")); + await bestEffort(() => + sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "pre-cleanup-openshell-sandbox-delete", + env: sandboxAccessEnv(), + timeoutMs: 60_000, + }), + ); + + const ready = await environment.assertReady({ + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-openclaw", + }); + + let instance: NemoClawInstance; + try { + instance = await onboard.from(ready, { + sandboxName: SANDBOX_NAME, + timeoutMs: ONBOARD_TIMEOUT_MS, + }); + } catch (error) { + const text = errorText(error); + if (isNvidiaEndpointValidationUnavailable(text)) { + await artifacts.writeJson("scenario-result.json", { + id: "state-backup-restore", + status: "skipped", + reason: "external-provider-validation-unavailable-before-state-backup-contract", + }); + skip("NVIDIA endpoint validation was unavailable/rate-limited during onboarding"); + } + throw error; + } + + const markerContent = `E2E_BACKUP_TEST_${Date.now()}`; + const expectations: BackupExpectation[] = WORKSPACE_FILES.map((file) => ({ + relativePath: file, + expected: `${markerContent}_${file}`, + })); + expectations.push({ + relativePath: MEMORY_FILE, + expected: `${markerContent}_daily`, + }); + + for (const expectation of expectations) { + await stateValidation.writeMarkerFile( + instance, + path.posix.join(WORKSPACE_PATH, expectation.relativePath), + expectation.expected, + { + artifactName: `phase-1-write-${expectation.relativePath.replace(/\//g, "-")}`, + env: sandboxAccessEnv(), + timeoutMs: 60_000, + }, + ); + } + await artifacts.writeJson("phase-1-marker-summary.json", { + workspaceFilesWritten: WORKSPACE_FILES.length, + memoryFilesWritten: 1, + }); + + const beforeBackupDirs = new Set(listBackupDirs()); + const backup = await host.command( + "bash", + [path.join(REPO_ROOT, "scripts", "backup-workspace.sh"), "backup", SANDBOX_NAME], + { + artifactName: "phase-2-backup-workspace", + cwd: REPO_ROOT, + env: backupRestoreEnv(), + timeoutMs: BACKUP_RESTORE_TIMEOUT_MS, + }, + ); + const backupText = resultText(backup); + if (commandFailed(backup) || !backupText.includes("Backup saved")) { + throw new Error( + `TC-STATE-01: Backup failed; backup-workspace.sh backup exited ${backup.exitCode}:\n${backupText}`, + ); + } + + const newBackupDirs = listBackupDirs().filter((dir) => !beforeBackupDirs.has(dir)); + createdBackupDir = latestBackupDir(newBackupDirs) ?? latestBackupDir(listBackupDirs()); + expect(createdBackupDir, "TC-STATE-01: Backup dir — no backup directory found").toBeTruthy(); + await artifacts.writeJson("phase-2-backup-summary.json", { + backupDir: createdBackupDir, + output: backupText, + }); + + let capturedFiles = 0; + for (const file of WORKSPACE_FILES) { + const expected = `${markerContent}_${file}`; + if (hostFileContains(path.join(createdBackupDir!, file), expected)) { + capturedFiles += 1; + } + } + expect( + capturedFiles, + `TC-STATE-01: BackupCaptureFiles — expected all 5 markdown files in host backup ${createdBackupDir}`, + ).toBe(WORKSPACE_FILES.length); + + const memoryBackupPath = path.join(createdBackupDir!, MEMORY_FILE); + expect( + fs.existsSync(memoryBackupPath), + `TC-STATE-01: BackupCaptureDir — ${memoryBackupPath} must exist in host backup`, + ).toBe(true); + expect( + hostFileContains(memoryBackupPath, `${markerContent}_daily`), + "TC-STATE-01: BackupCaptureDir — memory file must contain expected marker", + ).toBe(true); + + await destroySandboxUntilAbsent( + SANDBOX_NAME, + (artifactName) => onboard.destroySandbox(SANDBOX_NAME, artifactName), + (artifactName) => + host.nemoclaw(["list"], { + artifactName, + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }), + ); + await artifacts.writeJson("phase-3-destroy-summary.json", { + sandboxName: SANDBOX_NAME, + attempts: DESTROY_ATTEMPTS, + }); + + let restoredInstance: NemoClawInstance; + try { + restoredInstance = await onboard.from(ready, { + sandboxName: SANDBOX_NAME, + timeoutMs: ONBOARD_TIMEOUT_MS, + }); + } catch (error) { + const text = errorText(error); + if (isNvidiaEndpointValidationUnavailable(text)) { + await artifacts.writeJson("scenario-result.json", { + id: "state-backup-restore", + status: "skipped", + reason: "external-provider-validation-unavailable-during-reonboard", + }); + skip("NVIDIA endpoint validation was unavailable/rate-limited during re-onboard"); + } + throw error; + } + await artifacts.writeJson("phase-4-reonboard-summary.json", { + sandboxName: restoredInstance.sandboxName, + }); + + const restore = await host.command( + "bash", + [path.join(REPO_ROOT, "scripts", "backup-workspace.sh"), "restore", SANDBOX_NAME], + { + artifactName: "phase-5-restore-workspace", + cwd: REPO_ROOT, + env: backupRestoreEnv(), + timeoutMs: BACKUP_RESTORE_TIMEOUT_MS, + }, + ); + const restoreText = resultText(restore); + if (commandFailed(restore) || !restoreText.includes("Restored")) { + throw new Error( + `TC-STATE-01: Restore failed; backup-workspace.sh restore exited ${restore.exitCode}:\n${restoreText}`, + ); + } + await artifacts.writeText("phase-5-restore-output.txt", restoreText); + + let restoredFiles = 0; + const mismatches: Array<{ file: string; actual: string }> = []; + for (const file of WORKSPACE_FILES) { + const remotePath = path.posix.join(WORKSPACE_PATH, file); + const read = await sandbox.exec( + SANDBOX_NAME, + ["sh", "-c", 'cat "$1" 2>/dev/null', "sh", remotePath], + { + artifactName: `phase-6-read-${file}`, + env: sandboxAccessEnv(), + timeoutMs: 60_000, + }, + ); + const expected = `${markerContent}_${file}`; + if (read.exitCode === 0 && read.stdout.includes(expected)) { + restoredFiles += 1; + } else { + mismatches.push({ file, actual: resultText(read).slice(0, 200) }); + } + } + await artifacts.writeJson("phase-6-files-restore-summary.json", { + restoredFiles, + expectedFiles: WORKSPACE_FILES.length, + mismatches, + }); + expect( + restoredFiles, + "TC-STATE-01: FilesRestore — backup-workspace.sh must restore all 5 workspace files", + ).toBe(WORKSPACE_FILES.length); + + const memoryRemotePath = path.posix.join(WORKSPACE_PATH, MEMORY_FILE); + const memoryProbe = await sandbox.exec( + SANDBOX_NAME, + [ + "sh", + "-c", + 'if [ -f "$1" ]; then printf "STATE=EXISTS\\n"; cat "$1"; else printf "STATE=MISSING\\n"; fi', + "sh", + memoryRemotePath, + ], + { + artifactName: "phase-6-read-memory-directory-file", + env: sandboxAccessEnv(), + timeoutMs: 60_000, + }, + ); + const memoryText = resultText(memoryProbe); + await artifacts.writeText("phase-6-memory-probe.txt", memoryText); + if (memoryText.includes("STATE=MISSING")) { + await artifacts.writeText("phase-6-restore-output-for-memory-missing.txt", restoreText); + } + expect(memoryText).toContain("STATE=EXISTS"); + expect(memoryText).toContain(`${markerContent}_daily`); + }, +); diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index 450d18f0e70..a73d5ee98f6 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -235,6 +235,22 @@ describe("e2e-vitest-scenarios workflow boundary", () => { selectedFreeStandingJobs: ["rebuild-openclaw-vitest"], registryScenarios: [], }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "state-backup-restore" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["state-backup-restore-vitest"], + registryScenarios: [], + }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "state-backup-restore-vitest" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["state-backup-restore-vitest"], + registryScenarios: [], + }); expect( evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "model-router-provider-routed-inference", diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index a8c226c516b..9d3245be33a 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -1089,6 +1089,135 @@ function validateSandboxRebuildVitestJob(errors: string[], jobs: WorkflowRecord) } } +function validateStateBackupRestoreVitestJob(errors: string[], jobs: WorkflowRecord): void { + const jobName = "state-backup-restore-vitest"; + const scenarioName = "state-backup-restore"; + const job = asRecord(jobs[jobName]); + if (Object.keys(job).length === 0) { + errors.push("workflow missing state-backup-restore-vitest job"); + return; + } + + if (job["runs-on"] !== "ubuntu-latest") { + errors.push("state-backup-restore-vitest job must run on ubuntu-latest"); + } + validateFreeStandingJobSelector(errors, jobs, jobName, scenarioName); + if (job["timeout-minutes"] !== 60) { + errors.push("state-backup-restore-vitest job must keep the legacy 60 minute timeout"); + } + const jobEnv = asRecord(job.env); + if (jobEnv.NEMOCLAW_RUN_E2E_SCENARIOS !== "1") { + errors.push("state-backup-restore-vitest job must set NEMOCLAW_RUN_E2E_SCENARIOS=1"); + } + if ( + jobEnv.E2E_ARTIFACT_DIR !== + "${{ github.workspace }}/e2e-artifacts/vitest/state-backup-restore" + ) { + errors.push( + "state-backup-restore-vitest job must write artifacts under e2e-artifacts/vitest/state-backup-restore", + ); + } + if (jobEnv.NEMOCLAW_CLI_BIN !== "${{ github.workspace }}/bin/nemoclaw.js") { + errors.push("state-backup-restore-vitest job must point NEMOCLAW_CLI_BIN at the repo CLI"); + } + if (jobEnv.OPENSHELL_GATEWAY !== "nemoclaw") { + errors.push("state-backup-restore-vitest job must force OPENSHELL_GATEWAY=nemoclaw"); + } + if (jobEnv.NEMOCLAW_NON_INTERACTIVE !== "1") { + errors.push("state-backup-restore-vitest job must set NEMOCLAW_NON_INTERACTIVE=1"); + } + if (jobEnv.NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE !== "1") { + errors.push("state-backup-restore-vitest job must set NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1"); + } + if (jobEnv.NEMOCLAW_SANDBOX_NAME !== "e2e-state-backup") { + errors.push("state-backup-restore-vitest job must set NEMOCLAW_SANDBOX_NAME=e2e-state-backup"); + } + for (const secret of ["NVIDIA_API_KEY", "DOCKERHUB_USERNAME", "DOCKERHUB_TOKEN", "GITHUB_TOKEN"]) { + requireEnvDoesNotExposeSecret(errors, "state-backup-restore-vitest job", jobEnv, secret); + } + + const steps = asSteps(job.steps); + requireNoDispatchInputInterpolation(errors, steps); + for (const step of steps) { + const stepName = `state-backup-restore-vitest step '${step.name ?? step.uses ?? ""}'`; + const stepEnv = asRecord(step.env); + if (step.name !== "Run state backup restore live test") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "NVIDIA_API_KEY"); + } + if (step.name !== "Authenticate to Docker Hub") { + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_USERNAME"); + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "DOCKERHUB_TOKEN"); + requireNoDockerHubAuthInRun(errors, stepName, stringValue(step.run)); + } + requireEnvDoesNotExposeSecret(errors, stepName, stepEnv, "GITHUB_TOKEN"); + } + + const checkout = steps.find((step) => stringValue(step.uses).startsWith("actions/checkout@")); + if (!checkout) errors.push("state-backup-restore-vitest job missing checkout step"); + requireFullShaAction(errors, checkout, "state-backup-restore-vitest checkout"); + if (asRecord(checkout?.with)["persist-credentials"] !== false) { + errors.push("state-backup-restore-vitest checkout step must set persist-credentials=false"); + } + + const dockerHubAuth = requireJobStep(errors, jobName, steps, "Authenticate to Docker Hub"); + const dockerHubEnv = asRecord(dockerHubAuth?.env); + if (dockerHubEnv.DOCKERHUB_USERNAME !== "${{ secrets.DOCKERHUB_USERNAME }}") { + errors.push( + "state-backup-restore-vitest Docker Hub auth must receive DOCKERHUB_USERNAME from secrets", + ); + } + if (dockerHubEnv.DOCKERHUB_TOKEN !== "${{ secrets.DOCKERHUB_TOKEN }}") { + errors.push("state-backup-restore-vitest Docker Hub auth must receive DOCKERHUB_TOKEN from secrets"); + } + requireRunContains(errors, dockerHubAuth, "docker login docker.io"); + requireRunContains(errors, dockerHubAuth, "continuing with anonymous pulls"); + + const setupNode = namedStep(steps, "Set up Node"); + if (!setupNode) errors.push("state-backup-restore-vitest job missing step: Set up Node"); + requireFullShaAction(errors, setupNode, "state-backup-restore-vitest setup-node"); + + const installRootDependencies = requireJobStep(errors, jobName, steps, "Install root dependencies"); + requireRunContains(errors, installRootDependencies, "npm ci --ignore-scripts"); + + const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); + requireRunContains(errors, buildCli, "npm run build:cli"); + + const installOpenShell = requireJobStep(errors, jobName, steps, "Install OpenShell"); + requireRunContains(errors, installOpenShell, "bash scripts/install-openshell.sh"); + requireRunContains(errors, installOpenShell, "env -u DOCKER_CONFIG"); + requireRunContains(errors, installOpenShell, "-u DOCKERHUB_USERNAME"); + requireRunContains(errors, installOpenShell, "-u DOCKERHUB_TOKEN"); + requireRunContains(errors, installOpenShell, "-u NVIDIA_API_KEY"); + requireRunContains(errors, installOpenShell, "-u GITHUB_TOKEN"); + + const runVitest = requireJobStep(errors, jobName, steps, "Run state backup restore live test"); + const runVitestEnv = asRecord(runVitest?.env); + if (runVitestEnv.NVIDIA_API_KEY !== "${{ secrets.NVIDIA_API_KEY }}") { + errors.push("state-backup-restore-vitest step must receive NVIDIA_API_KEY from secrets"); + } + requireRunContains(errors, runVitest, "OPENSHELL_BIN"); + requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); + requireRunContains(errors, runVitest, "test/e2e-scenario/live/state-backup-restore.test.ts"); + + const upload = requireJobStep(errors, jobName, steps, "Upload state backup restore artifacts"); + requireFullShaAction(errors, upload, "state-backup-restore-vitest upload-artifact"); + const uploadWith = asRecord(upload?.with); + if (uploadWith.name !== "e2e-vitest-scenarios-state-backup-restore") { + errors.push("state-backup-restore-vitest artifact upload name must be stable"); + } + const uploadPath = stringValue(uploadWith.path); + requireUploadPathContains(errors, uploadPath, "e2e-artifacts/vitest/state-backup-restore/"); + if (uploadWith["include-hidden-files"] !== false) { + errors.push("state-backup-restore-vitest artifact upload must set include-hidden-files: false"); + } + if (uploadWith["if-no-files-found"] !== "ignore") { + errors.push("state-backup-restore-vitest artifact upload must ignore missing fixture artifacts"); + } + if (uploadWith["retention-days"] !== 14) { + errors.push("state-backup-restore-vitest artifact upload retention-days must be 14"); + } +} + function validateTokenRotationVitestJob(errors: string[], jobs: WorkflowRecord): void { const jobName = "token-rotation-vitest"; const job = asRecord(jobs[jobName]); @@ -2162,6 +2291,7 @@ export function validateE2eVitestScenariosWorkflowBoundary( validateShieldsConfigVitestJob(errors, jobs); validateRebuildOpenClawVitestJob(errors, jobs); validateSandboxRebuildVitestJob(errors, jobs); + validateStateBackupRestoreVitestJob(errors, jobs); validateTokenRotationVitestJob(errors, jobs); validateFreeStandingJobSelector( errors,