From 3e1ee55d371e6c68362d0ad3b2b05b7fddbbba0a Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 11 Jun 2026 09:05:40 -0400 Subject: [PATCH 01/11] test(e2e): add OpenClaw rebuild Vitest coverage --- .../live/rebuild-openclaw.test.ts | 635 ++++++++++++++++++ 1 file changed, 635 insertions(+) create mode 100644 test/e2e-scenario/live/rebuild-openclaw.test.ts diff --git a/test/e2e-scenario/live/rebuild-openclaw.test.ts b/test/e2e-scenario/live/rebuild-openclaw.test.ts new file mode 100644 index 00000000000..5a5f6e2561d --- /dev/null +++ b/test/e2e-scenario/live/rebuild-openclaw.test.ts @@ -0,0 +1,635 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; +import { expect, test } from "../fixtures/e2e-test.ts"; +import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; +import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; + +// Direct Vitest replacement coverage for test/e2e/test-rebuild-openclaw.sh. +// The contract stays intentionally local to this live test: build an older +// OpenClaw base image, create a sandbox from it through the real OpenShell CLI, +// seed workspace/policy/gateway-token state, run the real `nemoclaw rebuild`, +// and verify the rebuilt sandbox preserved state while rotating secrets. +// +// Simplicity boundary: no new registry, fixture family, or migration ledger. +// The legacy bash lane remains wired in nightly-e2e.yaml until #5098's cleanup +// phase intentionally retires converted shell entry points. + +const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); +const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); +const BLUEPRINT = path.join(REPO_ROOT, "nemoclaw-blueprint", "blueprint.yaml"); +const OLD_OPENCLAW_VERSION = "2026.3.11"; +const MARKER_FILE = "/sandbox/.openclaw/workspace/rebuild-marker.txt"; +const REGISTRY_FILE = path.join(os.homedir(), ".nemoclaw", "sandboxes.json"); +const SESSION_FILE = path.join(os.homedir(), ".nemoclaw", "onboard-session.json"); +const BACKUP_ROOT = path.join(os.homedir(), ".nemoclaw", "rebuild-backups"); +const DEFAULT_MODEL = "nvidia/nemotron-3-super-120b-a12b"; +const SANDBOX_NAME = + process.env.NEMOCLAW_SANDBOX_NAME ?? + ["e2e-rebuild-openclaw", process.env.GITHUB_RUN_ID, process.env.GITHUB_RUN_ATTEMPT] + .filter(Boolean) + .join("-"); +validateSandboxName(SANDBOX_NAME); + +const MARKER_CONTENT = `REBUILD_OC_E2E_${Date.now()}`; +const PRE_REBUILD_GATEWAY_TOKEN = `nemoclaw-e2e-old-gateway-token-${MARKER_CONTENT}`; +const OLD_BASE_TAG = `nemoclaw-old-base:${SANDBOX_NAME.toLowerCase().replace(/[^a-z0-9_.-]+/g, "-")}`; + +const ONBOARD_TIMEOUT_MS = 20 * 60_000; +const DOCKER_BUILD_TIMEOUT_MS = 35 * 60_000; +const REBUILD_TIMEOUT_MS = 30 * 60_000; +const OPENSHELL_TIMEOUT_MS = 2 * 60_000; + +interface SeedGatewayTokenResult { + seeded: boolean; + hashReferencesConfig: boolean; +} + +interface GatewayTokenRotationResult { + tokenPresent: boolean; + tokenRotated: boolean; + runtimeMatchesConfig: boolean; + runtimeStillOld: boolean; + hashReferencesConfig: boolean; + hashChanged: boolean; + hashValid: boolean; +} + +function resultText(result: ShellProbeResult): string { + return [result.stdout, result.stderr].filter(Boolean).join("\n"); +} + +function expectExitZero(result: ShellProbeResult, label: string): void { + expect(result.exitCode, `${label} failed:\n${resultText(result)}`).toBe(0); +} + +function readJsonFile(file: string, fallback: T): T { + if (!fs.existsSync(file)) return fallback; + return JSON.parse(fs.readFileSync(file, "utf8")) as T; +} + +function writeJsonFile(file: string, value: unknown): void { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`, "utf8"); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function dockerContextEnv(extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + ...extra, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1", + }; +} + +function cliEnv(apiKey: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return dockerContextEnv({ + NVIDIA_API_KEY: apiKey, + NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, + ...extra, + }); +} + +function updateBlueprintMinVersionForOldOpenClaw(): () => void { + const original = fs.readFileSync(BLUEPRINT, "utf8"); + const lowered = original.replace( + /min_openclaw_version:.*/, + `min_openclaw_version: "${OLD_OPENCLAW_VERSION}"`, + ); + expect(lowered, "blueprint min_openclaw_version line was not found").not.toBe(original); + fs.writeFileSync(BLUEPRINT, lowered, "utf8"); + return () => fs.writeFileSync(BLUEPRINT, original, "utf8"); +} + +async function waitForSandboxReady(sandbox: { + list(options?: object): Promise; +}): Promise { + for (let attempt = 0; attempt < 30; attempt += 1) { + const list = await sandbox.list({ + artifactName: `phase-3-sandbox-list-${attempt}`, + env: dockerContextEnv(), + timeoutMs: 30_000, + }); + if (new RegExp(`${SANDBOX_NAME}.*Ready`).test(list.stdout)) return; + await sleep(5_000); + } + throw new Error(`sandbox ${SANDBOX_NAME} did not become Ready`); +} + +function seedRegistryAndSession(): void { + const registry = { + sandboxes: { + [SANDBOX_NAME]: { + name: SANDBOX_NAME, + createdAt: new Date().toISOString(), + model: DEFAULT_MODEL, + provider: "nvidia-prod", + gpuEnabled: false, + policies: [], + policyTier: null, + agent: null, + agentVersion: OLD_OPENCLAW_VERSION, + }, + }, + defaultSandbox: SANDBOX_NAME, + }; + writeJsonFile(REGISTRY_FILE, registry); + + const now = new Date().toISOString(); + const complete = { status: "complete", startedAt: now, completedAt: now, error: null }; + const pending = { status: "pending", startedAt: null, completedAt: null, error: null }; + const session = readJsonFile>(SESSION_FILE, {}); + Object.assign(session, { + sandboxName: SANDBOX_NAME, + status: "complete", + resumable: true, + lastCompletedStep: "gateway", + failure: null, + provider: "nvidia-prod", + model: DEFAULT_MODEL, + credentialEnv: "NVIDIA_API_KEY", + agent: null, + steps: { + preflight: complete, + gateway: complete, + sandbox: pending, + provider_selection: pending, + inference: pending, + openclaw: pending, + agent_setup: pending, + policies: pending, + }, + }); + writeJsonFile(SESSION_FILE, session); +} + +function registrySandbox(): Record { + const data = readJsonFile<{ sandboxes?: Record> }>( + REGISTRY_FILE, + {}, + ); + const sandbox = data.sandboxes?.[SANDBOX_NAME]; + if (!sandbox) throw new Error(`registry entry missing for ${SANDBOX_NAME}`); + return sandbox; +} + +function latestRebuildManifest(): Record { + const sandboxBackupRoot = path.join(BACKUP_ROOT, SANDBOX_NAME); + expect(fs.existsSync(sandboxBackupRoot), `backup root missing: ${sandboxBackupRoot}`).toBe(true); + const latest = fs + .readdirSync(sandboxBackupRoot, { withFileTypes: true }) + .filter((entry) => entry.isDirectory()) + .map((entry) => entry.name) + .sort() + .at(-1); + expect(latest, `no timestamped backup directory under ${sandboxBackupRoot}`).toBeTruthy(); + const manifestPath = path.join(sandboxBackupRoot, latest!, "rebuild-manifest.json"); + expect(fs.existsSync(manifestPath), `backup manifest missing: ${manifestPath}`).toBe(true); + return JSON.parse(fs.readFileSync(manifestPath, "utf8")) as Record; +} + +function backupCredentialLeakPaths(oldGatewayToken: string): string[] { + const sandboxBackupRoot = path.join(BACKUP_ROOT, SANDBOX_NAME); + const leaks: string[] = []; + const skippedLockfiles = new Set([ + "package-lock.json", + "npm-shrinkwrap.json", + "yarn.lock", + "pnpm-lock.yaml", + "pnpm-lock.yml", + ]); + const candidatePattern = /(?:nvapi-|sk-|Bearer )/; + + function scan(dir: string): void { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + scan(fullPath); + continue; + } + if (!entry.isFile()) continue; + if (skippedLockfiles.has(entry.name)) continue; + if (!/\.json$|\.env$|^\.env$/i.test(entry.name)) continue; + const text = fs.readFileSync(fullPath, "utf8"); + if (candidatePattern.test(text) || text.includes(oldGatewayToken)) { + leaks.push(fullPath); + } + } + } + + if (fs.existsSync(sandboxBackupRoot)) scan(sandboxBackupRoot); + return leaks; +} + +// Gate this live test on NEMOCLAW_RUN_E2E_SCENARIOS=1. Accidental cli-test-shard +// discovery must not build Docker images, mutate ~/.nemoclaw, or call NVIDIA. +test.skipIf(!shouldRunLiveE2EScenarios())( + "rebuild-openclaw: old OpenClaw sandbox rebuild preserves state and rotates gateway token", + async ({ artifacts, cleanup, host, sandbox, secrets, skip }) => { + const apiKey = secrets.required("NVIDIA_API_KEY"); + expect(apiKey.startsWith("nvapi-"), "NVIDIA_API_KEY must start with nvapi-").toBe(true); + + expect( + fs.existsSync(CLI_ENTRYPOINT), + "bin/nemoclaw.js missing — run npm ci && npm run build:cli before live rebuild coverage", + ).toBe(true); + + const dockerInfo = await host.command("docker", ["info"], { + artifactName: "prereq-docker-info", + env: dockerContextEnv(), + timeoutMs: 30_000, + }); + if (dockerInfo.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error( + `Docker is required for rebuild-openclaw live coverage:\n${resultText(dockerInfo)}`, + ); + } + skip("Docker is required for rebuild-openclaw live coverage"); + } + + const openshellVersion = await host.command("openshell", ["--version"], { + artifactName: "prereq-openshell-version", + env: dockerContextEnv(), + timeoutMs: 30_000, + }); + if (openshellVersion.exitCode !== 0) { + if (process.env.GITHUB_ACTIONS === "true") { + throw new Error( + `OpenShell is required for rebuild-openclaw live coverage:\n${resultText(openshellVersion)}`, + ); + } + skip("OpenShell is required for rebuild-openclaw live coverage"); + } + + await artifacts.writeJson("contract.json", { + legacySource: "test/e2e/test-rebuild-openclaw.sh", + oldOpenClawVersion: OLD_OPENCLAW_VERSION, + sandboxName: SANDBOX_NAME, + markerFile: MARKER_FILE, + oldBaseTag: OLD_BASE_TAG, + preservedBoundaries: [ + "docker build Dockerfile.base with old OPENCLAW_VERSION", + "openshell sandbox create/exec/policy", + "real nemoclaw onboard and rebuild CLI", + "workspace marker, registry/session files, backup manifest, config hash", + ], + }); + + // Pre-clean any stale resources before registering final cleanup. These are + // best-effort and intentionally run after prereq/secret checks so a skipped + // test does not mutate local state. + await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "pre-cleanup-nemoclaw-destroy", + env: cliEnv(apiKey), + redactionValues: [apiKey], + timeoutMs: 2 * 60_000, + }); + await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "pre-cleanup-openshell-sandbox-delete", + env: dockerContextEnv(), + timeoutMs: OPENSHELL_TIMEOUT_MS, + }); + await sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: "pre-cleanup-openshell-gateway-destroy", + env: dockerContextEnv(), + timeoutMs: OPENSHELL_TIMEOUT_MS, + }); + await host.command("docker", ["rmi", OLD_BASE_TAG], { + artifactName: "pre-cleanup-docker-rmi-old-base", + env: dockerContextEnv(), + timeoutMs: OPENSHELL_TIMEOUT_MS, + }); + + cleanup.add(`destroy rebuilt sandbox ${SANDBOX_NAME}`, async () => { + await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { + artifactName: "cleanup-nemoclaw-destroy", + env: cliEnv(apiKey), + redactionValues: [apiKey], + timeoutMs: 2 * 60_000, + }); + await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "cleanup-openshell-sandbox-delete", + env: dockerContextEnv(), + timeoutMs: OPENSHELL_TIMEOUT_MS, + }); + await host.command("docker", ["rmi", OLD_BASE_TAG], { + artifactName: "cleanup-docker-rmi-old-base", + env: dockerContextEnv(), + timeoutMs: OPENSHELL_TIMEOUT_MS, + }); + }); + + // Phase 1: create a normal current sandbox first so the real gateway and + // session/credential scaffolding exist, matching the legacy install/onboard + // setup before it swaps in an old OpenClaw sandbox. + const onboard = await host.command("node", [CLI_ENTRYPOINT, "onboard", "--non-interactive"], { + artifactName: "phase-1-onboard-current", + env: cliEnv(apiKey, { NEMOCLAW_RECREATE_SANDBOX: "1" }), + redactionValues: [apiKey], + timeoutMs: ONBOARD_TIMEOUT_MS, + }); + expectExitZero(onboard, "initial current onboard"); + + const deleteCurrentSandbox = await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { + artifactName: "phase-1-delete-current-sandbox", + env: dockerContextEnv(), + timeoutMs: OPENSHELL_TIMEOUT_MS, + }); + expectExitZero(deleteCurrentSandbox, "openshell sandbox delete current sandbox"); + + // Phase 2: build the old base image while temporarily lowering the + // blueprint minimum-version gate, then restore the checkout file. + let restoreBlueprint: (() => void) | undefined; + try { + restoreBlueprint = updateBlueprintMinVersionForOldOpenClaw(); + const buildOldBase = await host.command( + "docker", + [ + "build", + "--build-arg", + `OPENCLAW_VERSION=${OLD_OPENCLAW_VERSION}`, + "-f", + path.join(REPO_ROOT, "Dockerfile.base"), + "-t", + OLD_BASE_TAG, + REPO_ROOT, + ], + { + artifactName: "phase-2-docker-build-old-openclaw-base", + env: dockerContextEnv(), + timeoutMs: DOCKER_BUILD_TIMEOUT_MS, + }, + ); + expectExitZero(buildOldBase, `docker build old OpenClaw ${OLD_OPENCLAW_VERSION}`); + } finally { + restoreBlueprint?.(); + } + + // Phase 3: create an OpenShell sandbox from the old base image. + const oldDockerfileDir = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-rebuild-openclaw-")); + const oldDockerfile = path.join(oldDockerfileDir, "Dockerfile"); + fs.writeFileSync( + oldDockerfile, + [ + `FROM ${OLD_BASE_TAG}`, + "USER sandbox", + "WORKDIR /sandbox", + "RUN mkdir -p /sandbox/.openclaw/workspace /sandbox/.openclaw && echo '{}' > /sandbox/.openclaw/openclaw.json", + '["/bin/bash"]', + ] + .map((line, index) => (index === 4 ? `CMD ${line}` : line)) + .join("\n"), + "utf8", + ); + try { + const createOldSandbox = await sandbox.openshell( + [ + "sandbox", + "create", + "--name", + SANDBOX_NAME, + "--from", + oldDockerfile, + "--gateway", + "nemoclaw", + "--no-tty", + "--", + "true", + ], + { + artifactName: "phase-3-create-old-openclaw-sandbox", + env: dockerContextEnv(), + timeoutMs: 10 * 60_000, + }, + ); + expectExitZero(createOldSandbox, "openshell sandbox create old OpenClaw sandbox"); + } finally { + fs.rmSync(oldDockerfileDir, { recursive: true, force: true }); + } + await waitForSandboxReady(sandbox); + + const oldVersion = await sandbox.exec(SANDBOX_NAME, ["openclaw", "--version"], { + artifactName: "phase-3-openclaw-old-version", + env: dockerContextEnv(), + timeoutMs: 30_000, + }); + expectExitZero(oldVersion, "old openclaw --version"); + expect(resultText(oldVersion)).toContain(OLD_OPENCLAW_VERSION); + + // Phase 4: seed workspace state, an existing gateway token, and registry / + // resume-session state so `nemoclaw rebuild --yes` drives the same + // user-visible rebuild path as the legacy script. + const markerWrite = await sandbox.exec( + SANDBOX_NAME, + [ + "sh", + "-c", + `mkdir -p /sandbox/.openclaw/workspace && printf '%s' '${MARKER_CONTENT}' > ${MARKER_FILE}`, + ], + { + artifactName: "phase-4-write-workspace-marker", + env: dockerContextEnv(), + timeoutMs: 30_000, + }, + ); + expectExitZero(markerWrite, "write workspace marker"); + + const seedGateway = await sandbox.exec( + SANDBOX_NAME, + [ + "env", + `PRE_REBUILD_GATEWAY_TOKEN=${PRE_REBUILD_GATEWAY_TOKEN}`, + "python3", + "-c", + `import json, os, subprocess\npath='/sandbox/.openclaw/openclaw.json'\ntry:\n cfg=json.load(open(path))\nexcept Exception:\n cfg={}\ncfg.setdefault('gateway', {}).setdefault('auth', {})['token']=os.environ['PRE_REBUILD_GATEWAY_TOKEN']\nwith open(path, 'w') as f:\n json.dump(cfg, f, indent=2)\n f.write('\\n')\nsubprocess.check_call(['bash','-lc','cd /sandbox/.openclaw && sha256sum openclaw.json > .config-hash'])\nsaved=json.load(open(path)).get('gateway',{}).get('auth',{}).get('token','')\nhash_text=open('/sandbox/.openclaw/.config-hash').read()\nprint(json.dumps({'seeded': saved == os.environ['PRE_REBUILD_GATEWAY_TOKEN'], 'hashReferencesConfig': 'openclaw.json' in hash_text}))`, + ], + { + artifactName: "phase-4-seed-gateway-token", + env: dockerContextEnv(), + redactionValues: [PRE_REBUILD_GATEWAY_TOKEN], + timeoutMs: 30_000, + }, + ); + expectExitZero(seedGateway, "seed old gateway token"); + const seedResult = JSON.parse(seedGateway.stdout.trim()) as SeedGatewayTokenResult; + expect(seedResult).toEqual({ seeded: true, hashReferencesConfig: true }); + + const preHashResult = await sandbox.exec( + SANDBOX_NAME, + ["cat", "/sandbox/.openclaw/.config-hash"], + { + artifactName: "phase-4-read-pre-rebuild-config-hash", + env: dockerContextEnv(), + timeoutMs: 30_000, + }, + ); + expectExitZero(preHashResult, "read pre-rebuild config hash"); + const preRebuildConfigHash = preHashResult.stdout.trim(); + expect(preRebuildConfigHash).toContain("openclaw.json"); + + seedRegistryAndSession(); + await artifacts.writeJson("phase-4-registry-session-summary.json", { + registry: registrySandbox(), + session: readJsonFile>(SESSION_FILE, {}), + }); + + // Phase 4.5: apply policy presets through the public CLI, then verify both + // registry persistence and the live OpenShell gateway policy. + for (const preset of ["npm", "pypi"]) { + const policyAdd = await host.command( + "node", + [CLI_ENTRYPOINT, "sandbox", "policy", "add", SANDBOX_NAME, preset, "--yes"], + { + artifactName: `phase-4-policy-add-${preset}`, + env: cliEnv(apiKey), + redactionValues: [apiKey], + timeoutMs: OPENSHELL_TIMEOUT_MS, + }, + ); + expectExitZero(policyAdd, `policy add ${preset}`); + } + + const prePolicy = await sandbox.openshell(["policy", "get", "--full", SANDBOX_NAME], { + artifactName: "phase-4-live-policy-before-rebuild", + env: dockerContextEnv(), + timeoutMs: OPENSHELL_TIMEOUT_MS, + }); + expectExitZero(prePolicy, "openshell policy get before rebuild"); + expect(prePolicy.stdout).toMatch(/npm|registry\.npmjs\.org/i); + expect(prePolicy.stdout).toMatch(/pypi|pypi\.org/i); + expect(registrySandbox().policies).toEqual(expect.arrayContaining(["npm", "pypi"])); + + // Phase 5: restore the current base image tag that rebuild consumes. + const buildCurrentBase = await host.command( + "docker", + [ + "build", + "-f", + path.join(REPO_ROOT, "Dockerfile.base"), + "-t", + "ghcr.io/nvidia/nemoclaw/sandbox-base:latest", + REPO_ROOT, + ], + { + artifactName: "phase-5-docker-build-current-base", + env: dockerContextEnv(), + timeoutMs: DOCKER_BUILD_TIMEOUT_MS, + }, + ); + expectExitZero(buildCurrentBase, "docker build current base image"); + + // Phase 6: run the real rebuild CLI. + const rebuild = await host.command( + "node", + [CLI_ENTRYPOINT, SANDBOX_NAME, "rebuild", "--yes", "--verbose"], + { + artifactName: "phase-6-nemoclaw-rebuild", + env: cliEnv(apiKey, { NEMOCLAW_REBUILD_VERBOSE: "1" }), + redactionValues: [apiKey, PRE_REBUILD_GATEWAY_TOKEN], + timeoutMs: REBUILD_TIMEOUT_MS, + }, + ); + expectExitZero(rebuild, "nemoclaw rebuild"); + + // Phase 7: state preservation, upgrade, token rotation, backup hygiene, and + // policy-preset preservation assertions. + const markerRead = await sandbox.exec(SANDBOX_NAME, ["cat", MARKER_FILE], { + artifactName: "phase-7-read-workspace-marker", + env: dockerContextEnv(), + timeoutMs: 30_000, + }); + expectExitZero(markerRead, "read workspace marker after rebuild"); + expect(markerRead.stdout).toBe(MARKER_CONTENT); + + const newVersion = await sandbox.exec(SANDBOX_NAME, ["openclaw", "--version"], { + artifactName: "phase-7-openclaw-new-version", + env: dockerContextEnv(), + timeoutMs: 30_000, + }); + expectExitZero(newVersion, "new openclaw --version"); + expect(resultText(newVersion)).not.toContain(OLD_OPENCLAW_VERSION); + expect(resultText(newVersion).trim()).not.toBe(""); + + const registryVersion = registrySandbox().agentVersion; + expect(registryVersion).not.toBe(OLD_OPENCLAW_VERSION); + expect(registryVersion).toEqual(expect.any(String)); + + const tokenCheck = await sandbox.exec( + SANDBOX_NAME, + [ + "env", + `PRE_REBUILD_GATEWAY_TOKEN=${PRE_REBUILD_GATEWAY_TOKEN}`, + `PRE_REBUILD_CONFIG_HASH=${preRebuildConfigHash}`, + "python3", + "-c", + `import json, os, subprocess\ncfg=json.load(open('/sandbox/.openclaw/openclaw.json'))\ntoken=cfg.get('gateway',{}).get('auth',{}).get('token','')\nruntime=subprocess.check_output(['bash','-lc','. /tmp/nemoclaw-proxy-env.sh >/dev/null 2>&1 || exit 1; printf "%s" "\${OPENCLAW_GATEWAY_TOKEN:-}"'], text=True)\nhash_text=open('/sandbox/.openclaw/.config-hash').read()\nhash_ok=subprocess.call(['bash','-lc','cd /sandbox/.openclaw && sha256sum -c .config-hash --status']) == 0\nold=os.environ['PRE_REBUILD_GATEWAY_TOKEN']\nprint(json.dumps({'tokenPresent': bool(token), 'tokenRotated': token != old, 'runtimeMatchesConfig': runtime == token, 'runtimeStillOld': runtime == old, 'hashReferencesConfig': 'openclaw.json' in hash_text, 'hashChanged': hash_text != os.environ['PRE_REBUILD_CONFIG_HASH'], 'hashValid': hash_ok}))`, + ], + { + artifactName: "phase-7-gateway-token-rotation-check", + env: dockerContextEnv(), + redactionValues: [PRE_REBUILD_GATEWAY_TOKEN], + timeoutMs: 30_000, + }, + ); + expectExitZero(tokenCheck, "gateway token rotation check"); + const tokenResult = JSON.parse(tokenCheck.stdout.trim()) as GatewayTokenRotationResult; + expect(tokenResult).toEqual({ + tokenPresent: true, + tokenRotated: true, + runtimeMatchesConfig: true, + runtimeStillOld: false, + hashReferencesConfig: true, + hashChanged: true, + hashValid: true, + }); + + const manifest = latestRebuildManifest(); + await artifacts.writeJson("phase-7-rebuild-manifest-summary.json", manifest); + expect(manifest.policyPresets).toEqual(expect.arrayContaining(["npm", "pypi"])); + expect(backupCredentialLeakPaths(PRE_REBUILD_GATEWAY_TOKEN)).toEqual([]); + + expect(registrySandbox().policies).toEqual(expect.arrayContaining(["npm", "pypi"])); + const postPolicy = await sandbox.openshell(["policy", "get", "--full", SANDBOX_NAME], { + artifactName: "phase-7-live-policy-after-rebuild", + env: dockerContextEnv(), + timeoutMs: OPENSHELL_TIMEOUT_MS, + }); + expectExitZero(postPolicy, "openshell policy get after rebuild"); + expect(postPolicy.stdout).toMatch(/npm|registry\.npmjs\.org/i); + expect(postPolicy.stdout).toMatch(/pypi|pypi\.org/i); + + // External API availability can make this inconclusive; keep it as a + // non-fatal artifact-producing probe like the legacy script did. + await sandbox.exec( + SANDBOX_NAME, + [ + "curl", + "-s", + "--max-time", + "60", + "https://inference.local/v1/chat/completions", + "-H", + "Content-Type: application/json", + "-d", + '{"model":"nvidia/nemotron-3-super-120b-a12b","messages":[{"role":"user","content":"Reply with exactly one word: PONG"}],"max_tokens":100}', + ], + { + artifactName: "phase-7-inference-after-rebuild-nonfatal", + env: dockerContextEnv(), + timeoutMs: 75_000, + }, + ); + }, + REBUILD_TIMEOUT_MS + 2 * DOCKER_BUILD_TIMEOUT_MS + ONBOARD_TIMEOUT_MS, +); From ac90914e2512148e3a35ac67384abbac49f1802d Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 11 Jun 2026 09:28:02 -0400 Subject: [PATCH 02/11] test(e2e): harden rebuild OpenClaw Vitest guard --- .github/workflows/e2e-vitest-scenarios.yaml | 73 +++++++++ .../live/rebuild-openclaw.test.ts | 150 +++++++++++++----- 2 files changed, 181 insertions(+), 42 deletions(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 07bb8c54f54..3f27f58a711 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -322,3 +322,76 @@ jobs: include-hidden-files: false if-no-files-found: ignore retention-days: 14 + + # Focused live migration of test/e2e/test-rebuild-openclaw.sh. This remains + # a standalone job because it builds old/current base images and mutates a + # real OpenShell sandbox rather than fitting the steady-state registry probe. + rebuild-openclaw-vitest: + if: ${{ inputs.scenarios == '' }} + runs-on: ubuntu-latest + timeout-minutes: 130 + env: + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/rebuild-openclaw + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Authenticate to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + login_succeeded=0 + for attempt in 1 2 3; do + if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then + login_succeeded=1 + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." + sleep 5 + fi + done + if [[ "$login_succeeded" -ne 1 ]]; then + echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." + fi + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Run OpenClaw rebuild live test + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/rebuild-openclaw.test.ts \ + --silent=false --reporter=default + + - name: Upload OpenClaw rebuild artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-rebuild-openclaw + path: e2e-artifacts/vitest/rebuild-openclaw/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 diff --git a/test/e2e-scenario/live/rebuild-openclaw.test.ts b/test/e2e-scenario/live/rebuild-openclaw.test.ts index 5a5f6e2561d..a59811033d3 100644 --- a/test/e2e-scenario/live/rebuild-openclaw.test.ts +++ b/test/e2e-scenario/live/rebuild-openclaw.test.ts @@ -23,7 +23,8 @@ import type { ShellProbeResult } from "../fixtures/shell-probe.ts"; const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const CLI_ENTRYPOINT = path.join(REPO_ROOT, "bin", "nemoclaw.js"); -const BLUEPRINT = path.join(REPO_ROOT, "nemoclaw-blueprint", "blueprint.yaml"); +const BLUEPRINT_RELPATH = path.join("nemoclaw-blueprint", "blueprint.yaml"); +const BLUEPRINT = path.join(REPO_ROOT, BLUEPRINT_RELPATH); const OLD_OPENCLAW_VERSION = "2026.3.11"; const MARKER_FILE = "/sandbox/.openclaw/workspace/rebuild-marker.txt"; const REGISTRY_FILE = path.join(os.homedir(), ".nemoclaw", "sandboxes.json"); @@ -32,7 +33,7 @@ const BACKUP_ROOT = path.join(os.homedir(), ".nemoclaw", "rebuild-backups"); const DEFAULT_MODEL = "nvidia/nemotron-3-super-120b-a12b"; const SANDBOX_NAME = process.env.NEMOCLAW_SANDBOX_NAME ?? - ["e2e-rebuild-openclaw", process.env.GITHUB_RUN_ID, process.env.GITHUB_RUN_ATTEMPT] + ["e2e-rebuild-openclaw", process.env.GITHUB_RUN_ID, process.env.GITHUB_RUN_ATTEMPT, process.pid] .filter(Boolean) .join("-"); validateSandboxName(SANDBOX_NAME); @@ -79,6 +80,26 @@ function writeJsonFile(file: string, value: unknown): void { fs.writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`, "utf8"); } +interface FileSnapshot { + exists: boolean; + content?: string; +} + +function snapshotFile(file: string): FileSnapshot { + return fs.existsSync(file) + ? { exists: true, content: fs.readFileSync(file, "utf8") } + : { exists: false }; +} + +function restoreFile(file: string, snapshot: FileSnapshot): void { + if (!snapshot.exists) { + fs.rmSync(file, { force: true }); + return; + } + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, snapshot.content ?? "", "utf8"); +} + function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -100,15 +121,17 @@ function cliEnv(apiKey: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEn }); } -function updateBlueprintMinVersionForOldOpenClaw(): () => void { +function createOldBaseBuildContext(): string { + const buildContext = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-rebuild-openclaw-base-")); + fs.mkdirSync(path.join(buildContext, path.dirname(BLUEPRINT_RELPATH)), { recursive: true }); const original = fs.readFileSync(BLUEPRINT, "utf8"); const lowered = original.replace( /min_openclaw_version:.*/, `min_openclaw_version: "${OLD_OPENCLAW_VERSION}"`, ); expect(lowered, "blueprint min_openclaw_version line was not found").not.toBe(original); - fs.writeFileSync(BLUEPRINT, lowered, "utf8"); - return () => fs.writeFileSync(BLUEPRINT, original, "utf8"); + fs.writeFileSync(path.join(buildContext, BLUEPRINT_RELPATH), lowered, "utf8"); + return buildContext; } async function waitForSandboxReady(sandbox: { @@ -127,22 +150,23 @@ async function waitForSandboxReady(sandbox: { } function seedRegistryAndSession(): void { - const registry = { - sandboxes: { - [SANDBOX_NAME]: { - name: SANDBOX_NAME, - createdAt: new Date().toISOString(), - model: DEFAULT_MODEL, - provider: "nvidia-prod", - gpuEnabled: false, - policies: [], - policyTier: null, - agent: null, - agentVersion: OLD_OPENCLAW_VERSION, - }, - }, - defaultSandbox: SANDBOX_NAME, + const registry = readJsonFile<{ + sandboxes?: Record>; + defaultSandbox?: string; + }>(REGISTRY_FILE, {}); + registry.sandboxes = registry.sandboxes ?? {}; + registry.sandboxes[SANDBOX_NAME] = { + name: SANDBOX_NAME, + createdAt: new Date().toISOString(), + model: DEFAULT_MODEL, + provider: "nvidia-prod", + gpuEnabled: false, + policies: [], + policyTier: null, + agent: null, + agentVersion: OLD_OPENCLAW_VERSION, }; + registry.defaultSandbox = SANDBOX_NAME; writeJsonFile(REGISTRY_FILE, registry); const now = new Date().toISOString(); @@ -183,7 +207,7 @@ function registrySandbox(): Record { return sandbox; } -function latestRebuildManifest(): Record { +function latestRebuildBackupDir(): string { const sandboxBackupRoot = path.join(BACKUP_ROOT, SANDBOX_NAME); expect(fs.existsSync(sandboxBackupRoot), `backup root missing: ${sandboxBackupRoot}`).toBe(true); const latest = fs @@ -193,13 +217,16 @@ function latestRebuildManifest(): Record { .sort() .at(-1); expect(latest, `no timestamped backup directory under ${sandboxBackupRoot}`).toBeTruthy(); - const manifestPath = path.join(sandboxBackupRoot, latest!, "rebuild-manifest.json"); + return path.join(sandboxBackupRoot, latest!); +} + +function latestRebuildManifest(backupDir: string): Record { + const manifestPath = path.join(backupDir, "rebuild-manifest.json"); expect(fs.existsSync(manifestPath), `backup manifest missing: ${manifestPath}`).toBe(true); return JSON.parse(fs.readFileSync(manifestPath, "utf8")) as Record; } -function backupCredentialLeakPaths(oldGatewayToken: string): string[] { - const sandboxBackupRoot = path.join(BACKUP_ROOT, SANDBOX_NAME); +function backupCredentialLeakPaths(backupDir: string, oldGatewayToken: string): string[] { const leaks: string[] = []; const skippedLockfiles = new Set([ "package-lock.json", @@ -227,7 +254,7 @@ function backupCredentialLeakPaths(oldGatewayToken: string): string[] { } } - if (fs.existsSync(sandboxBackupRoot)) scan(sandboxBackupRoot); + if (fs.existsSync(backupDir)) scan(backupDir); return leaks; } @@ -311,18 +338,36 @@ test.skipIf(!shouldRunLiveE2EScenarios())( timeoutMs: OPENSHELL_TIMEOUT_MS, }); + const registrySnapshot = snapshotFile(REGISTRY_FILE); + const sessionSnapshot = snapshotFile(SESSION_FILE); + const sandboxBackupRoot = path.join(BACKUP_ROOT, SANDBOX_NAME); + cleanup.add(`restore NemoClaw state files for ${SANDBOX_NAME}`, () => { + restoreFile(REGISTRY_FILE, registrySnapshot); + restoreFile(SESSION_FILE, sessionSnapshot); + fs.rmSync(sandboxBackupRoot, { recursive: true, force: true }); + }); + cleanup.add(`destroy rebuilt sandbox ${SANDBOX_NAME}`, async () => { - await host.command("node", [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes"], { - artifactName: "cleanup-nemoclaw-destroy", - env: cliEnv(apiKey), - redactionValues: [apiKey], - timeoutMs: 2 * 60_000, - }); + await host.command( + "node", + [CLI_ENTRYPOINT, SANDBOX_NAME, "destroy", "--yes", "--cleanup-gateway"], + { + artifactName: "cleanup-nemoclaw-destroy", + env: cliEnv(apiKey), + redactionValues: [apiKey], + timeoutMs: 2 * 60_000, + }, + ); await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { artifactName: "cleanup-openshell-sandbox-delete", env: dockerContextEnv(), timeoutMs: OPENSHELL_TIMEOUT_MS, }); + await sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { + artifactName: "cleanup-openshell-gateway-destroy", + env: dockerContextEnv(), + timeoutMs: OPENSHELL_TIMEOUT_MS, + }); await host.command("docker", ["rmi", OLD_BASE_TAG], { artifactName: "cleanup-docker-rmi-old-base", env: dockerContextEnv(), @@ -348,11 +393,11 @@ test.skipIf(!shouldRunLiveE2EScenarios())( }); expectExitZero(deleteCurrentSandbox, "openshell sandbox delete current sandbox"); - // Phase 2: build the old base image while temporarily lowering the - // blueprint minimum-version gate, then restore the checkout file. - let restoreBlueprint: (() => void) | undefined; + // Phase 2: build the old base image with a temporary build context that + // lowers only the blueprint minimum-version gate consumed by Dockerfile.base. + // The trusted checkout stays read-only. + const oldBaseBuildContext = createOldBaseBuildContext(); try { - restoreBlueprint = updateBlueprintMinVersionForOldOpenClaw(); const buildOldBase = await host.command( "docker", [ @@ -363,7 +408,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())( path.join(REPO_ROOT, "Dockerfile.base"), "-t", OLD_BASE_TAG, - REPO_ROOT, + oldBaseBuildContext, ], { artifactName: "phase-2-docker-build-old-openclaw-base", @@ -373,7 +418,7 @@ test.skipIf(!shouldRunLiveE2EScenarios())( ); expectExitZero(buildOldBase, `docker build old OpenClaw ${OLD_OPENCLAW_VERSION}`); } finally { - restoreBlueprint?.(); + fs.rmSync(oldBaseBuildContext, { recursive: true, force: true }); } // Phase 3: create an OpenShell sandbox from the old base image. @@ -479,9 +524,25 @@ test.skipIf(!shouldRunLiveE2EScenarios())( expect(preRebuildConfigHash).toContain("openclaw.json"); seedRegistryAndSession(); + const sessionAfterSeed = readJsonFile>(SESSION_FILE, {}); + const seededSteps = sessionAfterSeed.steps as Record | undefined; + const seededSandbox = registrySandbox(); await artifacts.writeJson("phase-4-registry-session-summary.json", { - registry: registrySandbox(), - session: readJsonFile>(SESSION_FILE, {}), + registry: { + name: seededSandbox.name, + provider: seededSandbox.provider, + agentVersion: seededSandbox.agentVersion, + policyCount: Array.isArray(seededSandbox.policies) ? seededSandbox.policies.length : 0, + }, + session: { + sandboxName: sessionAfterSeed.sandboxName, + status: sessionAfterSeed.status, + provider: sessionAfterSeed.provider, + model: sessionAfterSeed.model, + stepStatuses: Object.fromEntries( + Object.entries(seededSteps ?? {}).map(([step, value]) => [step, value.status]), + ), + }, }); // Phase 4.5: apply policy presets through the public CLI, then verify both @@ -594,10 +655,15 @@ test.skipIf(!shouldRunLiveE2EScenarios())( hashValid: true, }); - const manifest = latestRebuildManifest(); - await artifacts.writeJson("phase-7-rebuild-manifest-summary.json", manifest); + const backupDir = latestRebuildBackupDir(); + const manifest = latestRebuildManifest(backupDir); + await artifacts.writeJson("phase-7-rebuild-manifest-summary.json", { + backupDir, + stateDirCount: Array.isArray(manifest.stateDirs) ? manifest.stateDirs.length : undefined, + policyPresets: manifest.policyPresets, + }); expect(manifest.policyPresets).toEqual(expect.arrayContaining(["npm", "pypi"])); - expect(backupCredentialLeakPaths(PRE_REBUILD_GATEWAY_TOKEN)).toEqual([]); + expect(backupCredentialLeakPaths(backupDir, PRE_REBUILD_GATEWAY_TOKEN)).toEqual([]); expect(registrySandbox().policies).toEqual(expect.arrayContaining(["npm", "pypi"])); const postPolicy = await sandbox.openshell(["policy", "get", "--full", SANDBOX_NAME], { From 22795393cfd9af4bc82792180c472cfff03b3cc4 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 11 Jun 2026 09:33:26 -0400 Subject: [PATCH 03/11] test(e2e): make rebuild cleanup tolerate missing OpenShell --- .../live/rebuild-openclaw.test.ts | 76 ++++++++++--------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/test/e2e-scenario/live/rebuild-openclaw.test.ts b/test/e2e-scenario/live/rebuild-openclaw.test.ts index a59811033d3..4047ffebf89 100644 --- a/test/e2e-scenario/live/rebuild-openclaw.test.ts +++ b/test/e2e-scenario/live/rebuild-openclaw.test.ts @@ -6,6 +6,7 @@ import os from "node:os"; import path from "node:path"; import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts"; +import type { HostCliClient } from "../fixtures/clients/host.ts"; import { validateSandboxName } from "../fixtures/clients/sandbox.ts"; import { expect, test } from "../fixtures/e2e-test.ts"; import { shouldRunLiveE2EScenarios } from "../fixtures/live-project-gate.ts"; @@ -121,6 +122,27 @@ function cliEnv(apiKey: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEn }); } +function shellQuote(value: string): string { + return `'${value.replace(/'/g, `'"'"'`)}'`; +} + +function openshellBestEffort( + host: HostCliClient, + args: string[], + artifactName: string, +): Promise { + const quotedArgs = args.map(shellQuote).join(" "); + return host.command( + "bash", + ["-lc", `command -v openshell >/dev/null 2>&1 && openshell ${quotedArgs} || true`], + { + artifactName, + env: dockerContextEnv(), + timeoutMs: OPENSHELL_TIMEOUT_MS, + }, + ); +} + function createOldBaseBuildContext(): string { const buildContext = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-rebuild-openclaw-base-")); fs.mkdirSync(path.join(buildContext, path.dirname(BLUEPRINT_RELPATH)), { recursive: true }); @@ -285,20 +307,6 @@ test.skipIf(!shouldRunLiveE2EScenarios())( skip("Docker is required for rebuild-openclaw live coverage"); } - const openshellVersion = await host.command("openshell", ["--version"], { - artifactName: "prereq-openshell-version", - env: dockerContextEnv(), - timeoutMs: 30_000, - }); - if (openshellVersion.exitCode !== 0) { - if (process.env.GITHUB_ACTIONS === "true") { - throw new Error( - `OpenShell is required for rebuild-openclaw live coverage:\n${resultText(openshellVersion)}`, - ); - } - skip("OpenShell is required for rebuild-openclaw live coverage"); - } - await artifacts.writeJson("contract.json", { legacySource: "test/e2e/test-rebuild-openclaw.sh", oldOpenClawVersion: OLD_OPENCLAW_VERSION, @@ -322,16 +330,16 @@ test.skipIf(!shouldRunLiveE2EScenarios())( redactionValues: [apiKey], timeoutMs: 2 * 60_000, }); - await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { - artifactName: "pre-cleanup-openshell-sandbox-delete", - env: dockerContextEnv(), - timeoutMs: OPENSHELL_TIMEOUT_MS, - }); - await sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { - artifactName: "pre-cleanup-openshell-gateway-destroy", - env: dockerContextEnv(), - timeoutMs: OPENSHELL_TIMEOUT_MS, - }); + await openshellBestEffort( + host, + ["sandbox", "delete", SANDBOX_NAME], + "pre-cleanup-openshell-sandbox-delete", + ); + await openshellBestEffort( + host, + ["gateway", "destroy", "-g", "nemoclaw"], + "pre-cleanup-openshell-gateway-destroy", + ); await host.command("docker", ["rmi", OLD_BASE_TAG], { artifactName: "pre-cleanup-docker-rmi-old-base", env: dockerContextEnv(), @@ -358,16 +366,16 @@ test.skipIf(!shouldRunLiveE2EScenarios())( timeoutMs: 2 * 60_000, }, ); - await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { - artifactName: "cleanup-openshell-sandbox-delete", - env: dockerContextEnv(), - timeoutMs: OPENSHELL_TIMEOUT_MS, - }); - await sandbox.openshell(["gateway", "destroy", "-g", "nemoclaw"], { - artifactName: "cleanup-openshell-gateway-destroy", - env: dockerContextEnv(), - timeoutMs: OPENSHELL_TIMEOUT_MS, - }); + await openshellBestEffort( + host, + ["sandbox", "delete", SANDBOX_NAME], + "cleanup-openshell-sandbox-delete", + ); + await openshellBestEffort( + host, + ["gateway", "destroy", "-g", "nemoclaw"], + "cleanup-openshell-gateway-destroy", + ); await host.command("docker", ["rmi", OLD_BASE_TAG], { artifactName: "cleanup-docker-rmi-old-base", env: dockerContextEnv(), From 1cf016eb137cebcaa4637d53ebe2d35f3bdfd45d Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 11 Jun 2026 09:41:57 -0400 Subject: [PATCH 04/11] test(e2e): tolerate transient rebuild onboard validation --- .../live/rebuild-openclaw.test.ts | 37 +++++++++++++++---- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/test/e2e-scenario/live/rebuild-openclaw.test.ts b/test/e2e-scenario/live/rebuild-openclaw.test.ts index 4047ffebf89..a57091e51bd 100644 --- a/test/e2e-scenario/live/rebuild-openclaw.test.ts +++ b/test/e2e-scenario/live/rebuild-openclaw.test.ts @@ -71,6 +71,16 @@ function expectExitZero(result: ShellProbeResult, label: string): void { expect(result.exitCode, `${label} failed:\n${resultText(result)}`).toBe(0); } +function isRetryableOnboardEndpointFailure(result: ShellProbeResult): boolean { + const text = resultText(result); + return ( + /endpoint validation failed|Chat Completions API validation/i.test(text) && + /HTTP 429|timed? out|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN|ENOTFOUND|502|503|504|temporar/i.test( + text, + ) + ); +} + function readJsonFile(file: string, fallback: T): T { if (!fs.existsSync(file)) return fallback; return JSON.parse(fs.readFileSync(file, "utf8")) as T; @@ -392,14 +402,27 @@ test.skipIf(!shouldRunLiveE2EScenarios())( redactionValues: [apiKey], timeoutMs: ONBOARD_TIMEOUT_MS, }); - expectExitZero(onboard, "initial current onboard"); + if (onboard.exitCode !== 0) { + if (!isRetryableOnboardEndpointFailure(onboard)) { + expectExitZero(onboard, "initial current onboard"); + } + const gatewayProbe = await sandbox.list({ + artifactName: "phase-1-gateway-after-onboard-endpoint-transient", + env: dockerContextEnv(), + timeoutMs: OPENSHELL_TIMEOUT_MS, + }); + expectExitZero(gatewayProbe, "OpenShell gateway after tolerated onboard endpoint transient"); + await artifacts.writeJson("phase-1-onboard-transient-summary.json", { + tolerated: true, + reason: "retryable endpoint validation failure after gateway/provider setup", + }); + } - const deleteCurrentSandbox = await sandbox.openshell(["sandbox", "delete", SANDBOX_NAME], { - artifactName: "phase-1-delete-current-sandbox", - env: dockerContextEnv(), - timeoutMs: OPENSHELL_TIMEOUT_MS, - }); - expectExitZero(deleteCurrentSandbox, "openshell sandbox delete current sandbox"); + await openshellBestEffort( + host, + ["sandbox", "delete", SANDBOX_NAME], + "phase-1-delete-current-sandbox", + ); // Phase 2: build the old base image with a temporary build context that // lowers only the blueprint minimum-version gate consumed by Dockerfile.base. From a83dcfd21a5d70ff4c1377cf4516ed440a86d163 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 11 Jun 2026 09:49:46 -0400 Subject: [PATCH 05/11] test(e2e): allow current rebuild blueprint pin --- test/e2e-scenario/live/rebuild-openclaw.test.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/e2e-scenario/live/rebuild-openclaw.test.ts b/test/e2e-scenario/live/rebuild-openclaw.test.ts index a57091e51bd..74c0e065f2f 100644 --- a/test/e2e-scenario/live/rebuild-openclaw.test.ts +++ b/test/e2e-scenario/live/rebuild-openclaw.test.ts @@ -157,11 +157,12 @@ function createOldBaseBuildContext(): string { const buildContext = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-rebuild-openclaw-base-")); fs.mkdirSync(path.join(buildContext, path.dirname(BLUEPRINT_RELPATH)), { recursive: true }); const original = fs.readFileSync(BLUEPRINT, "utf8"); - const lowered = original.replace( - /min_openclaw_version:.*/, - `min_openclaw_version: "${OLD_OPENCLAW_VERSION}"`, - ); - expect(lowered, "blueprint min_openclaw_version line was not found").not.toBe(original); + const minOpenClawVersion = /^(\s*min_openclaw_version:\s*).*/m; + expect( + minOpenClawVersion.test(original), + "blueprint min_openclaw_version line was not found", + ).toBe(true); + const lowered = original.replace(minOpenClawVersion, `$1"${OLD_OPENCLAW_VERSION}"`); fs.writeFileSync(path.join(buildContext, BLUEPRINT_RELPATH), lowered, "utf8"); return buildContext; } From 396c52f8634a80a0f2a483cced844bb07e21cd2d Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 11 Jun 2026 10:02:31 -0400 Subject: [PATCH 06/11] test(e2e): avoid multiline OpenShell exec args --- .../live/rebuild-openclaw.test.ts | 34 +++++++++++++++---- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/test/e2e-scenario/live/rebuild-openclaw.test.ts b/test/e2e-scenario/live/rebuild-openclaw.test.ts index 74c0e065f2f..80ce91fddc7 100644 --- a/test/e2e-scenario/live/rebuild-openclaw.test.ts +++ b/test/e2e-scenario/live/rebuild-openclaw.test.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { Buffer } from "node:buffer"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -153,6 +154,11 @@ function openshellBestEffort( ); } +function pythonExecArgs(script: string): string[] { + const encoded = Buffer.from(script, "utf8").toString("base64"); + return ["python3", "-c", `import base64; exec(base64.b64decode('${encoded}'))`]; +} + function createOldBaseBuildContext(): string { const buildContext = fs.mkdtempSync(path.join(os.tmpdir(), "e2e-rebuild-openclaw-base-")); fs.mkdirSync(path.join(buildContext, path.dirname(BLUEPRINT_RELPATH)), { recursive: true }); @@ -527,9 +533,20 @@ test.skipIf(!shouldRunLiveE2EScenarios())( [ "env", `PRE_REBUILD_GATEWAY_TOKEN=${PRE_REBUILD_GATEWAY_TOKEN}`, - "python3", - "-c", - `import json, os, subprocess\npath='/sandbox/.openclaw/openclaw.json'\ntry:\n cfg=json.load(open(path))\nexcept Exception:\n cfg={}\ncfg.setdefault('gateway', {}).setdefault('auth', {})['token']=os.environ['PRE_REBUILD_GATEWAY_TOKEN']\nwith open(path, 'w') as f:\n json.dump(cfg, f, indent=2)\n f.write('\\n')\nsubprocess.check_call(['bash','-lc','cd /sandbox/.openclaw && sha256sum openclaw.json > .config-hash'])\nsaved=json.load(open(path)).get('gateway',{}).get('auth',{}).get('token','')\nhash_text=open('/sandbox/.openclaw/.config-hash').read()\nprint(json.dumps({'seeded': saved == os.environ['PRE_REBUILD_GATEWAY_TOKEN'], 'hashReferencesConfig': 'openclaw.json' in hash_text}))`, + ...pythonExecArgs(`import json, os, subprocess +path='/sandbox/.openclaw/openclaw.json' +try: + cfg=json.load(open(path)) +except Exception: + cfg={} +cfg.setdefault('gateway', {}).setdefault('auth', {})['token']=os.environ['PRE_REBUILD_GATEWAY_TOKEN'] +with open(path, 'w') as f: + json.dump(cfg, f, indent=2) + f.write('\\n') +subprocess.check_call(['bash','-lc','cd /sandbox/.openclaw && sha256sum openclaw.json > .config-hash']) +saved=json.load(open(path)).get('gateway',{}).get('auth',{}).get('token','') +hash_text=open('/sandbox/.openclaw/.config-hash').read() +print(json.dumps({'seeded': saved == os.environ['PRE_REBUILD_GATEWAY_TOKEN'], 'hashReferencesConfig': 'openclaw.json' in hash_text}))`), ], { artifactName: "phase-4-seed-gateway-token", @@ -664,9 +681,14 @@ test.skipIf(!shouldRunLiveE2EScenarios())( "env", `PRE_REBUILD_GATEWAY_TOKEN=${PRE_REBUILD_GATEWAY_TOKEN}`, `PRE_REBUILD_CONFIG_HASH=${preRebuildConfigHash}`, - "python3", - "-c", - `import json, os, subprocess\ncfg=json.load(open('/sandbox/.openclaw/openclaw.json'))\ntoken=cfg.get('gateway',{}).get('auth',{}).get('token','')\nruntime=subprocess.check_output(['bash','-lc','. /tmp/nemoclaw-proxy-env.sh >/dev/null 2>&1 || exit 1; printf "%s" "\${OPENCLAW_GATEWAY_TOKEN:-}"'], text=True)\nhash_text=open('/sandbox/.openclaw/.config-hash').read()\nhash_ok=subprocess.call(['bash','-lc','cd /sandbox/.openclaw && sha256sum -c .config-hash --status']) == 0\nold=os.environ['PRE_REBUILD_GATEWAY_TOKEN']\nprint(json.dumps({'tokenPresent': bool(token), 'tokenRotated': token != old, 'runtimeMatchesConfig': runtime == token, 'runtimeStillOld': runtime == old, 'hashReferencesConfig': 'openclaw.json' in hash_text, 'hashChanged': hash_text != os.environ['PRE_REBUILD_CONFIG_HASH'], 'hashValid': hash_ok}))`, + ...pythonExecArgs(`import json, os, subprocess +cfg=json.load(open('/sandbox/.openclaw/openclaw.json')) +token=cfg.get('gateway',{}).get('auth',{}).get('token','') +runtime=subprocess.check_output(['bash','-lc','. /tmp/nemoclaw-proxy-env.sh >/dev/null 2>&1 || exit 1; printf "%s" "\${OPENCLAW_GATEWAY_TOKEN:-}"'], text=True) +hash_text=open('/sandbox/.openclaw/.config-hash').read() +hash_ok=subprocess.call(['bash','-lc','cd /sandbox/.openclaw && sha256sum -c .config-hash --status']) == 0 +old=os.environ['PRE_REBUILD_GATEWAY_TOKEN'] +print(json.dumps({'tokenPresent': bool(token), 'tokenRotated': token != old, 'runtimeMatchesConfig': runtime == token, 'runtimeStillOld': runtime == old, 'hashReferencesConfig': 'openclaw.json' in hash_text, 'hashChanged': hash_text != os.environ['PRE_REBUILD_CONFIG_HASH'], 'hashValid': hash_ok}))`), ], { artifactName: "phase-7-gateway-token-rotation-check", From 6b03168ea474c0476293097c178b246ea1d726a2 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 11 Jun 2026 10:16:40 -0400 Subject: [PATCH 07/11] test(e2e): seed rebuild inference route --- .../live/rebuild-openclaw.test.ts | 37 +++++++++++++++++-- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/test/e2e-scenario/live/rebuild-openclaw.test.ts b/test/e2e-scenario/live/rebuild-openclaw.test.ts index 80ce91fddc7..db49dedac6a 100644 --- a/test/e2e-scenario/live/rebuild-openclaw.test.ts +++ b/test/e2e-scenario/live/rebuild-openclaw.test.ts @@ -188,6 +188,34 @@ async function waitForSandboxReady(sandbox: { throw new Error(`sandbox ${SANDBOX_NAME} did not become Ready`); } +async function configureGatewayInferenceRoute( + host: HostCliClient, + apiKey: string, +): Promise { + const model = shellQuote(DEFAULT_MODEL); + return host.command( + "bash", + [ + "-lc", + [ + "set -euo pipefail", + "if openshell provider get nvidia-prod >/dev/null 2>&1; then", + " openshell provider update nvidia-prod --credential NVIDIA_API_KEY", + "else", + " openshell provider create --name nvidia-prod --type nvidia --credential NVIDIA_API_KEY", + "fi", + `openshell inference set --no-verify --provider nvidia-prod --model ${model}`, + ].join("\n"), + ], + { + artifactName: "phase-4-configure-gateway-inference-route", + env: cliEnv(apiKey), + redactionValues: [apiKey], + timeoutMs: OPENSHELL_TIMEOUT_MS, + }, + ); +} + function seedRegistryAndSession(): void { const registry = readJsonFile<{ sandboxes?: Record>; @@ -216,7 +244,7 @@ function seedRegistryAndSession(): void { sandboxName: SANDBOX_NAME, status: "complete", resumable: true, - lastCompletedStep: "gateway", + lastCompletedStep: "inference", failure: null, provider: "nvidia-prod", model: DEFAULT_MODEL, @@ -226,8 +254,8 @@ function seedRegistryAndSession(): void { preflight: complete, gateway: complete, sandbox: pending, - provider_selection: pending, - inference: pending, + provider_selection: complete, + inference: complete, openclaw: pending, agent_setup: pending, policies: pending, @@ -594,6 +622,9 @@ print(json.dumps({'seeded': saved == os.environ['PRE_REBUILD_GATEWAY_TOKEN'], 'h }, }); + const routeResult = await configureGatewayInferenceRoute(host, apiKey); + expectExitZero(routeResult, "configure gateway inference route before rebuild"); + // Phase 4.5: apply policy presets through the public CLI, then verify both // registry persistence and the live OpenShell gateway policy. for (const preset of ["npm", "pypi"]) { From 8a41cbc90aa3a0accbb2b0bfcd003ee7bea14ec4 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 11 Jun 2026 10:41:24 -0400 Subject: [PATCH 08/11] ci(e2e): allow selective Vitest job dispatch --- .github/workflows/e2e-vitest-scenarios.yaml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 3f27f58a711..b97e1d204ff 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -11,12 +11,17 @@ on: required: false default: "" type: string + jobs: + description: "Optional comma-separated free-standing live Vitest job ids. Empty runs all enabled jobs." + required: false + default: "" + type: string permissions: contents: read concurrency: - group: e2e-vitest-scenarios-${{ github.ref }}-${{ inputs.scenarios || 'supported' }} + group: e2e-vitest-scenarios-${{ github.ref }}-${{ inputs.scenarios || 'supported' }}-${{ inputs.jobs || 'all-jobs' }} cancel-in-progress: false jobs: @@ -42,6 +47,7 @@ jobs: name: Generate Vitest scenario matrix env: SCENARIOS: ${{ inputs.scenarios }} + JOBS: ${{ inputs.jobs }} run: | set -euo pipefail args=(--emit-live-matrix) @@ -52,6 +58,10 @@ jobs: fi args+=(--scenarios "${SCENARIOS}") fi + if [ -n "${JOBS}" ] && [[ ! "${JOBS}" =~ ^[A-Za-z0-9_-]+(,[A-Za-z0-9_-]+)*$ ]]; then + echo "::error::Invalid jobs input: ${JOBS}" >&2 + exit 1 + fi matrix="$(npx tsx test/e2e-scenario/scenarios/run.ts "${args[@]}")" echo "matrix=${matrix}" >> "$GITHUB_OUTPUT" MATRIX_JSON="${matrix}" python - <<'PY' >> "$GITHUB_STEP_SUMMARY" @@ -69,6 +79,7 @@ jobs: live-scenarios: needs: generate-matrix + if: ${{ inputs.jobs == '' }} runs-on: ${{ matrix.runner }} timeout-minutes: 45 strategy: @@ -169,6 +180,7 @@ jobs: # because the matrix above only runs registry-scenarios.test.ts. Modeled on # #5049's free-standing pattern. openshell-version-pin-vitest: + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',openshell-version-pin-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 15 env: @@ -208,6 +220,7 @@ jobs: retention-days: 14 onboard-negative-paths-vitest: + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 15 env: @@ -254,7 +267,7 @@ jobs: # protocol/history contract. The retained legacy bash lane remains the # source for full closeout until a later PR proves replacement and deletes it. openclaw-tui-chat-correlation-vitest: - if: ${{ inputs.scenarios == '' }} + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',openclaw-tui-chat-correlation-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 75 env: @@ -327,7 +340,7 @@ jobs: # a standalone job because it builds old/current base images and mutates a # real OpenShell sandbox rather than fitting the steady-state registry probe. rebuild-openclaw-vitest: - if: ${{ inputs.scenarios == '' }} + if: ${{ inputs.jobs == '' || contains(format(',{0},', inputs.jobs), ',rebuild-openclaw-vitest,') }} runs-on: ubuntu-latest timeout-minutes: 130 env: From 9a4a76dfb9414d650ea026474341e1b019904a26 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 11 Jun 2026 12:34:17 -0400 Subject: [PATCH 09/11] ci(e2e): align rebuild-openclaw-vitest dispatch --- .github/workflows/e2e-vitest-scenarios.yaml | 74 ++++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index 65e06c7eda4..3ff6ec52961 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -40,7 +40,7 @@ jobs: SCENARIOS: ${{ inputs.scenarios }} run: | set -euo pipefail - allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery" + allowed_jobs="openshell-version-pin-vitest,onboard-negative-paths-vitest,openclaw-tui-chat-correlation-vitest,gateway-guard-recovery,rebuild-openclaw-vitest" if [ -n "${JOBS}" ] && [ -n "${SCENARIOS}" ]; then echo "::error::Use either scenarios or jobs, not both." >&2 exit 1 @@ -298,6 +298,77 @@ jobs: if-no-files-found: ignore retention-days: 14 + rebuild-openclaw-vitest: + needs: validate-jobs + if: ${{ (inputs.jobs == '' && inputs.scenarios == '') || contains(format(',{0},', inputs.jobs), ',rebuild-openclaw-vitest,') }} + runs-on: ubuntu-latest + timeout-minutes: 130 + env: + E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/vitest/rebuild-openclaw + NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js + NEMOCLAW_RUN_E2E_SCENARIOS: "1" + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Authenticate to Docker Hub + env: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + shell: bash + run: | + set -euo pipefail + if [[ -z "${DOCKERHUB_USERNAME}" || -z "${DOCKERHUB_TOKEN}" ]]; then + echo "::notice::Docker Hub credentials not configured; continuing with anonymous pulls." + exit 0 + fi + login_succeeded=0 + for attempt in 1 2 3; do + if echo "${DOCKERHUB_TOKEN}" | timeout 30s docker login docker.io --username "${DOCKERHUB_USERNAME}" --password-stdin; then + login_succeeded=1 + break + fi + if [[ "$attempt" -lt 3 ]]; then + echo "::warning::Docker Hub login attempt ${attempt} failed; retrying." + sleep 5 + fi + done + if [[ "$login_succeeded" -ne 1 ]]; then + echo "::warning::Docker Hub login failed after 3 attempts; continuing with anonymous pulls." + fi + + - name: Set up Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.0.0 + with: + node-version: 22 + cache: npm + + - name: Install root dependencies + run: npm ci --ignore-scripts + + - name: Build CLI + run: npm run build:cli + + - name: Run OpenClaw rebuild live test + env: + NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} + run: | + set -euo pipefail + npx vitest run --project e2e-scenarios-live \ + test/e2e-scenario/live/rebuild-openclaw.test.ts \ + --silent=false --reporter=default + + - name: Upload OpenClaw rebuild artifacts + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-vitest-scenarios-rebuild-openclaw + path: e2e-artifacts/vitest/rebuild-openclaw/ + include-hidden-files: false + if-no-files-found: ignore + retention-days: 14 + # Focused coverage slice for the #2603/#3145 OpenClaw websocket # protocol/history contract. The retained legacy bash lane remains the # source for full closeout until a later PR proves replacement and deletes it. @@ -481,6 +552,7 @@ jobs: live-scenarios, openshell-version-pin-vitest, onboard-negative-paths-vitest, + rebuild-openclaw-vitest, openclaw-tui-chat-correlation-vitest, gateway-guard-recovery, ] From 1f30142f76cbb01f3192afdfabc332deff1f78ab Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 11 Jun 2026 18:12:37 -0400 Subject: [PATCH 10/11] test(e2e): tighten rebuild openclaw coverage --- .github/workflows/e2e-vitest-scenarios.yaml | 21 ++++++ .../live/rebuild-openclaw.test.ts | 66 +++++++++++++++++-- tools/e2e-scenarios/workflow-boundary.mts | 10 +++ 3 files changed, 91 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e-vitest-scenarios.yaml b/.github/workflows/e2e-vitest-scenarios.yaml index bbb8a6b60bd..73f0df04f0d 100644 --- a/.github/workflows/e2e-vitest-scenarios.yaml +++ b/.github/workflows/e2e-vitest-scenarios.yaml @@ -607,11 +607,32 @@ jobs: - name: Build CLI run: npm run build:cli + - name: Install OpenShell + # Direct Vitest execution uses bin/nemoclaw.js instead of install.sh, + # so install OpenShell explicitly before onboard/rebuild commands. + env: + NEMOCLAW_NON_INTERACTIVE: "1" + run: | + set -euo pipefail + env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh + - name: Run OpenClaw rebuild live test env: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} run: | set -euo pipefail + export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH" + if command -v openshell >/dev/null 2>&1; then + OPENSHELL_BIN="$(command -v openshell)" + elif [ -x "$HOME/.local/bin/openshell" ]; then + OPENSHELL_BIN="$HOME/.local/bin/openshell" + else + echo "::error::OpenShell CLI not found after install" + ls -la /usr/local/bin/openshell "$HOME/.local/bin/openshell" 2>&1 || true + exit 1 + fi + export OPENSHELL_BIN + "$OPENSHELL_BIN" --version npx vitest run --project e2e-scenarios-live \ test/e2e-scenario/live/rebuild-openclaw.test.ts \ --silent=false --reporter=default diff --git a/test/e2e-scenario/live/rebuild-openclaw.test.ts b/test/e2e-scenario/live/rebuild-openclaw.test.ts index 87a138e8185..3dd4d6f6f1e 100644 --- a/test/e2e-scenario/live/rebuild-openclaw.test.ts +++ b/test/e2e-scenario/live/rebuild-openclaw.test.ts @@ -327,10 +327,14 @@ function backupCredentialLeakPaths(backupDir: string, oldGatewayToken: string): continue; } if (!entry.isFile()) continue; - if (skippedLockfiles.has(entry.name)) continue; const text = fs.readFileSync(fullPath, "utf8"); + if (text.includes(oldGatewayToken)) { + leaks.push(fullPath); + continue; + } + if (skippedLockfiles.has(entry.name)) continue; const isJsonOrEnv = /\.json$|\.env$|^\.env$/i.test(entry.name); - if (text.includes(oldGatewayToken) || (isJsonOrEnv && candidatePattern.test(text))) { + if (isJsonOrEnv && candidatePattern.test(text)) { leaks.push(fullPath); } } @@ -665,8 +669,21 @@ print(json.dumps({'seeded': saved == os.environ['PRE_REBUILD_GATEWAY_TOKEN'], 'h expectExitZero(prePolicy, "openshell policy get before rebuild"); expect(prePolicy.stdout).toMatch(/npm|registry\.npmjs\.org/i); expect(prePolicy.stdout).toMatch(/pypi|pypi\.org/i); - expect(prePolicy.stdout).toMatch(/telegram|api\.telegram\.org/i); + expect(prePolicy.stdout).toMatch(/telegram/i); + expect(prePolicy.stdout).toContain("api.telegram.org"); expect(registrySandbox().policies).toEqual(expect.arrayContaining([...POLICY_PRESETS])); + const prePolicyList = await host.command( + "node", + [CLI_ENTRYPOINT, SANDBOX_NAME, "policy-list"], + { + artifactName: "phase-4-nemoclaw-policy-list-before-rebuild", + env: cliEnv(apiKey), + redactionValues: [apiKey], + timeoutMs: OPENSHELL_TIMEOUT_MS, + }, + ); + expectExitZero(prePolicyList, "nemoclaw policy-list before rebuild"); + expect(prePolicyList.stdout).toMatch(/●\s+telegram/i); // Phase 5: restore the current base image tag that rebuild consumes. const buildCurrentBase = await host.command( @@ -763,7 +780,8 @@ print(json.dumps({'tokenPresent': bool(token), 'tokenRotated': token != old, 'ru backupDir, stateDirCount: Array.isArray(manifest.stateDirs) ? manifest.stateDirs.length : undefined, policyPresets: manifest.policyPresets, - telegramBridgeTraffic: "not exercised; real bot credentials/messages stay out of this rebuild migration", + telegramBridgeTraffic: + "real bot response remains owned by test/e2e/test-messaging-providers.sh M19b/future Phase 6 messaging-provider Vitest migration; this rebuild migration asserts restored telegram policy and api.telegram.org reachability", }); expect(manifest.policyPresets).toEqual(expect.arrayContaining([...POLICY_PRESETS])); expect(backupCredentialLeakPaths(backupDir, PRE_REBUILD_GATEWAY_TOKEN)).toEqual([]); @@ -777,9 +795,45 @@ print(json.dumps({'tokenPresent': bool(token), 'tokenRotated': token != old, 'ru expectExitZero(postPolicy, "openshell policy get after rebuild"); expect(postPolicy.stdout).toMatch(/npm|registry\.npmjs\.org/i); expect(postPolicy.stdout).toMatch(/pypi|pypi\.org/i); - expect(postPolicy.stdout).toMatch(/telegram|api\.telegram\.org/i); + expect(postPolicy.stdout).toMatch(/telegram/i); + expect(postPolicy.stdout).toContain("api.telegram.org"); + + const postPolicyList = await host.command( + "node", + [CLI_ENTRYPOINT, SANDBOX_NAME, "policy-list"], + { + artifactName: "phase-7-nemoclaw-policy-list-after-rebuild", + env: cliEnv(apiKey), + redactionValues: [apiKey], + timeoutMs: OPENSHELL_TIMEOUT_MS, + }, + ); + expectExitZero(postPolicyList, "nemoclaw policy-list after rebuild"); + expect(postPolicyList.stdout).toMatch(/●\s+telegram/i); + + // #1952's real bot-response clause is owned by the messaging provider E2E + // (`test/e2e/test-messaging-providers.sh` M19b, then the Phase 6 Vitest + // migration). This rebuild/state migration keeps the deterministic + // prerequisite: Telegram policy is restored and the gateway does not block + // api.telegram.org after rebuild. + const telegramApiReachability = await sandbox.exec( + SANDBOX_NAME, + [ + "node", + "-e", + "fetch('https://api.telegram.org/bot000000000:invalid/getMe', { signal: AbortSignal.timeout(15000) }).then((r) => console.log('STATUS_' + r.status)).catch((e) => { console.log('ERROR_' + (e.cause?.code || e.code || e.message)); process.exitCode = 1; })", + ], + { + artifactName: "phase-7-telegram-api-reachability-after-rebuild", + env: dockerContextEnv(), + timeoutMs: 30_000, + }, + ); + expectExitZero(telegramApiReachability, "api.telegram.org reachability after rebuild"); + expect(telegramApiReachability.stdout).toMatch(/STATUS_\d+/); + expect(telegramApiReachability.stdout).not.toMatch(/STATUS_403|Forbidden/i); - // External API availability can make this inconclusive; keep it as a + // External inference API availability can make this inconclusive; keep it as a // non-fatal artifact-producing probe like the legacy script did. await sandbox.exec( SANDBOX_NAME, diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index 3a666451832..2cbdd28266f 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -570,11 +570,21 @@ function validateRebuildOpenClawVitestJob(errors: string[], jobs: WorkflowRecord const buildCli = requireJobStep(errors, jobName, steps, "Build CLI"); requireRunContains(errors, buildCli, "npm run build:cli"); + const installOpenShell = requireJobStep(errors, jobName, steps, "Install OpenShell"); + requireEnvDoesNotExposeSecret(errors, "rebuild-openclaw-vitest step 'Install OpenShell'", asRecord(installOpenShell?.env), "GITHUB_TOKEN"); + requireRunContains(errors, installOpenShell, "bash scripts/install-openshell.sh"); + requireRunContains(errors, installOpenShell, "env -u DOCKER_CONFIG"); + requireRunContains(errors, installOpenShell, "-u DOCKERHUB_USERNAME"); + requireRunContains(errors, installOpenShell, "-u DOCKERHUB_TOKEN"); + requireRunContains(errors, installOpenShell, "-u NVIDIA_API_KEY"); + requireRunContains(errors, installOpenShell, "-u GITHUB_TOKEN"); + const runVitest = requireJobStep(errors, jobName, steps, "Run OpenClaw rebuild live test"); const runVitestEnv = asRecord(runVitest?.env); if (runVitestEnv.NVIDIA_API_KEY !== "${{ secrets.NVIDIA_API_KEY }}") { errors.push("rebuild-openclaw-vitest step must receive NVIDIA_API_KEY from secrets"); } + requireRunContains(errors, runVitest, "OPENSHELL_BIN"); requireRunContains(errors, runVitest, "npx vitest run --project e2e-scenarios-live"); requireRunContains(errors, runVitest, "test/e2e-scenario/live/rebuild-openclaw.test.ts"); From 0b919d1f1c973924abd5a1dbbcc4a4f0dce1b92e Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Thu, 11 Jun 2026 18:27:56 -0400 Subject: [PATCH 11/11] test(e2e): lock rebuild selector coverage --- .../e2e-scenarios-workflow.test.ts | 26 +++++++++++++++++++ tools/e2e-scenarios/workflow-boundary.mts | 1 + 2 files changed, 27 insertions(+) diff --git a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts index e835cac272c..eed801f20b0 100644 --- a/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts +++ b/test/e2e-scenario/support-tests/e2e-scenarios-workflow.test.ts @@ -136,6 +136,22 @@ describe("e2e-vitest-scenarios workflow boundary", () => { selectedFreeStandingJobs: ["hermes-e2e-vitest"], registryScenarios: [], }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ scenarios: "rebuild-openclaw" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["rebuild-openclaw-vitest"], + registryScenarios: [], + }); + expect( + evaluateE2eVitestWorkflowDispatchSelectors({ jobs: "rebuild-openclaw-vitest" }), + ).toMatchObject({ + valid: true, + liveScenariosRuns: false, + selectedFreeStandingJobs: ["rebuild-openclaw-vitest"], + registryScenarios: [], + }); }); it("keeps jobs-only dispatches from selecting the Hermes secret-bearing job", () => { @@ -165,6 +181,16 @@ describe("e2e-vitest-scenarios workflow boundary", () => { hermes_selected: "false", matrix: "[]", }); + expect( + generateMatrixForDispatch({ JOBS: "rebuild-openclaw-vitest", SCENARIOS: "" }), + ).toMatchObject({ + hermes_selected: "false", + matrix: "[]", + }); + expect(generateMatrixForDispatch({ JOBS: "", SCENARIOS: "rebuild-openclaw" })).toMatchObject({ + hermes_selected: "false", + matrix: "[]", + }); expect(generateMatrixForDispatch({ JOBS: "", SCENARIOS: "hermes-e2e" })).toMatchObject({ hermes_selected: "true", matrix: "[]", diff --git a/tools/e2e-scenarios/workflow-boundary.mts b/tools/e2e-scenarios/workflow-boundary.mts index ae3ae7ca36f..77e7763b895 100644 --- a/tools/e2e-scenarios/workflow-boundary.mts +++ b/tools/e2e-scenarios/workflow-boundary.mts @@ -281,6 +281,7 @@ function validateJobsSelector(errors: string[], jobs: WorkflowRecord): void { requireRunContains(errors, validate, "runtime-overrides-vitest"); requireRunContains(errors, validate, "hermes-e2e-vitest"); requireRunContains(errors, validate, "network-policy-vitest"); + requireRunContains(errors, validate, "rebuild-openclaw-vitest"); requireRunContains(errors, validate, "token-rotation-vitest"); requireRunContains(errors, validate, "openclaw-tui-chat-correlation-vitest"); requireRunContains(errors, validate, "gateway-guard-recovery");