From d0677ede839837b7c728d52b19249c91c7bfca85 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 9 Jun 2026 12:09:32 -0400 Subject: [PATCH 01/13] test(e2e): allow whitelisted lifecycle profiles in live runner `liveScenarioSupport` previously rejected any scenario that declared an `environment.lifecycle`, so post-onboard host mutations (reboot, rebuild, upgrade, drift) could not surface in the live Vitest matrix at all. Replace the unconditional reject with a `SUPPORTED_LIFECYCLES` whitelist that starts with the single profile the upcoming post-reboot-recovery fixture dispatches: `post-reboot-recovery`. Future profiles must land the dispatcher branch and an expected-state in the same change set, so the whitelist stays in lockstep with what the runner can actually execute. Prepares the runner for #4423's failing-test-first guard, which needs a post-reboot lifecycle scenario to demonstrate registry preservation + Docker-backed sandbox recovery on Linux/Spark Docker-driver hosts. Refs #4423 --- .../e2e-live-registry-discovery.test.ts | 35 +++++++++++++++++++ .../e2e-scenario/scenarios/runtime-support.ts | 8 ++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts b/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts index 21cb9b6329e..db1dea7f4d1 100644 --- a/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts @@ -50,4 +50,39 @@ describe("live Vitest registry discovery support", () => { reasons: ["runtime 'docker-missing' is not wired for live Vitest fixtures"], }); }); + + it("keeps unwhitelisted lifecycle profiles skipped with the lifecycle reason", () => { + const scenario = listScenarios().find((entry) => entry.id === "ubuntu-rebuild-openclaw"); + + expect(scenario).toBeTruthy(); + expect(liveScenarioSupport(scenario!)).toMatchObject({ + supported: false, + reasons: [ + "lifecycle 'rebuild-current-version' is not wired for live Vitest fixtures", + ], + }); + }); + + it("accepts whitelisted lifecycle profiles when the rest of the environment matches", () => { + // Synthesised scenario stands in for the not-yet-registered + // post-reboot-recovery scenario so this test pins the whitelist + // contract independently of when the scenario lands. Once the + // post-reboot-recovery scenario is registered, prefer asserting on + // the registry entry directly and remove this synthetic. + const supported = liveScenarioSupport({ + id: "synthetic-post-reboot-recovery", + assertionGroups: [], + expectedStateId: "cloud-openclaw-ready", + environment: { + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-openclaw", + lifecycle: "post-reboot-recovery", + }, + }); + + expect(supported.supported).toBe(true); + expect(supported.reasons).toEqual([]); + }); }); diff --git a/test/e2e-scenario/scenarios/runtime-support.ts b/test/e2e-scenario/scenarios/runtime-support.ts index 24b1e269242..6a9f874264d 100644 --- a/test/e2e-scenario/scenarios/runtime-support.ts +++ b/test/e2e-scenario/scenarios/runtime-support.ts @@ -7,6 +7,12 @@ const SUPPORTED_PLATFORMS = new Set(["ubuntu-local"]); const SUPPORTED_INSTALLS = new Set(["repo-current"]); const SUPPORTED_RUNTIMES = new Set(["docker-running"]); const SUPPORTED_ONBOARDING = new Set(["cloud-openclaw"]); +// Lifecycle profiles wired into the live Vitest driver. A profile is +// supported only after both (a) `LifecyclePhaseFixture.simulate(profile)` +// dispatches it, and (b) at least one expected-state declares the post- +// lifecycle host invariants the fixture creates. New profiles must add +// the dispatcher branch and an expected-state in the same change set. +const SUPPORTED_LIFECYCLES = new Set(["post-reboot-recovery"]); export interface LiveScenarioSupport { supported: boolean; @@ -43,7 +49,7 @@ export function liveScenarioSupport(scenario: ScenarioDefinition): LiveScenarioS if (!SUPPORTED_ONBOARDING.has(environment.onboarding)) { reasons.push(`onboarding '${environment.onboarding}' is not wired for live Vitest fixtures`); } - if (environment.lifecycle) { + if (environment.lifecycle && !SUPPORTED_LIFECYCLES.has(environment.lifecycle)) { reasons.push(`lifecycle '${environment.lifecycle}' is not wired for live Vitest fixtures`); } } From a220f87fe933ca5c9d20aa58303aedf6e68f52e3 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 9 Jun 2026 12:16:23 -0400 Subject: [PATCH 02/13] test(e2e): add host-side registry and Docker container probes Adds two host-side state-validation probes the live runner needs to express the regression target tracked by #4423: * `local-registry-entry-present` reads `~/.nemoclaw/sandboxes.json` and asserts the scenario's sandbox name is still recorded. This is deliberately orthogonal to `sandbox.expected`: post-reboot bugs can wipe the local registry while the live OpenShell gateway is healthy, and only a host-side probe catches the data-loss regression. * `docker-sandbox-container-present` runs `docker ps -a --filter label=openshell.ai/sandbox-name=` and accepts running, stopped, or `*-nemoclaw-gpu-backup-*` sibling containers. The label filter mirrors `OPENSHELL_SANDBOX_NAME_LABEL` used by `findOpenShellDockerSandboxContainerIds` in `src/lib/onboard/docker-gpu-patch.ts`, so the probe stays in lock- step with how OpenShell labels containers today. Probe wiring: * `StateProbeId` extended with the two new probe ids. * `ExpectedState` gains `localRegistry` and `dockerSandboxContainer` optional dimensions; `probesForState` emits the new probes only for `expected: "present"`. Negative-direction probes are intentionally omitted today and pinned by a probesForState test. * `StateValidationPhaseFixture.from()` now accepts either an expected-state ID or an inline `ExpectedState`, so unit tests can drive new probes without registering synthetic states in the typed registry. The live runner still calls `from(id, instance)`. * Fixture takes an optional `ProbeIO` injection so tests can stub the registry reader without touching `~/.nemoclaw`. No callers of the existing typed registry are affected: every shipped expected-state leaves `localRegistry` and `dockerSandboxContainer` unset, so `probesForState` returns the same probe lists as before. Refs #4423 --- .../e2e-expected-state.test.ts | 35 +++++ .../e2e-phase-state-validation.test.ts | 120 +++++++++++++++++- .../framework/phases/state-validation.ts | 113 ++++++++++++++++- .../e2e-scenario/scenarios/expected-states.ts | 10 ++ test/e2e-scenario/scenarios/types.ts | 25 +++- 5 files changed, 297 insertions(+), 6 deletions(-) diff --git a/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts b/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts index 2fe76802134..dcf15974923 100644 --- a/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts @@ -83,6 +83,41 @@ describe("probesForState maps typed expected-state into probe ids", () => { }; expect(probesForState(state)).toEqual([]); }); + + it("localRegistry.expected=present emits the local-registry-entry-present probe", () => { + const state: ExpectedState = { + id: "synthetic-local-registry", + cli: { installed: true }, + localRegistry: { expected: "present" }, + }; + expect(probesForState(state)).toEqual([ + "cli-installed", + "local-registry-entry-present", + ]); + }); + + it("dockerSandboxContainer.expected=present emits the docker-sandbox-container-present probe", () => { + const state: ExpectedState = { + id: "synthetic-docker-container", + cli: { installed: true }, + dockerSandboxContainer: { expected: "present" }, + }; + expect(probesForState(state)).toEqual([ + "cli-installed", + "docker-sandbox-container-present", + ]); + }); + + it("localRegistry/dockerSandboxContainer 'absent' emits no probe today", () => { + // Negative-direction probes haven't landed yet. Pin the gap so a + // future negative-scenario PR is forced to add the absent probes. + const state: ExpectedState = { + id: "synthetic-host-absent", + localRegistry: { expected: "absent" }, + dockerSandboxContainer: { expected: "absent" }, + }; + expect(probesForState(state)).toEqual([]); + }); }); describe("compiler emits state-validation phase actions from expected-state registry", () => { diff --git a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts index f2af06b99f5..50cf1306d59 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts @@ -108,9 +108,17 @@ function instance(overrides: Partial = {}): NemoClawInstance { }; } -function fixture(runner: FakeRunner): StateValidationPhaseFixture { +function fixture( + runner: FakeRunner, + io: ConstructorParameters[3] = {}, +): StateValidationPhaseFixture { const host = new HostCliClient(runner); - return new StateValidationPhaseFixture(host, new GatewayClient(host), new SandboxClient(runner)); + return new StateValidationPhaseFixture( + host, + new GatewayClient(host), + new SandboxClient(runner), + io, + ); } describe("state-validation phase fixture", () => { @@ -457,3 +465,111 @@ describe("state-validation phase fixture", () => { >().toEqualTypeOf(); }); }); + +describe("state-validation host-side probes", () => { + const localRegistryState = { + id: "synthetic-local-registry-present", + localRegistry: { expected: "present" as const }, + }; + const dockerContainerState = { + id: "synthetic-docker-container-present", + dockerSandboxContainer: { expected: "present" as const }, + }; + + it("local-registry-entry-present passes when the registry contains the sandbox name", async () => { + const runner = new FakeRunner(); + const fx = fixture(runner, { + readRegistry: () => ({ + entries: { "e2e-ubuntu-repo-cloud-openclaw": { name: "e2e-ubuntu-repo-cloud-openclaw" } }, + }), + }); + + const result = await fx.from(localRegistryState, instance()); + + expect(result.probes.map((probe) => probe.id)).toEqual(["local-registry-entry-present"]); + expect(runner.calls).toEqual([]); + }); + + it("local-registry-entry-present fails when the registry file is missing", async () => { + const runner = new FakeRunner(); + const fx = fixture(runner, { readRegistry: () => null }); + + await expect(fx.from(localRegistryState, instance())).rejects.toThrow( + /expected local registry entry for 'e2e-ubuntu-repo-cloud-openclaw'.*does not exist/, + ); + }); + + it("local-registry-entry-present fails when the sandbox name is missing from registry", async () => { + const runner = new FakeRunner(); + const fx = fixture(runner, { + readRegistry: () => ({ entries: { "some-other-sandbox": {} } }), + }); + + await expect(fx.from(localRegistryState, instance())).rejects.toThrow( + /registry contains: some-other-sandbox/, + ); + }); + + it("docker-sandbox-container-present passes when docker ps -a returns labeled names", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "e2e-ubuntu-repo-cloud-openclaw\n")); + const fx = fixture(runner); + + const result = await fx.from(dockerContainerState, instance()); + + expect(result.probes.map((probe) => probe.id)).toEqual(["docker-sandbox-container-present"]); + expect(runner.calls).toEqual([ + { + command: "docker", + args: [ + "ps", + "-a", + "--filter", + "label=openshell.ai/sandbox-name=e2e-ubuntu-repo-cloud-openclaw", + "--format", + "{{.Names}}", + ], + options: { + artifactName: "docker-sandbox-container-present-e2e-ubuntu-repo-cloud-openclaw", + env: expect.objectContaining({ PATH: expect.any(String) }), + timeoutMs: 15_000, + }, + }, + ]); + }); + + it("docker-sandbox-container-present matches *-nemoclaw-gpu-backup-* sibling containers", async () => { + const runner = new FakeRunner(); + runner.enqueue( + shellResult( + 0, + "e2e-ubuntu-repo-cloud-openclaw-nemoclaw-gpu-backup-1717280000000\n", + ), + ); + const fx = fixture(runner); + + const result = await fx.from(dockerContainerState, instance()); + + expect(result.probes.map((probe) => probe.id)).toEqual(["docker-sandbox-container-present"]); + }); + + it("docker-sandbox-container-present fails when docker ps -a returns no labeled container", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "\n")); + const fx = fixture(runner); + + await expect(fx.from(dockerContainerState, instance())).rejects.toThrow( + /docker ps -a returned none/, + ); + }); + + it("docker-sandbox-container-present fails when docker ps -a exits non-zero", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(1, "Cannot connect to the Docker daemon")); + const fx = fixture(runner); + + await expect(fx.from(dockerContainerState, instance())).rejects.toThrow( + /could not query Docker for label.*exit 1/, + ); + }); +}); diff --git a/test/e2e-scenario/framework/phases/state-validation.ts b/test/e2e-scenario/framework/phases/state-validation.ts index 4743be9fbe2..e919adc17de 100644 --- a/test/e2e-scenario/framework/phases/state-validation.ts +++ b/test/e2e-scenario/framework/phases/state-validation.ts @@ -1,6 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + import { buildAvailabilityProbeEnv } from "../availability-env.ts"; import { trustedProviderEndpoint, @@ -13,6 +17,36 @@ import { probesForState, requireExpectedState } from "../../scenarios/expected-s import type { ExpectedState, StateProbeId } from "../../scenarios/types.ts"; import type { NemoClawInstance } from "./onboarding.ts"; +// Mirror of `src/lib/state/registry.ts::REGISTRY_FILE`. The fixture +// owns its own copy because the framework code must not import from +// `src/lib/**` (CLI source) — that boundary keeps the live runner +// honest about probing only host-observable state. +const NEMOCLAW_REGISTRY_RELPATH = [".nemoclaw", "sandboxes.json"] as const; +const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; + +export interface ProbeIO { + readRegistry?(): { entries: Record } | null; +} + +function defaultRegistryPath(): string { + const home = process.env.HOME ?? os.homedir(); + return path.join(home, ...NEMOCLAW_REGISTRY_RELPATH); +} + +function defaultReadRegistry(): { entries: Record } | null { + const file = defaultRegistryPath(); + if (!fs.existsSync(file)) return null; + try { + const raw = fs.readFileSync(file, "utf8"); + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== "object") return { entries: {} }; + const entries = (parsed as { sandboxes?: Record }).sandboxes; + return { entries: entries && typeof entries === "object" ? entries : {} }; + } catch { + return { entries: {} }; + } +} + export interface StateValidationProbeResult { id: StateProbeId; status: "passed"; @@ -74,14 +108,23 @@ function isMissingOpenShellError(error: unknown): boolean { } export class StateValidationPhaseFixture { + private readonly io: ProbeIO; + constructor( private readonly host: HostCliClient, private readonly gateway: GatewayClient, private readonly sandbox: SandboxClient, - ) {} + io: ProbeIO = {}, + ) { + this.io = io; + } - async from(expectedStateId: string, instance?: NemoClawInstance): Promise { - const state = requireExpectedState(expectedStateId); + async from( + expectedState: string | ExpectedState, + instance?: NemoClawInstance, + ): Promise { + const state = + typeof expectedState === "string" ? requireExpectedState(expectedState) : expectedState; const probes: StateValidationProbeResult[] = []; for (const probe of probesForState(state)) { probes.push(await this.runProbe(probe, instance)); @@ -104,6 +147,10 @@ export class StateValidationPhaseFixture { return await this.expectSandboxRunning(requireInstance(probe, instance)); case "sandbox-absent": return await this.expectSandboxAbsent(requireInstance(probe, instance)); + case "local-registry-entry-present": + return this.expectLocalRegistryEntryPresent(requireInstance(probe, instance)); + case "docker-sandbox-container-present": + return await this.expectDockerSandboxContainerPresent(requireInstance(probe, instance)); default: { const _exhaustive: never = probe; throw new Error(`Unsupported state-validation probe '${_exhaustive}'.`); @@ -231,6 +278,66 @@ export class StateValidationPhaseFixture { return { id: "sandbox-running", status: "passed", results: [result] }; } + private expectLocalRegistryEntryPresent( + instance: NemoClawInstance, + ): StateValidationProbeResult { + const reader = this.io.readRegistry ?? defaultReadRegistry; + const registry = reader(); + if (!registry) { + throw new Error( + `state-validation expected local registry entry for '${instance.sandboxName}', ` + + `but ${defaultRegistryPath()} does not exist.`, + ); + } + if (!Object.prototype.hasOwnProperty.call(registry.entries, instance.sandboxName)) { + const present = Object.keys(registry.entries).sort().join(", ") || "(none)"; + throw new Error( + `state-validation expected local registry entry for '${instance.sandboxName}', ` + + `but the registry contains: ${present}.`, + ); + } + return { id: "local-registry-entry-present", status: "passed", results: [] }; + } + + private async expectDockerSandboxContainerPresent( + instance: NemoClawInstance, + ): Promise { + const result = await this.host.command( + "docker", + [ + "ps", + "-a", + "--filter", + `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${instance.sandboxName}`, + "--format", + "{{.Names}}", + ], + { + artifactName: `docker-sandbox-container-present-${instance.sandboxName}`, + env: statusProbeEnv(), + timeoutMs: 15_000, + }, + ); + if (result.exitCode !== 0) { + throw new Error( + `state-validation could not query Docker for label '${OPENSHELL_SANDBOX_NAME_LABEL}=${instance.sandboxName}' ` + + `(exit ${result.exitCode}).`, + ); + } + const names = result.stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + if (names.length === 0) { + throw new Error( + `state-validation expected at least one Docker container labeled ` + + `'${OPENSHELL_SANDBOX_NAME_LABEL}=${instance.sandboxName}' (running, stopped, or ` + + `*-nemoclaw-gpu-backup-* sibling), but docker ps -a returned none.`, + ); + } + return { id: "docker-sandbox-container-present", status: "passed", results: [result] }; + } + private async expectSandboxAbsent( instance: NemoClawInstance, ): Promise { diff --git a/test/e2e-scenario/scenarios/expected-states.ts b/test/e2e-scenario/scenarios/expected-states.ts index 2a740a38cfd..d79b32e185d 100644 --- a/test/e2e-scenario/scenarios/expected-states.ts +++ b/test/e2e-scenario/scenarios/expected-states.ts @@ -131,5 +131,15 @@ export function probesForState(state: ExpectedState): readonly StateProbeId[] { } else if (state.sandbox?.expected === "absent") { probes.push("sandbox-absent"); } + // Host-side aspects. "absent" deliberately emits no probe today: it + // would require asserting the registry/container does NOT exist, + // which has no scenario in flight. Add when a negative scenario + // needs it. + if (state.localRegistry?.expected === "present") { + probes.push("local-registry-entry-present"); + } + if (state.dockerSandboxContainer?.expected === "present") { + probes.push("docker-sandbox-container-present"); + } return probes; } diff --git a/test/e2e-scenario/scenarios/types.ts b/test/e2e-scenario/scenarios/types.ts index 693acff04e8..d363e4544a2 100644 --- a/test/e2e-scenario/scenarios/types.ts +++ b/test/e2e-scenario/scenarios/types.ts @@ -16,12 +16,23 @@ export type PhaseResultName = PhaseName | NegativeContractPhase; // nemoclaw_scenarios/probes/. Inference and credentials probes are // declared but not yet implemented; the compiler skips emitting actions // for them until the probe scripts land. +// +// `local-registry-entry-present` and `docker-sandbox-container-present` +// are host-side aspects of the sandbox: the local NemoClaw registry +// (`~/.nemoclaw/sandboxes.json`) and the Docker container labeled with +// `openshell.ai/sandbox-name=` (running OR stopped, including +// `*-nemoclaw-gpu-backup-*` siblings). These probes let scenarios +// assert preservation invariants that diverge from the live gateway +// view of the sandbox, which is precisely the regression class +// covered by the post-reboot recovery work tracked in #4423. export type StateProbeId = | "cli-installed" | "gateway-healthy" | "gateway-absent" | "sandbox-running" - | "sandbox-absent"; + | "sandbox-absent" + | "local-registry-entry-present" + | "docker-sandbox-container-present"; // User-facing phase the negative-scenario contract advertises. Wider // than PhaseName because manifests may declare "preflight" failures, @@ -74,6 +85,18 @@ export interface ExpectedState { credentials?: { expected: ExpectedPresence; }; + // Host-side registry entry for the scenario's sandbox name. + // "present" means `~/.nemoclaw/sandboxes.json` retains the entry, + // even if the live gateway can no longer see the sandbox. This is + // orthogonal to `sandbox.expected`: registry preservation is the + // user-visible regression target for #4423. + localRegistry?: { expected: ExpectedPresence }; + // Host-side Docker container labeled `openshell.ai/sandbox-name=`. + // "present" matches running OR stopped containers, including + // `*-nemoclaw-gpu-backup-*` siblings produced by the GPU patch path. + // Used to assert that recovery information remains available even + // when the live OpenShell gateway returns NotFound. + dockerSandboxContainer?: { expected: ExpectedPresence }; } export type TransientClassifier = From 051611262b96a6b43a9a5f602a983d13e10993df Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 9 Jun 2026 12:21:20 -0400 Subject: [PATCH 03/13] test(e2e): add LifecyclePhaseFixture for post-reboot recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a Vitest phase fixture that mutates host state between onboarding and state-validation, so live scenarios can express post-onboard invariants the legacy bash runner has no equivalent for. `LifecyclePhaseFixture.simulate("post-reboot-recovery", instance, opts)` reproduces the host-side conditions of a DGX Spark / Linux Docker-driver reboot in two modes: * `stop-original` (default) — `openshell gateway stop` + `docker stop` of the labeled sandbox container. Models the common reboot outcome where OpenShell forgets the sandbox while Docker keeps the container exited but labeled. * `rename-to-gpu-backup` — additionally `docker rename`s the container to a `*-nemoclaw-gpu- backup-` sibling, mirroring the GPU-patch reboot path in `src/lib/onboard/docker-gpu-patch.ts`. Both modes register cleanups (in reverse order) to restore the container so test teardown leaves Docker in a usable state. Wiring: * `framework/phases/index.ts` re-exports the fixture and types. * `framework/e2e-test.ts` registers a `lifecycle` Vitest fixture on `E2EScenarioFixtures`, wired with the shared `host`, `sandbox`, and `cleanup` registries. * `live/registry-scenarios.test.ts` invokes `lifecycle.simulate(profile, instance)` between `onboard.from(...)` and `stateValidation.from(...)` whenever the scenario declares a whitelisted `environment.lifecycle`. Scenarios that omit lifecycle are unaffected. A scenario whose lifecycle is whitelisted by `runtime-support.ts` but NOT dispatched by the fixture fails fast with a clear error so the whitelist and dispatcher stay in lock- step. Coverage in `e2e-phase-lifecycle.test.ts` exercises both modes, gateway-stop tolerance, the no-labeled-container failure case, the docker-discover failure case, the unsupported-profile rejection, the cleanup queue order, and `buildBackupContainerName` truncation. The fixture is intentionally narrow on profiles: only `post-reboot-recovery` is dispatched today. Adding rebuild, upgrade, or drift profiles is a separate, equally narrow change set that must land the dispatcher branch and `SUPPORTED_LIFECYCLES` whitelist together. Refs #4423 --- .../e2e-phase-lifecycle.test.ts | 239 ++++++++++++++++++ test/e2e-scenario/framework/e2e-test.ts | 5 + test/e2e-scenario/framework/phases/index.ts | 8 + .../framework/phases/lifecycle.ts | 207 +++++++++++++++ .../live/registry-scenarios.test.ts | 33 ++- 5 files changed, 491 insertions(+), 1 deletion(-) create mode 100644 test/e2e-scenario/framework-tests/e2e-phase-lifecycle.test.ts create mode 100644 test/e2e-scenario/framework/phases/lifecycle.ts diff --git a/test/e2e-scenario/framework-tests/e2e-phase-lifecycle.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-lifecycle.test.ts new file mode 100644 index 00000000000..b4cae772551 --- /dev/null +++ b/test/e2e-scenario/framework-tests/e2e-phase-lifecycle.test.ts @@ -0,0 +1,239 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, expectTypeOf, it } from "vitest"; + +import { + HostCliClient, + SandboxClient, + type CommandRunner, +} from "../framework/clients/index.ts"; +import type { E2EScenarioFixtures } from "../framework/e2e-test.ts"; +import { + buildBackupContainerName, + LifecyclePhaseFixture, + type LifecycleCleanup, +} from "../framework/phases/lifecycle.ts"; +import type { NemoClawInstance } from "../framework/phases/index.ts"; +import type { + ShellProbeResult, + ShellProbeRunOptions, + TrustedShellCommand, +} from "../framework/shell-probe.ts"; + +interface RunnerCall { + command: string; + args: string[]; + options?: ShellProbeRunOptions; +} + +interface CleanupCall { + name: string; + run: () => Promise | void; +} + +function shellResult(exitCode: number, output = ""): ShellProbeResult { + return { + command: [], + exitCode, + signal: null, + timedOut: false, + stdout: exitCode === 0 ? output : "", + stderr: exitCode === 0 ? "" : output, + artifacts: { + stdout: "/tmp/stdout.txt", + stderr: "/tmp/stderr.txt", + result: "/tmp/result.json", + }, + }; +} + +class FakeRunner implements CommandRunner { + readonly calls: RunnerCall[] = []; + private readonly responses: ShellProbeResult[] = []; + + enqueue(response: ShellProbeResult): void { + this.responses.push(response); + } + + async run( + command: TrustedShellCommand, + options?: ShellProbeRunOptions, + ): Promise { + this.calls.push({ command: command.command, args: [...command.args], options }); + const response = this.responses.shift(); + if (!response) { + throw new Error( + `FakeRunner response missing for command: ${command.command} ${command.args.join(" ")}`, + ); + } + return response; + } +} + +class FakeCleanup implements LifecycleCleanup { + readonly calls: CleanupCall[] = []; + + add(name: string, run: () => Promise | void): void { + this.calls.push({ name, run }); + } +} + +function instance(overrides: Partial = {}): NemoClawInstance { + return { + onboarding: "cloud-openclaw", + sandboxName: "e2e-ubuntu-repo-cloud-openclaw", + agent: "openclaw", + provider: "nvidia", + providerEnv: "cloud", + gatewayUrl: "http://127.0.0.1:18789", + result: shellResult(0), + ...overrides, + }; +} + +function fixture(runner: FakeRunner, cleanup: FakeCleanup): LifecyclePhaseFixture { + const host = new HostCliClient(runner); + const sandbox = new SandboxClient(runner); + return new LifecyclePhaseFixture(host, sandbox, cleanup); +} + +describe("LifecyclePhaseFixture.simulate post-reboot-recovery (stop-original)", () => { + it("stops the gateway, discovers the labeled container, and stops it", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0)); // openshell gateway stop + runner.enqueue(shellResult(0, "openshell-cluster-e2e-ubuntu-repo-cloud-openclaw\n")); // discover + runner.enqueue(shellResult(0)); // docker stop + const cleanup = new FakeCleanup(); + + const result = await fixture(runner, cleanup).simulate( + "post-reboot-recovery", + instance(), + ); + + expect(result.profile).toBe("post-reboot-recovery"); + expect(result.steps.map((step) => step.id)).toEqual([ + "gateway-stop", + "docker-stop:openshell-cluster-e2e-ubuntu-repo-cloud-openclaw", + ]); + expect(runner.calls.map((call) => ({ command: call.command, args: call.args }))).toEqual([ + { command: "openshell", args: ["gateway", "stop"] }, + { + command: "docker", + args: [ + "ps", + "-a", + "--filter", + "label=openshell.ai/sandbox-name=e2e-ubuntu-repo-cloud-openclaw", + "--format", + "{{.Names}}", + ], + }, + { command: "docker", args: ["stop", "openshell-cluster-e2e-ubuntu-repo-cloud-openclaw"] }, + ]); + expect(cleanup.calls.map((call) => call.name)).toEqual([ + "lifecycle.docker-start:openshell-cluster-e2e-ubuntu-repo-cloud-openclaw", + ]); + }); + + it("tolerates a non-zero gateway stop (post-reboot fresh runtime)", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(1, "no gateway runtime")); // gateway stop fails + runner.enqueue(shellResult(0, "container-1\n")); + runner.enqueue(shellResult(0)); // docker stop + const cleanup = new FakeCleanup(); + + const result = await fixture(runner, cleanup).simulate( + "post-reboot-recovery", + instance(), + ); + + expect(result.steps.find((step) => step.id === "gateway-stop")).toBeTruthy(); + }); + + it("fails when no Docker container carries the OpenShell sandbox-name label", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0)); // gateway stop + runner.enqueue(shellResult(0, "\n")); // discover returns nothing + const cleanup = new FakeCleanup(); + + await expect( + fixture(runner, cleanup).simulate("post-reboot-recovery", instance()), + ).rejects.toThrow(/expected at least one Docker container labeled/); + }); + + it("fails when docker discover returns non-zero", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0)); // gateway stop + runner.enqueue(shellResult(1, "Cannot connect to the Docker daemon")); + const cleanup = new FakeCleanup(); + + await expect( + fixture(runner, cleanup).simulate("post-reboot-recovery", instance()), + ).rejects.toThrow(/could not query Docker for label/); + }); +}); + +describe("LifecyclePhaseFixture.simulate post-reboot-recovery (rename-to-gpu-backup)", () => { + it("stops, then renames the labeled container to a *-nemoclaw-gpu-backup-* sibling", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0)); // openshell gateway stop + runner.enqueue(shellResult(0, "openshell-cluster-e2e-x\n")); // discover + runner.enqueue(shellResult(0)); // docker stop + runner.enqueue(shellResult(0)); // docker rename + const cleanup = new FakeCleanup(); + + const result = await fixture(runner, cleanup).simulate( + "post-reboot-recovery", + instance({ sandboxName: "e2e-x" }), + { mode: "rename-to-gpu-backup" }, + ); + + expect(result.steps.map((step) => step.id.split("->")[0])).toContain( + "docker-rename:openshell-cluster-e2e-x", + ); + const renameCall = runner.calls.find( + (call) => call.command === "docker" && call.args[0] === "rename", + ); + expect(renameCall).toBeTruthy(); + expect(renameCall!.args[1]).toBe("openshell-cluster-e2e-x"); + expect(renameCall!.args[2]).toMatch(/^openshell-cluster-e2e-x-nemoclaw-gpu-backup-\d+$/); + + // Cleanup queue now has both docker-start and docker-rename-back. + expect(cleanup.calls.map((call) => call.name.split(":")[0])).toEqual([ + "lifecycle.docker-start", + "lifecycle.docker-rename-back", + ]); + }); +}); + +describe("LifecyclePhaseFixture profile dispatch", () => { + it("rejects unknown lifecycle profiles", async () => { + const runner = new FakeRunner(); + const cleanup = new FakeCleanup(); + + await expect( + // @ts-expect-error — exhaustiveness check + fixture(runner, cleanup).simulate("not-a-profile", instance()), + ).rejects.toThrow(/Unsupported lifecycle profile/); + }); + + it("exposes the lifecycle phase on the Vitest scenario context", () => { + expectTypeOf().toEqualTypeOf(); + }); +}); + +describe("buildBackupContainerName", () => { + it("appends -nemoclaw-gpu-backup- to the original name", () => { + expect(buildBackupContainerName("openshell-cluster-foo", 1717280000000)).toBe( + "openshell-cluster-foo-nemoclaw-gpu-backup-1717280000000", + ); + }); + + it("truncates the original name to fit within Docker's 253-char limit", () => { + const longName = "a".repeat(253); + const result = buildBackupContainerName(longName, 1717280000000); + expect(result.length).toBeLessThanOrEqual(253); + expect(result.endsWith("-nemoclaw-gpu-backup-1717280000000")).toBe(true); + }); +}); diff --git a/test/e2e-scenario/framework/e2e-test.ts b/test/e2e-scenario/framework/e2e-test.ts index af31e4560a9..75891b38fa7 100644 --- a/test/e2e-scenario/framework/e2e-test.ts +++ b/test/e2e-scenario/framework/e2e-test.ts @@ -14,6 +14,7 @@ import { import { assertCleanupPassed, CleanupRegistry } from "./cleanup.ts"; import { EnvironmentPhaseFixture, + LifecyclePhaseFixture, OnboardingPhaseFixture, StateValidationPhaseFixture, } from "./phases/index.ts"; @@ -32,6 +33,7 @@ export interface E2EScenarioFixtures { state: StateClient; environment: EnvironmentPhaseFixture; onboard: OnboardingPhaseFixture; + lifecycle: LifecyclePhaseFixture; stateValidation: StateValidationPhaseFixture; } @@ -91,6 +93,9 @@ export const test = base.extend({ onboard: async ({ cleanup, host, secrets }, use) => { await use(new OnboardingPhaseFixture(host, secrets, cleanup)); }, + lifecycle: async ({ cleanup, host, sandbox }, use) => { + await use(new LifecyclePhaseFixture(host, sandbox, cleanup)); + }, stateValidation: async ({ host, gateway, sandbox }, use) => { await use(new StateValidationPhaseFixture(host, gateway, sandbox)); }, diff --git a/test/e2e-scenario/framework/phases/index.ts b/test/e2e-scenario/framework/phases/index.ts index f6345ba476f..1905028a0c0 100644 --- a/test/e2e-scenario/framework/phases/index.ts +++ b/test/e2e-scenario/framework/phases/index.ts @@ -7,6 +7,14 @@ export { type DockerRuntimeReady, type EnvironmentReady, } from "./environment.ts"; +export { + LifecyclePhaseFixture, + type LifecycleCleanup, + type LifecycleProfile, + type LifecycleResult, + type PostRebootMode, + type PostRebootOptions, +} from "./lifecycle.ts"; export { OnboardingPhaseFixture, type NemoClawInstance, diff --git a/test/e2e-scenario/framework/phases/lifecycle.ts b/test/e2e-scenario/framework/phases/lifecycle.ts new file mode 100644 index 00000000000..75225c04811 --- /dev/null +++ b/test/e2e-scenario/framework/phases/lifecycle.ts @@ -0,0 +1,207 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { buildAvailabilityProbeEnv } from "../availability-env.ts"; +import { assertExitZero } from "../clients/command.ts"; +import type { HostCliClient } from "../clients/host.ts"; +import type { SandboxClient } from "../clients/sandbox.ts"; +import type { ShellProbeResult } from "../shell-probe.ts"; +import type { NemoClawInstance } from "./onboarding.ts"; + +// Mirror of `OPENSHELL_SANDBOX_NAME_LABEL` in +// `src/lib/onboard/docker-gpu-patch.ts`. Duplicated here because the +// fixture layer must not import from `src/lib/**` (CLI source) — that +// boundary keeps the live runner honest about probing only host- +// observable state. Drift is caught by the integration test that wires +// a real onboarded sandbox through the docker-sandbox-container-present +// probe. +const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; +const DOCKER_PROBE_TIMEOUT_MS = 15_000; +const GATEWAY_STOP_TIMEOUT_MS = 60_000; + +export type LifecycleProfile = "post-reboot-recovery"; + +export interface LifecycleCleanup { + add(name: string, run: () => Promise | void): void; +} + +/** + * How the post-reboot-recovery profile leaves Docker before the test + * exits the lifecycle phase: + * + * - `stop-original` — `docker stop` the labeled container in place. + * Matches the common Spark reboot path: the + * container exists, is exited, retains its + * OpenShell labels, but is no longer running. + * + * - `rename-to-gpu-backup` — stop the labeled container, then + * `docker rename` it to `-nemoclaw- + * gpu-backup-`. Reproduces the rarer GPU- + * patch reboot path where only the backup + * sibling survives and recovery has to rename + * it back. Mirrors `buildBackupContainerName()` + * in `src/lib/onboard/docker-gpu-patch.ts`. + */ +export type PostRebootMode = "stop-original" | "rename-to-gpu-backup"; + +export interface PostRebootOptions { + mode?: PostRebootMode; +} + +export interface LifecycleResult { + profile: LifecycleProfile; + steps: Array<{ id: string; results: ShellProbeResult[] }>; +} + +export class LifecyclePhaseFixture { + constructor( + private readonly host: HostCliClient, + private readonly sandbox: SandboxClient, + private readonly cleanup: LifecycleCleanup, + ) {} + + async simulate( + profile: LifecycleProfile, + instance: NemoClawInstance, + options: PostRebootOptions = {}, + ): Promise { + switch (profile) { + case "post-reboot-recovery": + return await this.simulatePostReboot(instance, options); + default: { + const _exhaustive: never = profile; + throw new Error(`Unsupported lifecycle profile '${_exhaustive}'.`); + } + } + } + + /** + * Reproduce the host-side conditions of a DGX Spark / Linux Docker-driver + * reboot: + * + * 1. Ask OpenShell to stop its gateway runtime so the in-memory + * sandbox view drops to NotFound. The actual sandbox container + * is unaffected — that is the entire point of the bug class + * tracked by #4423. + * + * 2. Locate the OpenShell-labeled Docker container for the + * scenario's sandbox name and either stop it (default) or + * stop+rename it to a `*-nemoclaw-gpu-backup-*` sibling. + * + * Cleanups (run in reverse order at end of test): + * - rename the backup sibling back to the original name (if we + * created one); + * - `docker start` the labeled container so the sandbox returns + * to a usable state for any teardown that expects it live. + */ + async simulatePostReboot( + instance: NemoClawInstance, + options: PostRebootOptions = {}, + ): Promise { + const mode: PostRebootMode = options.mode ?? "stop-original"; + const steps: LifecycleResult["steps"] = []; + + const gatewayStop = await this.sandbox.openshell(["gateway", "stop"], { + artifactName: "lifecycle-post-reboot-gateway-stop", + env: buildAvailabilityProbeEnv(), + timeoutMs: GATEWAY_STOP_TIMEOUT_MS, + }); + // gateway stop is best-effort: a fresh-start/no-runtime gateway + // will exit non-zero with NoSuchProcess, which is exactly the + // post-reboot state we want to simulate. Don't fail the lifecycle + // phase on it. + steps.push({ id: "gateway-stop", results: [gatewayStop] }); + + const containerNames = await this.discoverLabeledContainerNames(instance); + if (containerNames.length === 0) { + throw new Error( + `lifecycle.post-reboot-recovery expected at least one Docker container labeled ` + + `'${OPENSHELL_SANDBOX_NAME_LABEL}=${instance.sandboxName}', but docker ps -a returned none. ` + + `Did onboarding create the sandbox?`, + ); + } + const originalName = containerNames[0]; + + const stop = await this.host.command( + "docker", + ["stop", originalName], + { + artifactName: `lifecycle-post-reboot-docker-stop-${originalName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: DOCKER_PROBE_TIMEOUT_MS, + }, + ); + assertExitZero(stop, `docker stop ${originalName}`); + steps.push({ id: `docker-stop:${originalName}`, results: [stop] }); + this.cleanup.add(`lifecycle.docker-start:${originalName}`, async () => { + await this.host.command("docker", ["start", originalName], { + artifactName: `lifecycle-cleanup-docker-start-${originalName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: DOCKER_PROBE_TIMEOUT_MS, + }); + }); + + if (mode === "rename-to-gpu-backup") { + const backupName = buildBackupContainerName(originalName, Date.now()); + const rename = await this.host.command( + "docker", + ["rename", originalName, backupName], + { + artifactName: `lifecycle-post-reboot-docker-rename-${originalName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: DOCKER_PROBE_TIMEOUT_MS, + }, + ); + assertExitZero(rename, `docker rename ${originalName} ${backupName}`); + steps.push({ id: `docker-rename:${originalName}->${backupName}`, results: [rename] }); + this.cleanup.add(`lifecycle.docker-rename-back:${backupName}`, async () => { + await this.host.command("docker", ["rename", backupName, originalName], { + artifactName: `lifecycle-cleanup-docker-rename-back-${backupName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: DOCKER_PROBE_TIMEOUT_MS, + }); + }); + } + + return { profile: "post-reboot-recovery", steps }; + } + + private async discoverLabeledContainerNames(instance: NemoClawInstance): Promise { + const result = await this.host.command( + "docker", + [ + "ps", + "-a", + "--filter", + `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${instance.sandboxName}`, + "--format", + "{{.Names}}", + ], + { + artifactName: `lifecycle-post-reboot-docker-discover-${instance.sandboxName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: DOCKER_PROBE_TIMEOUT_MS, + }, + ); + if (result.exitCode !== 0) { + throw new Error( + `lifecycle.post-reboot-recovery could not query Docker for label ` + + `'${OPENSHELL_SANDBOX_NAME_LABEL}=${instance.sandboxName}' (exit ${result.exitCode}).`, + ); + } + return result.stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + } +} + +// Mirror of `MAX_DOCKER_CONTAINER_NAME_LENGTH` in +// `src/lib/onboard/docker-gpu-patch.ts`. +const MAX_DOCKER_CONTAINER_NAME_LENGTH = 253; + +export function buildBackupContainerName(originalName: string, nowMs: number): string { + const suffix = `-nemoclaw-gpu-backup-${String(nowMs)}`; + const maxOriginalLength = MAX_DOCKER_CONTAINER_NAME_LENGTH - suffix.length; + return `${originalName.slice(0, Math.max(1, maxOriginalLength))}${suffix}`; +} diff --git a/test/e2e-scenario/live/registry-scenarios.test.ts b/test/e2e-scenario/live/registry-scenarios.test.ts index 892cadb8124..3395ac553a8 100644 --- a/test/e2e-scenario/live/registry-scenarios.test.ts +++ b/test/e2e-scenario/live/registry-scenarios.test.ts @@ -5,9 +5,16 @@ import fs from "node:fs"; import path from "node:path"; import { expect, test } from "../framework/e2e-test.ts"; +import type { LifecycleProfile } from "../framework/phases/index.ts"; import { listScenarios } from "../scenarios/registry.ts"; import { liveScenarioSupport, liveScenarioTestName } from "../scenarios/runtime-support.ts"; +const LIFECYCLE_PROFILES: ReadonlySet = new Set(["post-reboot-recovery"]); + +function isLifecycleProfile(value: string | undefined): value is LifecycleProfile { + return value !== undefined && LIFECYCLE_PROFILES.has(value as LifecycleProfile); +} + const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const CLI_DIST_ENTRYPOINT = path.join(REPO_ROOT, "dist", "nemoclaw.js"); process.env.NEMOCLAW_CLI_BIN ??= path.join(REPO_ROOT, "bin", "nemoclaw.js"); @@ -30,7 +37,7 @@ for (const scenario of listScenarios()) { test( liveScenarioTestName(scenario), - async ({ artifacts, environment, onboard, secrets, stateValidation }) => { + async ({ artifacts, environment, lifecycle, onboard, secrets, stateValidation }) => { for (const secret of scenario.requiredSecrets ?? []) { secrets.required(secret); } @@ -55,6 +62,27 @@ for (const scenario of listScenarios()) { const ready = await environment.assertReady(scenario.environment); const instance = await onboard.from(ready, { sandboxName: `e2e-${scenario.id}` }); + + // Lifecycle phase runs between onboard and state-validation. + // Scenarios opt in by setting `environment.lifecycle` to a + // whitelisted profile (see SUPPORTED_LIFECYCLES in + // runtime-support.ts). Today only `post-reboot-recovery` is + // wired, and it dispatches through `LifecyclePhaseFixture` to + // mutate host state (gateway runtime, Docker container) before + // the state-validation probes assert preservation invariants. + let lifecycleResult: Awaited> | undefined; + const profile = scenario.environment.lifecycle; + if (profile) { + if (!isLifecycleProfile(profile)) { + throw new Error( + `scenario '${scenario.id}' declares lifecycle '${profile}' which is not ` + + `dispatched by LifecyclePhaseFixture; update the fixture and the ` + + `SUPPORTED_LIFECYCLES whitelist together.`, + ); + } + lifecycleResult = await lifecycle.simulate(profile, instance); + } + const validation = await stateValidation.from(scenario.expectedStateId, instance); await artifacts.writeJson("scenario-result.json", { @@ -62,6 +90,9 @@ for (const scenario of listScenarios()) { expectedStateId: validation.state.id, probes: validation.probes.map((probe) => probe.id), pendingRuntimeSuites: support.pendingRuntimeSuites, + lifecycle: lifecycleResult + ? { profile: lifecycleResult.profile, steps: lifecycleResult.steps.map((s) => s.id) } + : undefined, }); }, ); From 5d132a728bfe32b8c9b2be9f57293f265594625e Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 9 Jun 2026 13:54:36 -0400 Subject: [PATCH 04/13] test(e2e): add post-reboot-recovery scenario as #4423 regression guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Registers the failing-test-first guard for #4423 in the typed scenario registry so the live Vitest matrix from #5006 fans it out as a dedicated CI job. Builds on the framework primitives added earlier in this PR (lifecycle phase fixture, host-side probes, lifecycle whitelist). Additions: * `post-reboot-recovery-ready` expected-state in `scenarios/expected-states.ts` declaring the user-visible invariants that must hold after a `nemoclaw status` call on a freshly-rebooted DGX Spark / Linux Docker-driver host: - cli installed, - gateway healthy (the user-systemd unit from #4580 brings it back up before status runs), - sandbox running (recovery completed in time), - localRegistry entry preserved (the user-visible regression target — destroyed on unfixed `main`), - dockerSandboxContainer present (recovery didn't delete the labeled container or its `*-nemoclaw-gpu-backup-*` sibling). * `ubuntu-repo-docker-post-reboot-recovery` scenario in `scenarios/scenarios/baseline.ts` wiring `ubuntuRepoDockerLifecycle("cloud-openclaw", "post-reboot-recovery")` against the new expected-state and a smoke suite. Carries a description that explains the RED/GREEN contract and points to the PR-A fix landing in `src/lib/`. * `manifests/openclaw-nvidia-post-reboot-recovery.yaml` declares `lifecycle: post-reboot-recovery` and the same NVIDIA_API_KEY credential ref the cloud-openclaw scenarios use. * `.github/workflows/e2e-scenarios.yaml` ROUTES table gains the new scenario so the workflow-boundary test (`e2e-scenarios-workflow.test.ts`) routes every typed id. Test pinning: * `e2e-scenario-matrix.test.ts` updated from a 1-entry to a 2-entry live matrix expectation. The new entry asserts on `expectedStateId: "post-reboot-recovery-ready"` so a future accidental dropped-lifecycle change to the scenario regresses loudly. * `e2e-live-registry-discovery.test.ts` swaps the synthetic whitelist-coverage test for an assertion against the real `ubuntu-repo-docker-post-reboot-recovery` registry entry. Behavior: * On unfixed `main`, the live runner's lifecycle phase stops the OpenShell gateway runtime and `docker stop`s the labeled sandbox container. State-validation then runs `nemoclaw status` (which restarts the gateway via systemd) and the destructive `missing` branch in `src/lib/actions/sandbox/status.ts` wipes the local registry entry. The `local-registry-entry-present` probe fails. Scenario goes RED. * On the PR-A fix branch, the new Docker-driver sandbox recovery helper restarts the labeled container before stale-removal can fire, registry survives, all five probes pass. Scenario flips GREEN. The bash-side legacy compiler emits a `lifecycle.profile.post-reboot-recovery` PhaseAction pointing at `nemoclaw_scenarios/lifecycle/dispatch.sh`, but the legacy bash worker is intentionally not provided: this scenario is Vitest-only. The typed runner's `LifecyclePhaseFixture` handles dispatch directly. If the legacy runner is invoked against this scenario it errors out at the dispatcher; that's the right failure mode while the bash side stays on its own retirement clock. Refs #4423 --- .github/workflows/e2e-scenarios.yaml | 1 + .../e2e-live-registry-discovery.test.ts | 30 +++++--------- .../e2e-scenario-matrix.test.ts | 22 ++++++++++- .../openclaw-nvidia-post-reboot-recovery.yaml | 39 +++++++++++++++++++ .../e2e-scenario/scenarios/expected-states.ts | 26 +++++++++++++ .../scenarios/scenarios/baseline.ts | 32 +++++++++++++++ 6 files changed, 129 insertions(+), 21 deletions(-) create mode 100644 test/e2e-scenario/manifests/openclaw-nvidia-post-reboot-recovery.yaml diff --git a/.github/workflows/e2e-scenarios.yaml b/.github/workflows/e2e-scenarios.yaml index 49f317caff2..804426f1331 100644 --- a/.github/workflows/e2e-scenarios.yaml +++ b/.github/workflows/e2e-scenarios.yaml @@ -78,6 +78,7 @@ jobs: [ubuntu-repo-cloud-openclaw-slack]=ubuntu-latest [ubuntu-repo-cloud-openclaw-telegram]=ubuntu-latest [ubuntu-repo-cloud-openclaw-token-rotation]=ubuntu-latest + [ubuntu-repo-docker-post-reboot-recovery]=ubuntu-latest [ubuntu-repo-openai-compatible-openclaw]=ubuntu-latest ) selected="" diff --git a/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts b/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts index db1dea7f4d1..35f51b78741 100644 --- a/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts @@ -63,26 +63,16 @@ describe("live Vitest registry discovery support", () => { }); }); - it("accepts whitelisted lifecycle profiles when the rest of the environment matches", () => { - // Synthesised scenario stands in for the not-yet-registered - // post-reboot-recovery scenario so this test pins the whitelist - // contract independently of when the scenario lands. Once the - // post-reboot-recovery scenario is registered, prefer asserting on - // the registry entry directly and remove this synthetic. - const supported = liveScenarioSupport({ - id: "synthetic-post-reboot-recovery", - assertionGroups: [], - expectedStateId: "cloud-openclaw-ready", - environment: { - platform: "ubuntu-local", - install: "repo-current", - runtime: "docker-running", - onboarding: "cloud-openclaw", - lifecycle: "post-reboot-recovery", - }, - }); + it("accepts the whitelisted post-reboot-recovery lifecycle scenario", () => { + const scenario = listScenarios().find( + (entry) => entry.id === "ubuntu-repo-docker-post-reboot-recovery", + ); - expect(supported.supported).toBe(true); - expect(supported.reasons).toEqual([]); + expect(scenario).toBeTruthy(); + expect(scenario!.environment?.lifecycle).toBe("post-reboot-recovery"); + expect(liveScenarioSupport(scenario!)).toMatchObject({ + supported: true, + reasons: [], + }); }); }); diff --git a/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts b/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts index a6a9bf9d18e..78eb216d53d 100644 --- a/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts @@ -131,6 +131,7 @@ describe("typed scenario matrix", () => { it("builds the default live Vitest matrix from fixture-supported scenarios only", () => { expect(buildLiveScenarioMatrix().map((entry) => entry.id)).toEqual([ "ubuntu-repo-cloud-openclaw", + "ubuntu-repo-docker-post-reboot-recovery", ]); expect(buildLiveScenarioMatrix()[0]).toMatchObject({ id: "ubuntu-repo-cloud-openclaw", @@ -145,6 +146,22 @@ describe("typed scenario matrix", () => { supportReasons: [], pendingRuntimeSuites: ["smoke", "inference", "credentials"], }); + // Failing-test-first guard for #4423. Pinned in the matrix to + // confirm the lifecycle whitelist + post-reboot-recovery scenario + // are wired together; the actual RED/GREEN behavior is exercised + // by the live runner (gates on the fix landing in src/lib/). + expect(buildLiveScenarioMatrix()[1]).toMatchObject({ + id: "ubuntu-repo-docker-post-reboot-recovery", + runner: "ubuntu-latest", + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-openclaw", + expectedStateId: "post-reboot-recovery-ready", + requiredSecrets: ["NVIDIA_API_KEY"], + supported: true, + supportReasons: [], + }); }); it("keeps explicitly selected unsupported live scenarios in the matrix with skip reasons", () => { @@ -163,7 +180,10 @@ describe("typed scenario matrix", () => { const lines = result.stdout.trim().split("\n"); expect(lines.length, "live matrix output must be a single line").toBe(1); const parsed = JSON.parse(lines[0]); - expect(parsed.map((entry: { id: string }) => entry.id)).toEqual(["ubuntu-repo-cloud-openclaw"]); + expect(parsed.map((entry: { id: string }) => entry.id)).toEqual([ + "ubuntu-repo-cloud-openclaw", + "ubuntu-repo-docker-post-reboot-recovery", + ]); }); it("--emit-live-matrix honors explicit scenario selections", () => { diff --git a/test/e2e-scenario/manifests/openclaw-nvidia-post-reboot-recovery.yaml b/test/e2e-scenario/manifests/openclaw-nvidia-post-reboot-recovery.yaml new file mode 100644 index 00000000000..571ec5b2f5f --- /dev/null +++ b/test/e2e-scenario/manifests/openclaw-nvidia-post-reboot-recovery.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: openclaw-nvidia-post-reboot-recovery +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: running + platform: + os: ubuntu + executionTarget: local + onboarding: + agent: openclaw + provider: nvidia + modelRoute: inference-local + policyTier: balanced + messaging: [] + # Lifecycle phase opt-in. The Vitest live runner dispatches this + # profile through `LifecyclePhaseFixture.simulate(...)`, which: + # 1. stops the OpenShell gateway runtime, and + # 2. `docker stop`s the labeled sandbox container. + # The host-side state-validation probes + # (`local-registry-entry-present`, `docker-sandbox-container-present`) + # then assert that `nemoclaw status` recovers the sandbox + # without destroying registry state. On unfixed code the registry + # entry is wiped by the `missing` branch in + # `src/lib/actions/sandbox/{status,gateway-state}.ts` and the + # `local-registry-entry-present` probe fails. See #4423. + lifecycle: post-reboot-recovery + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY diff --git a/test/e2e-scenario/scenarios/expected-states.ts b/test/e2e-scenario/scenarios/expected-states.ts index d79b32e185d..75027f5e238 100644 --- a/test/e2e-scenario/scenarios/expected-states.ts +++ b/test/e2e-scenario/scenarios/expected-states.ts @@ -77,6 +77,31 @@ const onboardingFailureGatewayPortConflict: ExpectedState = { sandbox: { expected: "absent" }, }; +// Post-reboot recovery contract for #4423. After the lifecycle phase +// stops the OpenShell gateway runtime + the labeled sandbox container, +// the user-visible invariants are: +// +// * `cli` still installed. +// * `gateway` healthy: the user-systemd unit from #4580 brings the +// gateway back up before status runs. +// * `sandbox` running by the time validation completes: a correct +// fix performs Docker-backed recovery before responding. +// * `localRegistry` entry preserved: this is the user-visible +// regression target. On unfixed code, the destructive `missing` +// branch wipes the entry; on fixed code it survives because +// Docker corroborated the sandbox container existence. +// * `dockerSandboxContainer` still present: the recovery path must +// not delete the labeled container or its `*-nemoclaw-gpu-backup-*` +// sibling as a side effect. +const postRebootRecoveryReady: ExpectedState = { + id: "post-reboot-recovery-ready", + cli: { installed: true }, + gateway: { expected: "present", health: "healthy" }, + sandbox: { expected: "present", status: "running", agent: "openclaw" }, + localRegistry: { expected: "present" }, + dockerSandboxContainer: { expected: "present" }, +}; + const REGISTRY: readonly ExpectedState[] = [ cloudOpenclawReady, cloudOpenclawCustomPoliciesReady, @@ -86,6 +111,7 @@ const REGISTRY: readonly ExpectedState[] = [ preflightFailureNoSandbox, onboardingFailureInvalidNvidiaKey, onboardingFailureGatewayPortConflict, + postRebootRecoveryReady, ]; const BY_ID: ReadonlyMap = new Map( diff --git a/test/e2e-scenario/scenarios/scenarios/baseline.ts b/test/e2e-scenario/scenarios/scenarios/baseline.ts index a5b0e050406..4e5befad679 100644 --- a/test/e2e-scenario/scenarios/scenarios/baseline.ts +++ b/test/e2e-scenario/scenarios/scenarios/baseline.ts @@ -148,6 +148,38 @@ const canonicalScenarioInputs: CanonicalScenarioInput[] = [ suiteIds: ["smoke", "rebuild", "upgrade"], requiredSecrets: ["NVIDIA_API_KEY"], }, + { + // Failing-test-first regression guard for #4423. After onboarding, + // the lifecycle phase reproduces the host-side conditions of a + // DGX Spark / Linux Docker-driver reboot: stop the OpenShell + // gateway runtime + `docker stop` the labeled sandbox container. + // The state-validation phase then runs `nemoclaw status` + // and asserts the post-recovery invariants declared by the + // `post-reboot-recovery-ready` expected-state. + // + // On unfixed `main`, the destructive `missing` branch in + // `src/lib/actions/sandbox/status.ts` (and the parallel branch + // reached through `ensureLiveSandboxOrExit` in + // `src/lib/actions/sandbox/gateway-state.ts`) wipes the local + // registry entry once the gateway returns to `healthy_named`, + // so the `local-registry-entry-present` probe fails and this + // scenario goes RED. + // + // The fix lands in PR-A (parts 2 & 3 of ericksoa's plan): add a + // Docker-driver sandbox recovery helper, then tighten + // stale-removal in active paths to require Docker-corroborated + // absence before destroying the registry. PR-A flips this guard + // 🔴 → 🟢. + id: "ubuntu-repo-docker-post-reboot-recovery", + manifestName: "openclaw-nvidia-post-reboot-recovery", + environment: ubuntuRepoDockerLifecycle("cloud-openclaw", "post-reboot-recovery"), + expectedStateId: "post-reboot-recovery-ready", + suiteIds: ["smoke"], + requiredSecrets: ["NVIDIA_API_KEY"], + description: + "Failing-test-first guard for #4423: post-reboot recovery must preserve " + + "the local registry entry and restart the labeled Docker container.", + }, { id: "ubuntu-repo-openai-compatible-openclaw", manifestName: "openclaw-openai-compatible", From 34466dd271422e450d9038f2bb826494f3a9c090 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 11:06:13 -0700 Subject: [PATCH 05/13] chore(e2e): scaffold inventory internals migration draft Signed-off-by: Carlos Villela From 48019e6039b2f858ad2d4e65230c783ef849050c Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 11:07:30 -0700 Subject: [PATCH 06/13] chore(e2e): scaffold fan-out draft 02 Signed-off-by: Carlos Villela From 06b84f36b6ef637259468f86e824f1ca188517ce Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 9 Jun 2026 14:22:08 -0400 Subject: [PATCH 07/13] chore(e2e): apply biome formatting Prek hook auto-fixed formatting in 6 files added/touched by this PR. No behavior change. --- node_modules | 1 + .../e2e-expected-state.test.ts | 10 ++----- .../e2e-live-registry-discovery.test.ts | 4 +-- .../e2e-phase-lifecycle.test.ts | 16 ++--------- .../e2e-phase-state-validation.test.ts | 5 +--- .../framework/phases/lifecycle.ts | 28 +++++++------------ .../framework/phases/state-validation.ts | 4 +-- 7 files changed, 19 insertions(+), 49 deletions(-) create mode 120000 node_modules diff --git a/node_modules b/node_modules new file mode 120000 index 00000000000..8b4b1610468 --- /dev/null +++ b/node_modules @@ -0,0 +1 @@ +/Users/jyaunches/Development/NemoClaw/node_modules \ No newline at end of file diff --git a/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts b/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts index dcf15974923..173a32a0070 100644 --- a/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts @@ -90,10 +90,7 @@ describe("probesForState maps typed expected-state into probe ids", () => { cli: { installed: true }, localRegistry: { expected: "present" }, }; - expect(probesForState(state)).toEqual([ - "cli-installed", - "local-registry-entry-present", - ]); + expect(probesForState(state)).toEqual(["cli-installed", "local-registry-entry-present"]); }); it("dockerSandboxContainer.expected=present emits the docker-sandbox-container-present probe", () => { @@ -102,10 +99,7 @@ describe("probesForState maps typed expected-state into probe ids", () => { cli: { installed: true }, dockerSandboxContainer: { expected: "present" }, }; - expect(probesForState(state)).toEqual([ - "cli-installed", - "docker-sandbox-container-present", - ]); + expect(probesForState(state)).toEqual(["cli-installed", "docker-sandbox-container-present"]); }); it("localRegistry/dockerSandboxContainer 'absent' emits no probe today", () => { diff --git a/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts b/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts index 35f51b78741..7158039d2c8 100644 --- a/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts @@ -57,9 +57,7 @@ describe("live Vitest registry discovery support", () => { expect(scenario).toBeTruthy(); expect(liveScenarioSupport(scenario!)).toMatchObject({ supported: false, - reasons: [ - "lifecycle 'rebuild-current-version' is not wired for live Vitest fixtures", - ], + reasons: ["lifecycle 'rebuild-current-version' is not wired for live Vitest fixtures"], }); }); diff --git a/test/e2e-scenario/framework-tests/e2e-phase-lifecycle.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-lifecycle.test.ts index b4cae772551..5590016373d 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-lifecycle.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-lifecycle.test.ts @@ -3,11 +3,7 @@ import { describe, expect, expectTypeOf, it } from "vitest"; -import { - HostCliClient, - SandboxClient, - type CommandRunner, -} from "../framework/clients/index.ts"; +import { HostCliClient, SandboxClient, type CommandRunner } from "../framework/clients/index.ts"; import type { E2EScenarioFixtures } from "../framework/e2e-test.ts"; import { buildBackupContainerName, @@ -106,10 +102,7 @@ describe("LifecyclePhaseFixture.simulate post-reboot-recovery (stop-original)", runner.enqueue(shellResult(0)); // docker stop const cleanup = new FakeCleanup(); - const result = await fixture(runner, cleanup).simulate( - "post-reboot-recovery", - instance(), - ); + const result = await fixture(runner, cleanup).simulate("post-reboot-recovery", instance()); expect(result.profile).toBe("post-reboot-recovery"); expect(result.steps.map((step) => step.id)).toEqual([ @@ -143,10 +136,7 @@ describe("LifecyclePhaseFixture.simulate post-reboot-recovery (stop-original)", runner.enqueue(shellResult(0)); // docker stop const cleanup = new FakeCleanup(); - const result = await fixture(runner, cleanup).simulate( - "post-reboot-recovery", - instance(), - ); + const result = await fixture(runner, cleanup).simulate("post-reboot-recovery", instance()); expect(result.steps.find((step) => step.id === "gateway-stop")).toBeTruthy(); }); diff --git a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts index 50cf1306d59..2d4011061b7 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts @@ -541,10 +541,7 @@ describe("state-validation host-side probes", () => { it("docker-sandbox-container-present matches *-nemoclaw-gpu-backup-* sibling containers", async () => { const runner = new FakeRunner(); runner.enqueue( - shellResult( - 0, - "e2e-ubuntu-repo-cloud-openclaw-nemoclaw-gpu-backup-1717280000000\n", - ), + shellResult(0, "e2e-ubuntu-repo-cloud-openclaw-nemoclaw-gpu-backup-1717280000000\n"), ); const fx = fixture(runner); diff --git a/test/e2e-scenario/framework/phases/lifecycle.ts b/test/e2e-scenario/framework/phases/lifecycle.ts index 75225c04811..3b130f789fa 100644 --- a/test/e2e-scenario/framework/phases/lifecycle.ts +++ b/test/e2e-scenario/framework/phases/lifecycle.ts @@ -122,15 +122,11 @@ export class LifecyclePhaseFixture { } const originalName = containerNames[0]; - const stop = await this.host.command( - "docker", - ["stop", originalName], - { - artifactName: `lifecycle-post-reboot-docker-stop-${originalName}`, - env: buildAvailabilityProbeEnv(), - timeoutMs: DOCKER_PROBE_TIMEOUT_MS, - }, - ); + const stop = await this.host.command("docker", ["stop", originalName], { + artifactName: `lifecycle-post-reboot-docker-stop-${originalName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: DOCKER_PROBE_TIMEOUT_MS, + }); assertExitZero(stop, `docker stop ${originalName}`); steps.push({ id: `docker-stop:${originalName}`, results: [stop] }); this.cleanup.add(`lifecycle.docker-start:${originalName}`, async () => { @@ -143,15 +139,11 @@ export class LifecyclePhaseFixture { if (mode === "rename-to-gpu-backup") { const backupName = buildBackupContainerName(originalName, Date.now()); - const rename = await this.host.command( - "docker", - ["rename", originalName, backupName], - { - artifactName: `lifecycle-post-reboot-docker-rename-${originalName}`, - env: buildAvailabilityProbeEnv(), - timeoutMs: DOCKER_PROBE_TIMEOUT_MS, - }, - ); + const rename = await this.host.command("docker", ["rename", originalName, backupName], { + artifactName: `lifecycle-post-reboot-docker-rename-${originalName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: DOCKER_PROBE_TIMEOUT_MS, + }); assertExitZero(rename, `docker rename ${originalName} ${backupName}`); steps.push({ id: `docker-rename:${originalName}->${backupName}`, results: [rename] }); this.cleanup.add(`lifecycle.docker-rename-back:${backupName}`, async () => { diff --git a/test/e2e-scenario/framework/phases/state-validation.ts b/test/e2e-scenario/framework/phases/state-validation.ts index e919adc17de..d7a9e5de2d1 100644 --- a/test/e2e-scenario/framework/phases/state-validation.ts +++ b/test/e2e-scenario/framework/phases/state-validation.ts @@ -278,9 +278,7 @@ export class StateValidationPhaseFixture { return { id: "sandbox-running", status: "passed", results: [result] }; } - private expectLocalRegistryEntryPresent( - instance: NemoClawInstance, - ): StateValidationProbeResult { + private expectLocalRegistryEntryPresent(instance: NemoClawInstance): StateValidationProbeResult { const reader = this.io.readRegistry ?? defaultReadRegistry; const registry = reader(); if (!registry) { From 839b7bfd874669aed369bcac4ba42cfe7b5ed843 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 9 Jun 2026 14:22:49 -0400 Subject: [PATCH 08/13] chore(e2e): drop accidental node_modules symlink The biome-format commit accidentally added a node_modules symlink alongside the formatting fixes. Remove it; the directory is already in .gitignore. --- node_modules | 1 - 1 file changed, 1 deletion(-) delete mode 120000 node_modules diff --git a/node_modules b/node_modules deleted file mode 120000 index 8b4b1610468..00000000000 --- a/node_modules +++ /dev/null @@ -1 +0,0 @@ -/Users/jyaunches/Development/NemoClaw/node_modules \ No newline at end of file From 5ed17356bc3b4717b39614824c5382383fc641e7 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 11:06:13 -0700 Subject: [PATCH 09/13] test(e2e): extend migration inventory to runner internals --- test/e2e-scenario/docs/MIGRATION.md | 14 +- test/e2e-scenario/docs/README.md | 9 +- .../e2e-migration-inventory.test.ts | 148 +++++++++++++++--- .../migration/legacy-inventory.json | 70 +++++++++ 4 files changed, 209 insertions(+), 32 deletions(-) diff --git a/test/e2e-scenario/docs/MIGRATION.md b/test/e2e-scenario/docs/MIGRATION.md index 0da12a97e59..170b3871d0d 100644 --- a/test/e2e-scenario/docs/MIGRATION.md +++ b/test/e2e-scenario/docs/MIGRATION.md @@ -84,18 +84,22 @@ that owns the work instead. The one repo-local exception is the machine-readable deletion gate inventory at `test/e2e-scenario/migration/legacy-inventory.json`. Keep that file focused on -script-level migration state that prevents accidental legacy E2E deletion. It +deletion-readiness evidence that prevents accidental legacy E2E deletion. It must cover every direct legacy shell entrypoint under `test/e2e/test-*.sh`, -plus any explicitly retained bridge entrypoints such as Brev. It is not a -progress dashboard or owner queue: +plus any explicitly retained bridge entrypoints such as Brev. It also tracks +coarse internal legacy runner surfaces such as the YAML/bash scenario workers, +validation suites, TypeScript shell-runner orchestrators, and runtime helper +libraries so those surfaces cannot be removed without #4357 evidence. It is not +a progress dashboard or owner queue: - `not-migrated`: legacy coverage still has no equivalent Vitest scenario. - `bridge-probe`: coverage is temporarily represented by a bridge path. - `covered`: equivalent Vitest live scenario coverage exists. - `retired`: maintainers agreed the legacy coverage is no longer required. -Do not set `deletionReady: true` unless the entry is `covered` or `retired` and -the deletion approval is recorded through #4357. +Do not set `deletionReady: true` on a script entry or internal surface unless +the record is `covered` or `retired` and the deletion approval is recorded +through #4357. After #4357 completes final legacy E2E reconciliation, remove the inventory if there are no remaining legacy entrypoints to guard. If maintainers keep it, keep diff --git a/test/e2e-scenario/docs/README.md b/test/e2e-scenario/docs/README.md index 10893cb5d03..8f0f12f704b 100644 --- a/test/e2e-scenario/docs/README.md +++ b/test/e2e-scenario/docs/README.md @@ -203,9 +203,12 @@ is tracked in #4941. The narrow repo-local exception is `test/e2e-scenario/migration/legacy-inventory.json`, a machine-readable deletion gate for direct legacy `test/e2e/test-*.sh` entrypoints and explicit bridge -entrypoints. It should prevent accidental deletions, not become a parallel -status table. Remove it after #4357 completes final legacy E2E reconciliation, -or keep it only as an audit artifact if maintainers still need that record. +entrypoints. It also tracks coarse internal legacy runner surfaces such as +scenario shell workers, validation suites, shell-runner orchestrators, and +runtime helper libraries so they cannot be removed without #4357 evidence. It +should prevent accidental deletions, not become a parallel status table. Remove +it after #4357 completes final legacy E2E reconciliation, or keep it only as an +audit artifact if maintainers still need that record. The old workflow-level parity report has been removed. Use scenario framework tests, the coverage report, PR review, and the audit issues to decide what to diff --git a/test/e2e-scenario/framework-tests/e2e-migration-inventory.test.ts b/test/e2e-scenario/framework-tests/e2e-migration-inventory.test.ts index fe43f3ca724..14998a690a5 100644 --- a/test/e2e-scenario/framework-tests/e2e-migration-inventory.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-migration-inventory.test.ts @@ -10,6 +10,14 @@ const INVENTORY_PATH = path.resolve(import.meta.dirname, "../migration/legacy-in const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const LEGACY_E2E_DIR = path.join(REPO_ROOT, "test/e2e"); const EXPECTED_STATUS_VALUES = ["not-migrated", "bridge-probe", "covered", "retired"] as const; +const INTERNAL_SURFACE_ROOTS = [ + "test/e2e-scenario/nemoclaw_scenarios", + "test/e2e-scenario/onboarding_assertions", + "test/e2e-scenario/runtime/lib", + "test/e2e-scenario/runtime/reports", + "test/e2e-scenario/scenarios/orchestrators", + "test/e2e-scenario/validation_suites", +] as const; type MigrationStatus = "not-migrated" | "bridge-probe" | "covered" | "retired"; @@ -26,6 +34,21 @@ interface LegacyInventoryEntry { notes: string; } +interface LegacyInternalSurface { + id: string; + paths: string[]; + domain: string; + ownerIssue: string; + status: MigrationStatus; + replacementSurface: string; + targetVitestScenarios: string[]; + bridgeProbes: string[]; + retiredReason: string; + deletionReady: boolean; + deletionApprovalIssue?: string; + notes: string; +} + interface LegacyInventory { version: number; statusValues: MigrationStatus[]; @@ -33,6 +56,7 @@ interface LegacyInventory { requires: string[]; }; entries: LegacyInventoryEntry[]; + internalSurfaces: LegacyInternalSurface[]; } function loadInventory(): LegacyInventory { @@ -54,16 +78,86 @@ function listLegacyShellEntrypoints(): string[] { .sort(); } +function listRepoFilesUnder(repoRelativeDir: string): string[] { + const absoluteDir = path.join(REPO_ROOT, repoRelativeDir); + const files: string[] = []; + const visit = (dir: string) => { + for (const dirent of fs.readdirSync(dir, { withFileTypes: true })) { + const absolutePath = path.join(dir, dirent.name); + if (dirent.isDirectory()) { + visit(absolutePath); + } else if (dirent.isFile()) { + files.push(path.relative(REPO_ROOT, absolutePath).split(path.sep).join("/")); + } + } + }; + visit(absoluteDir); + return files.sort(); +} + +function isCoveredByInventoryPath(filePath: string, inventoryPath: string): boolean { + return filePath === inventoryPath || filePath.startsWith(`${inventoryPath}/`); +} + +function expectPathListIsRepoRelative(paths: readonly string[]) { + expect(paths.length).toBeGreaterThan(0); + for (const repoRelativePath of paths) { + expect(repoRelativePath).not.toBe(""); + expect(repoPathExists(repoRelativePath)).toBe(true); + } +} + +function expectMigrationRecordDeletionGate( + record: Pick< + LegacyInventoryEntry | LegacyInternalSurface, + | "status" + | "targetVitestScenarios" + | "bridgeProbes" + | "retiredReason" + | "deletionReady" + | "deletionApprovalIssue" + >, +) { + if (record.status === "covered") { + expect(record.targetVitestScenarios.length).toBeGreaterThan(0); + for (const scenario of record.targetVitestScenarios) { + expect(scenario).toMatch(/^test\/e2e-scenario\/live\/.+\.test\.ts$/); + expect(repoPathExists(scenario)).toBe(true); + } + } + + if (record.status === "bridge-probe") { + expect(record.bridgeProbes.length).toBeGreaterThan(0); + for (const probe of record.bridgeProbes) { + expect(repoPathExists(probe)).toBe(true); + } + } + + if (record.status === "retired") { + expect(record.retiredReason).not.toBe(""); + } + + if (record.deletionReady) { + expect(["covered", "retired"]).toContain(record.status); + expect(record.deletionApprovalIssue).toBe("#4357"); + expect( + record.status === "retired" ? record.retiredReason : record.targetVitestScenarios.length, + ).toBeTruthy(); + } +} + describe("E2E migration inventory deletion gates", () => { it("uses a constrained migration vocabulary with owning issues", () => { const inventory = loadInventory(); const statuses = new Set(inventory.statusValues); const legacyScripts = new Set(); + const internalSurfaceIds = new Set(); expect(inventory.version).toBe(1); expect(inventory.statusValues).toEqual([...EXPECTED_STATUS_VALUES]); expect(inventory.deletionReadiness.requires.length).toBeGreaterThan(0); expect(inventory.entries.length).toBeGreaterThan(0); + expect(inventory.internalSurfaces.length).toBeGreaterThan(0); for (const entry of inventory.entries) { expect(statuses.has(entry.status)).toBe(true); @@ -75,6 +169,18 @@ describe("E2E migration inventory deletion gates", () => { expect(entry.ownerIssue).toMatch(/^#(?:3588|434[7-9]|435[0-7]|4941)$/); expect(entry.notes).not.toBe(""); } + + for (const surface of inventory.internalSurfaces) { + expect(statuses.has(surface.status)).toBe(true); + expect(surface.id).toMatch(/^[a-z0-9-]+$/); + expect(internalSurfaceIds.has(surface.id)).toBe(false); + internalSurfaceIds.add(surface.id); + expectPathListIsRepoRelative(surface.paths); + expect(surface.domain).not.toBe(""); + expect(surface.ownerIssue).toMatch(/^#(?:3588|434[7-9]|435[0-7]|4941)$/); + expect(surface.replacementSurface).not.toBe(""); + expect(surface.notes).not.toBe(""); + } }); it("covers every current direct legacy shell entrypoint", () => { @@ -87,36 +193,30 @@ describe("E2E migration inventory deletion gates", () => { expect(inventoriedShellScripts).toEqual(listLegacyShellEntrypoints()); }); - it("requires coverage, retirement evidence, and #4357 approval before deletion", () => { + it("covers legacy scenario runner internal surfaces by path", () => { const inventory = loadInventory(); + const surfacePaths = inventory.internalSurfaces.flatMap((surface) => surface.paths); - for (const entry of inventory.entries) { - if (entry.status === "covered") { - expect(entry.targetVitestScenarios.length).toBeGreaterThan(0); - for (const scenario of entry.targetVitestScenarios) { - expect(scenario).toMatch(/^test\/e2e-scenario\/live\/.+\.test\.ts$/); - expect(repoPathExists(scenario)).toBe(true); - } + for (const root of INTERNAL_SURFACE_ROOTS) { + const files = listRepoFilesUnder(root); + expect(files.length).toBeGreaterThan(0); + for (const file of files) { + expect( + surfacePaths.some((surfacePath) => isCoveredByInventoryPath(file, surfacePath)), + ).toBe(true); } + } + }); - if (entry.status === "bridge-probe") { - expect(entry.bridgeProbes.length).toBeGreaterThan(0); - for (const probe of entry.bridgeProbes) { - expect(repoPathExists(probe)).toBe(true); - } - } + it("requires coverage, retirement evidence, and #4357 approval before deletion", () => { + const inventory = loadInventory(); - if (entry.status === "retired") { - expect(entry.retiredReason).not.toBe(""); - } + for (const entry of inventory.entries) { + expectMigrationRecordDeletionGate(entry); + } - if (entry.deletionReady) { - expect(["covered", "retired"]).toContain(entry.status); - expect(entry.deletionApprovalIssue).toBe("#4357"); - expect( - entry.status === "retired" ? entry.retiredReason : entry.targetVitestScenarios.length, - ).toBeTruthy(); - } + for (const surface of inventory.internalSurfaces) { + expectMigrationRecordDeletionGate(surface); } }); }); diff --git a/test/e2e-scenario/migration/legacy-inventory.json b/test/e2e-scenario/migration/legacy-inventory.json index 154bdf76c67..e102478d913 100644 --- a/test/e2e-scenario/migration/legacy-inventory.json +++ b/test/e2e-scenario/migration/legacy-inventory.json @@ -816,5 +816,75 @@ "deletionReady": false, "notes": "Already uses Vitest, but still dispatches legacy remote shell suites; keep as a bridge until remote execution uses shared fixtures." } + ], + "internalSurfaces": [ + { + "id": "typed-shell-orchestrators", + "paths": ["test/e2e-scenario/scenarios/orchestrators"], + "domain": "scenario-runner", + "ownerIssue": "#4357", + "status": "not-migrated", + "replacementSurface": "test/e2e-scenario/framework/phases", + "targetVitestScenarios": [], + "bridgeProbes": [], + "retiredReason": "", + "deletionReady": false, + "notes": "Retire after the registry-driven Vitest runner owns phase ordering, expected-failure matching, cleanup, redaction, and artifact evidence." + }, + { + "id": "legacy-bash-scenario-workers", + "paths": ["test/e2e-scenario/nemoclaw_scenarios"], + "domain": "scenario-runner", + "ownerIssue": "#4357", + "status": "not-migrated", + "replacementSurface": "test/e2e-scenario/framework/phases", + "targetVitestScenarios": [], + "bridgeProbes": [], + "retiredReason": "", + "deletionReady": false, + "notes": "Install, onboarding, lifecycle, probe, fixture, and context workers are bridge adapters until equivalent Vitest fixtures or CI setup actions own those phases." + }, + { + "id": "legacy-onboarding-assertion-workers", + "paths": ["test/e2e-scenario/onboarding_assertions"], + "domain": "smoke-onboarding", + "ownerIssue": "#4348", + "status": "not-migrated", + "replacementSurface": "test/e2e-scenario/framework/phases/onboarding.ts", + "targetVitestScenarios": [], + "bridgeProbes": [], + "retiredReason": "", + "deletionReady": false, + "notes": "Retire after onboarding phase fixtures emit equivalent pass/fail evidence for base install and preflight assertions." + }, + { + "id": "legacy-validation-suites", + "paths": ["test/e2e-scenario/validation_suites"], + "domain": "runtime-suites", + "ownerIssue": "#4357", + "status": "not-migrated", + "replacementSurface": "test/e2e-scenario/framework/phases", + "targetVitestScenarios": [], + "bridgeProbes": [], + "retiredReason": "", + "deletionReady": false, + "notes": "Runtime suite assertions migrate one family at a time into typed Vitest runtime helpers before this shell suite tree can be removed." + }, + { + "id": "legacy-runtime-helper-libraries", + "paths": [ + "test/e2e-scenario/runtime/lib", + "test/e2e-scenario/runtime/reports" + ], + "domain": "scenario-runner", + "ownerIssue": "#4357", + "status": "not-migrated", + "replacementSurface": "test/e2e-scenario/framework", + "targetVitestScenarios": [], + "bridgeProbes": [], + "retiredReason": "", + "deletionReady": false, + "notes": "Runtime helper libraries stay only while bridge shell workers need shared environment, logging, context, teardown, or report rendering behavior." + } ] } From 24b7e890cf50c57a09b65b76a8f7818e7ece37af Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 11:40:15 -0700 Subject: [PATCH 10/13] chore(e2e): apply static formatting --- src/commands/sandbox/agents/list.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/commands/sandbox/agents/list.ts b/src/commands/sandbox/agents/list.ts index c2f373b6f91..c5d2027cc72 100644 --- a/src/commands/sandbox/agents/list.ts +++ b/src/commands/sandbox/agents/list.ts @@ -24,7 +24,12 @@ export default class SandboxAgentsListCommand extends NemoClawCommand { public async run(): Promise { this.parsed = true; const [sandboxName, ...extraArgs] = this.argv; - if (!sandboxName || sandboxName.trim() === "" || sandboxName === "--help" || sandboxName === "-h") { + if ( + !sandboxName || + sandboxName.trim() === "" || + sandboxName === "--help" || + sandboxName === "-h" + ) { printAgentsPassthroughHelp("list"); return; } From 57ef73d7a54e10d88d09e856108388da9f229e8b Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 12:01:29 -0700 Subject: [PATCH 11/13] test(e2e): wire onboarding variant fixtures --- .../e2e-phase-onboarding.test.ts | 310 +++++++++++++- .../e2e-phase-orchestrators.test.ts | 21 +- .../framework/phases/onboarding.ts | 384 +++++++++++++++++- test/e2e-scenario/scenarios/compiler.ts | 4 +- .../scenarios/scenarios/baseline.ts | 1 - 5 files changed, 701 insertions(+), 19 deletions(-) diff --git a/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts index 51ca4d7f0c4..867ff0c30d5 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts @@ -1,15 +1,15 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, expectTypeOf, it } from "vitest"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { describe, expect, expectTypeOf, it } from "vitest"; -import { HostCliClient, type CommandRunner } from "../framework/clients/index.ts"; +import { type CommandRunner, HostCliClient } from "../framework/clients/index.ts"; import type { E2EScenarioFixtures } from "../framework/e2e-test.ts"; -import { OnboardingPhaseFixture, type OnboardingSecrets } from "../framework/phases/index.ts"; import type { EnvironmentReady } from "../framework/phases/index.ts"; +import { OnboardingPhaseFixture, type OnboardingSecrets } from "../framework/phases/index.ts"; import type { ShellProbeResult, ShellProbeRunOptions, @@ -259,6 +259,259 @@ describe("onboarding phase fixture", () => { }); }); + it("runs cloud OpenClaw onboarding with custom model and policy presets", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "onboarded\n")); + const onboard = new OnboardingPhaseFixture( + new HostCliClient(runner), + new FakeSecrets({ NVIDIA_API_KEY: "secret-token" }), + ); + + const instance = await onboard.from(ready({ onboarding: "cloud-openclaw-custom-policies" }), { + sandboxName: "e2e-custom-policies", + }); + + expect(instance).toMatchObject({ + onboarding: "cloud-openclaw-custom-policies", + model: "nvidia/nemotron-3-super-120b-a12b", + policyPresets: ["npm", "pypi"], + }); + expect(runner.calls[0]).toMatchObject({ + command: "nemoclaw", + args: ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], + options: { + artifactName: "onboard-cloud-openclaw-custom-policies", + env: expect.objectContaining({ + NEMOCLAW_MODEL: "nvidia/nemotron-3-super-120b-a12b", + NEMOCLAW_POLICY_MODE: "custom", + NEMOCLAW_POLICY_PRESETS: "npm,pypi", + NEMOCLAW_SANDBOX_NAME: "e2e-custom-policies", + NVIDIA_API_KEY: "secret-token", + }), + redactionValues: ["secret-token"], + timeoutMs: 900_000, + }, + }); + }); + + it("injects the invalid NVIDIA key fixture without requiring a live secret", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(1, "Invalid NVIDIA API key. Must start with nvapi-")); + const secrets = new FakeSecrets(); + const cleanup = new FakeCleanup(); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), secrets, cleanup); + + const instance = await onboard.from( + ready({ onboarding: "cloud-openclaw-invalid-nvidia-key" }), + { sandboxName: "e2e-invalid-key" }, + ); + + expect(instance.expectedFailure).toEqual({ + phase: "onboarding", + errorClass: "invalid-nvidia-api-key", + }); + expect(secrets.requiredCalls).toEqual([]); + expect(cleanup.calls).toHaveLength(1); + expect(runner.calls[0]).toMatchObject({ + command: "nemoclaw", + args: ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], + options: { + artifactName: "onboard-cloud-openclaw-invalid-nvidia-key", + env: expect.objectContaining({ + NEMOCLAW_POLICY_MODE: "skip", + NEMOCLAW_SANDBOX_NAME: "e2e-invalid-key", + NVIDIA_API_KEY: "not-a-nvidia-key", + }), + redactionValues: ["not-a-nvidia-key"], + timeoutMs: 900_000, + }, + }); + }); + + it("rejects invalid NVIDIA key failures that include a stack trace", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(1, "Invalid NVIDIA API key\n at validateKey")); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), new FakeSecrets()); + + await expect( + onboard.from(ready({ onboarding: "cloud-openclaw-invalid-nvidia-key" })), + ).rejects.toThrow(/printed a stack trace/); + }); + + it("runs the gateway port conflict negative path with a local port holder", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(1, "listen tcp 127.0.0.1:18080: bind: address already in use")); + const onboard = new OnboardingPhaseFixture( + new HostCliClient(runner), + new FakeSecrets({ NVIDIA_API_KEY: "secret-token" }), + ); + + const instance = await onboard.from( + ready({ onboarding: "cloud-openclaw-gateway-port-conflict" }), + { sandboxName: "e2e-port-conflict" }, + ); + + expect(instance).toMatchObject({ + onboarding: "cloud-openclaw-gateway-port-conflict", + gatewayPort: 18_080, + expectedFailure: { + phase: "onboarding", + errorClass: "gateway-port-conflict", + }, + }); + expect(runner.calls[0]).toMatchObject({ + command: "nemoclaw", + args: ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], + options: { + artifactName: "onboard-cloud-openclaw-gateway-port-conflict", + env: expect.objectContaining({ + NEMOCLAW_GATEWAY_PORT: "18080", + NEMOCLAW_SANDBOX_NAME: "e2e-port-conflict", + NVIDIA_API_KEY: "secret-token", + }), + redactionValues: ["secret-token"], + timeoutMs: 900_000, + }, + }); + }); + + it("runs resume onboarding after an injected policy-step interruption", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(1, "Forced onboarding failure at step 'policies'")); + runner.enqueue(shellResult(0, "resumed\n")); + const onboard = new OnboardingPhaseFixture( + new HostCliClient(runner), + new FakeSecrets({ NVIDIA_API_KEY: "secret-token" }), + ); + + const instance = await onboard.from( + ready({ onboarding: "cloud-nvidia-openclaw-resume-after-interrupt" }), + { sandboxName: "e2e-resume" }, + ); + + expect(instance.result).toBe(instance.results?.resume); + expect(instance.results?.initial?.exitCode).toBe(1); + expect(runner.calls).toHaveLength(2); + expect(runner.calls[0]).toMatchObject({ + command: "nemoclaw", + args: ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], + options: { + artifactName: "onboard-cloud-nvidia-openclaw-resume-after-interrupt-interrupted", + env: expect.objectContaining({ + NEMOCLAW_E2E_FAILURE_INJECTION: "1", + NEMOCLAW_E2E_FORCE_FAIL_AT_STEP: "policies", + NEMOCLAW_POLICY_MODE: "suggested", + NEMOCLAW_SANDBOX_NAME: "e2e-resume", + NVIDIA_API_KEY: "secret-token", + }), + }, + }); + expect(runner.calls[1]).toMatchObject({ + command: "nemoclaw", + args: [ + "onboard", + "--resume", + "--non-interactive", + "--yes", + "--yes-i-accept-third-party-software", + ], + options: { + artifactName: "onboard-cloud-nvidia-openclaw-resume-after-interrupt-resume", + env: expect.objectContaining({ + NEMOCLAW_POLICY_MODE: "skip", + NEMOCLAW_SANDBOX_NAME: "e2e-resume", + }), + }, + }); + expect(runner.calls[1]?.options?.env).not.toHaveProperty("NVIDIA_API_KEY"); + }); + + it("repairs existing onboarding config before resuming", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(1, "Forced onboarding failure at step 'policies'")); + runner.enqueue(shellResult(0, "sandbox deleted\n")); + runner.enqueue(shellResult(0, "forward stopped\n")); + runner.enqueue(shellResult(0, "resumed\n")); + const onboard = new OnboardingPhaseFixture( + new HostCliClient(runner), + new FakeSecrets({ NVIDIA_API_KEY: "secret-token" }), + ); + + const instance = await onboard.from( + ready({ onboarding: "cloud-nvidia-openclaw-repair-existing-config" }), + { sandboxName: "e2e-repair" }, + ); + + expect(instance.result).toBe(instance.results?.resume); + expect(instance.results?.repairDelete?.stdout).toContain("sandbox deleted"); + expect(instance.results?.repairForwardStop?.stdout).toContain("forward stopped"); + expect(runner.calls.map((call) => [call.command, call.args])).toEqual([ + [ + "nemoclaw", + ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], + ], + ["openshell", ["sandbox", "delete", "e2e-repair"]], + ["openshell", ["forward", "stop", "18789"]], + [ + "nemoclaw", + [ + "onboard", + "--resume", + "--non-interactive", + "--yes", + "--yes-i-accept-third-party-software", + ], + ], + ]); + expect(runner.calls[1]?.options).toMatchObject({ + artifactName: "onboard-cloud-nvidia-openclaw-repair-delete-sandbox", + timeoutMs: 60_000, + }); + expect(runner.calls[2]?.options).toMatchObject({ + artifactName: "onboard-cloud-nvidia-openclaw-repair-stop-forward", + timeoutMs: 30_000, + }); + }); + + it("reruns OpenClaw onboarding for the same provider with sandbox recreation enabled", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "onboarded\n")); + runner.enqueue(shellResult(0, "recreated\n")); + const onboard = new OnboardingPhaseFixture( + new HostCliClient(runner), + new FakeSecrets({ NVIDIA_API_KEY: "secret-token" }), + ); + + const instance = await onboard.from( + ready({ onboarding: "cloud-nvidia-openclaw-double-same-provider" }), + { sandboxName: "e2e-double" }, + ); + + expect(instance.result).toBe(instance.results?.second); + expect(instance.results?.initial?.stdout).toContain("onboarded"); + expect(runner.calls).toHaveLength(2); + expect(runner.calls[0]?.options?.env).toMatchObject({ + NEMOCLAW_POLICY_MODE: "skip", + NEMOCLAW_SANDBOX_NAME: "e2e-double", + NVIDIA_API_KEY: "secret-token", + }); + expect(runner.calls[1]).toMatchObject({ + command: "nemoclaw", + args: ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], + options: { + artifactName: "onboard-cloud-nvidia-openclaw-double-same-provider-recreate", + env: expect.objectContaining({ + NEMOCLAW_POLICY_MODE: "skip", + NEMOCLAW_RECREATE_SANDBOX: "1", + NEMOCLAW_SANDBOX_NAME: "e2e-double", + NVIDIA_API_KEY: "secret-token", + }), + redactionValues: ["secret-token"], + timeoutMs: 900_000, + }, + }); + }); + it("runs the no-Docker negative path with a failing Docker shim", async () => { const runner = new FakeRunner(); runner.enqueue(shellResult(7, "Cannot connect to the Docker daemon")); @@ -466,6 +719,57 @@ describe("onboarding phase fixture", () => { ).rejects.toThrow(/without Docker-missing preflight signature/); }); + it("rejects no-Docker onboarding failures that include a stack trace", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(9, "Docker is required before onboarding\n at preflight")); + const onboard = new OnboardingPhaseFixture( + new HostCliClient(runner), + new FakeSecrets({ NVIDIA_API_KEY: "secret" }), + ); + + await expect( + onboard.from( + ready({ + runtime: "docker-missing", + onboarding: "cloud-openclaw-no-docker", + docker: { id: "docker-missing", expectation: "missing", available: false }, + }), + ), + ).rejects.toThrow(/printed a stack trace/); + }); + + it("requires Docker before running resume, repair, and double-onboard variants", async () => { + const onboard = new OnboardingPhaseFixture( + new HostCliClient(new FakeRunner()), + new FakeSecrets({ NVIDIA_API_KEY: "secret" }), + ); + + for (const onboarding of [ + "cloud-nvidia-openclaw-resume-after-interrupt", + "cloud-nvidia-openclaw-repair-existing-config", + "cloud-nvidia-openclaw-double-same-provider", + ]) { + await expect( + onboard.from( + ready({ + onboarding, + docker: { id: "docker-running", expectation: "required", available: false }, + }), + ), + ).rejects.toThrow(/requires an available Docker runtime/); + } + }); + + it("rejects unexpected success for negative onboarding variants", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "onboarded\n")); + const onboard = new OnboardingPhaseFixture(new HostCliClient(runner), new FakeSecrets()); + + await expect( + onboard.from(ready({ onboarding: "cloud-openclaw-invalid-nvidia-key" })), + ).rejects.toThrow(/unexpectedly succeeded/); + }); + it("rejects unsupported onboarding profiles", async () => { const onboard = new OnboardingPhaseFixture( new HostCliClient(new FakeRunner()), diff --git a/test/e2e-scenario/framework-tests/e2e-phase-orchestrators.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-orchestrators.test.ts index 7460664a3c2..23de3642bbe 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-orchestrators.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-orchestrators.test.ts @@ -1,10 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { describe, expect, it } from "vitest"; import { HostCliClient } from "../scenarios/clients/host-cli.ts"; import { compileRunPlans } from "../scenarios/compiler.ts"; @@ -1007,6 +1007,25 @@ describe("framework-owned secret hygiene at the spawn boundary", () => { expect(cloudOnboard?.secretEnv).toEqual(["NVIDIA_API_KEY"]); expect(localOnboard?.secretEnv).toEqual([]); }); + + it("should not require a live NVIDIA API key for the invalid-key negative fixture", async () => { + const { compileRunPlans } = await import("../scenarios/compiler.ts"); + const [invalidKeyPlan, portConflictPlan] = compileRunPlans([ + "ubuntu-invalid-nvidia-key-negative", + "ubuntu-gateway-port-conflict-negative", + ]); + const invalidKeyOnboard = invalidKeyPlan.phases + .find((p) => p.name === "onboarding") + ?.actions.find((a) => a.id.startsWith("onboarding.profile.")); + const portConflictOnboard = portConflictPlan.phases + .find((p) => p.name === "onboarding") + ?.actions.find((a) => a.id.startsWith("onboarding.profile.")); + + expect(invalidKeyPlan.requiredSecrets).toEqual([]); + expect(invalidKeyOnboard?.secretEnv).toEqual([]); + expect(portConflictPlan.requiredSecrets).toEqual(["NVIDIA_API_KEY"]); + expect(portConflictOnboard?.secretEnv).toEqual(["NVIDIA_API_KEY"]); + }); }); describe("clients are pass/fail/policy free", () => { diff --git a/test/e2e-scenario/framework/phases/onboarding.ts b/test/e2e-scenario/framework/phases/onboarding.ts index f5802b49b90..0cc78c3eb55 100644 --- a/test/e2e-scenario/framework/phases/onboarding.ts +++ b/test/e2e-scenario/framework/phases/onboarding.ts @@ -2,15 +2,15 @@ // SPDX-License-Identifier: Apache-2.0 import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { createServer, type Server } from "node:net"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; - +import { redactString } from "../../scenarios/orchestrators/redaction.ts"; import { buildAvailabilityProbeEnv } from "../availability-env.ts"; import { artifactLabel, assertExitZero } from "../clients/command.ts"; import type { HostCliClient } from "../clients/host.ts"; import { validateSandboxName } from "../clients/sandbox.ts"; import type { ShellProbeResult } from "../shell-probe.ts"; -import { redactString } from "../../scenarios/orchestrators/redaction.ts"; import type { EnvironmentReady } from "./environment.ts"; const ONBOARD_ARGS = [ @@ -19,9 +19,20 @@ const ONBOARD_ARGS = [ "--yes", "--yes-i-accept-third-party-software", ]; +const RESUME_ONBOARD_ARGS = [ + "onboard", + "--resume", + "--non-interactive", + "--yes", + "--yes-i-accept-third-party-software", +]; const DEFAULT_TIMEOUT_MS = 15 * 60_000; const OPENCLAW_GATEWAY_URL = "http://127.0.0.1:18789"; const NEGATIVE_PREFLIGHT_LOG = "negative-preflight.log"; +const DEFAULT_CUSTOM_POLICY_MODEL = "nvidia/nemotron-3-super-120b-a12b"; +const DEFAULT_CUSTOM_POLICY_PRESETS = Object.freeze(["npm", "pypi"]); +const INVALID_NVIDIA_API_KEY = "not-a-nvidia-key"; +const GATEWAY_PORT_CONFLICT_PORT = 18_080; const DOCKER_MISSING_PATTERNS = [ /Cannot connect to the Docker daemon/i, /Is the docker daemon running\??/i, @@ -40,6 +51,23 @@ const MISSING_SANDBOX_DELETE_PATTERNS = [ /sandbox does not exist/i, /no such sandbox/i, ]; +const INVALID_NVIDIA_API_KEY_PATTERNS = [ + /Invalid NVIDIA API key/i, + /Must start with nvapi-/i, + /invalid .*NVIDIA.*api key/i, +]; +const GATEWAY_PORT_CONFLICT_PATTERNS = [ + /address already in use/i, + /port .*18080.*(?:in use|occupied|unavailable)/i, + /gateway port .*18080/i, + /port conflict/i, +]; +const E2E_FORCED_POLICY_FAILURE_PATTERNS = [ + /Forced onboarding failure at step 'policies'/i, + /forced.*polic/i, + /policy failure/i, +]; +const STACK_TRACE_PATTERNS = [/(^|\s)(TypeError|ReferenceError|SyntaxError):/m, /^\s+at /m]; export interface OnboardingSecrets { required(name: string): string; @@ -55,9 +83,22 @@ export interface OnboardingOptions { timeoutMs?: number; } -export interface OnboardingExpectedFailure { - phase: "preflight"; - errorClass: "docker-missing"; +export type OnboardingExpectedFailure = + | { + phase: "preflight"; + errorClass: "docker-missing"; + } + | { + phase: "onboarding"; + errorClass: "invalid-nvidia-api-key" | "gateway-port-conflict"; + }; + +export interface OnboardingResultSet { + initial?: ShellProbeResult; + resume?: ShellProbeResult; + repairDelete?: ShellProbeResult; + repairForwardStop?: ShellProbeResult; + second?: ShellProbeResult; } export interface NemoClawInstance { @@ -69,6 +110,10 @@ export interface NemoClawInstance { platformOs?: "ubuntu" | "macos" | "windows"; gatewayUrl: string; result: ShellProbeResult; + results?: OnboardingResultSet; + model?: string; + policyPresets?: readonly string[]; + gatewayPort?: number; expectedFailure?: OnboardingExpectedFailure; } @@ -92,6 +137,16 @@ function commandEnv(sandboxName: string, extra: NodeJS.ProcessEnv = {}): NodeJS. }; } +function resumeCommandEnv(sandboxName: string, extra: NodeJS.ProcessEnv = {}): NodeJS.ProcessEnv { + return { + ...buildAvailabilityProbeEnv(), + ...extra, + NEMOCLAW_AGENT: "openclaw", + NEMOCLAW_PROVIDER: "cloud", + NEMOCLAW_SANDBOX_NAME: sandboxName, + }; +} + function noDockerShim(): string { // Migration source of truth for the typed fixture path: simulate the invalid // state where the Docker client exists but the daemon is unreachable. The @@ -125,13 +180,42 @@ function legacyNegativePreflightLogPath(): string | undefined { } function hasDockerMissingSignature(result: ShellProbeResult): boolean { - const text = resultText(result); - return DOCKER_MISSING_PATTERNS.some((pattern) => pattern.test(text)); + return hasSignature(result, DOCKER_MISSING_PATTERNS); } function hasMissingSandboxDeleteSignature(result: ShellProbeResult): boolean { + return hasSignature(result, MISSING_SANDBOX_DELETE_PATTERNS); +} + +function hasSignature(result: ShellProbeResult, patterns: readonly RegExp[]): boolean { const text = resultText(result); - return MISSING_SANDBOX_DELETE_PATTERNS.some((pattern) => pattern.test(text)); + return patterns.some((pattern) => pattern.test(text)); +} + +function hasStackTrace(result: ShellProbeResult): boolean { + return hasSignature(result, STACK_TRACE_PATTERNS); +} + +function assertDockerAvailable(environment: EnvironmentReady, onboarding: string): void { + if (!environment.docker.available) { + throw new Error(`${onboarding} onboarding requires an available Docker runtime.`); + } +} + +function assertExpectedFailureSignature( + result: ShellProbeResult, + patterns: readonly RegExp[], + label: string, +): void { + if (result.exitCode === 0) { + throw new Error(`${label} unexpectedly succeeded.`); + } + if (hasStackTrace(result)) { + throw new Error(`${label} printed a stack trace: ${resultText(result)}`); + } + if (!hasSignature(result, patterns)) { + throw new Error(`${label} failed without expected failure signature: ${resultText(result)}`); + } } export class OnboardingPhaseFixture { @@ -148,8 +232,20 @@ export class OnboardingPhaseFixture { switch (environment.onboarding) { case "cloud-openclaw": return await this.cloudOpenClaw(environment, options); + case "cloud-openclaw-custom-policies": + return await this.cloudOpenClawCustomPolicies(environment, options); + case "cloud-openclaw-invalid-nvidia-key": + return await this.cloudOpenClawInvalidNvidiaKey(environment, options); + case "cloud-openclaw-gateway-port-conflict": + return await this.cloudOpenClawGatewayPortConflict(environment, options); case "cloud-openclaw-no-docker": return await this.cloudOpenClawNoDocker(environment, options); + case "cloud-nvidia-openclaw-resume-after-interrupt": + return await this.cloudNvidiaOpenClawResumeAfterInterrupt(environment, options); + case "cloud-nvidia-openclaw-repair-existing-config": + return await this.cloudNvidiaOpenClawRepairExistingConfig(environment, options); + case "cloud-nvidia-openclaw-double-same-provider": + return await this.cloudNvidiaOpenClawDoubleSameProvider(environment, options); default: throw new Error(`Unsupported onboarding profile '${environment.onboarding}'.`); } @@ -159,19 +255,189 @@ export class OnboardingPhaseFixture { environment: EnvironmentReady, options: OnboardingOptions = {}, ): Promise { - if (!environment.docker.available) { - throw new Error("cloud-openclaw onboarding requires an available Docker runtime."); - } + assertDockerAvailable(environment, environment.onboarding); const sandboxName = sandboxNameFromOptions(environment.onboarding, options); const apiKey = this.secrets.required("NVIDIA_API_KEY"); this.registerSandboxCleanup(sandboxName); - const result = await this.host.nemoclaw(ONBOARD_ARGS, { + const result = await this.runOnboard({ artifactName: "onboard-cloud-openclaw", env: commandEnv(sandboxName, { NVIDIA_API_KEY: apiKey }), redactionValues: [apiKey], timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, }); assertExitZero(result, "cloud-openclaw onboarding"); + return this.instance(environment, sandboxName, result); + } + + async cloudOpenClawCustomPolicies( + environment: EnvironmentReady, + options: OnboardingOptions = {}, + ): Promise { + assertDockerAvailable(environment, environment.onboarding); + const sandboxName = sandboxNameFromOptions(environment.onboarding, options); + const apiKey = this.secrets.required("NVIDIA_API_KEY"); + const policyPresets = DEFAULT_CUSTOM_POLICY_PRESETS; + this.registerSandboxCleanup(sandboxName); + const result = await this.runOnboard({ + artifactName: "onboard-cloud-openclaw-custom-policies", + env: commandEnv(sandboxName, { + NVIDIA_API_KEY: apiKey, + NEMOCLAW_MODEL: DEFAULT_CUSTOM_POLICY_MODEL, + NEMOCLAW_POLICY_MODE: "custom", + NEMOCLAW_POLICY_PRESETS: policyPresets.join(","), + }), + redactionValues: [apiKey], + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }); + assertExitZero(result, "cloud-openclaw-custom-policies onboarding"); + return this.instance(environment, sandboxName, result, { + model: DEFAULT_CUSTOM_POLICY_MODEL, + policyPresets, + }); + } + + async cloudOpenClawInvalidNvidiaKey( + environment: EnvironmentReady, + options: OnboardingOptions = {}, + ): Promise { + assertDockerAvailable(environment, environment.onboarding); + const sandboxName = sandboxNameFromOptions(environment.onboarding, options); + this.registerSandboxCleanup(sandboxName); + const result = await this.runOnboard({ + artifactName: "onboard-cloud-openclaw-invalid-nvidia-key", + env: commandEnv(sandboxName, { + NVIDIA_API_KEY: INVALID_NVIDIA_API_KEY, + NEMOCLAW_POLICY_MODE: "skip", + }), + redactionValues: [INVALID_NVIDIA_API_KEY], + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }); + assertExpectedFailureSignature( + result, + INVALID_NVIDIA_API_KEY_PATTERNS, + "cloud-openclaw-invalid-nvidia-key onboarding", + ); + return this.instance(environment, sandboxName, result, { + expectedFailure: { + phase: "onboarding", + errorClass: "invalid-nvidia-api-key", + }, + }); + } + + async cloudOpenClawGatewayPortConflict( + environment: EnvironmentReady, + options: OnboardingOptions = {}, + ): Promise { + assertDockerAvailable(environment, environment.onboarding); + const sandboxName = sandboxNameFromOptions(environment.onboarding, options); + const apiKey = this.secrets.required("NVIDIA_API_KEY"); + this.registerSandboxCleanup(sandboxName); + const result = await this.withPortHolder(GATEWAY_PORT_CONFLICT_PORT, async () => + this.runOnboard({ + artifactName: "onboard-cloud-openclaw-gateway-port-conflict", + env: commandEnv(sandboxName, { + NVIDIA_API_KEY: apiKey, + NEMOCLAW_GATEWAY_PORT: String(GATEWAY_PORT_CONFLICT_PORT), + }), + redactionValues: [apiKey], + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }), + ); + assertExpectedFailureSignature( + result, + GATEWAY_PORT_CONFLICT_PATTERNS, + "cloud-openclaw-gateway-port-conflict onboarding", + ); + return this.instance(environment, sandboxName, result, { + gatewayPort: GATEWAY_PORT_CONFLICT_PORT, + expectedFailure: { + phase: "onboarding", + errorClass: "gateway-port-conflict", + }, + }); + } + + async cloudNvidiaOpenClawResumeAfterInterrupt( + environment: EnvironmentReady, + options: OnboardingOptions = {}, + ): Promise { + const sandboxName = sandboxNameFromOptions(environment.onboarding, options); + const apiKey = this.secrets.required("NVIDIA_API_KEY"); + const initial = await this.interruptAtPolicyStep(environment, sandboxName, apiKey, options); + const resume = await this.resumeOnboard(environment, sandboxName, options, { + artifactName: "onboard-cloud-nvidia-openclaw-resume-after-interrupt-resume", + }); + return this.instance(environment, sandboxName, resume, { + results: { initial, resume }, + }); + } + + async cloudNvidiaOpenClawRepairExistingConfig( + environment: EnvironmentReady, + options: OnboardingOptions = {}, + ): Promise { + const sandboxName = sandboxNameFromOptions(environment.onboarding, options); + const apiKey = this.secrets.required("NVIDIA_API_KEY"); + const initial = await this.interruptAtPolicyStep(environment, sandboxName, apiKey, options); + const repairDelete = await this.host.command("openshell", ["sandbox", "delete", sandboxName], { + artifactName: "onboard-cloud-nvidia-openclaw-repair-delete-sandbox", + env: buildAvailabilityProbeEnv(), + timeoutMs: 60_000, + }); + const repairForwardStop = await this.host.command("openshell", ["forward", "stop", "18789"], { + artifactName: "onboard-cloud-nvidia-openclaw-repair-stop-forward", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }); + const resume = await this.resumeOnboard(environment, sandboxName, options, { + artifactName: "onboard-cloud-nvidia-openclaw-repair-existing-config-resume", + }); + return this.instance(environment, sandboxName, resume, { + results: { initial, repairDelete, repairForwardStop, resume }, + }); + } + + async cloudNvidiaOpenClawDoubleSameProvider( + environment: EnvironmentReady, + options: OnboardingOptions = {}, + ): Promise { + assertDockerAvailable(environment, environment.onboarding); + const sandboxName = sandboxNameFromOptions(environment.onboarding, options); + const apiKey = this.secrets.required("NVIDIA_API_KEY"); + this.registerSandboxCleanup(sandboxName); + const initial = await this.runOnboard({ + artifactName: "onboard-cloud-nvidia-openclaw-double-same-provider-initial", + env: commandEnv(sandboxName, { + NVIDIA_API_KEY: apiKey, + NEMOCLAW_POLICY_MODE: "skip", + }), + redactionValues: [apiKey], + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }); + assertExitZero(initial, "cloud-nvidia-openclaw-double-same-provider initial onboarding"); + const second = await this.runOnboard({ + artifactName: "onboard-cloud-nvidia-openclaw-double-same-provider-recreate", + env: commandEnv(sandboxName, { + NVIDIA_API_KEY: apiKey, + NEMOCLAW_POLICY_MODE: "skip", + NEMOCLAW_RECREATE_SANDBOX: "1", + }), + redactionValues: [apiKey], + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }); + assertExitZero(second, "cloud-nvidia-openclaw-double-same-provider recreate onboarding"); + return this.instance(environment, sandboxName, second, { + results: { initial, second }, + }); + } + + private instance( + environment: EnvironmentReady, + sandboxName: string, + result: ShellProbeResult, + extra: Partial = {}, + ): NemoClawInstance { return { onboarding: environment.onboarding, sandboxName, @@ -180,6 +446,7 @@ export class OnboardingPhaseFixture { providerEnv: "cloud", gatewayUrl: OPENCLAW_GATEWAY_URL, result, + ...extra, }; } @@ -202,7 +469,7 @@ export class OnboardingPhaseFixture { await chmod(shimPath, 0o700); const env = commandEnv(sandboxName, { NVIDIA_API_KEY: apiKey }); env.PATH = prependPath(shimDir, env.PATH); - const result = await this.host.nemoclaw(ONBOARD_ARGS, { + const result = await this.runOnboard({ artifactName: "onboard-cloud-openclaw-no-docker", env, redactionValues: [apiKey], @@ -212,6 +479,11 @@ export class OnboardingPhaseFixture { if (result.exitCode === 0) { throw new Error("cloud-openclaw-no-docker onboarding unexpectedly succeeded."); } + if (hasStackTrace(result)) { + throw new Error( + `cloud-openclaw-no-docker onboarding printed a stack trace: ${resultText(result)}`, + ); + } if (!hasDockerMissingSignature(result)) { throw new Error( `cloud-openclaw-no-docker onboarding failed without Docker-missing preflight signature: ${resultText(result)}`, @@ -235,6 +507,92 @@ export class OnboardingPhaseFixture { } } + private async interruptAtPolicyStep( + environment: EnvironmentReady, + sandboxName: string, + apiKey: string, + options: OnboardingOptions, + ): Promise { + assertDockerAvailable(environment, environment.onboarding); + this.registerSandboxCleanup(sandboxName); + const result = await this.runOnboard({ + artifactName: `onboard-${artifactLabel(environment.onboarding)}-interrupted`, + env: commandEnv(sandboxName, { + NVIDIA_API_KEY: apiKey, + NEMOCLAW_E2E_FAILURE_INJECTION: "1", + NEMOCLAW_E2E_FORCE_FAIL_AT_STEP: "policies", + NEMOCLAW_POLICY_MODE: "suggested", + }), + redactionValues: [apiKey], + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }); + assertExpectedFailureSignature( + result, + E2E_FORCED_POLICY_FAILURE_PATTERNS, + `${environment.onboarding} interrupted onboarding`, + ); + return result; + } + + private async resumeOnboard( + environment: EnvironmentReady, + sandboxName: string, + options: OnboardingOptions, + settings: { artifactName: string }, + ): Promise { + assertDockerAvailable(environment, environment.onboarding); + const result = await this.runOnboard({ + args: RESUME_ONBOARD_ARGS, + artifactName: settings.artifactName, + env: resumeCommandEnv(sandboxName, { + NEMOCLAW_POLICY_MODE: "skip", + }), + timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS, + }); + assertExitZero(result, `${environment.onboarding} resume onboarding`); + return result; + } + + private async runOnboard(options: { + args?: string[]; + artifactName: string; + env: NodeJS.ProcessEnv; + redactionValues?: string[]; + timeoutMs: number; + }): Promise { + return await this.host.nemoclaw(options.args ?? ONBOARD_ARGS, { + artifactName: options.artifactName, + env: options.env, + redactionValues: options.redactionValues, + timeoutMs: options.timeoutMs, + }); + } + + private async withPortHolder(port: number, run: () => Promise): Promise { + const server = await this.tryStartPortHolder(port); + try { + return await run(); + } finally { + if (server) { + await new Promise((resolve) => server.close(() => resolve())); + } + } + } + + private async tryStartPortHolder(port: number): Promise { + const server = createServer((socket) => socket.end()); + return await new Promise((resolve, reject) => { + server.once("error", (error: NodeJS.ErrnoException) => { + if (error.code === "EADDRINUSE") { + resolve(null); + return; + } + reject(error); + }); + server.listen(port, "127.0.0.1", () => resolve(server)); + }); + } + private registerSandboxCleanup(sandboxName: string): void { if (!this.cleanup) return; this.cleanup.add(`destroy NemoClaw sandbox ${sandboxName}`, async () => { diff --git a/test/e2e-scenario/scenarios/compiler.ts b/test/e2e-scenario/scenarios/compiler.ts index 3f478ac0a77..9ca0ce11a5c 100644 --- a/test/e2e-scenario/scenarios/compiler.ts +++ b/test/e2e-scenario/scenarios/compiler.ts @@ -129,7 +129,9 @@ const ONBOARD_PROFILE_SECRET_ENV: Readonly> = // NVIDIA cloud provider via NVIDIA_API_KEY. "cloud-openclaw": ["NVIDIA_API_KEY"], "cloud-openclaw-custom-policies": ["NVIDIA_API_KEY"], - "cloud-openclaw-invalid-nvidia-key": ["NVIDIA_API_KEY"], + // The invalid-key negative path injects its own bad credential in the + // fixture. Do not pass a real parent-env key into that child process. + "cloud-openclaw-invalid-nvidia-key": [], "cloud-openclaw-gateway-port-conflict": ["NVIDIA_API_KEY"], // Negative scenario: nemoclaw onboard runs against a docker shim that // exits non-zero. Onboard never reaches the cloud auth step, but the diff --git a/test/e2e-scenario/scenarios/scenarios/baseline.ts b/test/e2e-scenario/scenarios/scenarios/baseline.ts index 4e5befad679..d174d3da497 100644 --- a/test/e2e-scenario/scenarios/scenarios/baseline.ts +++ b/test/e2e-scenario/scenarios/scenarios/baseline.ts @@ -299,7 +299,6 @@ const canonicalScenarioInputs: CanonicalScenarioInput[] = [ expectedStateId: "onboarding-failure-invalid-nvidia-key", onboardingAssertionIds: ["base-installed"], suiteIds: [], - requiredSecrets: ["NVIDIA_API_KEY"], expectedFailure: { phase: "onboarding", errorClass: "invalid-nvidia-api-key", From 22c0a58be3371b347bc2825afb6f99d89dc939e2 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 12:11:24 -0700 Subject: [PATCH 12/13] test(e2e): validate onboarding repair cleanup --- .../e2e-phase-onboarding.test.ts | 72 +++++++++++++++++++ .../framework/phases/onboarding.ts | 36 ++++++++++ 2 files changed, 108 insertions(+) diff --git a/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts index 867ff0c30d5..4b71c51884d 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-onboarding.test.ts @@ -473,6 +473,78 @@ describe("onboarding phase fixture", () => { }); }); + it("fails repair onboarding before resume when sandbox deletion fails unexpectedly", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(1, "Forced onboarding failure at step 'policies'")); + runner.enqueue(shellResult(2, "permission denied")); + const onboard = new OnboardingPhaseFixture( + new HostCliClient(runner), + new FakeSecrets({ NVIDIA_API_KEY: "secret-token" }), + ); + + await expect( + onboard.from(ready({ onboarding: "cloud-nvidia-openclaw-repair-existing-config" }), { + sandboxName: "e2e-repair-delete-fail", + }), + ).rejects.toThrow(/delete sandbox failed: permission denied/); + + expect(runner.calls.map((call) => [call.command, call.args])).toEqual([ + [ + "nemoclaw", + ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], + ], + ["openshell", ["sandbox", "delete", "e2e-repair-delete-fail"]], + ]); + }); + + it("fails repair onboarding before resume when forward stop fails unexpectedly", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(1, "Forced onboarding failure at step 'policies'")); + runner.enqueue(shellResult(0, "sandbox deleted\n")); + runner.enqueue(shellResult(2, "permission denied")); + const onboard = new OnboardingPhaseFixture( + new HostCliClient(runner), + new FakeSecrets({ NVIDIA_API_KEY: "secret-token" }), + ); + + await expect( + onboard.from(ready({ onboarding: "cloud-nvidia-openclaw-repair-existing-config" }), { + sandboxName: "e2e-repair-forward-fail", + }), + ).rejects.toThrow(/stop forward failed: permission denied/); + + expect(runner.calls.map((call) => [call.command, call.args])).toEqual([ + [ + "nemoclaw", + ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"], + ], + ["openshell", ["sandbox", "delete", "e2e-repair-forward-fail"]], + ["openshell", ["forward", "stop", "18789"]], + ]); + }); + + it("tolerates already-missing repair cleanup before resuming", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(1, "Forced onboarding failure at step 'policies'")); + runner.enqueue(shellResult(1, "sandbox e2e-repair-missing not found")); + runner.enqueue(shellResult(1, "no active forward for port 18789")); + runner.enqueue(shellResult(0, "resumed\n")); + const onboard = new OnboardingPhaseFixture( + new HostCliClient(runner), + new FakeSecrets({ NVIDIA_API_KEY: "secret-token" }), + ); + + const instance = await onboard.from( + ready({ onboarding: "cloud-nvidia-openclaw-repair-existing-config" }), + { sandboxName: "e2e-repair-missing" }, + ); + + expect(instance.result).toBe(instance.results?.resume); + expect(instance.results?.repairDelete?.exitCode).toBe(1); + expect(instance.results?.repairForwardStop?.exitCode).toBe(1); + expect(runner.calls).toHaveLength(4); + }); + it("reruns OpenClaw onboarding for the same provider with sandbox recreation enabled", async () => { const runner = new FakeRunner(); runner.enqueue(shellResult(0, "onboarded\n")); diff --git a/test/e2e-scenario/framework/phases/onboarding.ts b/test/e2e-scenario/framework/phases/onboarding.ts index 0cc78c3eb55..bc7699398cd 100644 --- a/test/e2e-scenario/framework/phases/onboarding.ts +++ b/test/e2e-scenario/framework/phases/onboarding.ts @@ -51,6 +51,15 @@ const MISSING_SANDBOX_DELETE_PATTERNS = [ /sandbox does not exist/i, /no such sandbox/i, ]; +const MISSING_FORWARD_STOP_PATTERNS = [ + /\bNotFound\b/i, + /\bNot Found\b/i, + /forward .*not found/i, + /forward .*not running/i, + /no active forward/i, + /no such forward/i, + /port .*not forwarded/i, +]; const INVALID_NVIDIA_API_KEY_PATTERNS = [ /Invalid NVIDIA API key/i, /Must start with nvapi-/i, @@ -187,6 +196,10 @@ function hasMissingSandboxDeleteSignature(result: ShellProbeResult): boolean { return hasSignature(result, MISSING_SANDBOX_DELETE_PATTERNS); } +function hasMissingForwardStopSignature(result: ShellProbeResult): boolean { + return hasSignature(result, MISSING_FORWARD_STOP_PATTERNS); +} + function hasSignature(result: ShellProbeResult, patterns: readonly RegExp[]): boolean { const text = resultText(result); return patterns.some((pattern) => pattern.test(text)); @@ -218,6 +231,19 @@ function assertExpectedFailureSignature( } } +function assertRepairStepResult( + result: ShellProbeResult, + label: string, + isBenignMissing: (result: ShellProbeResult) => boolean, +): void { + if (result.exitCode === 0) return; + if (hasStackTrace(result)) { + throw new Error(`${label} printed a stack trace: ${resultText(result)}`); + } + if (isBenignMissing(result)) return; + assertExitZero(result, label); +} + export class OnboardingPhaseFixture { constructor( private readonly host: HostCliClient, @@ -385,11 +411,21 @@ export class OnboardingPhaseFixture { env: buildAvailabilityProbeEnv(), timeoutMs: 60_000, }); + assertRepairStepResult( + repairDelete, + "cloud-nvidia-openclaw-repair-existing-config delete sandbox", + hasMissingSandboxDeleteSignature, + ); const repairForwardStop = await this.host.command("openshell", ["forward", "stop", "18789"], { artifactName: "onboard-cloud-nvidia-openclaw-repair-stop-forward", env: buildAvailabilityProbeEnv(), timeoutMs: 30_000, }); + assertRepairStepResult( + repairForwardStop, + "cloud-nvidia-openclaw-repair-existing-config stop forward", + hasMissingForwardStopSignature, + ); const resume = await this.resumeOnboard(environment, sandboxName, options, { artifactName: "onboard-cloud-nvidia-openclaw-repair-existing-config-resume", }); From 6a4ebd4baca8a4ec17634b006be9e752e9221ce4 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Tue, 9 Jun 2026 12:22:51 -0700 Subject: [PATCH 13/13] test(e2e): document onboarding repair fixture scope --- .../e2e-phase-orchestrators.test.ts | 19 ------------------- .../framework/phases/onboarding.ts | 6 ++++++ test/e2e-scenario/scenarios/compiler.ts | 4 +--- .../scenarios/scenarios/baseline.ts | 1 + 4 files changed, 8 insertions(+), 22 deletions(-) diff --git a/test/e2e-scenario/framework-tests/e2e-phase-orchestrators.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-orchestrators.test.ts index 23de3642bbe..f9292dabf09 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-orchestrators.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-orchestrators.test.ts @@ -1007,25 +1007,6 @@ describe("framework-owned secret hygiene at the spawn boundary", () => { expect(cloudOnboard?.secretEnv).toEqual(["NVIDIA_API_KEY"]); expect(localOnboard?.secretEnv).toEqual([]); }); - - it("should not require a live NVIDIA API key for the invalid-key negative fixture", async () => { - const { compileRunPlans } = await import("../scenarios/compiler.ts"); - const [invalidKeyPlan, portConflictPlan] = compileRunPlans([ - "ubuntu-invalid-nvidia-key-negative", - "ubuntu-gateway-port-conflict-negative", - ]); - const invalidKeyOnboard = invalidKeyPlan.phases - .find((p) => p.name === "onboarding") - ?.actions.find((a) => a.id.startsWith("onboarding.profile.")); - const portConflictOnboard = portConflictPlan.phases - .find((p) => p.name === "onboarding") - ?.actions.find((a) => a.id.startsWith("onboarding.profile.")); - - expect(invalidKeyPlan.requiredSecrets).toEqual([]); - expect(invalidKeyOnboard?.secretEnv).toEqual([]); - expect(portConflictPlan.requiredSecrets).toEqual(["NVIDIA_API_KEY"]); - expect(portConflictOnboard?.secretEnv).toEqual(["NVIDIA_API_KEY"]); - }); }); describe("clients are pass/fail/policy free", () => { diff --git a/test/e2e-scenario/framework/phases/onboarding.ts b/test/e2e-scenario/framework/phases/onboarding.ts index bc7699398cd..e4a95b7900d 100644 --- a/test/e2e-scenario/framework/phases/onboarding.ts +++ b/test/e2e-scenario/framework/phases/onboarding.ts @@ -406,6 +406,12 @@ export class OnboardingPhaseFixture { const sandboxName = sandboxNameFromOptions(environment.onboarding, options); const apiKey = this.secrets.required("NVIDIA_API_KEY"); const initial = await this.interruptAtPolicyStep(environment, sandboxName, apiKey, options); + // Permanent scenario semantics for the repair-existing-config profile: + // the fixture creates an interrupted onboarding session, then removes the + // live OpenShell artifacts that a user may have already cleaned up. The + // product behavior under test is `nemoclaw onboard --resume` repairing + // that stale recorded state; remove this cleanup only if the scenario is + // replaced by a lower-level product fixture that creates the same state. const repairDelete = await this.host.command("openshell", ["sandbox", "delete", sandboxName], { artifactName: "onboard-cloud-nvidia-openclaw-repair-delete-sandbox", env: buildAvailabilityProbeEnv(), diff --git a/test/e2e-scenario/scenarios/compiler.ts b/test/e2e-scenario/scenarios/compiler.ts index 9ca0ce11a5c..3f478ac0a77 100644 --- a/test/e2e-scenario/scenarios/compiler.ts +++ b/test/e2e-scenario/scenarios/compiler.ts @@ -129,9 +129,7 @@ const ONBOARD_PROFILE_SECRET_ENV: Readonly> = // NVIDIA cloud provider via NVIDIA_API_KEY. "cloud-openclaw": ["NVIDIA_API_KEY"], "cloud-openclaw-custom-policies": ["NVIDIA_API_KEY"], - // The invalid-key negative path injects its own bad credential in the - // fixture. Do not pass a real parent-env key into that child process. - "cloud-openclaw-invalid-nvidia-key": [], + "cloud-openclaw-invalid-nvidia-key": ["NVIDIA_API_KEY"], "cloud-openclaw-gateway-port-conflict": ["NVIDIA_API_KEY"], // Negative scenario: nemoclaw onboard runs against a docker shim that // exits non-zero. Onboard never reaches the cloud auth step, but the diff --git a/test/e2e-scenario/scenarios/scenarios/baseline.ts b/test/e2e-scenario/scenarios/scenarios/baseline.ts index d174d3da497..4e5befad679 100644 --- a/test/e2e-scenario/scenarios/scenarios/baseline.ts +++ b/test/e2e-scenario/scenarios/scenarios/baseline.ts @@ -299,6 +299,7 @@ const canonicalScenarioInputs: CanonicalScenarioInput[] = [ expectedStateId: "onboarding-failure-invalid-nvidia-key", onboardingAssertionIds: ["base-installed"], suiteIds: [], + requiredSecrets: ["NVIDIA_API_KEY"], expectedFailure: { phase: "onboarding", errorClass: "invalid-nvidia-api-key",