diff --git a/.github/workflows/e2e-scenarios.yaml b/.github/workflows/e2e-scenarios.yaml index 49f317caff2..804426f1331 100644 --- a/.github/workflows/e2e-scenarios.yaml +++ b/.github/workflows/e2e-scenarios.yaml @@ -78,6 +78,7 @@ jobs: [ubuntu-repo-cloud-openclaw-slack]=ubuntu-latest [ubuntu-repo-cloud-openclaw-telegram]=ubuntu-latest [ubuntu-repo-cloud-openclaw-token-rotation]=ubuntu-latest + [ubuntu-repo-docker-post-reboot-recovery]=ubuntu-latest [ubuntu-repo-openai-compatible-openclaw]=ubuntu-latest ) selected="" diff --git a/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts b/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts index 2fe76802134..dcf15974923 100644 --- a/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-expected-state.test.ts @@ -83,6 +83,41 @@ describe("probesForState maps typed expected-state into probe ids", () => { }; expect(probesForState(state)).toEqual([]); }); + + it("localRegistry.expected=present emits the local-registry-entry-present probe", () => { + const state: ExpectedState = { + id: "synthetic-local-registry", + cli: { installed: true }, + localRegistry: { expected: "present" }, + }; + expect(probesForState(state)).toEqual([ + "cli-installed", + "local-registry-entry-present", + ]); + }); + + it("dockerSandboxContainer.expected=present emits the docker-sandbox-container-present probe", () => { + const state: ExpectedState = { + id: "synthetic-docker-container", + cli: { installed: true }, + dockerSandboxContainer: { expected: "present" }, + }; + expect(probesForState(state)).toEqual([ + "cli-installed", + "docker-sandbox-container-present", + ]); + }); + + it("localRegistry/dockerSandboxContainer 'absent' emits no probe today", () => { + // Negative-direction probes haven't landed yet. Pin the gap so a + // future negative-scenario PR is forced to add the absent probes. + const state: ExpectedState = { + id: "synthetic-host-absent", + localRegistry: { expected: "absent" }, + dockerSandboxContainer: { expected: "absent" }, + }; + expect(probesForState(state)).toEqual([]); + }); }); describe("compiler emits state-validation phase actions from expected-state registry", () => { diff --git a/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts b/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts index 21cb9b6329e..35f51b78741 100644 --- a/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts @@ -50,4 +50,29 @@ describe("live Vitest registry discovery support", () => { reasons: ["runtime 'docker-missing' is not wired for live Vitest fixtures"], }); }); + + it("keeps unwhitelisted lifecycle profiles skipped with the lifecycle reason", () => { + const scenario = listScenarios().find((entry) => entry.id === "ubuntu-rebuild-openclaw"); + + expect(scenario).toBeTruthy(); + expect(liveScenarioSupport(scenario!)).toMatchObject({ + supported: false, + reasons: [ + "lifecycle 'rebuild-current-version' is not wired for live Vitest fixtures", + ], + }); + }); + + it("accepts the whitelisted post-reboot-recovery lifecycle scenario", () => { + const scenario = listScenarios().find( + (entry) => entry.id === "ubuntu-repo-docker-post-reboot-recovery", + ); + + expect(scenario).toBeTruthy(); + expect(scenario!.environment?.lifecycle).toBe("post-reboot-recovery"); + expect(liveScenarioSupport(scenario!)).toMatchObject({ + supported: true, + reasons: [], + }); + }); }); diff --git a/test/e2e-scenario/framework-tests/e2e-phase-lifecycle.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-lifecycle.test.ts new file mode 100644 index 00000000000..b4cae772551 --- /dev/null +++ b/test/e2e-scenario/framework-tests/e2e-phase-lifecycle.test.ts @@ -0,0 +1,239 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, expectTypeOf, it } from "vitest"; + +import { + HostCliClient, + SandboxClient, + type CommandRunner, +} from "../framework/clients/index.ts"; +import type { E2EScenarioFixtures } from "../framework/e2e-test.ts"; +import { + buildBackupContainerName, + LifecyclePhaseFixture, + type LifecycleCleanup, +} from "../framework/phases/lifecycle.ts"; +import type { NemoClawInstance } from "../framework/phases/index.ts"; +import type { + ShellProbeResult, + ShellProbeRunOptions, + TrustedShellCommand, +} from "../framework/shell-probe.ts"; + +interface RunnerCall { + command: string; + args: string[]; + options?: ShellProbeRunOptions; +} + +interface CleanupCall { + name: string; + run: () => Promise | void; +} + +function shellResult(exitCode: number, output = ""): ShellProbeResult { + return { + command: [], + exitCode, + signal: null, + timedOut: false, + stdout: exitCode === 0 ? output : "", + stderr: exitCode === 0 ? "" : output, + artifacts: { + stdout: "/tmp/stdout.txt", + stderr: "/tmp/stderr.txt", + result: "/tmp/result.json", + }, + }; +} + +class FakeRunner implements CommandRunner { + readonly calls: RunnerCall[] = []; + private readonly responses: ShellProbeResult[] = []; + + enqueue(response: ShellProbeResult): void { + this.responses.push(response); + } + + async run( + command: TrustedShellCommand, + options?: ShellProbeRunOptions, + ): Promise { + this.calls.push({ command: command.command, args: [...command.args], options }); + const response = this.responses.shift(); + if (!response) { + throw new Error( + `FakeRunner response missing for command: ${command.command} ${command.args.join(" ")}`, + ); + } + return response; + } +} + +class FakeCleanup implements LifecycleCleanup { + readonly calls: CleanupCall[] = []; + + add(name: string, run: () => Promise | void): void { + this.calls.push({ name, run }); + } +} + +function instance(overrides: Partial = {}): NemoClawInstance { + return { + onboarding: "cloud-openclaw", + sandboxName: "e2e-ubuntu-repo-cloud-openclaw", + agent: "openclaw", + provider: "nvidia", + providerEnv: "cloud", + gatewayUrl: "http://127.0.0.1:18789", + result: shellResult(0), + ...overrides, + }; +} + +function fixture(runner: FakeRunner, cleanup: FakeCleanup): LifecyclePhaseFixture { + const host = new HostCliClient(runner); + const sandbox = new SandboxClient(runner); + return new LifecyclePhaseFixture(host, sandbox, cleanup); +} + +describe("LifecyclePhaseFixture.simulate post-reboot-recovery (stop-original)", () => { + it("stops the gateway, discovers the labeled container, and stops it", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0)); // openshell gateway stop + runner.enqueue(shellResult(0, "openshell-cluster-e2e-ubuntu-repo-cloud-openclaw\n")); // discover + runner.enqueue(shellResult(0)); // docker stop + const cleanup = new FakeCleanup(); + + const result = await fixture(runner, cleanup).simulate( + "post-reboot-recovery", + instance(), + ); + + expect(result.profile).toBe("post-reboot-recovery"); + expect(result.steps.map((step) => step.id)).toEqual([ + "gateway-stop", + "docker-stop:openshell-cluster-e2e-ubuntu-repo-cloud-openclaw", + ]); + expect(runner.calls.map((call) => ({ command: call.command, args: call.args }))).toEqual([ + { command: "openshell", args: ["gateway", "stop"] }, + { + command: "docker", + args: [ + "ps", + "-a", + "--filter", + "label=openshell.ai/sandbox-name=e2e-ubuntu-repo-cloud-openclaw", + "--format", + "{{.Names}}", + ], + }, + { command: "docker", args: ["stop", "openshell-cluster-e2e-ubuntu-repo-cloud-openclaw"] }, + ]); + expect(cleanup.calls.map((call) => call.name)).toEqual([ + "lifecycle.docker-start:openshell-cluster-e2e-ubuntu-repo-cloud-openclaw", + ]); + }); + + it("tolerates a non-zero gateway stop (post-reboot fresh runtime)", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(1, "no gateway runtime")); // gateway stop fails + runner.enqueue(shellResult(0, "container-1\n")); + runner.enqueue(shellResult(0)); // docker stop + const cleanup = new FakeCleanup(); + + const result = await fixture(runner, cleanup).simulate( + "post-reboot-recovery", + instance(), + ); + + expect(result.steps.find((step) => step.id === "gateway-stop")).toBeTruthy(); + }); + + it("fails when no Docker container carries the OpenShell sandbox-name label", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0)); // gateway stop + runner.enqueue(shellResult(0, "\n")); // discover returns nothing + const cleanup = new FakeCleanup(); + + await expect( + fixture(runner, cleanup).simulate("post-reboot-recovery", instance()), + ).rejects.toThrow(/expected at least one Docker container labeled/); + }); + + it("fails when docker discover returns non-zero", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0)); // gateway stop + runner.enqueue(shellResult(1, "Cannot connect to the Docker daemon")); + const cleanup = new FakeCleanup(); + + await expect( + fixture(runner, cleanup).simulate("post-reboot-recovery", instance()), + ).rejects.toThrow(/could not query Docker for label/); + }); +}); + +describe("LifecyclePhaseFixture.simulate post-reboot-recovery (rename-to-gpu-backup)", () => { + it("stops, then renames the labeled container to a *-nemoclaw-gpu-backup-* sibling", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0)); // openshell gateway stop + runner.enqueue(shellResult(0, "openshell-cluster-e2e-x\n")); // discover + runner.enqueue(shellResult(0)); // docker stop + runner.enqueue(shellResult(0)); // docker rename + const cleanup = new FakeCleanup(); + + const result = await fixture(runner, cleanup).simulate( + "post-reboot-recovery", + instance({ sandboxName: "e2e-x" }), + { mode: "rename-to-gpu-backup" }, + ); + + expect(result.steps.map((step) => step.id.split("->")[0])).toContain( + "docker-rename:openshell-cluster-e2e-x", + ); + const renameCall = runner.calls.find( + (call) => call.command === "docker" && call.args[0] === "rename", + ); + expect(renameCall).toBeTruthy(); + expect(renameCall!.args[1]).toBe("openshell-cluster-e2e-x"); + expect(renameCall!.args[2]).toMatch(/^openshell-cluster-e2e-x-nemoclaw-gpu-backup-\d+$/); + + // Cleanup queue now has both docker-start and docker-rename-back. + expect(cleanup.calls.map((call) => call.name.split(":")[0])).toEqual([ + "lifecycle.docker-start", + "lifecycle.docker-rename-back", + ]); + }); +}); + +describe("LifecyclePhaseFixture profile dispatch", () => { + it("rejects unknown lifecycle profiles", async () => { + const runner = new FakeRunner(); + const cleanup = new FakeCleanup(); + + await expect( + // @ts-expect-error — exhaustiveness check + fixture(runner, cleanup).simulate("not-a-profile", instance()), + ).rejects.toThrow(/Unsupported lifecycle profile/); + }); + + it("exposes the lifecycle phase on the Vitest scenario context", () => { + expectTypeOf().toEqualTypeOf(); + }); +}); + +describe("buildBackupContainerName", () => { + it("appends -nemoclaw-gpu-backup- to the original name", () => { + expect(buildBackupContainerName("openshell-cluster-foo", 1717280000000)).toBe( + "openshell-cluster-foo-nemoclaw-gpu-backup-1717280000000", + ); + }); + + it("truncates the original name to fit within Docker's 253-char limit", () => { + const longName = "a".repeat(253); + const result = buildBackupContainerName(longName, 1717280000000); + expect(result.length).toBeLessThanOrEqual(253); + expect(result.endsWith("-nemoclaw-gpu-backup-1717280000000")).toBe(true); + }); +}); diff --git a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts index f2af06b99f5..50cf1306d59 100644 --- a/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-phase-state-validation.test.ts @@ -108,9 +108,17 @@ function instance(overrides: Partial = {}): NemoClawInstance { }; } -function fixture(runner: FakeRunner): StateValidationPhaseFixture { +function fixture( + runner: FakeRunner, + io: ConstructorParameters[3] = {}, +): StateValidationPhaseFixture { const host = new HostCliClient(runner); - return new StateValidationPhaseFixture(host, new GatewayClient(host), new SandboxClient(runner)); + return new StateValidationPhaseFixture( + host, + new GatewayClient(host), + new SandboxClient(runner), + io, + ); } describe("state-validation phase fixture", () => { @@ -457,3 +465,111 @@ describe("state-validation phase fixture", () => { >().toEqualTypeOf(); }); }); + +describe("state-validation host-side probes", () => { + const localRegistryState = { + id: "synthetic-local-registry-present", + localRegistry: { expected: "present" as const }, + }; + const dockerContainerState = { + id: "synthetic-docker-container-present", + dockerSandboxContainer: { expected: "present" as const }, + }; + + it("local-registry-entry-present passes when the registry contains the sandbox name", async () => { + const runner = new FakeRunner(); + const fx = fixture(runner, { + readRegistry: () => ({ + entries: { "e2e-ubuntu-repo-cloud-openclaw": { name: "e2e-ubuntu-repo-cloud-openclaw" } }, + }), + }); + + const result = await fx.from(localRegistryState, instance()); + + expect(result.probes.map((probe) => probe.id)).toEqual(["local-registry-entry-present"]); + expect(runner.calls).toEqual([]); + }); + + it("local-registry-entry-present fails when the registry file is missing", async () => { + const runner = new FakeRunner(); + const fx = fixture(runner, { readRegistry: () => null }); + + await expect(fx.from(localRegistryState, instance())).rejects.toThrow( + /expected local registry entry for 'e2e-ubuntu-repo-cloud-openclaw'.*does not exist/, + ); + }); + + it("local-registry-entry-present fails when the sandbox name is missing from registry", async () => { + const runner = new FakeRunner(); + const fx = fixture(runner, { + readRegistry: () => ({ entries: { "some-other-sandbox": {} } }), + }); + + await expect(fx.from(localRegistryState, instance())).rejects.toThrow( + /registry contains: some-other-sandbox/, + ); + }); + + it("docker-sandbox-container-present passes when docker ps -a returns labeled names", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "e2e-ubuntu-repo-cloud-openclaw\n")); + const fx = fixture(runner); + + const result = await fx.from(dockerContainerState, instance()); + + expect(result.probes.map((probe) => probe.id)).toEqual(["docker-sandbox-container-present"]); + expect(runner.calls).toEqual([ + { + command: "docker", + args: [ + "ps", + "-a", + "--filter", + "label=openshell.ai/sandbox-name=e2e-ubuntu-repo-cloud-openclaw", + "--format", + "{{.Names}}", + ], + options: { + artifactName: "docker-sandbox-container-present-e2e-ubuntu-repo-cloud-openclaw", + env: expect.objectContaining({ PATH: expect.any(String) }), + timeoutMs: 15_000, + }, + }, + ]); + }); + + it("docker-sandbox-container-present matches *-nemoclaw-gpu-backup-* sibling containers", async () => { + const runner = new FakeRunner(); + runner.enqueue( + shellResult( + 0, + "e2e-ubuntu-repo-cloud-openclaw-nemoclaw-gpu-backup-1717280000000\n", + ), + ); + const fx = fixture(runner); + + const result = await fx.from(dockerContainerState, instance()); + + expect(result.probes.map((probe) => probe.id)).toEqual(["docker-sandbox-container-present"]); + }); + + it("docker-sandbox-container-present fails when docker ps -a returns no labeled container", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(0, "\n")); + const fx = fixture(runner); + + await expect(fx.from(dockerContainerState, instance())).rejects.toThrow( + /docker ps -a returned none/, + ); + }); + + it("docker-sandbox-container-present fails when docker ps -a exits non-zero", async () => { + const runner = new FakeRunner(); + runner.enqueue(shellResult(1, "Cannot connect to the Docker daemon")); + const fx = fixture(runner); + + await expect(fx.from(dockerContainerState, instance())).rejects.toThrow( + /could not query Docker for label.*exit 1/, + ); + }); +}); diff --git a/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts b/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts index a6a9bf9d18e..78eb216d53d 100644 --- a/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts +++ b/test/e2e-scenario/framework-tests/e2e-scenario-matrix.test.ts @@ -131,6 +131,7 @@ describe("typed scenario matrix", () => { it("builds the default live Vitest matrix from fixture-supported scenarios only", () => { expect(buildLiveScenarioMatrix().map((entry) => entry.id)).toEqual([ "ubuntu-repo-cloud-openclaw", + "ubuntu-repo-docker-post-reboot-recovery", ]); expect(buildLiveScenarioMatrix()[0]).toMatchObject({ id: "ubuntu-repo-cloud-openclaw", @@ -145,6 +146,22 @@ describe("typed scenario matrix", () => { supportReasons: [], pendingRuntimeSuites: ["smoke", "inference", "credentials"], }); + // Failing-test-first guard for #4423. Pinned in the matrix to + // confirm the lifecycle whitelist + post-reboot-recovery scenario + // are wired together; the actual RED/GREEN behavior is exercised + // by the live runner (gates on the fix landing in src/lib/). + expect(buildLiveScenarioMatrix()[1]).toMatchObject({ + id: "ubuntu-repo-docker-post-reboot-recovery", + runner: "ubuntu-latest", + platform: "ubuntu-local", + install: "repo-current", + runtime: "docker-running", + onboarding: "cloud-openclaw", + expectedStateId: "post-reboot-recovery-ready", + requiredSecrets: ["NVIDIA_API_KEY"], + supported: true, + supportReasons: [], + }); }); it("keeps explicitly selected unsupported live scenarios in the matrix with skip reasons", () => { @@ -163,7 +180,10 @@ describe("typed scenario matrix", () => { const lines = result.stdout.trim().split("\n"); expect(lines.length, "live matrix output must be a single line").toBe(1); const parsed = JSON.parse(lines[0]); - expect(parsed.map((entry: { id: string }) => entry.id)).toEqual(["ubuntu-repo-cloud-openclaw"]); + expect(parsed.map((entry: { id: string }) => entry.id)).toEqual([ + "ubuntu-repo-cloud-openclaw", + "ubuntu-repo-docker-post-reboot-recovery", + ]); }); it("--emit-live-matrix honors explicit scenario selections", () => { diff --git a/test/e2e-scenario/framework/e2e-test.ts b/test/e2e-scenario/framework/e2e-test.ts index af31e4560a9..75891b38fa7 100644 --- a/test/e2e-scenario/framework/e2e-test.ts +++ b/test/e2e-scenario/framework/e2e-test.ts @@ -14,6 +14,7 @@ import { import { assertCleanupPassed, CleanupRegistry } from "./cleanup.ts"; import { EnvironmentPhaseFixture, + LifecyclePhaseFixture, OnboardingPhaseFixture, StateValidationPhaseFixture, } from "./phases/index.ts"; @@ -32,6 +33,7 @@ export interface E2EScenarioFixtures { state: StateClient; environment: EnvironmentPhaseFixture; onboard: OnboardingPhaseFixture; + lifecycle: LifecyclePhaseFixture; stateValidation: StateValidationPhaseFixture; } @@ -91,6 +93,9 @@ export const test = base.extend({ onboard: async ({ cleanup, host, secrets }, use) => { await use(new OnboardingPhaseFixture(host, secrets, cleanup)); }, + lifecycle: async ({ cleanup, host, sandbox }, use) => { + await use(new LifecyclePhaseFixture(host, sandbox, cleanup)); + }, stateValidation: async ({ host, gateway, sandbox }, use) => { await use(new StateValidationPhaseFixture(host, gateway, sandbox)); }, diff --git a/test/e2e-scenario/framework/phases/index.ts b/test/e2e-scenario/framework/phases/index.ts index f6345ba476f..1905028a0c0 100644 --- a/test/e2e-scenario/framework/phases/index.ts +++ b/test/e2e-scenario/framework/phases/index.ts @@ -7,6 +7,14 @@ export { type DockerRuntimeReady, type EnvironmentReady, } from "./environment.ts"; +export { + LifecyclePhaseFixture, + type LifecycleCleanup, + type LifecycleProfile, + type LifecycleResult, + type PostRebootMode, + type PostRebootOptions, +} from "./lifecycle.ts"; export { OnboardingPhaseFixture, type NemoClawInstance, diff --git a/test/e2e-scenario/framework/phases/lifecycle.ts b/test/e2e-scenario/framework/phases/lifecycle.ts new file mode 100644 index 00000000000..75225c04811 --- /dev/null +++ b/test/e2e-scenario/framework/phases/lifecycle.ts @@ -0,0 +1,207 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { buildAvailabilityProbeEnv } from "../availability-env.ts"; +import { assertExitZero } from "../clients/command.ts"; +import type { HostCliClient } from "../clients/host.ts"; +import type { SandboxClient } from "../clients/sandbox.ts"; +import type { ShellProbeResult } from "../shell-probe.ts"; +import type { NemoClawInstance } from "./onboarding.ts"; + +// Mirror of `OPENSHELL_SANDBOX_NAME_LABEL` in +// `src/lib/onboard/docker-gpu-patch.ts`. Duplicated here because the +// fixture layer must not import from `src/lib/**` (CLI source) — that +// boundary keeps the live runner honest about probing only host- +// observable state. Drift is caught by the integration test that wires +// a real onboarded sandbox through the docker-sandbox-container-present +// probe. +const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; +const DOCKER_PROBE_TIMEOUT_MS = 15_000; +const GATEWAY_STOP_TIMEOUT_MS = 60_000; + +export type LifecycleProfile = "post-reboot-recovery"; + +export interface LifecycleCleanup { + add(name: string, run: () => Promise | void): void; +} + +/** + * How the post-reboot-recovery profile leaves Docker before the test + * exits the lifecycle phase: + * + * - `stop-original` — `docker stop` the labeled container in place. + * Matches the common Spark reboot path: the + * container exists, is exited, retains its + * OpenShell labels, but is no longer running. + * + * - `rename-to-gpu-backup` — stop the labeled container, then + * `docker rename` it to `-nemoclaw- + * gpu-backup-`. Reproduces the rarer GPU- + * patch reboot path where only the backup + * sibling survives and recovery has to rename + * it back. Mirrors `buildBackupContainerName()` + * in `src/lib/onboard/docker-gpu-patch.ts`. + */ +export type PostRebootMode = "stop-original" | "rename-to-gpu-backup"; + +export interface PostRebootOptions { + mode?: PostRebootMode; +} + +export interface LifecycleResult { + profile: LifecycleProfile; + steps: Array<{ id: string; results: ShellProbeResult[] }>; +} + +export class LifecyclePhaseFixture { + constructor( + private readonly host: HostCliClient, + private readonly sandbox: SandboxClient, + private readonly cleanup: LifecycleCleanup, + ) {} + + async simulate( + profile: LifecycleProfile, + instance: NemoClawInstance, + options: PostRebootOptions = {}, + ): Promise { + switch (profile) { + case "post-reboot-recovery": + return await this.simulatePostReboot(instance, options); + default: { + const _exhaustive: never = profile; + throw new Error(`Unsupported lifecycle profile '${_exhaustive}'.`); + } + } + } + + /** + * Reproduce the host-side conditions of a DGX Spark / Linux Docker-driver + * reboot: + * + * 1. Ask OpenShell to stop its gateway runtime so the in-memory + * sandbox view drops to NotFound. The actual sandbox container + * is unaffected — that is the entire point of the bug class + * tracked by #4423. + * + * 2. Locate the OpenShell-labeled Docker container for the + * scenario's sandbox name and either stop it (default) or + * stop+rename it to a `*-nemoclaw-gpu-backup-*` sibling. + * + * Cleanups (run in reverse order at end of test): + * - rename the backup sibling back to the original name (if we + * created one); + * - `docker start` the labeled container so the sandbox returns + * to a usable state for any teardown that expects it live. + */ + async simulatePostReboot( + instance: NemoClawInstance, + options: PostRebootOptions = {}, + ): Promise { + const mode: PostRebootMode = options.mode ?? "stop-original"; + const steps: LifecycleResult["steps"] = []; + + const gatewayStop = await this.sandbox.openshell(["gateway", "stop"], { + artifactName: "lifecycle-post-reboot-gateway-stop", + env: buildAvailabilityProbeEnv(), + timeoutMs: GATEWAY_STOP_TIMEOUT_MS, + }); + // gateway stop is best-effort: a fresh-start/no-runtime gateway + // will exit non-zero with NoSuchProcess, which is exactly the + // post-reboot state we want to simulate. Don't fail the lifecycle + // phase on it. + steps.push({ id: "gateway-stop", results: [gatewayStop] }); + + const containerNames = await this.discoverLabeledContainerNames(instance); + if (containerNames.length === 0) { + throw new Error( + `lifecycle.post-reboot-recovery expected at least one Docker container labeled ` + + `'${OPENSHELL_SANDBOX_NAME_LABEL}=${instance.sandboxName}', but docker ps -a returned none. ` + + `Did onboarding create the sandbox?`, + ); + } + const originalName = containerNames[0]; + + const stop = await this.host.command( + "docker", + ["stop", originalName], + { + artifactName: `lifecycle-post-reboot-docker-stop-${originalName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: DOCKER_PROBE_TIMEOUT_MS, + }, + ); + assertExitZero(stop, `docker stop ${originalName}`); + steps.push({ id: `docker-stop:${originalName}`, results: [stop] }); + this.cleanup.add(`lifecycle.docker-start:${originalName}`, async () => { + await this.host.command("docker", ["start", originalName], { + artifactName: `lifecycle-cleanup-docker-start-${originalName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: DOCKER_PROBE_TIMEOUT_MS, + }); + }); + + if (mode === "rename-to-gpu-backup") { + const backupName = buildBackupContainerName(originalName, Date.now()); + const rename = await this.host.command( + "docker", + ["rename", originalName, backupName], + { + artifactName: `lifecycle-post-reboot-docker-rename-${originalName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: DOCKER_PROBE_TIMEOUT_MS, + }, + ); + assertExitZero(rename, `docker rename ${originalName} ${backupName}`); + steps.push({ id: `docker-rename:${originalName}->${backupName}`, results: [rename] }); + this.cleanup.add(`lifecycle.docker-rename-back:${backupName}`, async () => { + await this.host.command("docker", ["rename", backupName, originalName], { + artifactName: `lifecycle-cleanup-docker-rename-back-${backupName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: DOCKER_PROBE_TIMEOUT_MS, + }); + }); + } + + return { profile: "post-reboot-recovery", steps }; + } + + private async discoverLabeledContainerNames(instance: NemoClawInstance): Promise { + const result = await this.host.command( + "docker", + [ + "ps", + "-a", + "--filter", + `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${instance.sandboxName}`, + "--format", + "{{.Names}}", + ], + { + artifactName: `lifecycle-post-reboot-docker-discover-${instance.sandboxName}`, + env: buildAvailabilityProbeEnv(), + timeoutMs: DOCKER_PROBE_TIMEOUT_MS, + }, + ); + if (result.exitCode !== 0) { + throw new Error( + `lifecycle.post-reboot-recovery could not query Docker for label ` + + `'${OPENSHELL_SANDBOX_NAME_LABEL}=${instance.sandboxName}' (exit ${result.exitCode}).`, + ); + } + return result.stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + } +} + +// Mirror of `MAX_DOCKER_CONTAINER_NAME_LENGTH` in +// `src/lib/onboard/docker-gpu-patch.ts`. +const MAX_DOCKER_CONTAINER_NAME_LENGTH = 253; + +export function buildBackupContainerName(originalName: string, nowMs: number): string { + const suffix = `-nemoclaw-gpu-backup-${String(nowMs)}`; + const maxOriginalLength = MAX_DOCKER_CONTAINER_NAME_LENGTH - suffix.length; + return `${originalName.slice(0, Math.max(1, maxOriginalLength))}${suffix}`; +} diff --git a/test/e2e-scenario/framework/phases/state-validation.ts b/test/e2e-scenario/framework/phases/state-validation.ts index 4743be9fbe2..e919adc17de 100644 --- a/test/e2e-scenario/framework/phases/state-validation.ts +++ b/test/e2e-scenario/framework/phases/state-validation.ts @@ -1,6 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + import { buildAvailabilityProbeEnv } from "../availability-env.ts"; import { trustedProviderEndpoint, @@ -13,6 +17,36 @@ import { probesForState, requireExpectedState } from "../../scenarios/expected-s import type { ExpectedState, StateProbeId } from "../../scenarios/types.ts"; import type { NemoClawInstance } from "./onboarding.ts"; +// Mirror of `src/lib/state/registry.ts::REGISTRY_FILE`. The fixture +// owns its own copy because the framework code must not import from +// `src/lib/**` (CLI source) — that boundary keeps the live runner +// honest about probing only host-observable state. +const NEMOCLAW_REGISTRY_RELPATH = [".nemoclaw", "sandboxes.json"] as const; +const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; + +export interface ProbeIO { + readRegistry?(): { entries: Record } | null; +} + +function defaultRegistryPath(): string { + const home = process.env.HOME ?? os.homedir(); + return path.join(home, ...NEMOCLAW_REGISTRY_RELPATH); +} + +function defaultReadRegistry(): { entries: Record } | null { + const file = defaultRegistryPath(); + if (!fs.existsSync(file)) return null; + try { + const raw = fs.readFileSync(file, "utf8"); + const parsed = JSON.parse(raw); + if (!parsed || typeof parsed !== "object") return { entries: {} }; + const entries = (parsed as { sandboxes?: Record }).sandboxes; + return { entries: entries && typeof entries === "object" ? entries : {} }; + } catch { + return { entries: {} }; + } +} + export interface StateValidationProbeResult { id: StateProbeId; status: "passed"; @@ -74,14 +108,23 @@ function isMissingOpenShellError(error: unknown): boolean { } export class StateValidationPhaseFixture { + private readonly io: ProbeIO; + constructor( private readonly host: HostCliClient, private readonly gateway: GatewayClient, private readonly sandbox: SandboxClient, - ) {} + io: ProbeIO = {}, + ) { + this.io = io; + } - async from(expectedStateId: string, instance?: NemoClawInstance): Promise { - const state = requireExpectedState(expectedStateId); + async from( + expectedState: string | ExpectedState, + instance?: NemoClawInstance, + ): Promise { + const state = + typeof expectedState === "string" ? requireExpectedState(expectedState) : expectedState; const probes: StateValidationProbeResult[] = []; for (const probe of probesForState(state)) { probes.push(await this.runProbe(probe, instance)); @@ -104,6 +147,10 @@ export class StateValidationPhaseFixture { return await this.expectSandboxRunning(requireInstance(probe, instance)); case "sandbox-absent": return await this.expectSandboxAbsent(requireInstance(probe, instance)); + case "local-registry-entry-present": + return this.expectLocalRegistryEntryPresent(requireInstance(probe, instance)); + case "docker-sandbox-container-present": + return await this.expectDockerSandboxContainerPresent(requireInstance(probe, instance)); default: { const _exhaustive: never = probe; throw new Error(`Unsupported state-validation probe '${_exhaustive}'.`); @@ -231,6 +278,66 @@ export class StateValidationPhaseFixture { return { id: "sandbox-running", status: "passed", results: [result] }; } + private expectLocalRegistryEntryPresent( + instance: NemoClawInstance, + ): StateValidationProbeResult { + const reader = this.io.readRegistry ?? defaultReadRegistry; + const registry = reader(); + if (!registry) { + throw new Error( + `state-validation expected local registry entry for '${instance.sandboxName}', ` + + `but ${defaultRegistryPath()} does not exist.`, + ); + } + if (!Object.prototype.hasOwnProperty.call(registry.entries, instance.sandboxName)) { + const present = Object.keys(registry.entries).sort().join(", ") || "(none)"; + throw new Error( + `state-validation expected local registry entry for '${instance.sandboxName}', ` + + `but the registry contains: ${present}.`, + ); + } + return { id: "local-registry-entry-present", status: "passed", results: [] }; + } + + private async expectDockerSandboxContainerPresent( + instance: NemoClawInstance, + ): Promise { + const result = await this.host.command( + "docker", + [ + "ps", + "-a", + "--filter", + `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${instance.sandboxName}`, + "--format", + "{{.Names}}", + ], + { + artifactName: `docker-sandbox-container-present-${instance.sandboxName}`, + env: statusProbeEnv(), + timeoutMs: 15_000, + }, + ); + if (result.exitCode !== 0) { + throw new Error( + `state-validation could not query Docker for label '${OPENSHELL_SANDBOX_NAME_LABEL}=${instance.sandboxName}' ` + + `(exit ${result.exitCode}).`, + ); + } + const names = result.stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + if (names.length === 0) { + throw new Error( + `state-validation expected at least one Docker container labeled ` + + `'${OPENSHELL_SANDBOX_NAME_LABEL}=${instance.sandboxName}' (running, stopped, or ` + + `*-nemoclaw-gpu-backup-* sibling), but docker ps -a returned none.`, + ); + } + return { id: "docker-sandbox-container-present", status: "passed", results: [result] }; + } + private async expectSandboxAbsent( instance: NemoClawInstance, ): Promise { diff --git a/test/e2e-scenario/live/registry-scenarios.test.ts b/test/e2e-scenario/live/registry-scenarios.test.ts index 892cadb8124..3395ac553a8 100644 --- a/test/e2e-scenario/live/registry-scenarios.test.ts +++ b/test/e2e-scenario/live/registry-scenarios.test.ts @@ -5,9 +5,16 @@ import fs from "node:fs"; import path from "node:path"; import { expect, test } from "../framework/e2e-test.ts"; +import type { LifecycleProfile } from "../framework/phases/index.ts"; import { listScenarios } from "../scenarios/registry.ts"; import { liveScenarioSupport, liveScenarioTestName } from "../scenarios/runtime-support.ts"; +const LIFECYCLE_PROFILES: ReadonlySet = new Set(["post-reboot-recovery"]); + +function isLifecycleProfile(value: string | undefined): value is LifecycleProfile { + return value !== undefined && LIFECYCLE_PROFILES.has(value as LifecycleProfile); +} + const REPO_ROOT = path.resolve(import.meta.dirname, "../../.."); const CLI_DIST_ENTRYPOINT = path.join(REPO_ROOT, "dist", "nemoclaw.js"); process.env.NEMOCLAW_CLI_BIN ??= path.join(REPO_ROOT, "bin", "nemoclaw.js"); @@ -30,7 +37,7 @@ for (const scenario of listScenarios()) { test( liveScenarioTestName(scenario), - async ({ artifacts, environment, onboard, secrets, stateValidation }) => { + async ({ artifacts, environment, lifecycle, onboard, secrets, stateValidation }) => { for (const secret of scenario.requiredSecrets ?? []) { secrets.required(secret); } @@ -55,6 +62,27 @@ for (const scenario of listScenarios()) { const ready = await environment.assertReady(scenario.environment); const instance = await onboard.from(ready, { sandboxName: `e2e-${scenario.id}` }); + + // Lifecycle phase runs between onboard and state-validation. + // Scenarios opt in by setting `environment.lifecycle` to a + // whitelisted profile (see SUPPORTED_LIFECYCLES in + // runtime-support.ts). Today only `post-reboot-recovery` is + // wired, and it dispatches through `LifecyclePhaseFixture` to + // mutate host state (gateway runtime, Docker container) before + // the state-validation probes assert preservation invariants. + let lifecycleResult: Awaited> | undefined; + const profile = scenario.environment.lifecycle; + if (profile) { + if (!isLifecycleProfile(profile)) { + throw new Error( + `scenario '${scenario.id}' declares lifecycle '${profile}' which is not ` + + `dispatched by LifecyclePhaseFixture; update the fixture and the ` + + `SUPPORTED_LIFECYCLES whitelist together.`, + ); + } + lifecycleResult = await lifecycle.simulate(profile, instance); + } + const validation = await stateValidation.from(scenario.expectedStateId, instance); await artifacts.writeJson("scenario-result.json", { @@ -62,6 +90,9 @@ for (const scenario of listScenarios()) { expectedStateId: validation.state.id, probes: validation.probes.map((probe) => probe.id), pendingRuntimeSuites: support.pendingRuntimeSuites, + lifecycle: lifecycleResult + ? { profile: lifecycleResult.profile, steps: lifecycleResult.steps.map((s) => s.id) } + : undefined, }); }, ); diff --git a/test/e2e-scenario/manifests/openclaw-nvidia-post-reboot-recovery.yaml b/test/e2e-scenario/manifests/openclaw-nvidia-post-reboot-recovery.yaml new file mode 100644 index 00000000000..571ec5b2f5f --- /dev/null +++ b/test/e2e-scenario/manifests/openclaw-nvidia-post-reboot-recovery.yaml @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: nemoclaw.io/v1 +kind: NemoClawInstance +metadata: + name: openclaw-nvidia-post-reboot-recovery +spec: + setup: + install: + source: repo-current + runtime: + containerEngine: docker + containerDaemon: running + platform: + os: ubuntu + executionTarget: local + onboarding: + agent: openclaw + provider: nvidia + modelRoute: inference-local + policyTier: balanced + messaging: [] + # Lifecycle phase opt-in. The Vitest live runner dispatches this + # profile through `LifecyclePhaseFixture.simulate(...)`, which: + # 1. stops the OpenShell gateway runtime, and + # 2. `docker stop`s the labeled sandbox container. + # The host-side state-validation probes + # (`local-registry-entry-present`, `docker-sandbox-container-present`) + # then assert that `nemoclaw status` recovers the sandbox + # without destroying registry state. On unfixed code the registry + # entry is wiped by the `missing` branch in + # `src/lib/actions/sandbox/{status,gateway-state}.ts` and the + # `local-registry-entry-present` probe fails. See #4423. + lifecycle: post-reboot-recovery + state: + workspaceRef: default + credentialRefs: + - NVIDIA_API_KEY diff --git a/test/e2e-scenario/scenarios/expected-states.ts b/test/e2e-scenario/scenarios/expected-states.ts index 2a740a38cfd..75027f5e238 100644 --- a/test/e2e-scenario/scenarios/expected-states.ts +++ b/test/e2e-scenario/scenarios/expected-states.ts @@ -77,6 +77,31 @@ const onboardingFailureGatewayPortConflict: ExpectedState = { sandbox: { expected: "absent" }, }; +// Post-reboot recovery contract for #4423. After the lifecycle phase +// stops the OpenShell gateway runtime + the labeled sandbox container, +// the user-visible invariants are: +// +// * `cli` still installed. +// * `gateway` healthy: the user-systemd unit from #4580 brings the +// gateway back up before status runs. +// * `sandbox` running by the time validation completes: a correct +// fix performs Docker-backed recovery before responding. +// * `localRegistry` entry preserved: this is the user-visible +// regression target. On unfixed code, the destructive `missing` +// branch wipes the entry; on fixed code it survives because +// Docker corroborated the sandbox container existence. +// * `dockerSandboxContainer` still present: the recovery path must +// not delete the labeled container or its `*-nemoclaw-gpu-backup-*` +// sibling as a side effect. +const postRebootRecoveryReady: ExpectedState = { + id: "post-reboot-recovery-ready", + cli: { installed: true }, + gateway: { expected: "present", health: "healthy" }, + sandbox: { expected: "present", status: "running", agent: "openclaw" }, + localRegistry: { expected: "present" }, + dockerSandboxContainer: { expected: "present" }, +}; + const REGISTRY: readonly ExpectedState[] = [ cloudOpenclawReady, cloudOpenclawCustomPoliciesReady, @@ -86,6 +111,7 @@ const REGISTRY: readonly ExpectedState[] = [ preflightFailureNoSandbox, onboardingFailureInvalidNvidiaKey, onboardingFailureGatewayPortConflict, + postRebootRecoveryReady, ]; const BY_ID: ReadonlyMap = new Map( @@ -131,5 +157,15 @@ export function probesForState(state: ExpectedState): readonly StateProbeId[] { } else if (state.sandbox?.expected === "absent") { probes.push("sandbox-absent"); } + // Host-side aspects. "absent" deliberately emits no probe today: it + // would require asserting the registry/container does NOT exist, + // which has no scenario in flight. Add when a negative scenario + // needs it. + if (state.localRegistry?.expected === "present") { + probes.push("local-registry-entry-present"); + } + if (state.dockerSandboxContainer?.expected === "present") { + probes.push("docker-sandbox-container-present"); + } return probes; } diff --git a/test/e2e-scenario/scenarios/runtime-support.ts b/test/e2e-scenario/scenarios/runtime-support.ts index 24b1e269242..6a9f874264d 100644 --- a/test/e2e-scenario/scenarios/runtime-support.ts +++ b/test/e2e-scenario/scenarios/runtime-support.ts @@ -7,6 +7,12 @@ const SUPPORTED_PLATFORMS = new Set(["ubuntu-local"]); const SUPPORTED_INSTALLS = new Set(["repo-current"]); const SUPPORTED_RUNTIMES = new Set(["docker-running"]); const SUPPORTED_ONBOARDING = new Set(["cloud-openclaw"]); +// Lifecycle profiles wired into the live Vitest driver. A profile is +// supported only after both (a) `LifecyclePhaseFixture.simulate(profile)` +// dispatches it, and (b) at least one expected-state declares the post- +// lifecycle host invariants the fixture creates. New profiles must add +// the dispatcher branch and an expected-state in the same change set. +const SUPPORTED_LIFECYCLES = new Set(["post-reboot-recovery"]); export interface LiveScenarioSupport { supported: boolean; @@ -43,7 +49,7 @@ export function liveScenarioSupport(scenario: ScenarioDefinition): LiveScenarioS if (!SUPPORTED_ONBOARDING.has(environment.onboarding)) { reasons.push(`onboarding '${environment.onboarding}' is not wired for live Vitest fixtures`); } - if (environment.lifecycle) { + if (environment.lifecycle && !SUPPORTED_LIFECYCLES.has(environment.lifecycle)) { reasons.push(`lifecycle '${environment.lifecycle}' is not wired for live Vitest fixtures`); } } diff --git a/test/e2e-scenario/scenarios/scenarios/baseline.ts b/test/e2e-scenario/scenarios/scenarios/baseline.ts index a5b0e050406..4e5befad679 100644 --- a/test/e2e-scenario/scenarios/scenarios/baseline.ts +++ b/test/e2e-scenario/scenarios/scenarios/baseline.ts @@ -148,6 +148,38 @@ const canonicalScenarioInputs: CanonicalScenarioInput[] = [ suiteIds: ["smoke", "rebuild", "upgrade"], requiredSecrets: ["NVIDIA_API_KEY"], }, + { + // Failing-test-first regression guard for #4423. After onboarding, + // the lifecycle phase reproduces the host-side conditions of a + // DGX Spark / Linux Docker-driver reboot: stop the OpenShell + // gateway runtime + `docker stop` the labeled sandbox container. + // The state-validation phase then runs `nemoclaw status` + // and asserts the post-recovery invariants declared by the + // `post-reboot-recovery-ready` expected-state. + // + // On unfixed `main`, the destructive `missing` branch in + // `src/lib/actions/sandbox/status.ts` (and the parallel branch + // reached through `ensureLiveSandboxOrExit` in + // `src/lib/actions/sandbox/gateway-state.ts`) wipes the local + // registry entry once the gateway returns to `healthy_named`, + // so the `local-registry-entry-present` probe fails and this + // scenario goes RED. + // + // The fix lands in PR-A (parts 2 & 3 of ericksoa's plan): add a + // Docker-driver sandbox recovery helper, then tighten + // stale-removal in active paths to require Docker-corroborated + // absence before destroying the registry. PR-A flips this guard + // 🔴 → 🟢. + id: "ubuntu-repo-docker-post-reboot-recovery", + manifestName: "openclaw-nvidia-post-reboot-recovery", + environment: ubuntuRepoDockerLifecycle("cloud-openclaw", "post-reboot-recovery"), + expectedStateId: "post-reboot-recovery-ready", + suiteIds: ["smoke"], + requiredSecrets: ["NVIDIA_API_KEY"], + description: + "Failing-test-first guard for #4423: post-reboot recovery must preserve " + + "the local registry entry and restart the labeled Docker container.", + }, { id: "ubuntu-repo-openai-compatible-openclaw", manifestName: "openclaw-openai-compatible", diff --git a/test/e2e-scenario/scenarios/types.ts b/test/e2e-scenario/scenarios/types.ts index 693acff04e8..d363e4544a2 100644 --- a/test/e2e-scenario/scenarios/types.ts +++ b/test/e2e-scenario/scenarios/types.ts @@ -16,12 +16,23 @@ export type PhaseResultName = PhaseName | NegativeContractPhase; // nemoclaw_scenarios/probes/. Inference and credentials probes are // declared but not yet implemented; the compiler skips emitting actions // for them until the probe scripts land. +// +// `local-registry-entry-present` and `docker-sandbox-container-present` +// are host-side aspects of the sandbox: the local NemoClaw registry +// (`~/.nemoclaw/sandboxes.json`) and the Docker container labeled with +// `openshell.ai/sandbox-name=` (running OR stopped, including +// `*-nemoclaw-gpu-backup-*` siblings). These probes let scenarios +// assert preservation invariants that diverge from the live gateway +// view of the sandbox, which is precisely the regression class +// covered by the post-reboot recovery work tracked in #4423. export type StateProbeId = | "cli-installed" | "gateway-healthy" | "gateway-absent" | "sandbox-running" - | "sandbox-absent"; + | "sandbox-absent" + | "local-registry-entry-present" + | "docker-sandbox-container-present"; // User-facing phase the negative-scenario contract advertises. Wider // than PhaseName because manifests may declare "preflight" failures, @@ -74,6 +85,18 @@ export interface ExpectedState { credentials?: { expected: ExpectedPresence; }; + // Host-side registry entry for the scenario's sandbox name. + // "present" means `~/.nemoclaw/sandboxes.json` retains the entry, + // even if the live gateway can no longer see the sandbox. This is + // orthogonal to `sandbox.expected`: registry preservation is the + // user-visible regression target for #4423. + localRegistry?: { expected: ExpectedPresence }; + // Host-side Docker container labeled `openshell.ai/sandbox-name=`. + // "present" matches running OR stopped containers, including + // `*-nemoclaw-gpu-backup-*` siblings produced by the GPU patch path. + // Used to assert that recovery information remains available even + // when the live OpenShell gateway returns NotFound. + dockerSandboxContainer?: { expected: ExpectedPresence }; } export type TransientClassifier =