test(e2e): add post-reboot-recovery regression guard for #4423

.github/workflows/e2e-scenarios.yaml

@@ -78,6 +78,7 @@ jobs:
             [ubuntu-repo-cloud-openclaw-slack]=ubuntu-latest
             [ubuntu-repo-cloud-openclaw-telegram]=ubuntu-latest
             [ubuntu-repo-cloud-openclaw-token-rotation]=ubuntu-latest
+            [ubuntu-repo-docker-post-reboot-recovery]=ubuntu-latest
             [ubuntu-repo-openai-compatible-openclaw]=ubuntu-latest
           )
           selected=""

src/commands/sandbox/agents/list.ts

@@ -24,7 +24,12 @@ export default class SandboxAgentsListCommand extends NemoClawCommand {
   public async run(): Promise<void> {
     this.parsed = true;
     const [sandboxName, ...extraArgs] = this.argv;
-    if (!sandboxName || sandboxName.trim() === "" || sandboxName === "--help" || sandboxName === "-h") {
+    if (
+      !sandboxName ||
+      sandboxName.trim() === "" ||
+      sandboxName === "--help" ||
+      sandboxName === "-h"
+    ) {
       printAgentsPassthroughHelp("list");
       return;
     }

test/e2e-scenario/framework-tests/e2e-expected-state.test.ts

@@ -83,6 +83,35 @@ describe("probesForState maps typed expected-state into probe ids", () => {
     };
     expect(probesForState(state)).toEqual([]);
   });
+
+  it("localRegistry.expected=present emits the local-registry-entry-present probe", () => {
+    const state: ExpectedState = {
+      id: "synthetic-local-registry",
+      cli: { installed: true },
+      localRegistry: { expected: "present" },
+    };
+    expect(probesForState(state)).toEqual(["cli-installed", "local-registry-entry-present"]);
+  });
+
+  it("dockerSandboxContainer.expected=present emits the docker-sandbox-container-present probe", () => {
+    const state: ExpectedState = {
+      id: "synthetic-docker-container",
+      cli: { installed: true },
+      dockerSandboxContainer: { expected: "present" },
+    };
+    expect(probesForState(state)).toEqual(["cli-installed", "docker-sandbox-container-present"]);
+  });
+
+  it("localRegistry/dockerSandboxContainer 'absent' emits no probe today", () => {
+    // Negative-direction probes haven't landed yet. Pin the gap so a
+    // future negative-scenario PR is forced to add the absent probes.
+    const state: ExpectedState = {
+      id: "synthetic-host-absent",
+      localRegistry: { expected: "absent" },
+      dockerSandboxContainer: { expected: "absent" },
+    };
+    expect(probesForState(state)).toEqual([]);
+  });
 });
 
 describe("compiler emits state-validation phase actions from expected-state registry", () => {

test/e2e-scenario/framework-tests/e2e-live-registry-discovery.test.ts

@@ -50,4 +50,27 @@ describe("live Vitest registry discovery support", () => {
       reasons: ["runtime 'docker-missing' is not wired for live Vitest fixtures"],
     });
   });
+
+  it("keeps unwhitelisted lifecycle profiles skipped with the lifecycle reason", () => {
+    const scenario = listScenarios().find((entry) => entry.id === "ubuntu-rebuild-openclaw");
+
+    expect(scenario).toBeTruthy();
+    expect(liveScenarioSupport(scenario!)).toMatchObject({
+      supported: false,
+      reasons: ["lifecycle 'rebuild-current-version' is not wired for live Vitest fixtures"],
+    });
+  });
+
+  it("accepts the whitelisted post-reboot-recovery lifecycle scenario", () => {
+    const scenario = listScenarios().find(
+      (entry) => entry.id === "ubuntu-repo-docker-post-reboot-recovery",
+    );
+
+    expect(scenario).toBeTruthy();
+    expect(scenario!.environment?.lifecycle).toBe("post-reboot-recovery");
+    expect(liveScenarioSupport(scenario!)).toMatchObject({
+      supported: true,
+      reasons: [],
+    });
+  });
 });

test/e2e-scenario/framework-tests/e2e-phase-lifecycle.test.ts

@@ -0,0 +1,249 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, expectTypeOf, it } from "vitest";
+
+import { HostCliClient, SandboxClient, type CommandRunner } from "../framework/clients/index.ts";
+import type { E2EScenarioFixtures } from "../framework/e2e-test.ts";
+import {
+  buildBackupContainerName,
+  LifecyclePhaseFixture,
+  type LifecycleCleanup,
+} from "../framework/phases/lifecycle.ts";
+import type { NemoClawInstance } from "../framework/phases/index.ts";
+import type {
+  ShellProbeResult,
+  ShellProbeRunOptions,
+  TrustedShellCommand,
+} from "../framework/shell-probe.ts";
+
+interface RunnerCall {
+  command: string;
+  args: string[];
+  options?: ShellProbeRunOptions;
+}
+
+interface CleanupCall {
+  name: string;
+  run: () => Promise<void> | void;
+}
+
+function shellResult(exitCode: number, output = ""): ShellProbeResult {
+  return {
+    command: [],
+    exitCode,
+    signal: null,
+    timedOut: false,
+    stdout: exitCode === 0 ? output : "",
+    stderr: exitCode === 0 ? "" : output,
+    artifacts: {
+      stdout: "/tmp/stdout.txt",
+      stderr: "/tmp/stderr.txt",
+      result: "/tmp/result.json",
+    },
+  };
+}
+
+class FakeRunner implements CommandRunner {
+  readonly calls: RunnerCall[] = [];
+  private readonly responses: ShellProbeResult[] = [];
+
+  enqueue(response: ShellProbeResult): void {
+    this.responses.push(response);
+  }
+
+  async run(
+    command: TrustedShellCommand,
+    options?: ShellProbeRunOptions,
+  ): Promise<ShellProbeResult> {
+    this.calls.push({ command: command.command, args: [...command.args], options });
+    const response = this.responses.shift();
+    if (!response) {
+      throw new Error(
+        `FakeRunner response missing for command: ${command.command} ${command.args.join(" ")}`,
+      );
+    }
+    return response;
+  }
+}
+
+class FakeCleanup implements LifecycleCleanup {
+  readonly calls: CleanupCall[] = [];
+
+  add(name: string, run: () => Promise<void> | void): void {
+    this.calls.push({ name, run });
+  }
+}
+
+function instance(overrides: Partial<NemoClawInstance> = {}): NemoClawInstance {
+  return {
+    onboarding: "cloud-openclaw",
+    sandboxName: "e2e-ubuntu-repo-cloud-openclaw",
+    agent: "openclaw",
+    provider: "nvidia",
+    providerEnv: "cloud",
+    gatewayUrl: "http://127.0.0.1:18789",
+    result: shellResult(0),
+    ...overrides,
+  };
+}
+
+function fixture(runner: FakeRunner, cleanup: FakeCleanup): LifecyclePhaseFixture {
+  const host = new HostCliClient(runner);
+  const sandbox = new SandboxClient(runner);
+  return new LifecyclePhaseFixture(host, sandbox, cleanup);
+}
+
+describe("LifecyclePhaseFixture.simulate post-reboot-recovery (stop-original)", () => {
+  it("stops the gateway, the labeled container, then drives `nemoclaw <name> status`", async () => {
+    const runner = new FakeRunner();
+    runner.enqueue(shellResult(0)); // openshell gateway stop
+    runner.enqueue(shellResult(0, "openshell-cluster-e2e-ubuntu-repo-cloud-openclaw\n")); // discover
+    runner.enqueue(shellResult(0)); // docker stop
+    runner.enqueue(shellResult(1, "Removed stale local registry entry.\n")); // status (non-zero on unfixed)
+    const cleanup = new FakeCleanup();
+
+    const result = await fixture(runner, cleanup).simulate("post-reboot-recovery", instance());
+
+    expect(result.profile).toBe("post-reboot-recovery");
+    expect(result.steps.map((step) => step.id)).toEqual([
+      "gateway-stop",
+      "docker-stop:openshell-cluster-e2e-ubuntu-repo-cloud-openclaw",
+      "nemoclaw-status:e2e-ubuntu-repo-cloud-openclaw",
+    ]);
+    expect(runner.calls.map((call) => ({ command: call.command, args: call.args }))).toEqual([
+      { command: "openshell", args: ["gateway", "stop"] },
+      {
+        command: "docker",
+        args: [
+          "ps",
+          "-a",
+          "--filter",
+          "label=openshell.ai/sandbox-name=e2e-ubuntu-repo-cloud-openclaw",
+          "--format",
+          "{{.Names}}",
+        ],
+      },
+      { command: "docker", args: ["stop", "openshell-cluster-e2e-ubuntu-repo-cloud-openclaw"] },
+      { command: "nemoclaw", args: ["e2e-ubuntu-repo-cloud-openclaw", "status"] },
+    ]);
+    expect(cleanup.calls.map((call) => call.name)).toEqual([
+      "lifecycle.docker-start:openshell-cluster-e2e-ubuntu-repo-cloud-openclaw",
+    ]);
+  });
+
+  it("tolerates a non-zero status exit (the bug succeeds at destroying state)", async () => {
+    const runner = new FakeRunner();
+    runner.enqueue(shellResult(0)); // gateway stop
+    runner.enqueue(shellResult(0, "container-1\n")); // discover
+    runner.enqueue(shellResult(0)); // docker stop
+    runner.enqueue(shellResult(1, "Removed stale local registry entry.\n")); // status non-zero
+    const cleanup = new FakeCleanup();
+
+    const result = await fixture(runner, cleanup).simulate("post-reboot-recovery", instance());
+
+    // simulate() does not throw; the post-status invariants belong
+    // to the state-validation phase that runs after.
+    expect(result.steps.find((step) => step.id.startsWith("nemoclaw-status:"))).toBeTruthy();
+  });
+
+  it("tolerates a non-zero gateway stop (post-reboot fresh runtime)", async () => {
+    const runner = new FakeRunner();
+    runner.enqueue(shellResult(1, "no gateway runtime")); // gateway stop fails
+    runner.enqueue(shellResult(0, "container-1\n"));
+    runner.enqueue(shellResult(0)); // docker stop
+    runner.enqueue(shellResult(0)); // status
+    const cleanup = new FakeCleanup();
+
+    const result = await fixture(runner, cleanup).simulate("post-reboot-recovery", instance());
+
+    expect(result.steps.find((step) => step.id === "gateway-stop")).toBeTruthy();
+  });
+
+  it("fails when no Docker container carries the OpenShell sandbox-name label", async () => {
+    const runner = new FakeRunner();
+    runner.enqueue(shellResult(0)); // gateway stop
+    runner.enqueue(shellResult(0, "\n")); // discover returns nothing
+    const cleanup = new FakeCleanup();
+
+    await expect(
+      fixture(runner, cleanup).simulate("post-reboot-recovery", instance()),
+    ).rejects.toThrow(/expected at least one Docker container labeled/);
+  });
+
+  it("fails when docker discover returns non-zero", async () => {
+    const runner = new FakeRunner();
+    runner.enqueue(shellResult(0)); // gateway stop
+    runner.enqueue(shellResult(1, "Cannot connect to the Docker daemon"));
+    const cleanup = new FakeCleanup();
+
+    await expect(
+      fixture(runner, cleanup).simulate("post-reboot-recovery", instance()),
+    ).rejects.toThrow(/could not query Docker for label/);
+  });
+});
+
+describe("LifecyclePhaseFixture.simulate post-reboot-recovery (rename-to-gpu-backup)", () => {
+  it("stops, then renames the labeled container to a *-nemoclaw-gpu-backup-* sibling", async () => {
+    const runner = new FakeRunner();
+    runner.enqueue(shellResult(0)); // openshell gateway stop
+    runner.enqueue(shellResult(0, "openshell-cluster-e2e-x\n")); // discover
+    runner.enqueue(shellResult(0)); // docker stop
+    runner.enqueue(shellResult(0)); // docker rename
+    runner.enqueue(shellResult(1, "Removed stale local registry entry.\n")); // status
+    const cleanup = new FakeCleanup();
+
+    const result = await fixture(runner, cleanup).simulate(
+      "post-reboot-recovery",
+      instance({ sandboxName: "e2e-x" }),
+      { mode: "rename-to-gpu-backup" },
+    );
+
+    expect(result.steps.map((step) => step.id.split("->")[0])).toContain(
+      "docker-rename:openshell-cluster-e2e-x",
+    );
+    const renameCall = runner.calls.find(
+      (call) => call.command === "docker" && call.args[0] === "rename",
+    );
+    expect(renameCall).toBeTruthy();
+    expect(renameCall!.args[1]).toBe("openshell-cluster-e2e-x");
+    expect(renameCall!.args[2]).toMatch(/^openshell-cluster-e2e-x-nemoclaw-gpu-backup-\d+$/);
+
+    // Cleanup queue now has both docker-start and docker-rename-back.
+    expect(cleanup.calls.map((call) => call.name.split(":")[0])).toEqual([
+      "lifecycle.docker-start",
+      "lifecycle.docker-rename-back",
+    ]);
+  });
+});
+
+describe("LifecyclePhaseFixture profile dispatch", () => {
+  it("rejects unknown lifecycle profiles", async () => {
+    const runner = new FakeRunner();
+    const cleanup = new FakeCleanup();
+
+    await expect(
+      // @ts-expect-error — exhaustiveness check
+      fixture(runner, cleanup).simulate("not-a-profile", instance()),
+    ).rejects.toThrow(/Unsupported lifecycle profile/);
+  });
+
+  it("exposes the lifecycle phase on the Vitest scenario context", () => {
+    expectTypeOf<E2EScenarioFixtures["lifecycle"]>().toEqualTypeOf<LifecyclePhaseFixture>();
+  });
+});
+
+describe("buildBackupContainerName", () => {
+  it("appends -nemoclaw-gpu-backup-<ts> to the original name", () => {
+    expect(buildBackupContainerName("openshell-cluster-foo", 1717280000000)).toBe(
+      "openshell-cluster-foo-nemoclaw-gpu-backup-1717280000000",
+    );
+  });
+
+  it("truncates the original name to fit within Docker's 253-char limit", () => {
+    const longName = "a".repeat(253);
+    const result = buildBackupContainerName(longName, 1717280000000);
+    expect(result.length).toBeLessThanOrEqual(253);
+    expect(result.endsWith("-nemoclaw-gpu-backup-1717280000000")).toBe(true);
+  });
+});