fix(onboard): preserve PVC across host stop/start by recovering stopped gateway (#4187)

src/lib/onboard.ts

@@ -428,6 +428,8 @@ const { destroyGatewayWithVolumeCleanup } =
 const {
   gatewayCliSupportsLifecycleCommands,
 } = require("./onboard/gateway-lifecycle") as typeof import("./onboard/gateway-lifecycle");
+const { reconcilePreflightGatewayReuseState } =
+  require("./onboard/preflight-gateway-reuse") as typeof import("./onboard/preflight-gateway-reuse");
 const {
   getGatewayReuseHealthWaitConfig,
   isDockerDriverGatewayHttpReady,
@@ -2080,65 +2082,22 @@ async function preflight(
   // metadata can be stale after a manual `docker rm`. See #2020. Newer
   // package-managed OpenShell gateways do not have an openshell-cluster-*
   // Docker container, so the live CLI health check is the source of truth.
-  if (gatewayReuseState === "healthy" && gatewayCliSupportsLifecycleCommands(runCaptureOpenshell)) {
-    const containerState = verifyGatewayContainerRunning(GATEWAY_NAME);
-    if (containerState === "missing") {
-      console.log("  Gateway metadata is stale (container not running). Cleaning up...");
-      runOpenshell(["forward", "stop", String(DASHBOARD_PORT)], { ignoreError: true });
-      gatewayReuseState = destroyGatewayForReuse(
-        destroyGateway,
-        "  ✓ Stale gateway metadata cleaned up",
-        "  ! Stale gateway metadata cleanup failed; leaving registry state intact.",
-      );
-    } else if (containerState === "unknown") {
-      // Docker probe failed but cached metadata says healthy. Try the host-level
-      // HTTP probe — it doesn't depend on Docker, so it can confirm the gateway
-      // is genuinely serving even when the daemon is flaky.
-      //
-      // Per #2020 the "unknown" state must stay non-destructive end-to-end:
-      // do not downgrade to "missing" in preflight even when HTTP probe fails.
-      // Doing so would feed the orphan-cleanup block below, and a transient
-      // `docker inspect` failure plus an HTTP warm-up miss would delete a
-      // live gateway. The main onboard "unknown" branch makes the abort/reuse
-      // decision once preflight has surfaced the warning to the user.
-      if (await waitForGatewayHttpReady()) {
-        console.log(
-          "  Warning: could not verify gateway container state (Docker may be unavailable), but the gateway is responding on HTTP. Proceeding with reuse.",
-        );
-      } else {
-        console.log(
-          "  Warning: could not verify gateway container state and the gateway is not responding on HTTP. Onboard will abort before reuse if this persists; restart Docker and re-run.",
-        );
-      }
-    } else if (!(await waitForGatewayHttpReady())) {
-      // Container is running but the gateway HTTP endpoint is not responding.
-      // Common immediately after a Docker daemon restart — the container comes
-      // back before the OpenShell gateway upstream finishes warming up. Safe to
-      // recreate because Docker is functional. See #3258.
-      console.log(
-        `  Gateway container is running but ${getGatewayLocalEndpoint()}/ is not responding. Recreating...`,
-      );
-      runOpenshell(["forward", "stop", String(DASHBOARD_PORT)], { ignoreError: true });
-      gatewayReuseState = destroyGatewayForReuse(
-        destroyGateway,
-        "  ✓ Stale gateway cleaned up",
-        "  ! Stale gateway cleanup failed; leaving registry state intact.",
-      );
-    } else {
-      const imageDrift = getGatewayClusterImageDrift();
-      if (imageDrift) {
-        console.log(
-          `  Gateway image ${imageDrift.currentVersion} does not match openshell ${imageDrift.expectedVersion}. Recreating...`,
-        );
-        stopAllDashboardForwards();
-        gatewayReuseState = destroyGatewayForReuse(
-          destroyGateway,
-          "  ✓ Previous gateway cleaned up",
-          "  ! Previous gateway cleanup failed; leaving registry state intact.",
-        );
-      }
-    }
-  }
+  gatewayReuseState = await reconcilePreflightGatewayReuseState({
+    gatewayReuseState,
+    supportsLifecycleCommands: gatewayCliSupportsLifecycleCommands(runCaptureOpenshell),
+    gatewayName: GATEWAY_NAME,
+    verifyGatewayContainerRunning,
+    recoverGatewayRuntime,
+    waitForGatewayHttpReady,
+    getGatewayLocalEndpoint,
+    stopDashboardForward: () =>
+      runOpenshell(["forward", "stop", String(DASHBOARD_PORT)], { ignoreError: true }),
+    stopAllDashboardForwards,
+    destroyGateway,
+    destroyGatewayForReuse,
+    getGatewayClusterImageDrift,
+    exitProcess: (code) => process.exit(code),
+  });
 
   if (gatewayReuseState === "stale" || gatewayReuseState === "active-unnamed") {
     console.log(`  Cleaning up previous ${cliDisplayName()} session...`);
@@ -7134,6 +7093,7 @@ async function onboard(opts: OnboardOptions = {}): Promise<void> {
         gatewayCliSupportsLifecycleCommands: () => gatewayCliSupportsLifecycleCommands(runCaptureOpenshell),
         verifyGatewayContainerRunning,
         waitForGatewayHttpReady,
+        recoverGatewayRuntime,
         getGatewayLocalEndpoint,
         stopDashboardForward: () => bestEffortForwardStop(runOpenshell, DASHBOARD_PORT),
         destroyGateway,

src/lib/onboard/gateway-container-running.test.ts

@@ -20,10 +20,10 @@ describe("verifyGatewayContainerRunning", () => {
     );
   });
 
-  it("returns missing for existing but stopped containers", () => {
+  it("returns stopped for existing but stopped containers (#4187)", () => {
     const dockerInspect = vi.fn(() => dockerInspectResult(0, "false\n"));
 
-    expect(verifyGatewayContainerRunning("nemoclaw", { dockerInspect })).toBe("missing");
+    expect(verifyGatewayContainerRunning("nemoclaw", { dockerInspect })).toBe("stopped");
   });
 
   it("returns missing when Docker reports the gateway container is absent", () => {

src/lib/onboard/gateway-container-running.ts

@@ -1,13 +1,21 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-export type GatewayContainerState = "running" | "missing" | "unknown";
+export type GatewayContainerState = "running" | "stopped" | "missing" | "unknown";
 
 type DockerInspect = (
   args: string[],
   opts: { ignoreError: true; suppressOutput: true },
 ) => { status: number | null; stdout?: unknown; stderr?: unknown };
 
+// Distinguishes a stopped-but-existing legacy gateway container from a truly
+// absent one. Conflating the two caused #4187: after a host VM stop/start the
+// k3s-in-Docker `openshell-cluster-${gatewayName}` container is stopped but
+// still holds the PVC volume. Treating that as "missing" routed onboarding
+// through the destructive cleanup branch (`destroyGatewayWithVolumeCleanup`)
+// which removes `openshell-cluster-${gatewayName}*` Docker volumes — i.e. the
+// PVC backing store — before sandbox recreation, so the next `createSandbox`
+// provisioned a fresh, empty workspace.
 export function verifyGatewayContainerRunning(
   gatewayName: string,
   deps: { dockerInspect?: DockerInspect } = {},
@@ -26,7 +34,7 @@ export function verifyGatewayContainerRunning(
   }
 
   if (result.status === 0) {
-    return "missing";
+    return "stopped";
   }
 
   const stderr = (result.stderr || "").toString();

src/lib/onboard/machine/handlers/gateway.test.ts

@@ -16,6 +16,7 @@ function createDeps(overrides: Partial<GatewayStateOptions<Gpu>["deps"]> = {}) {
     lifecycle: vi.fn(() => false),
     verifyContainer: vi.fn((_gatewayName: string): GatewayContainerState => "running"),
     waitHttp: vi.fn(async () => true),
+    recoverGateway: vi.fn(async () => true),
     stopDashboardForward: vi.fn(),
     destroy: vi.fn(() => true),
     destroyForReuse: vi.fn(() => "missing" as GatewayReuseState),
@@ -42,6 +43,7 @@ function createDeps(overrides: Partial<GatewayStateOptions<Gpu>["deps"]> = {}) {
       gatewayCliSupportsLifecycleCommands: calls.lifecycle,
       verifyGatewayContainerRunning: calls.verifyContainer,
       waitForGatewayHttpReady: calls.waitHttp,
+      recoverGatewayRuntime: calls.recoverGateway,
       getGatewayLocalEndpoint: () => "http://127.0.0.1:31818",
       stopDashboardForward: calls.stopDashboardForward,
       destroyGateway: calls.destroy,
@@ -159,6 +161,65 @@ describe("handleGatewayState", () => {
     expect(calls.startGateway).toHaveBeenCalled();
   });
 
+  it("recovers a stopped lifecycle gateway without destroying volumes (#4187)", async () => {
+    const recoverGateway = vi.fn(async () => true);
+    const { deps, calls } = createDeps({
+      gatewayCliSupportsLifecycleCommands: vi.fn(() => true),
+      verifyGatewayContainerRunning: vi.fn(() => "stopped" as GatewayContainerState),
+      recoverGatewayRuntime: recoverGateway,
+    });
+
+    await handleGatewayState(baseOptions(deps, "healthy"));
+
+    expect(recoverGateway).toHaveBeenCalledOnce();
+    expect(calls.stopDashboardForward).not.toHaveBeenCalled();
+    expect(calls.destroyForReuse).not.toHaveBeenCalled();
+    expect(calls.exit).not.toHaveBeenCalled();
+    expect(calls.startGateway).not.toHaveBeenCalled();
+    expect(calls.skipped).toHaveBeenCalledWith("gateway", "running", "reuse");
+    expect(calls.complete).toHaveBeenCalledWith("gateway");
+  });
+
+  it("refuses to destroy volumes when stopped-container recovery fails (#4187)", async () => {
+    const recoverGateway = vi.fn(async () => false);
+    const { deps, calls } = createDeps({
+      gatewayCliSupportsLifecycleCommands: vi.fn(() => true),
+      verifyGatewayContainerRunning: vi.fn(() => "stopped" as GatewayContainerState),
+      recoverGatewayRuntime: recoverGateway,
+    });
+
+    await expect(handleGatewayState(baseOptions(deps, "healthy"))).rejects.toThrow("exit 1");
+
+    expect(recoverGateway).toHaveBeenCalledOnce();
+    expect(calls.exit).toHaveBeenCalledWith(1);
+    expect(calls.destroyForReuse).not.toHaveBeenCalled();
+    expect(calls.stopDashboardForward).not.toHaveBeenCalled();
+  });
+
+  it("still recreates a recovered stopped gateway when image drift is detected (#4187)", async () => {
+    const recoverGateway = vi.fn(async () => true);
+    const { deps, calls } = createDeps({
+      gatewayCliSupportsLifecycleCommands: vi.fn(() => true),
+      verifyGatewayContainerRunning: vi.fn(() => "stopped" as GatewayContainerState),
+      recoverGatewayRuntime: recoverGateway,
+      getGatewayClusterImageDrift: vi.fn(() => ({
+        currentVersion: "0.0.38",
+        expectedVersion: "0.0.39",
+      })),
+      destroyGatewayForReuse: vi.fn(() => "missing" as GatewayReuseState),
+    });
+
+    await handleGatewayState(baseOptions(deps, "healthy"));
+
+    expect(recoverGateway).toHaveBeenCalledOnce();
+    expect(calls.stopForwards).toHaveBeenCalledOnce();
+    expect(deps.destroyGatewayForReuse).toHaveBeenCalledWith(
+      deps.destroyGateway,
+      "  ✓ Previous gateway cleaned up",
+      "  ! Previous gateway cleanup failed; leaving registry state intact.",
+    );
+  });
+
   it("refuses to destroy an unknown container state when HTTP is also unavailable", async () => {
     const { deps, calls } = createDeps({
       gatewayCliSupportsLifecycleCommands: vi.fn(() => true),

src/lib/onboard/machine/handlers/gateway.ts

@@ -21,6 +21,7 @@ export interface GatewayStateOptions<Gpu> {
     gatewayCliSupportsLifecycleCommands(): boolean;
     verifyGatewayContainerRunning(gatewayName: string): GatewayContainerState;
     waitForGatewayHttpReady(): Promise<boolean>;
+    recoverGatewayRuntime(): Promise<boolean>;
     getGatewayLocalEndpoint(): string;
     stopDashboardForward(): void;
     destroyGateway(
@@ -85,6 +86,7 @@ export async function handleGatewayState<Gpu>({
 
   if (gatewayReuseState === "healthy" && supportsLifecycleCommands) {
     const containerState = deps.verifyGatewayContainerRunning(gatewayName);
+    let checkImageDrift = false;
     if (containerState === "missing") {
       console.log("  Gateway metadata is stale (container not running). Cleaning up...");
       deps.stopDashboardForward();
@@ -93,6 +95,30 @@ export async function handleGatewayState<Gpu>({
         "  ✓ Stale gateway metadata cleaned up",
         "  ! Stale gateway metadata cleanup failed; leaving registry state intact.",
       );
+    } else if (containerState === "stopped") {
+      // #4187: a stopped legacy `openshell-cluster-*` container after a host
+      // VM stop/start still holds the PVC volume. Attempt non-destructive
+      // recovery (openshell gateway start) before any destructive path so we
+      // never delete the k3s local-path PVC backing data.
+      console.log(
+        "  Gateway container is stopped (likely host or Docker restart). Attempting non-destructive recovery...",
+      );
+      const recovered = await deps.recoverGatewayRuntime();
+      if (recovered) {
+        console.log("  ✓ Gateway recovered without removing volumes; existing sandbox PVC preserved.");
+        checkImageDrift = true;
+      } else {
+        console.log(
+          `  Could not start the stopped NemoClaw gateway and ${deps.getGatewayLocalEndpoint()}/ is not responding.`,
+        );
+        console.log(
+          "  Refusing to delete openshell-cluster-* volumes — they may hold the existing PVC/workspace data.",
+        );
+        console.log(
+          "  Restart Docker, free the gateway port if held by another process, and re-run `nemoclaw onboard`. See #4187.",
+        );
+        deps.exitProcess(1);
+      }
     } else if (containerState === "unknown") {
       if (await deps.waitForGatewayHttpReady()) {
         console.log(
@@ -118,6 +144,10 @@ export async function handleGatewayState<Gpu>({
         "  ! Stale gateway cleanup failed; leaving registry state intact.",
       );
     } else {
+      checkImageDrift = true;
+    }
+
+    if (checkImageDrift) {
       const imageDrift = deps.getGatewayClusterImageDrift();
       if (imageDrift) {
         console.log(