fix(onboard): wait for final lifecycle release

docs/reference/commands.mdx

@@ -1070,7 +1070,8 @@ On Jetson/Tegra hosts, the compatibility path uses the NVIDIA runtime and adds e
 These include selected `/dev/nvmap`, `/dev/nvhost-*`, and `/dev/nvgpu/igpu0/*` nodes plus real `/dev/dri/renderD*` character devices.
 After compatibility recreation starts, onboarding keeps the pre-patch container as a rollback backup until the replacement passes the Ready, GPU, and applicable local-inference checks.
 If one of those checks fails before backup removal, onboarding prints failure diagnostics and attempts to restore the pre-patch container.
-To commit the replacement, NemoClaw stops it, removes the rollback backup, starts the replacement as the final container lifecycle event, and verifies OpenShell supervisor readiness again.
+To commit the replacement, NemoClaw stops it, removes the rollback backup, and waits until a successful OpenShell sandbox list has no row with that sandbox name.
+NemoClaw then starts the replacement as the final container lifecycle event and verifies OpenShell supervisor readiness again within the same handoff deadline.
 If that final handoff cannot be confirmed, onboarding exits with the container diagnostics and cleanup guidance instead of reporting success.
 If rollback fails, onboarding reports that the pre-patch container was not restored and prints container-cleanup guidance.
 GPU-proof diagnostics are captured before rollback and can print that guidance before the final container state is known, so inspect the sandbox and its labeled Docker containers before running a deletion command.

src/lib/actions/sandbox/process-recovery.ts

@@ -10,6 +10,7 @@ import {
   captureSandboxSshConfig,
   getOpenshellBinary,
   isCommandTimeout,
+  runOpenshell,
 } from "../../adapters/openshell/runtime";
 import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts";
 import {
@@ -498,6 +499,9 @@ function finalRelaunchContainerFailureDetail(
   if (completion.replacementStoppedForCommit === false) {
     return "Docker could not stop the replacement container for the final recovery handoff. NemoClaw did not start the primary dashboard/API host forward";
   }
+  if (completion.lifecycleReleaseObserved === false) {
+    return "OpenShell did not release the sandbox name before the final recovery handoff. NemoClaw did not restart the replacement container or start the primary dashboard/API host forward";
+  }
   if (completion.replacementRestarted === false) {
     return "Docker could not start the replacement container to complete the final recovery handoff. NemoClaw did not start the primary dashboard/API host forward";
   }
@@ -748,6 +752,7 @@ function recoverSandboxProcesses(
       const relaunch = relaunchManagedSupervisorSessionImpl(sandboxName, {
         quiet,
         deps: {
+          runOpenshell,
           confirmMissingSupervisor: (containerId) =>
             isExactlyMissingManagedSupervisor(
               requestPinnedGatewaySupervisorAction(sandboxName, "probe", 210000, containerId),

src/lib/actions/sandbox/supervisor-relaunch.test.ts

@@ -74,6 +74,7 @@ function baseDeps(overrides: ManagedSupervisorRelaunchDeps = {}) {
       failedFiles: [],
     })),
     removeBackup: vi.fn(() => true),
+    runOpenshell: vi.fn(() => ({ status: 0, stdout: "No sandboxes found.\n" })),
     recreate: vi.fn(() => patchResult()),
     finalize: vi.fn(({ supervisorReady }) =>
       supervisorReady
@@ -153,10 +154,15 @@ describe("relaunchManagedSupervisorSession", () => {
     });
     expect(deps.restoreState).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery");
     expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery");
-    expect(deps.finalize).toHaveBeenCalledWith({
-      result: expect.objectContaining({ newContainerId: "new-container-id" }),
-      supervisorReady: true,
-    });
+    expect(deps.finalize).toHaveBeenCalledWith(
+      {
+        lifecycleReleaseTimeoutSecs: 900,
+        result: expect.objectContaining({ newContainerId: "new-container-id" }),
+        sandboxName: "alpha",
+        supervisorReady: true,
+      },
+      { runOpenshell: deps.runOpenshell },
+    );
   });
 
   it("retries only transport-level state backup failures after a container restart", () => {
@@ -381,9 +387,26 @@ describe("relaunchManagedSupervisorSession", () => {
     });
     expect(order).toEqual(["restore-state", "restart-restored-gateway", "commit-container"]);
     expect(deps.restartRestoredManagedGateway).toHaveBeenCalledWith("new-container-id");
-    expect(deps.finalize).toHaveBeenCalledWith({
-      result: expect.objectContaining({ newContainerId: "new-container-id" }),
-      supervisorReady: true,
+    expect(deps.finalize).toHaveBeenCalledWith(
+      {
+        lifecycleReleaseTimeoutSecs: 900,
+        result: expect.objectContaining({ newContainerId: "new-container-id" }),
+        sandboxName: "alpha",
+        supervisorReady: true,
+      },
+      { runOpenshell: deps.runOpenshell },
+    );
+  });
+
+  it("uses only an injected host sleep for lifecycle polling after recreation (#9531)", () => {
+    const sleep = vi.fn();
+    const deps = baseDeps({ sleep });
+    const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps });
+
+    expect(relaunch?.finalize(true)).toMatchObject({ backupRemoved: true, rolledBack: false });
+    expect(deps.finalize).toHaveBeenCalledWith(expect.objectContaining({ supervisorReady: true }), {
+      runOpenshell: deps.runOpenshell,
+      sleep,
     });
   });
 

src/lib/actions/sandbox/supervisor-relaunch.ts

@@ -13,6 +13,7 @@ import {
   type DockerGpuPatchFinalizeOutcome,
   finalizeDockerGpuPatchBackup,
 } from "../../onboard/docker-gpu-patch-finalize";
+import { getDockerGpuSupervisorReconnectTimeoutSecs } from "../../onboard/docker-gpu-supervisor-reconnect";
 import { recreateOpenShellDockerSandboxWithStartupCommand } from "../../onboard/docker-startup-command-patch";
 import { buildSandboxRuntimeEnvArgs } from "../../onboard/sandbox-create-launch";
 import { resolveDirectSandboxContainer } from "../../sandbox/privileged-exec";
@@ -57,6 +58,7 @@ export type ManagedSupervisorRelaunchDeps = {
   removeBackup?: typeof sandboxState.removeSandboxStateBackup;
   recreate?: typeof recreateOpenShellDockerSandboxWithStartupCommand;
   finalize?: typeof finalizeDockerGpuPatchBackup;
+  runOpenshell?: NonNullable<Parameters<typeof finalizeDockerGpuPatchBackup>[1]>["runOpenshell"];
 };
 
 function inspectContainer(containerId: string): DockerContainerInspect {
@@ -284,8 +286,22 @@ export function relaunchManagedSupervisorSession(
           // both succeed.
           return finalizeFailure();
         }
+        const runLifecycleProbe = deps.runOpenshell;
+        if (!runLifecycleProbe) return finalizeFailure();
+        const lifecycleDeps = {
+          runOpenshell: runLifecycleProbe,
+          ...(deps.sleep ? { sleep: deps.sleep } : {}),
+        };
         const outcome = {
-          ...finalize({ result, supervisorReady: true }),
+          ...finalize(
+            {
+              result,
+              supervisorReady: true,
+              sandboxName,
+              lifecycleReleaseTimeoutSecs: getDockerGpuSupervisorReconnectTimeoutSecs(1),
+            },
+            lifecycleDeps,
+          ),
           stateRestored: true,
           stateBackupRemoved: removeSettledStateBackup(),
         };

src/lib/onboard/docker-gpu-patch-finalize.test.ts

@@ -68,14 +68,25 @@ describe("finalizeDockerGpuPatchBackup", () => {
     const dockerRm = vi.fn((_name: string) => ({ status: 0 }));
     const dockerStart = vi.fn(() => ({ status: 0 }));
     const outcome = finalizeDockerGpuPatchBackup(
-      { result: deferredCreateResult(), supervisorReady: true },
-      { dockerStop, dockerRm, dockerStart },
+      {
+        result: deferredCreateResult(),
+        supervisorReady: true,
+        sandboxName: "alpha",
+        lifecycleReleaseTimeoutSecs: 60,
+      },
+      {
+        dockerStop,
+        dockerRm,
+        dockerStart,
+        runOpenshell: vi.fn(() => ({ status: 0, stdout: "No sandboxes found.\n" })),
+      },
     );
     expect(outcome).toEqual({
       backupRemoved: true,
       rolledBack: false,
       replacementStoppedForCommit: true,
       replacementRestarted: true,
+      lifecycleReleaseObserved: true,
     });
     expect(dockerStop).toHaveBeenCalledWith(
       "new-container-id",
@@ -97,6 +108,124 @@ describe("finalizeDockerGpuPatchBackup", () => {
     );
   });
 
+  it("waits for the sandbox name to disappear before restarting the replacement (#9531)", () => {
+    const events: string[] = [];
+    const dockerStop = vi.fn(() => {
+      events.push("stop replacement");
+      return { status: 0 };
+    });
+    const dockerRm = vi.fn(() => {
+      events.push("remove backup");
+      return { status: 0 };
+    });
+    const dockerStart = vi.fn(() => {
+      events.push("start replacement");
+      return { status: 0 };
+    });
+    const runOpenshell = vi
+      .fn()
+      .mockImplementationOnce(() => {
+        events.push("observe deleting");
+        return { status: 0, stdout: "alpha  2026-08-21 05:53:16  Deleting\n" };
+      })
+      .mockImplementationOnce(() => {
+        events.push("observe error");
+        return { status: 0, stdout: "alpha  2026-08-21 05:53:18  Error\n" };
+      })
+      .mockImplementationOnce(() => {
+        events.push("observe name absence");
+        return { status: 0, stdout: "beta  2026-08-21 05:53:20  Ready\n" };
+      });
+
+    const outcome = finalizeDockerGpuPatchBackup(
+      {
+        result: deferredCreateResult(),
+        supervisorReady: true,
+        sandboxName: "alpha",
+        lifecycleReleaseTimeoutSecs: 60,
+      },
+      { dockerStop, dockerRm, dockerStart, runOpenshell, sleep: vi.fn() },
+    );
+
+    expect(outcome).toMatchObject({
+      backupRemoved: true,
+      lifecycleReleaseObserved: true,
+      replacementRestarted: true,
+    });
+    expect(events).toEqual([
+      "stop replacement",
+      "remove backup",
+      "observe deleting",
+      "observe error",
+      "observe name absence",
+      "start replacement",
+    ]);
+  });
+
+  it("does not treat failed lifecycle probes as a release receipt (#9531)", () => {
+    const runOpenshell = vi
+      .fn()
+      .mockReturnValueOnce({ status: 0, stdout: "Error: gateway unavailable" })
+      .mockReturnValueOnce({ status: 1, stderr: "gateway unavailable" });
+    const dockerStart = vi.fn(() => ({ status: 0 }));
+
+    const outcome = finalizeDockerGpuPatchBackup(
+      {
+        result: deferredCreateResult(),
+        supervisorReady: true,
+        sandboxName: "alpha",
+        lifecycleReleaseTimeoutSecs: 1,
+      },
+      {
+        dockerStop: vi.fn(() => ({ status: 0 })),
+        dockerRm: vi.fn(() => ({ status: 0 })),
+        dockerStart,
+        runOpenshell,
+        sleep: vi.fn(),
+      },
+    );
+
+    expect(outcome).toMatchObject({
+      backupRemoved: true,
+      lifecycleReleaseObserved: false,
+      replacementRestarted: false,
+    });
+    expect(runOpenshell).toHaveBeenCalledTimes(2);
+    expect(runOpenshell.mock.calls[0]?.[1]?.timeout).toBeGreaterThan(0);
+    expect(runOpenshell.mock.calls[0]?.[1]?.timeout).toBeLessThanOrEqual(1000);
+    expect(runOpenshell.mock.calls[1]?.[1]?.timeout).toBeGreaterThan(0);
+    expect(runOpenshell.mock.calls[1]?.[1]?.timeout).toBeLessThanOrEqual(1000);
+    expect(dockerStart).not.toHaveBeenCalled();
+  });
+
+  it("does not treat an unrelated terminal lifecycle phase as the stopped replacement (#9531)", () => {
+    const runOpenshell = vi.fn(() => ({
+      status: 0,
+      stdout: "alpha  2026-08-21 05:53:18  Failed\n",
+    }));
+
+    const dockerStart = vi.fn(() => ({ status: 0 }));
+    const outcome = finalizeDockerGpuPatchBackup(
+      {
+        result: deferredCreateResult(),
+        supervisorReady: true,
+        sandboxName: "alpha",
+        lifecycleReleaseTimeoutSecs: 1,
+      },
+      {
+        dockerStop: vi.fn(() => ({ status: 0 })),
+        dockerRm: vi.fn(() => ({ status: 0 })),
+        dockerStart,
+        runOpenshell,
+        sleep: vi.fn(),
+      },
+    );
+
+    expect(outcome.lifecycleReleaseObserved).toBe(false);
+    expect(runOpenshell).toHaveBeenCalledTimes(2);
+    expect(dockerStart).not.toHaveBeenCalled();
+  });
+
   it("rolls back to the backup container when supervisor reconnect failed", () => {
     const dockerStop = vi.fn(() => ({ status: 0 }));
     const dockerRm = vi.fn((_name: string) => ({ status: 0 }));
@@ -186,7 +315,15 @@ describe("finalizeDockerGpuPatchBackup", () => {
   it("is a no-op when the backup was already removed by the patch helper", () => {
     const dockerRm = vi.fn((_name: string) => ({ status: 0 }));
     const result = { ...deferredCreateResult(), backupRemoved: true };
-    const outcome = finalizeDockerGpuPatchBackup({ result, supervisorReady: true }, { dockerRm });
+    const outcome = finalizeDockerGpuPatchBackup(
+      {
+        result,
+        supervisorReady: true,
+        sandboxName: "alpha",
+        lifecycleReleaseTimeoutSecs: 60,
+      },
+      { dockerRm },
+    );
     expect(outcome).toEqual({ backupRemoved: true, rolledBack: false });
     expect(dockerRm).not.toHaveBeenCalled();
   });
@@ -199,35 +336,49 @@ describe("finalizeDockerGpuPatchBackup", () => {
     }));
     const dockerStart = vi.fn(() => ({ status: 0 }));
     const outcome = finalizeDockerGpuPatchBackup(
-      { result: deferredCreateResult(), supervisorReady: true },
+      {
+        result: deferredCreateResult(),
+        supervisorReady: true,
+        sandboxName: "alpha",
+        lifecycleReleaseTimeoutSecs: 60,
+      },
       { dockerStop, dockerRm, dockerStart },
     );
     expect(outcome).toEqual({
       backupRemoved: false,
       rolledBack: false,
       replacementStoppedForCommit: true,
-      replacementRestarted: true,
+      replacementRestarted: false,
+      lifecycleReleaseObserved: false,
     });
     expect(dockerRm).toHaveBeenCalledWith(
       "openshell-alpha-nemoclaw-gpu-backup-1780491860342",
       expect.objectContaining({ ignoreError: true }),
     );
+    expect(dockerStart).not.toHaveBeenCalled();
   });
 
   it("fails closed when backup removal has no exit status", () => {
     const dockerStop = vi.fn(() => ({ status: 0 }));
     const dockerRm = vi.fn((_name: string) => ({ status: null, stderr: "timed out" }));
     const dockerStart = vi.fn(() => ({ status: 0 }));
     const outcome = finalizeDockerGpuPatchBackup(
-      { result: deferredCreateResult(), supervisorReady: true },
+      {
+        result: deferredCreateResult(),
+        supervisorReady: true,
+        sandboxName: "alpha",
+        lifecycleReleaseTimeoutSecs: 60,
+      },
       { dockerStop, dockerRm, dockerStart },
     );
     expect(outcome).toEqual({
       backupRemoved: false,
       rolledBack: false,
       replacementStoppedForCommit: true,
-      replacementRestarted: true,
+      replacementRestarted: false,
+      lifecycleReleaseObserved: false,
     });
+    expect(dockerStart).not.toHaveBeenCalled();
   });
 
   it("retains the backup when the replacement cannot be stopped for the final handoff", () => {
@@ -236,7 +387,12 @@ describe("finalizeDockerGpuPatchBackup", () => {
     const dockerStart = vi.fn(() => ({ status: 0 }));
 
     const outcome = finalizeDockerGpuPatchBackup(
-      { result: deferredCreateResult(), supervisorReady: true },
+      {
+        result: deferredCreateResult(),
+        supervisorReady: true,
+        sandboxName: "alpha",
+        lifecycleReleaseTimeoutSecs: 60,
+      },
       { dockerStop, dockerRm, dockerStart },
     );
 
@@ -251,11 +407,17 @@ describe("finalizeDockerGpuPatchBackup", () => {
 
   it("reports a failed replacement restart after the backup is removed", () => {
     const outcome = finalizeDockerGpuPatchBackup(
-      { result: deferredCreateResult(), supervisorReady: true },
+      {
+        result: deferredCreateResult(),
+        supervisorReady: true,
+        sandboxName: "alpha",
+        lifecycleReleaseTimeoutSecs: 60,
+      },
       {
         dockerStop: vi.fn(() => ({ status: 0 })),
         dockerRm: vi.fn(() => ({ status: 0 })),
         dockerStart: vi.fn(() => ({ status: 1 })),
+        runOpenshell: vi.fn(() => ({ status: 0, stdout: "No sandboxes found.\n" })),
       },
     );
 
@@ -264,6 +426,7 @@ describe("finalizeDockerGpuPatchBackup", () => {
       rolledBack: false,
       replacementStoppedForCommit: true,
       replacementRestarted: false,
+      lifecycleReleaseObserved: true,
     });
   });