fix(onboard): wait for final lifecycle release

docs/reference/commands.mdx

@@ -1070,7 +1070,7 @@ On Jetson/Tegra hosts, the compatibility path uses the NVIDIA runtime and adds e
 These include selected `/dev/nvmap`, `/dev/nvhost-*`, and `/dev/nvgpu/igpu0/*` nodes plus real `/dev/dri/renderD*` character devices.
 After compatibility recreation starts, onboarding keeps the pre-patch container as a rollback backup until the replacement passes the Ready, GPU, and applicable local-inference checks.
 If one of those checks fails before backup removal, onboarding prints failure diagnostics and attempts to restore the pre-patch container.
-To commit the replacement, NemoClaw stops it, removes the rollback backup, starts the replacement as the final container lifecycle event, and verifies OpenShell supervisor readiness again.
+To commit the replacement, NemoClaw stops it, removes the rollback backup, waits for OpenShell to retire the previous lifecycle record, starts the replacement as the final container lifecycle event, and verifies OpenShell supervisor readiness again within the same handoff deadline.
 If that final handoff cannot be confirmed, onboarding exits with the container diagnostics and cleanup guidance instead of reporting success.
 If rollback fails, onboarding reports that the pre-patch container was not restored and prints container-cleanup guidance.
 GPU-proof diagnostics are captured before rollback and can print that guidance before the final container state is known, so inspect the sandbox and its labeled Docker containers before running a deletion command.

src/lib/actions/sandbox/process-recovery.ts

@@ -10,6 +10,7 @@ import {
   captureSandboxSshConfig,
   getOpenshellBinary,
   isCommandTimeout,
+  runOpenshell,
 } from "../../adapters/openshell/runtime";
 import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts";
 import {
@@ -748,6 +749,7 @@ function recoverSandboxProcesses(
       const relaunch = relaunchManagedSupervisorSessionImpl(sandboxName, {
         quiet,
         deps: {
+          runOpenshell,
           confirmMissingSupervisor: (containerId) =>
             isExactlyMissingManagedSupervisor(
               requestPinnedGatewaySupervisorAction(sandboxName, "probe", 210000, containerId),

src/lib/actions/sandbox/supervisor-relaunch.test.ts

@@ -74,6 +74,7 @@ function baseDeps(overrides: ManagedSupervisorRelaunchDeps = {}) {
       failedFiles: [],
     })),
     removeBackup: vi.fn(() => true),
+    runOpenshell: vi.fn(() => ({ status: 0, stdout: "No sandboxes found.\n" })),
     recreate: vi.fn(() => patchResult()),
     finalize: vi.fn(({ supervisorReady }) =>
       supervisorReady
@@ -153,10 +154,15 @@ describe("relaunchManagedSupervisorSession", () => {
     });
     expect(deps.restoreState).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery");
     expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery");
-    expect(deps.finalize).toHaveBeenCalledWith({
-      result: expect.objectContaining({ newContainerId: "new-container-id" }),
-      supervisorReady: true,
-    });
+    expect(deps.finalize).toHaveBeenCalledWith(
+      {
+        lifecycleReleaseTimeoutSecs: 900,
+        result: expect.objectContaining({ newContainerId: "new-container-id" }),
+        sandboxName: "alpha",
+        supervisorReady: true,
+      },
+      expect.objectContaining({ runOpenshell: deps.runOpenshell, sleep: expect.any(Function) }),
+    );
   });
 
   it("retries only transport-level state backup failures after a container restart", () => {
@@ -381,10 +387,15 @@ describe("relaunchManagedSupervisorSession", () => {
     });
     expect(order).toEqual(["restore-state", "restart-restored-gateway", "commit-container"]);
     expect(deps.restartRestoredManagedGateway).toHaveBeenCalledWith("new-container-id");
-    expect(deps.finalize).toHaveBeenCalledWith({
-      result: expect.objectContaining({ newContainerId: "new-container-id" }),
-      supervisorReady: true,
-    });
+    expect(deps.finalize).toHaveBeenCalledWith(
+      {
+        lifecycleReleaseTimeoutSecs: 900,
+        result: expect.objectContaining({ newContainerId: "new-container-id" }),
+        sandboxName: "alpha",
+        supervisorReady: true,
+      },
+      expect.objectContaining({ runOpenshell: deps.runOpenshell, sleep: expect.any(Function) }),
+    );
   });
 
   it("rolls back when managed health fails after state restore", () => {

src/lib/actions/sandbox/supervisor-relaunch.ts

@@ -13,6 +13,7 @@ import {
   type DockerGpuPatchFinalizeOutcome,
   finalizeDockerGpuPatchBackup,
 } from "../../onboard/docker-gpu-patch-finalize";
+import { getDockerGpuSupervisorReconnectTimeoutSecs } from "../../onboard/docker-gpu-supervisor-reconnect";
 import { recreateOpenShellDockerSandboxWithStartupCommand } from "../../onboard/docker-startup-command-patch";
 import { buildSandboxRuntimeEnvArgs } from "../../onboard/sandbox-create-launch";
 import { resolveDirectSandboxContainer } from "../../sandbox/privileged-exec";
@@ -57,6 +58,7 @@ export type ManagedSupervisorRelaunchDeps = {
   removeBackup?: typeof sandboxState.removeSandboxStateBackup;
   recreate?: typeof recreateOpenShellDockerSandboxWithStartupCommand;
   finalize?: typeof finalizeDockerGpuPatchBackup;
+  runOpenshell?: NonNullable<Parameters<typeof finalizeDockerGpuPatchBackup>[1]>["runOpenshell"];
 };
 
 function inspectContainer(containerId: string): DockerContainerInspect {
@@ -284,8 +286,21 @@ export function relaunchManagedSupervisorSession(
           // both succeed.
           return finalizeFailure();
         }
+        const runLifecycleProbe = deps.runOpenshell;
+        if (!runLifecycleProbe) return finalizeFailure();
         const outcome = {
-          ...finalize({ result, supervisorReady: true }),
+          ...finalize(
+            {
+              result,
+              supervisorReady: true,
+              sandboxName,
+              lifecycleReleaseTimeoutSecs: getDockerGpuSupervisorReconnectTimeoutSecs(1),
+            },
+            {
+              runOpenshell: runLifecycleProbe,
+              sleep,
+            },
+          ),
           stateRestored: true,
           stateBackupRemoved: removeSettledStateBackup(),
         };

src/lib/onboard/docker-gpu-patch-finalize.test.ts

@@ -68,14 +68,25 @@ describe("finalizeDockerGpuPatchBackup", () => {
     const dockerRm = vi.fn((_name: string) => ({ status: 0 }));
     const dockerStart = vi.fn(() => ({ status: 0 }));
     const outcome = finalizeDockerGpuPatchBackup(
-      { result: deferredCreateResult(), supervisorReady: true },
-      { dockerStop, dockerRm, dockerStart },
+      {
+        result: deferredCreateResult(),
+        supervisorReady: true,
+        sandboxName: "alpha",
+        lifecycleReleaseTimeoutSecs: 60,
+      },
+      {
+        dockerStop,
+        dockerRm,
+        dockerStart,
+        runOpenshell: vi.fn(() => ({ status: 0, stdout: "No sandboxes found.\n" })),
+      },
     );
     expect(outcome).toEqual({
       backupRemoved: true,
       rolledBack: false,
       replacementStoppedForCommit: true,
       replacementRestarted: true,
+      lifecycleReleaseObserved: true,
     });
     expect(dockerStop).toHaveBeenCalledWith(
       "new-container-id",
@@ -97,6 +108,119 @@ describe("finalizeDockerGpuPatchBackup", () => {
     );
   });
 
+  it("waits for the deleting lifecycle record to clear before restarting the replacement (#9531)", () => {
+    const events: string[] = [];
+    const dockerStop = vi.fn(() => {
+      events.push("stop replacement");
+      return { status: 0 };
+    });
+    const dockerRm = vi.fn(() => {
+      events.push("remove backup");
+      return { status: 0 };
+    });
+    const dockerStart = vi.fn(() => {
+      events.push("start replacement");
+      return { status: 0 };
+    });
+    const runOpenshell = vi
+      .fn()
+      .mockImplementationOnce(() => {
+        events.push("observe deleting");
+        return { status: 0, stdout: "alpha  2026-08-21 05:53:16  Deleting\n" };
+      })
+      .mockImplementationOnce(() => {
+        events.push("observe stopped replacement");
+        return { status: 0, stdout: "alpha  2026-08-21 05:53:18  Error\n" };
+      });
+
+    const outcome = finalizeDockerGpuPatchBackup(
+      {
+        result: deferredCreateResult(),
+        supervisorReady: true,
+        sandboxName: "alpha",
+        lifecycleReleaseTimeoutSecs: 60,
+      },
+      { dockerStop, dockerRm, dockerStart, runOpenshell, sleep: vi.fn() },
+    );
+
+    expect(outcome).toMatchObject({
+      backupRemoved: true,
+      lifecycleReleaseObserved: true,
+      replacementRestarted: true,
+    });
+    expect(events).toEqual([
+      "stop replacement",
+      "remove backup",
+      "observe deleting",
+      "observe stopped replacement",
+      "start replacement",
+    ]);
+  });
+
+  it("does not treat failed lifecycle probes as a release receipt (#9531)", () => {
+    const runOpenshell = vi
+      .fn()
+      .mockReturnValueOnce({ status: 0, stdout: "Error: gateway unavailable" })
+      .mockReturnValueOnce({ status: 1, stderr: "gateway unavailable" });
+    const dockerStart = vi.fn(() => ({ status: 0 }));
+
+    const outcome = finalizeDockerGpuPatchBackup(
+      {
+        result: deferredCreateResult(),
+        supervisorReady: true,
+        sandboxName: "alpha",
+        lifecycleReleaseTimeoutSecs: 1,
+      },
+      {
+        dockerStop: vi.fn(() => ({ status: 0 })),
+        dockerRm: vi.fn(() => ({ status: 0 })),
+        dockerStart,
+        runOpenshell,
+        sleep: vi.fn(),
+      },
+    );
+
+    expect(outcome).toMatchObject({
+      backupRemoved: true,
+      lifecycleReleaseObserved: false,
+      replacementRestarted: false,
+    });
+    expect(runOpenshell).toHaveBeenCalledTimes(2);
+    expect(runOpenshell.mock.calls[0]?.[1]?.timeout).toBeGreaterThan(0);
+    expect(runOpenshell.mock.calls[0]?.[1]?.timeout).toBeLessThanOrEqual(1000);
+    expect(runOpenshell.mock.calls[1]?.[1]?.timeout).toBeGreaterThan(0);
+    expect(runOpenshell.mock.calls[1]?.[1]?.timeout).toBeLessThanOrEqual(1000);
+    expect(dockerStart).not.toHaveBeenCalled();
+  });
+
+  it("does not treat an unrelated terminal lifecycle phase as the stopped replacement (#9531)", () => {
+    const runOpenshell = vi.fn(() => ({
+      status: 0,
+      stdout: "alpha  2026-08-21 05:53:18  Failed\n",
+    }));
+
+    const dockerStart = vi.fn(() => ({ status: 0 }));
+    const outcome = finalizeDockerGpuPatchBackup(
+      {
+        result: deferredCreateResult(),
+        supervisorReady: true,
+        sandboxName: "alpha",
+        lifecycleReleaseTimeoutSecs: 1,
+      },
+      {
+        dockerStop: vi.fn(() => ({ status: 0 })),
+        dockerRm: vi.fn(() => ({ status: 0 })),
+        dockerStart,
+        runOpenshell,
+        sleep: vi.fn(),
+      },
+    );
+
+    expect(outcome.lifecycleReleaseObserved).toBe(false);
+    expect(runOpenshell).toHaveBeenCalledTimes(2);
+    expect(dockerStart).not.toHaveBeenCalled();
+  });
+
   it("rolls back to the backup container when supervisor reconnect failed", () => {
     const dockerStop = vi.fn(() => ({ status: 0 }));
     const dockerRm = vi.fn((_name: string) => ({ status: 0 }));
@@ -186,7 +310,15 @@ describe("finalizeDockerGpuPatchBackup", () => {
   it("is a no-op when the backup was already removed by the patch helper", () => {
     const dockerRm = vi.fn((_name: string) => ({ status: 0 }));
     const result = { ...deferredCreateResult(), backupRemoved: true };
-    const outcome = finalizeDockerGpuPatchBackup({ result, supervisorReady: true }, { dockerRm });
+    const outcome = finalizeDockerGpuPatchBackup(
+      {
+        result,
+        supervisorReady: true,
+        sandboxName: "alpha",
+        lifecycleReleaseTimeoutSecs: 60,
+      },
+      { dockerRm },
+    );
     expect(outcome).toEqual({ backupRemoved: true, rolledBack: false });
     expect(dockerRm).not.toHaveBeenCalled();
   });
@@ -199,35 +331,49 @@ describe("finalizeDockerGpuPatchBackup", () => {
     }));
     const dockerStart = vi.fn(() => ({ status: 0 }));
     const outcome = finalizeDockerGpuPatchBackup(
-      { result: deferredCreateResult(), supervisorReady: true },
+      {
+        result: deferredCreateResult(),
+        supervisorReady: true,
+        sandboxName: "alpha",
+        lifecycleReleaseTimeoutSecs: 60,
+      },
       { dockerStop, dockerRm, dockerStart },
     );
     expect(outcome).toEqual({
       backupRemoved: false,
       rolledBack: false,
       replacementStoppedForCommit: true,
-      replacementRestarted: true,
+      replacementRestarted: false,
+      lifecycleReleaseObserved: false,
     });
     expect(dockerRm).toHaveBeenCalledWith(
       "openshell-alpha-nemoclaw-gpu-backup-1780491860342",
       expect.objectContaining({ ignoreError: true }),
     );
+    expect(dockerStart).not.toHaveBeenCalled();
   });
 
   it("fails closed when backup removal has no exit status", () => {
     const dockerStop = vi.fn(() => ({ status: 0 }));
     const dockerRm = vi.fn((_name: string) => ({ status: null, stderr: "timed out" }));
     const dockerStart = vi.fn(() => ({ status: 0 }));
     const outcome = finalizeDockerGpuPatchBackup(
-      { result: deferredCreateResult(), supervisorReady: true },
+      {
+        result: deferredCreateResult(),
+        supervisorReady: true,
+        sandboxName: "alpha",
+        lifecycleReleaseTimeoutSecs: 60,
+      },
       { dockerStop, dockerRm, dockerStart },
     );
     expect(outcome).toEqual({
       backupRemoved: false,
       rolledBack: false,
       replacementStoppedForCommit: true,
-      replacementRestarted: true,
+      replacementRestarted: false,
+      lifecycleReleaseObserved: false,
     });
+    expect(dockerStart).not.toHaveBeenCalled();
   });
 
   it("retains the backup when the replacement cannot be stopped for the final handoff", () => {
@@ -236,7 +382,12 @@ describe("finalizeDockerGpuPatchBackup", () => {
     const dockerStart = vi.fn(() => ({ status: 0 }));
 
     const outcome = finalizeDockerGpuPatchBackup(
-      { result: deferredCreateResult(), supervisorReady: true },
+      {
+        result: deferredCreateResult(),
+        supervisorReady: true,
+        sandboxName: "alpha",
+        lifecycleReleaseTimeoutSecs: 60,
+      },
       { dockerStop, dockerRm, dockerStart },
     );
 
@@ -251,11 +402,17 @@ describe("finalizeDockerGpuPatchBackup", () => {
 
   it("reports a failed replacement restart after the backup is removed", () => {
     const outcome = finalizeDockerGpuPatchBackup(
-      { result: deferredCreateResult(), supervisorReady: true },
+      {
+        result: deferredCreateResult(),
+        supervisorReady: true,
+        sandboxName: "alpha",
+        lifecycleReleaseTimeoutSecs: 60,
+      },
       {
         dockerStop: vi.fn(() => ({ status: 0 })),
         dockerRm: vi.fn(() => ({ status: 0 })),
         dockerStart: vi.fn(() => ({ status: 1 })),
+        runOpenshell: vi.fn(() => ({ status: 0, stdout: "No sandboxes found.\n" })),
       },
     );
 
@@ -264,6 +421,7 @@ describe("finalizeDockerGpuPatchBackup", () => {
       rolledBack: false,
       replacementStoppedForCommit: true,
       replacementRestarted: false,
+      lifecycleReleaseObserved: true,
     });
   });