From 4be72a782af6aa88285f80eef8ab4c15ae7aa547 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Tue, 25 Aug 2026 20:45:25 -0700 Subject: [PATCH 01/12] test(recovery): reproduce final handoff capture failure --- .../onboard/docker-gpu-patch-finalize.test.ts | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index 5c917ebe1cb..4add26c888d 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -311,6 +311,58 @@ describe("finalizeDockerGpuPatchBackup", () => { ]); }); + it("captures the retiring lifecycle row before restarting the exact replacement (#10153)", () => { + const result = exactDeferredCreateResult(); + const dockerRunResults = { + inspect: { status: 0, stdout: "true\n" }, + ps: { status: 0, stdout: `${result.newContainerId}\n` }, + } as const; + const dockerStart = vi.fn(() => ({ status: 0 })); + const runCaptureOpenshell = vi + .fn() + .mockReturnValueOnce("alpha 2026-08-23 10:00:00 Error\n") + .mockReturnValueOnce("alpha 2026-08-23 10:00:02 Ready\n"); + const runOpenshell = vi.fn((args: readonly string[]) => + args[1] === "list" ? { status: 0 } : { status: 0 }, + ); + + const outcome = finalizeDockerGpuPatchBackup( + { + result, + supervisorReady: true, + sandboxName: "alpha", + finalHandoffTimeoutSecs: 1, + }, + { + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + dockerRun: vi.fn( + (args: readonly string[]) => + dockerRunResults[String(args[0]) as keyof typeof dockerRunResults], + ), + dockerStart, + runCaptureOpenshell, + runOpenshell, + sleep: vi.fn(), + }, + ); + + expect(outcome).toMatchObject({ + backupRemoved: true, + lifecycleReleaseObserved: true, + replacementRestarted: true, + finalHandoffAcknowledged: true, + }); + expect(dockerStart).toHaveBeenCalledWith( + result.newContainerId, + expect.objectContaining({ ignoreError: true }), + ); + expect(runOpenshell).not.toHaveBeenCalledWith( + ["sandbox", "list"], + expect.any(Object), + ); + }); + it.each([ ["a failed query", { status: 1, stderr: "daemon unavailable" }], ["no labeled replacement", { status: 0, stdout: "" }], From dfd113923ac4332e11571336bb49a8df05b741f8 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Date: Tue, 25 Aug 2026 21:01:42 -0700 Subject: [PATCH 02/12] fix(recovery): capture final handoff lifecycle state --- .../onboard/docker-gpu-patch-finalize.test.ts | 79 ++++++++----------- src/lib/onboard/docker-gpu-patch-finalize.ts | 2 +- .../onboard/docker-gpu-patch-recreate.test.ts | 10 ++- .../docker-gpu-supervisor-reconnect.test.ts | 33 +++++--- .../docker-gpu-supervisor-reconnect.ts | 28 ++++--- ...ocess-recovery-supervisor-relaunch.test.ts | 36 ++++++--- 6 files changed, 108 insertions(+), 80 deletions(-) diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index 4add26c888d..59343a64b2b 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -40,12 +40,12 @@ function exactDeferredCreateResult(): DockerGpuPatchResult { function readyHandoffDeps() { return { - runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 10:00:00 Ready\n"), - runOpenshell: vi.fn((args: readonly string[]) => - args[1] === "list" - ? { status: 0, stdout: "beta 2026-08-23 10:00:00 Ready\n" } - : { status: 0 }, - ), + runCaptureOpenshell: vi + .fn() + .mockReturnValueOnce("beta 2026-08-23 10:00:00 Ready\n") + .mockReturnValue("alpha 2026-08-23 10:00:02 Ready\n"), + runOpenshell: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), }; } @@ -124,21 +124,19 @@ describe("finalizeDockerGpuPatchBackup", () => { events.push("start replacement"); return { status: 0 }; }); - const runCaptureOpenshell = vi.fn(() => { - events.push("observe ready"); - return "alpha 2026-08-23 10:00:00 Ready\n"; - }); - const runOpenshellResults = { - exec: { event: "exec ready", result: { status: 0 } }, - list: { - event: "observe lifecycle release", - result: { status: 0, stdout: "beta 2026-08-23 10:00:00 Ready\n" }, - }, - } as const; - const runOpenshell = vi.fn((args: readonly string[]) => { - const response = runOpenshellResults[args[1] === "list" ? "list" : "exec"]; - events.push(response.event); - return response.result; + const runCaptureOpenshell = vi + .fn() + .mockImplementationOnce(() => { + events.push("observe lifecycle release"); + return "beta 2026-08-23 10:00:00 Ready\n"; + }) + .mockImplementation(() => { + events.push("observe ready"); + return "alpha 2026-08-23 10:00:02 Ready\n"; + }); + const runOpenshell = vi.fn(() => { + events.push("exec ready"); + return { status: 0 }; }); const dockerResults = { ps: { @@ -228,15 +226,14 @@ describe("finalizeDockerGpuPatchBackup", () => { events.push("start replacement"); return { status: 0 }; }), - runCaptureOpenshell: vi.fn(() => { - events.push("observe deleting"); - return "alpha 2026-08-23 10:00:00 Deleting\n"; - }), - runOpenshell: vi.fn((args: readonly string[]) => - args[1] === "list" - ? { status: 0, stdout: "beta 2026-08-23 10:00:00 Ready\n" } - : { status: 1 }, - ), + runCaptureOpenshell: vi + .fn() + .mockReturnValueOnce("beta 2026-08-23 10:00:00 Ready\n") + .mockImplementation(() => { + events.push("observe deleting"); + return "alpha 2026-08-23 10:00:02 Deleting\n"; + }), + runOpenshell: vi.fn(() => ({ status: 1 })), dockerRun: vi.fn(() => ({ status: 0, stdout: `${result.newContainerId}\n` })), sleep, }, @@ -283,12 +280,11 @@ describe("finalizeDockerGpuPatchBackup", () => { dockerRm: vi.fn(() => ({ status: 0 })), dockerStart: vi.fn(() => ({ status: 0 })), dockerRun, - runCaptureOpenshell: vi.fn(() => "restored-name 2026-08-23 01:40:35 Ready\n"), - runOpenshell: vi.fn((args: readonly string[]) => - args[1] === "list" - ? { status: 0, stdout: "restored-name 2026-08-23 01:40:35 Error\n" } - : { status: 0 }, - ), + runCaptureOpenshell: vi + .fn() + .mockReturnValueOnce("restored-name 2026-08-23 01:40:35 Error\n") + .mockReturnValue("restored-name 2026-08-23 01:40:37 Ready\n"), + runOpenshell: vi.fn(() => ({ status: 0 })), sleep: vi.fn(), }, ); @@ -322,9 +318,7 @@ describe("finalizeDockerGpuPatchBackup", () => { .fn() .mockReturnValueOnce("alpha 2026-08-23 10:00:00 Error\n") .mockReturnValueOnce("alpha 2026-08-23 10:00:02 Ready\n"); - const runOpenshell = vi.fn((args: readonly string[]) => - args[1] === "list" ? { status: 0 } : { status: 0 }, - ); + const runOpenshell = vi.fn(() => ({ status: 0 })); const outcome = finalizeDockerGpuPatchBackup( { @@ -387,11 +381,8 @@ describe("finalizeDockerGpuPatchBackup", () => { dockerRm: vi.fn(() => ({ status: 0 })), dockerRun: vi.fn(() => query), dockerStart, - runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 01:40:35 Ready\n"), - runOpenshell: vi.fn(() => ({ - status: 0, - stdout: "alpha 2026-08-23 01:40:35 Error\n", - })), + runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 01:40:35 Error\n"), + runOpenshell: vi.fn(() => ({ status: 0 })), sleep: vi.fn(), }, ); diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index 7a1c09fe3c1..5aeb231e41f 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -172,7 +172,7 @@ export function finalizeDockerGpuPatchBackup( options.sandboxName, options.finalHandoffTimeoutSecs, { - runOpenshell: deps.runOpenshell, + runCaptureOpenshell: deps.runCaptureOpenshell, sleep: deps.sleep, soleLabeledReplacementCorroboratesRetiringPhase: (remainingMs) => { const expectedContainerId = fullDockerContainerId(options.result.newContainerId); diff --git a/src/lib/onboard/docker-gpu-patch-recreate.test.ts b/src/lib/onboard/docker-gpu-patch-recreate.test.ts index 98867a8a3d4..a58c2ef3d28 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.test.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.test.ts @@ -42,7 +42,10 @@ describe("Docker GPU recreate orchestration", () => { const runOpenshell = vi.fn((args: readonly string[]) => args[1] === "list" ? { status: 0, stdout: "No sandboxes found.\n" } : { status: 0 }, ); - const runCaptureOpenshell = vi.fn(() => "alpha 2026-08-23 10:00:00 Ready\n"); + const runCaptureOpenshell = vi + .fn() + .mockReturnValueOnce("No sandboxes found.\n") + .mockReturnValue("alpha 2026-08-23 10:00:02 Ready\n"); const result = recreateOpenShellDockerSandboxWithGpu( { sandboxName: "alpha", timeoutSecs: 1 }, @@ -125,7 +128,10 @@ describe("Docker GPU recreate orchestration", () => { dockerStop: vi.fn(() => ({ status: 0 })), dockerRm: vi.fn(() => ({ status: 0 })), dockerStart: vi.fn(() => ({ status: 0 })), - runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 10:00:00 Deleting\n"), + runCaptureOpenshell: vi + .fn() + .mockReturnValueOnce("No sandboxes found.\n") + .mockReturnValue("alpha 2026-08-23 10:00:02 Deleting\n"), runOpenshell: vi.fn((args: readonly string[]) => args[1] === "list" ? { status: 0, stdout: "No sandboxes found.\n" } : { status: 0 }, ), diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts index 86a84d9cee2..503149498f7 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts @@ -14,20 +14,19 @@ import { describe("Docker GPU final lifecycle release", () => { it("requires corroboration for a retiring lifecycle row (#9962)", () => { const corroborate = vi.fn(() => true); - const runOpenshell = vi.fn(() => ({ - status: 0, - stdout: "alpha 2026-08-23 01:40:35 Deleting\n", - })); + const runCaptureOpenshell = vi.fn( + () => "alpha 2026-08-23 01:40:35 Deleting\n", + ); expect( waitForOpenShellSandboxLifecycleRelease("alpha", 1, { - runOpenshell, + runCaptureOpenshell, sleep: vi.fn(), soleLabeledReplacementCorroboratesRetiringPhase: corroborate, }), ).toBe(true); expect(corroborate).toHaveBeenCalledOnce(); - expect(runOpenshell).toHaveBeenCalledWith( + expect(runCaptureOpenshell).toHaveBeenCalledWith( ["sandbox", "list"], expect.objectContaining({ killProcessTreeOnTimeout: true, @@ -38,17 +37,29 @@ describe("Docker GPU final lifecycle release", () => { }); it("does not accept an uncorroborated retiring lifecycle row (#9962)", () => { - const runOpenshell = vi.fn(() => ({ - status: 0, - stdout: "alpha 2026-08-23 01:40:35 Error\n", - })); + const runCaptureOpenshell = vi.fn(() => "alpha 2026-08-23 01:40:35 Error\n"); expect( waitForOpenShellSandboxLifecycleRelease("alpha", 1, { - runOpenshell, + runCaptureOpenshell, + sleep: vi.fn(), + }), + ).toBe(false); + }); + + it("does not release the lifecycle when the captured sandbox list is unavailable (#10153)", () => { + const corroborate = vi.fn(() => true); + + expect( + waitForOpenShellSandboxLifecycleRelease("alpha", 1, { + runCaptureOpenshell: vi.fn(() => { + throw new Error("sandbox list transport unavailable"); + }), sleep: vi.fn(), + soleLabeledReplacementCorroboratesRetiringPhase: corroborate, }), ).toBe(false); + expect(corroborate).not.toHaveBeenCalled(); }); it("does not report reconnect without an OpenShell execution boundary (#9531)", () => { diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts index 9f41580f7ee..bad9cd26943 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts @@ -75,7 +75,7 @@ export type DockerGpuSupervisorReconnectDeps = { type DockerLifecycleReleaseDeps = Pick< DockerGpuSupervisorReconnectDeps, - "runOpenshell" | "sleep" + "runCaptureOpenshell" | "sleep" > & { /** Corroborates a retiring lifecycle row with the stopped exact replacement. */ soleLabeledReplacementCorroboratesRetiringPhase?: (remainingMs: number) => boolean; @@ -114,7 +114,7 @@ export function waitForOpenShellSandboxLifecycleRelease( timeoutSecs: number, deps: DockerLifecycleReleaseDeps, ): boolean { - if (!deps.runOpenshell) return false; + if (!deps.runCaptureOpenshell) return false; const sleep = deps.sleep ?? defaultSleep; const deadline = Date.now() + Math.max(1, Math.round(timeoutSecs)) * 1000; const maxAttempts = Math.max(1, Math.ceil(Math.max(1, Math.round(timeoutSecs)) / 2) + 1); @@ -122,14 +122,22 @@ export function waitForOpenShellSandboxLifecycleRelease( for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { const remainingMs = deadline - Date.now(); if (remainingMs <= 0) break; - const result = deps.runOpenshell(["sandbox", "list"], { - ignoreError: true, - ...PROCESS_TREE_BOUNDED_OPENSHELL_OPTIONS, - suppressOutput: true, - timeout: Math.min(DOCKER_GPU_PATCH_TIMEOUT_MS, remainingMs), - }); - if (hasZeroDockerExitStatus(result)) { - const output = String(result.stdout ?? "").trim(); + let output = ""; + try { + // The streaming runner does not return sandbox-list stdout. Capture the + // row so a successful command cannot hide a retiring lifecycle phase. + output = deps + .runCaptureOpenshell(["sandbox", "list"], { + ignoreError: true, + ...PROCESS_TREE_BOUNDED_OPENSHELL_OPTIONS, + suppressOutput: true, + timeout: Math.min(DOCKER_GPU_PATCH_TIMEOUT_MS, remainingMs), + }) + .trim(); + } catch { + output = ""; + } + if (output) { const entries = parseLiveSandboxEntries(output); const sandboxPresent = entries.some((entry) => entry.name === sandboxName); const retiring = entries.some( diff --git a/test/process-recovery/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery/process-recovery-supervisor-relaunch.test.ts index e110f53b2a6..f944cdfced6 100644 --- a/test/process-recovery/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery/process-recovery-supervisor-relaunch.test.ts @@ -82,9 +82,16 @@ function composedRelaunchTransaction( .mockReturnValueOnce(containerIds.old) .mockReturnValue(containerIds.replacement); const runOpenshell = vi.fn(() => ({ status: 0, stdout: "No sandboxes found.\n" })); + let activeSandboxName = ""; + const runCaptureOpenshell = vi.fn(() => + runCaptureOpenshell.mock.calls.length === 1 + ? "No sandboxes found.\n" + : `${activeSandboxName} 2026-08-23 10:00:02 Ready\n`, + ); const relaunchManagedSupervisorSessionImpl = vi.fn( - (sandboxName: string, options: Parameters[1]) => - relaunchManagedSupervisorSession(sandboxName, { + (sandboxName: string, options: Parameters[1]) => { + activeSandboxName = sandboxName; + return relaunchManagedSupervisorSession(sandboxName, { quiet: options.quiet, deps: { ...options.deps, @@ -114,6 +121,7 @@ function composedRelaunchTransaction( }; }), removeBackup: vi.fn(() => true), + runCaptureOpenshell, runOpenshell, recreate: vi.fn(() => ({ applied: true as const, @@ -131,9 +139,15 @@ function composedRelaunchTransaction( })), finalize: finalizeTransaction, }, - }), + }); + }, ); - return { finalizeTransaction, relaunchManagedSupervisorSessionImpl, runOpenshell }; + return { + finalizeTransaction, + relaunchManagedSupervisorSessionImpl, + runCaptureOpenshell, + runOpenshell, + }; } function scriptedPinnedGatewayRecovery( @@ -497,11 +511,11 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { dockerStop, }), ); - const { relaunchManagedSupervisorSessionImpl } = composedRelaunchTransaction( - order, - finalizeTransaction, - { old: oldContainerId, replacement: replacementContainerId }, - ); + const { relaunchManagedSupervisorSessionImpl, runCaptureOpenshell } = + composedRelaunchTransaction(order, finalizeTransaction, { + old: oldContainerId, + replacement: replacementContainerId, + }); const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, ); @@ -550,7 +564,6 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { forwardStarted ||= args.join(" ") === "forward start --background 18789 legacy-handoff-box"; return { status: 0 } as never; }); - const result = checkAndRecoverSandboxProcesses("legacy-handoff-box", { quiet: true, isSandboxGatewayRunningImpl: () => false, @@ -559,7 +572,6 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { relaunchManagedSupervisorSessionImpl, waitForRecreatedSandboxOpenShellReadyImpl, }); - expect(result).toMatchObject({ checked: true, wasRunning: false, @@ -589,7 +601,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { ["forward", "start", "--background", "18789", "legacy-handoff-box"], expect.objectContaining({ ignoreError: true }), ); - expect(captureOpenshell).toHaveBeenCalledWith( + expect(runCaptureOpenshell).toHaveBeenCalledWith( ["sandbox", "list"], expect.objectContaining({ killProcessTreeOnTimeout: true, From 1176511437c6bfc0e3bd57f694ff870f38603061 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 26 Aug 2026 11:44:47 -0700 Subject: [PATCH 03/12] test(onboard): align rollback lifecycle capture fixture Signed-off-by: Prekshi Vyas --- src/lib/onboard/docker-gpu-patch-rollback.test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lib/onboard/docker-gpu-patch-rollback.test.ts b/src/lib/onboard/docker-gpu-patch-rollback.test.ts index 6c6af7c26ac..d541200ae48 100644 --- a/src/lib/onboard/docker-gpu-patch-rollback.test.ts +++ b/src/lib/onboard/docker-gpu-patch-rollback.test.ts @@ -276,7 +276,11 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { runOpenshell: vi.fn((args: readonly string[]) => args[1] === "list" ? { status: 0, stdout: "No sandboxes found.\n" } : { status: 0 }, ), - runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 10:00:00 Ready\n"), + runCaptureOpenshell: vi.fn(() => + !restoredPresent && !startTargets.includes(retryId) + ? "No sandboxes found.\n" + : "alpha 2026-08-23 10:00:02 Ready\n", + ), sleep: vi.fn(), homedir: () => tmpDir, now: () => new Date("2026-07-03T00:00:00Z"), From c1e53def63f49eb45eb859bc81cfaadfd6e157d8 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 26 Aug 2026 14:07:38 -0700 Subject: [PATCH 04/12] test(e2e): parse legacy handoff receipt after progress Signed-off-by: Prekshi Vyas --- .../gateway-guard-legacy-keepalive-fixture.ts | 40 ++++++++++++++++ test/e2e/live/gateway-guard-recovery.test.ts | 5 +- ...way-guard-legacy-keepalive-fixture.test.ts | 46 +++++++++++++++++++ 3 files changed, 88 insertions(+), 3 deletions(-) diff --git a/test/e2e/live/gateway-guard-legacy-keepalive-fixture.ts b/test/e2e/live/gateway-guard-legacy-keepalive-fixture.ts index c96798c7eb4..6198ac1cffa 100644 --- a/test/e2e/live/gateway-guard-legacy-keepalive-fixture.ts +++ b/test/e2e/live/gateway-guard-legacy-keepalive-fixture.ts @@ -69,6 +69,12 @@ export type LegacyKeepaliveFixtureOptions = { timeoutSecs?: number; }; +export type LegacyKeepaliveHandoffReceipt = { + readonly oldContainerId: string; + readonly newContainerId: string; + readonly startupCommand: "sleep infinity"; +}; + export type LegacyKeepaliveFixtureDeps = { recreate: StartupCommandRecreate; dockerCapture: DockerCapture; @@ -92,6 +98,40 @@ function requireFixtureInput(condition: boolean, message: string): asserts condi if (!condition) throw new Error(message); } +/** Read the final machine receipt without treating recreation progress as JSON. */ +export function parseLegacyKeepaliveHandoffReceipt( + output: string, +): LegacyKeepaliveHandoffReceipt { + const receiptLine = output + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean) + .at(-1); + let parsed: unknown; + try { + parsed = JSON.parse(receiptLine ?? ""); + } catch { + throw new Error("legacy keepalive fixture did not emit a final JSON handoff receipt"); + } + requireFixtureInput( + typeof parsed === "object" && parsed !== null && !Array.isArray(parsed), + "legacy keepalive fixture handoff receipt must be an object", + ); + const receipt = parsed as Record; + requireFixtureInput( + DOCKER_CONTAINER_ID_PATTERN.test(String(receipt.oldContainerId ?? "")) && + DOCKER_CONTAINER_ID_PATTERN.test(String(receipt.newContainerId ?? "")) && + receipt.oldContainerId !== receipt.newContainerId && + receipt.startupCommand === LEGACY_KEEPALIVE_COMMAND.join(" "), + "legacy keepalive fixture handoff receipt is invalid", + ); + return { + oldContainerId: String(receipt.oldContainerId), + newContainerId: String(receipt.newContainerId), + startupCommand: "sleep infinity", + }; +} + function hasExactTokens(value: unknown, expected: readonly string[]): boolean { return ( Array.isArray(value) && diff --git a/test/e2e/live/gateway-guard-recovery.test.ts b/test/e2e/live/gateway-guard-recovery.test.ts index 8df89fe17bb..bcc7bda6360 100644 --- a/test/e2e/live/gateway-guard-recovery.test.ts +++ b/test/e2e/live/gateway-guard-recovery.test.ts @@ -54,6 +54,7 @@ import { expect, test } from "../fixtures/e2e-test.ts"; import { pollUntil } from "../fixtures/polling.ts"; import type { TestProgress } from "../fixtures/progress.ts"; import { ubuntuRepoDocker } from "../registry/matrix.ts"; +import { parseLegacyKeepaliveHandoffReceipt } from "./gateway-guard-legacy-keepalive-fixture.ts"; // Reuses the standard ubuntu-repo-docker environment with the // `cloud-openclaw` onboarding profile (the only one the framework's @@ -574,9 +575,7 @@ test( }, ); expect(createLegacyKeepalive.exitCode, resultText(createLegacyKeepalive)).toBe(0); - const handoffReceipt = JSON.parse(createLegacyKeepalive.stdout) as { - newContainerId?: unknown; - }; + const handoffReceipt = parseLegacyKeepaliveHandoffReceipt(createLegacyKeepalive.stdout); expect(handoffReceipt.newContainerId).toMatch(/^[0-9a-f]{64}$/iu); // Do not overlap the fixture's recreation with the restart below. The // fixture runs in its own process, so the host must observe the replacement diff --git a/test/e2e/support/gateway-guard-legacy-keepalive-fixture.test.ts b/test/e2e/support/gateway-guard-legacy-keepalive-fixture.test.ts index b14ed1124b2..7ea8f360d3e 100644 --- a/test/e2e/support/gateway-guard-legacy-keepalive-fixture.test.ts +++ b/test/e2e/support/gateway-guard-legacy-keepalive-fixture.test.ts @@ -13,6 +13,7 @@ import { import { createLegacyKeepaliveFixture, type LegacyKeepaliveFixtureDeps, + parseLegacyKeepaliveHandoffReceipt, rewriteManagedInspectForLegacyKeepalive, } from "../live/gateway-guard-legacy-keepalive-fixture.ts"; @@ -126,6 +127,51 @@ function managedRuntimeInspectWithoutOciImageUser( } describe("gateway guard legacy keepalive fixture", () => { + it("parses the final handoff receipt after recreation progress output", () => { + expect( + parseLegacyKeepaliveHandoffReceipt( + [ + " ✓ Sandbox 'e2e-2701' became Ready", + " Waiting for final replacement handoff...", + JSON.stringify({ + oldContainerId: OLD_CONTAINER_ID, + newContainerId: NEW_CONTAINER_ID, + startupCommand: "sleep infinity", + }), + "", + ].join("\n"), + ), + ).toEqual({ + oldContainerId: OLD_CONTAINER_ID, + newContainerId: NEW_CONTAINER_ID, + startupCommand: "sleep infinity", + }); + }); + + it.each([ + { name: "missing receipt", output: "Waiting for handoff\n" }, + { + name: "unchanged container identity", + output: JSON.stringify({ + oldContainerId: OLD_CONTAINER_ID, + newContainerId: OLD_CONTAINER_ID, + startupCommand: "sleep infinity", + }), + }, + { + name: "unexpected startup command", + output: JSON.stringify({ + oldContainerId: OLD_CONTAINER_ID, + newContainerId: NEW_CONTAINER_ID, + startupCommand: "unreviewed", + }), + }, + ])("rejects $name in the handoff receipt", ({ output }) => { + expect(() => parseLegacyKeepaliveHandoffReceipt(output)).toThrow( + /final JSON handoff receipt|handoff receipt is invalid/u, + ); + }); + it("recreates only the pinned sandbox container with the reviewed supervisor and legacy workload (#9364)", () => { const dockerCapture = vi.fn(() => managedRuntimeInspect()); const recreate = vi.fn((_, deps: Parameters[1]) => { From 1093a53a52e1527c563120f98c6c2f0e583f9b14 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 26 Aug 2026 15:17:52 -0700 Subject: [PATCH 05/12] fix(recovery): retry incomplete supervisor discovery Signed-off-by: Prekshi Vyas --- docs/reference/troubleshooting.mdx | 10 ++++-- scripts/managed-gateway-control.py | 24 +++++++++---- .../actions/sandbox/gateway-restart.test.ts | 1 + src/lib/actions/sandbox/gateway-restart.ts | 6 ++++ .../process-recovery-managed-startup.test.ts | 35 ++++++++++++++++++- src/lib/actions/sandbox/process-recovery.ts | 14 ++++++++ .../managed/managed-gateway-control.test.ts | 12 ++++--- 7 files changed, 86 insertions(+), 16 deletions(-) diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 5c7d9b1652a..093cb2b6c9c 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -1737,6 +1737,7 @@ $$nemoclaw status When `recover` repairs a stopped built-in gateway, NemoClaw repeats the recovery action only for these exact transient results: +- Status `1` with blank stdout and exactly one stderr line: `SUPERVISOR_NOT_RUNNING`, `SUPERVISOR_DISCOVERY_PENDING`, `PRIVILEGED_CONTROL_UNAVAILABLE`, or `GATEWAY_HEALTH_TIMEOUT`. - Status `1` with blank stdout and exactly one stderr line, `SUPERVISOR_BUSY`. - Status `137` with blank stdout and stderr. - Status `1` with blank stdout and exactly one stderr line, `Error response from daemon: Container is restarting, wait until the container is running`. @@ -1744,13 +1745,16 @@ When `recover` repairs a stopped built-in gateway, NemoClaw repeats the recovery For the Docker result, `` must be a 64-character lowercase hexadecimal ID that matches the selected registry-owned container. Recovery makes at most 11 controller attempts in total. It stops after 3 of those attempts return `SUPERVISOR_BUSY`. +The managed controller emits `SUPERVISOR_DISCOVERY_PENDING` only when an incomplete process-table scan during startup cannot yet prove either one exact supervisor or clean supervisor absence. +That result does not authorize container recreation or accept a supervisor identity; a later controller request must perform the full identity proof again. Managed settle confirmation treats exact `SUPERVISOR_BUSY` as inconclusive within its configured window. Status `137` and the Docker restart result remain terminal during that confirmation. -The managed supervisor startup waiter accepts the two container-transition results within its separate 11-attempt bound. +The managed supervisor startup waiter accepts the four exact startup results, `SUPERVISOR_BUSY`, and the two container-transition results within its separate 11-attempt bound. Unbound container IDs, reformatted Docker errors, status `137` with nonblank output, and other diagnostic results stop immediately. NemoClaw treats `SUPERVISOR_UNAVAILABLE` as terminal because it can report unreadable or untrusted supervisor state, ambiguous discovery, or a process-identity change. -`SUPERVISOR_NOT_RUNNING` is a separate result that requires two zero-supervisor scans with a stable PID 1 and does not enter that retry loop. -On a supported local Docker-driver sandbox with the legacy keepalive startup, it can authorize a container-identity-pinned recreation that commits only after managed health and settle checks pass. +`SUPERVISOR_NOT_RUNNING` is a separate result that requires two zero-supervisor scans with a stable PID 1. +It enters the bounded startup retry first; only an exact missing-supervisor result that remains after the bound can authorize a container-identity-pinned recreation on a supported local Docker-driver sandbox with the legacy keepalive startup. +That recreation commits only after managed health and settle checks pass. To bypass that trusted recreation while troubleshooting, run `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH=1 $$nemoclaw recover`; NemoClaw leaves the container unchanged and returns rebuild or re-onboard guidance. If recovery stops after 3 `SUPERVISOR_BUSY` results, or if `gateway restart` reports `SUPERVISOR_BUSY`, wait for the active request to finish and retry the command. If recovery exhausts the transition bound after status `137` or the Docker restart result, wait for the container to finish restarting and retry the command. diff --git a/scripts/managed-gateway-control.py b/scripts/managed-gateway-control.py index efe81c23bca..a8af06156b2 100755 --- a/scripts/managed-gateway-control.py +++ b/scripts/managed-gateway-control.py @@ -1010,21 +1010,31 @@ def _discover_supervisor(reader: ProcReader) -> ProcessIdentity: matches, inconclusive = _supervisor_candidates(reader, pid1, sandbox_uid) if inconclusive: # Busy agents can create or reap an unrelated short-lived process while - # /proc is being read. Retry only when one exact supervisor was already - # proven, and require that same pinned identity on every scan. A missing, - # changing, or duplicate supervisor still fails closed immediately. - if len(matches) != 1: + # /proc is being read. A restart can expose this churn before the + # supervisor's argv is observable, so zero matches plus an incomplete + # scan is not the clean two-scan absence proof below. Keep one exact + # supervisor pinned when it is already visible. If no supervisor was + # visible, require a fresh controller request after one appears rather + # than accepting an identity born during this ambiguous scan. The host + # retries only the exact SUPERVISOR_DISCOVERY_PENDING marker within its + # existing bound; duplicate or changing identities remain terminal. + if len(matches) > 1: raise ControlError("SUPERVISOR_UNAVAILABLE") - expected = matches[0].stable_key() + expected = matches[0].stable_key() if matches else None deadline = time.monotonic() + PROCESS_PROOF_GRACE_SECONDS while inconclusive: remaining = deadline - time.monotonic() if remaining <= 0: - raise ControlError("SUPERVISOR_UNAVAILABLE") + raise ControlError("SUPERVISOR_DISCOVERY_PENDING") time.sleep(min(PROCESS_PROOF_RETRY_SECONDS, remaining)) _recapture_exact_identity(reader, pid1, deadline=deadline) matches, inconclusive = _supervisor_candidates(reader, pid1, sandbox_uid) - if len(matches) != 1 or matches[0].stable_key() != expected: + if len(matches) > 1: + raise ControlError("SUPERVISOR_UNAVAILABLE") + if expected is None: + if matches: + raise ControlError("SUPERVISOR_DISCOVERY_PENDING") + elif len(matches) != 1 or matches[0].stable_key() != expected: raise ControlError("SUPERVISOR_UNAVAILABLE") if len(matches) == 0: # A zero-match scan is the only absence signal that may authorize the diff --git a/src/lib/actions/sandbox/gateway-restart.test.ts b/src/lib/actions/sandbox/gateway-restart.test.ts index d1058a7957a..a0485efa932 100644 --- a/src/lib/actions/sandbox/gateway-restart.test.ts +++ b/src/lib/actions/sandbox/gateway-restart.test.ts @@ -21,6 +21,7 @@ const supervisorFailureMarkers: Array< ["SUPERVISOR_UNAVAILABLE", "privileged control unavailable"], ["SUPERVISOR_UNAVAILABLE\nNEMOCLAW_CONTROL_STAGE=await-replacement", "supervisor unavailable"], ["SUPERVISOR_NOT_RUNNING", "supervisor not running"], + ["SUPERVISOR_DISCOVERY_PENDING", "supervisor unavailable"], ["SUPERVISOR_REBUILD_REQUIRED", "privileged control unavailable"], ["SUPERVISOR_BUSY", "privileged control unavailable"], [MARKERS.SECRET_BOUNDARY_REFUSED, "secret-boundary refusal"], diff --git a/src/lib/actions/sandbox/gateway-restart.ts b/src/lib/actions/sandbox/gateway-restart.ts index f3fe9817f64..24c5fa2a5e8 100644 --- a/src/lib/actions/sandbox/gateway-restart.ts +++ b/src/lib/actions/sandbox/gateway-restart.ts @@ -206,6 +206,12 @@ export function classifyGatewayRestartFailure(result: GatewayRestartCommandResul detail: detail || "the in-sandbox gateway supervisor is not running", }; } + if (output.includes("SUPERVISOR_DISCOVERY_PENDING")) { + return { + layer: "supervisor unavailable", + detail: detail || "the managed gateway supervisor is still starting", + }; + } if (output.includes("SUPERVISOR_UNAVAILABLE") && output.includes("NEMOCLAW_CONTROL_STAGE=")) { return { layer: "supervisor unavailable", diff --git a/src/lib/actions/sandbox/process-recovery-managed-startup.test.ts b/src/lib/actions/sandbox/process-recovery-managed-startup.test.ts index e5fab9f4492..a10ced832c3 100644 --- a/src/lib/actions/sandbox/process-recovery-managed-startup.test.ts +++ b/src/lib/actions/sandbox/process-recovery-managed-startup.test.ts @@ -48,7 +48,12 @@ afterEach(() => { }); describe("checkAndRecoverSandboxProcesses managed startup", () => { - it.each(["SUPERVISOR_NOT_RUNNING", "PRIVILEGED_CONTROL_UNAVAILABLE", "GATEWAY_HEALTH_TIMEOUT"])( + it.each([ + "SUPERVISOR_NOT_RUNNING", + "SUPERVISOR_DISCOVERY_PENDING", + "PRIVILEGED_CONTROL_UNAVAILABLE", + "GATEWAY_HEALTH_TIMEOUT", + ])( "waits through the exact %s startup transition (#9466)", (startupMarker) => { const sandboxName = "startup-box"; @@ -81,6 +86,34 @@ describe("checkAndRecoverSandboxProcesses managed startup", () => { }, ); + it("does not retry a diagnostic-bearing supervisor-discovery result", () => { + const sandboxName = "diagnostic-start"; + mockOpenClawSandbox(sandboxName); + vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS", "0"); + const requestGatewaySupervisorAction = vi.fn(() => ({ + status: 1, + stdout: "", + stderr: "SUPERVISOR_DISCOVERY_PENDING\nunexpected diagnostic", + })); + const relaunchManagedSupervisorSessionImpl = vi.fn(() => null); + + const result = checkAndRecoverSandboxProcesses(sandboxName, { + quiet: true, + isSandboxGatewayRunningImpl: () => false, + requestGatewaySupervisorAction, + relaunchManagedSupervisorSessionImpl, + }); + + expect(result).toMatchObject({ + checked: true, + wasRunning: false, + recovered: false, + forwardRecovered: false, + }); + expect(requestGatewaySupervisorAction).toHaveBeenCalledOnce(); + expect(relaunchManagedSupervisorSessionImpl).not.toHaveBeenCalled(); + }); + it("does not retry a managed-container identity mismatch (#9466)", () => { const sandboxName = "identity-box"; mockOpenClawSandbox(sandboxName); diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 1240c665d95..6054c371bcf 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -398,6 +398,19 @@ function isExactlyPendingManagedSupervisorControl(result: SandboxCommandResult | return lines.length === 1 && lines[0] === "PRIVILEGED_CONTROL_UNAVAILABLE"; } +function isExactlyPendingManagedSupervisorDiscovery(result: SandboxCommandResult | null): boolean { + if (result === null || result.status !== 1 || result.stdout.trim() !== "") return false; + const lines = result.stderr + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + // The installed managed controller emits this only before selecting a + // supervisor, when an incomplete process-table scan cannot yet prove either + // one exact supervisor or clean absence. Retrying this exact bare marker can + // delay recovery but cannot authorize a relaunch or accept an identity. + return lines.length === 1 && lines[0] === "SUPERVISOR_DISCOVERY_PENDING"; +} + function isExactlyPendingManagedGatewayHealth(result: SandboxCommandResult | null): boolean { if (result === null || result.status !== 1 || result.stdout.trim() !== "") return false; const lines = result.stderr @@ -416,6 +429,7 @@ function isExactlyManagedGatewayStartupTransition( return ( isExactlyMissingManagedSupervisor(result) || isExactlyPendingManagedSupervisorControl(result) || + isExactlyPendingManagedSupervisorDiscovery(result) || isExactlyPendingManagedGatewayHealth(result) ); } diff --git a/test/inference/managed/managed-gateway-control.test.ts b/test/inference/managed/managed-gateway-control.test.ts index e2b788d12ea..843e3168e13 100644 --- a/test/inference/managed/managed-gateway-control.test.ts +++ b/test/inference/managed/managed-gateway-control.test.ts @@ -1178,7 +1178,8 @@ with tempfile.TemporaryDirectory() as root: control._read_start_log_diagnostic_excerpt = real_start_log_reader health_diagnostic = [health_status, health_stderr.getvalue().splitlines()] exact_failure_diagnostics = [] - for failure_code in ("SUPERVISOR_BUSY", "SUPERVISOR_NOT_RUNNING"): + for failure_code in ("SUPERVISOR_BUSY", "SUPERVISOR_NOT_RUNNING", + "SUPERVISOR_DISCOVERY_PENDING"): def fail_with_exact_marker(*_args, code=failure_code): with control._control_stage("discover-supervisor"): raise control.ControlError(code) @@ -1316,11 +1317,11 @@ describe("managed gateway root control", () => { gateway: [41, "333", 40], healthy: true, }, - zombie_leader_with_live_sibling: "SUPERVISOR_UNAVAILABLE", + zombie_leader_with_live_sibling: "SUPERVISOR_DISCOVERY_PENDING", state_key_behavior: [true, false], mixed_namespace_rejected: true, transient_supervisor_retry: [40, 6], - persistent_supervisor_churn: ["SUPERVISOR_UNAVAILABLE", expect.any(Number), 1], + persistent_supervisor_churn: ["SUPERVISOR_DISCOVERY_PENDING", expect.any(Number), 1], transient_gateway_candidates: [41, 5], namespace_denied: true, preflight: [ @@ -1347,8 +1348,8 @@ describe("managed gateway root control", () => { runtime_validation: "in-process", missing_supervisor: "SUPERVISOR_NOT_RUNNING", appearing_supervisor: "SUPERVISOR_UNAVAILABLE", - unreadable_process: "SUPERVISOR_UNAVAILABLE", - empty_live_process: "SUPERVISOR_UNAVAILABLE", + unreadable_process: "SUPERVISOR_DISCOVERY_PENDING", + empty_live_process: "SUPERVISOR_DISCOVERY_PENDING", duplicate_supervisor: "SUPERVISOR_UNAVAILABLE", duplicate: "SUPERVISOR_UNAVAILABLE", signals: [15, 9], @@ -1469,6 +1470,7 @@ describe("managed gateway root control", () => { exact_failure_diagnostics: [ [1, ["SUPERVISOR_BUSY"]], [1, ["SUPERVISOR_NOT_RUNNING"]], + [1, ["SUPERVISOR_DISCOVERY_PENDING"]], ], }); expect(output.timeout_refresh[2][2][1]).toBeGreaterThan(0); From 80e8f0b8e94b03bc2af1e72d4ce2370283a54d08 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 26 Aug 2026 16:27:06 -0700 Subject: [PATCH 06/12] fix(recovery): wait for lifecycle name absence Signed-off-by: Prekshi Vyas --- docs/reference/commands.mdx | 13 ++- docs/reference/troubleshooting.mdx | 5 +- .../actions/sandbox/process-recovery.test.ts | 4 +- src/lib/actions/sandbox/process-recovery.ts | 79 +++++++------------ .../onboard/docker-gpu-patch-finalize.test.ts | 77 +++++++++++------- src/lib/onboard/docker-gpu-patch-finalize.ts | 47 ----------- .../docker-gpu-supervisor-reconnect.test.ts | 44 +++++++---- .../docker-gpu-supervisor-reconnect.ts | 44 +++-------- .../gateway-guard-legacy-keepalive-fixture.ts | 40 +++++++--- ...way-guard-legacy-keepalive-fixture.test.ts | 17 +++- 10 files changed, 174 insertions(+), 196 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 9a6bc8c912f..ef1497f123a 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1844,6 +1844,9 @@ NemoClaw waits for the exact replacement to pass managed gateway health and Open After state restoration, it restarts the gateway in the exact replacement container and requires an authenticated `ok` result. It then runs the managed settle check. It commits only after the replacement identity, state restoration, gateway restart, and settle check pass. +At the final commit handoff, NemoClaw stops the replacement, removes the rollback container, and waits for a captured, phase-bearing `openshell sandbox list` result that omits the selected sandbox name before it restarts the replacement. +An `Error` or `Deleting` row for the selected sandbox does not prove lifecycle release, even when Docker still exposes the exact replacement container. +If that release proof does not arrive after the rollback container has been removed, NemoClaw leaves the replacement stopped and reports that automatic rollback is unavailable. If OpenShell re-registration, state restoration, or a later gateway check fails, NemoClaw attempts to roll back the replacement. The primary dashboard or API host forward stays stopped. NemoClaw removes the temporary state backup after a successful restore or rollback. @@ -1852,20 +1855,24 @@ Mounted state remains available, but a committed swap does not retain other writ It is idempotent. When `recover` repairs a stopped built-in OpenClaw or Hermes gateway, it repeats the recovery action only for these exact transient results: -- Status `1` with blank stdout and exactly one stderr line, `SUPERVISOR_BUSY`. +- Status `1` with blank stdout and exactly one stderr line: `SUPERVISOR_NOT_RUNNING`, `SUPERVISOR_DISCOVERY_PENDING`, `PRIVILEGED_CONTROL_UNAVAILABLE`, `GATEWAY_HEALTH_TIMEOUT`, or `SUPERVISOR_BUSY`. - Status `137` with blank stdout and stderr. - Status `1` with blank stdout and exactly one stderr line, `Error response from daemon: Container is restarting, wait until the container is running`. For the Docker result, `` must be a 64-character lowercase hexadecimal ID that matches the selected registry-owned container. Recovery makes at most 11 controller attempts in total. It stops after 3 of those attempts return `SUPERVISOR_BUSY`. -Managed settle confirmation treats exact `SUPERVISOR_BUSY` as inconclusive within its configured window. +The managed controller emits `SUPERVISOR_DISCOVERY_PENDING` only when an incomplete process-table scan during startup cannot yet prove either one exact supervisor or clean supervisor absence. +That result delays recovery but cannot authorize container recreation or accept a supervisor identity; a later request must perform the complete identity proof again. +Managed settle confirmation treats exact `SUPERVISOR_BUSY` and `SUPERVISOR_DISCOVERY_PENDING` results as inconclusive within its configured window. Status `137` and the Docker restart result remain terminal during that confirmation. -The managed supervisor startup waiter accepts the two container-transition results within its separate 11-attempt bound. +The managed supervisor startup waiter accepts the four exact startup results, `SUPERVISOR_BUSY`, and the two container-transition results within its separate 11-attempt bound. Unbound container IDs, reformatted Docker errors, status `137` with nonblank output, and other diagnostic results are terminal. NemoClaw treats `SUPERVISOR_UNAVAILABLE` as terminal because the managed controller uses it for integrity refusals, ambiguous discovery, and process-identity changes. It does not repeat the recovery action or treat the settle probe as inconclusive, and instead prints host-side restart and rebuild guidance. Other controller failures also stop immediately. +Only an exact `SUPERVISOR_NOT_RUNNING` result that remains after the bounded startup retries can enter transactional legacy keepalive recreation. +The pinned controller probe must then confirm the missing supervisor before recreation proceeds. If the gateway is already running, the command exits zero without force-restarting it; it can still re-evaluate supported safety checks and check or recover host-side forwards. Use [`$$nemoclaw gateway restart`](#$$nemoclaw-name-gateway-restart) when you deliberately need a running gateway to reload runtime configuration or plugins. diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index 093cb2b6c9c..f923e3f82fd 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -1747,7 +1747,7 @@ Recovery makes at most 11 controller attempts in total. It stops after 3 of those attempts return `SUPERVISOR_BUSY`. The managed controller emits `SUPERVISOR_DISCOVERY_PENDING` only when an incomplete process-table scan during startup cannot yet prove either one exact supervisor or clean supervisor absence. That result does not authorize container recreation or accept a supervisor identity; a later controller request must perform the full identity proof again. -Managed settle confirmation treats exact `SUPERVISOR_BUSY` as inconclusive within its configured window. +Managed settle confirmation treats exact `SUPERVISOR_BUSY` and `SUPERVISOR_DISCOVERY_PENDING` results as inconclusive within its configured window. Status `137` and the Docker restart result remain terminal during that confirmation. The managed supervisor startup waiter accepts the four exact startup results, `SUPERVISOR_BUSY`, and the two container-transition results within its separate 11-attempt bound. Unbound container IDs, reformatted Docker errors, status `137` with nonblank output, and other diagnostic results stop immediately. @@ -1755,6 +1755,9 @@ NemoClaw treats `SUPERVISOR_UNAVAILABLE` as terminal because it can report unrea `SUPERVISOR_NOT_RUNNING` is a separate result that requires two zero-supervisor scans with a stable PID 1. It enters the bounded startup retry first; only an exact missing-supervisor result that remains after the bound can authorize a container-identity-pinned recreation on a supported local Docker-driver sandbox with the legacy keepalive startup. That recreation commits only after managed health and settle checks pass. +At the final commit handoff, NemoClaw stops the replacement, removes the rollback container, and waits for a captured, phase-bearing `openshell sandbox list` result that omits the selected sandbox name before it restarts the replacement. +An `Error` or `Deleting` row for the selected sandbox does not prove lifecycle release, even when Docker still exposes the exact replacement container. +If the release proof does not arrive after the rollback container has been removed, NemoClaw leaves the replacement stopped and reports that automatic rollback is unavailable. To bypass that trusted recreation while troubleshooting, run `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH=1 $$nemoclaw recover`; NemoClaw leaves the container unchanged and returns rebuild or re-onboard guidance. If recovery stops after 3 `SUPERVISOR_BUSY` results, or if `gateway restart` reports `SUPERVISOR_BUSY`, wait for the active request to finish and retry the command. If recovery exhausts the transition bound after status `137` or the Docker restart result, wait for the container to finish restarting and retry the command. diff --git a/src/lib/actions/sandbox/process-recovery.test.ts b/src/lib/actions/sandbox/process-recovery.test.ts index 28826ef3ecf..2d2132a9bc9 100644 --- a/src/lib/actions/sandbox/process-recovery.test.ts +++ b/src/lib/actions/sandbox/process-recovery.test.ts @@ -650,7 +650,7 @@ describe("confirmRecoveredSandboxGatewayManaged scope", () => { ).toBe(false); }); - it("keeps unavailable supervisor results terminal while lease contention stays transient", () => { + it("keeps unavailable results terminal while exact transient results stay inconclusive", () => { const confirm = (stderr: string) => confirmRecoveredSandboxGatewayManaged("my-sandbox", { getSandboxImpl: () => openClawEntry, @@ -660,6 +660,8 @@ describe("confirmRecoveredSandboxGatewayManaged scope", () => { expect(confirm("SUPERVISOR_UNAVAILABLE")).toBe(false); expect(confirm("SUPERVISOR_BUSY")).toBeNull(); + expect(confirm("SUPERVISOR_DISCOVERY_PENDING")).toBeNull(); + expect(confirm("SUPERVISOR_DISCOVERY_PENDING\nunexpected diagnostic")).toBe(false); }); }); diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 6054c371bcf..28a9fadc814 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -369,7 +369,10 @@ function hasGatewayRecoveryMarker(result: SandboxCommandResult | null): boolean // definitive. Retry only the exact lease-contention marker within the // existing bounded window. Removal condition: delete this classifier and its // retry cases once the installed controller waits through contention itself. -function isExactlyRetryableManagedRecoveryFailure(result: SandboxCommandResult | null): boolean { +function isExactlyManagedControlMarker( + result: SandboxCommandResult | null, + marker: string, +): boolean { if (result === null) return false; if (result.status !== 1) return false; if (result.stdout.trim() !== "") return false; @@ -377,61 +380,26 @@ function isExactlyRetryableManagedRecoveryFailure(result: SandboxCommandResult | .split(/\r?\n/) .map((line) => line.trim()) .filter(Boolean); - return lines.length === 1 && lines[0] === "SUPERVISOR_BUSY"; -} - -function isExactlyMissingManagedSupervisor(result: SandboxCommandResult | null): boolean { - if (result === null || result.status !== 1 || result.stdout.trim() !== "") return false; - const lines = result.stderr - .split(/\r?\n/) - .map((line) => line.trim()) - .filter(Boolean); - return lines.length === 1 && lines[0] === "SUPERVISOR_NOT_RUNNING"; + return lines.length === 1 && lines[0] === marker; } -function isExactlyPendingManagedSupervisorControl(result: SandboxCommandResult | null): boolean { - if (result === null || result.status !== 1 || result.stdout.trim() !== "") return false; - const lines = result.stderr - .split(/\r?\n/) - .map((line) => line.trim()) - .filter(Boolean); - return lines.length === 1 && lines[0] === "PRIVILEGED_CONTROL_UNAVAILABLE"; -} - -function isExactlyPendingManagedSupervisorDiscovery(result: SandboxCommandResult | null): boolean { - if (result === null || result.status !== 1 || result.stdout.trim() !== "") return false; - const lines = result.stderr - .split(/\r?\n/) - .map((line) => line.trim()) - .filter(Boolean); - // The installed managed controller emits this only before selecting a - // supervisor, when an incomplete process-table scan cannot yet prove either - // one exact supervisor or clean absence. Retrying this exact bare marker can - // delay recovery but cannot authorize a relaunch or accept an identity. - return lines.length === 1 && lines[0] === "SUPERVISOR_DISCOVERY_PENDING"; -} - -function isExactlyPendingManagedGatewayHealth(result: SandboxCommandResult | null): boolean { - if (result === null || result.status !== 1 || result.stdout.trim() !== "") return false; - const lines = result.stderr - .split(/\r?\n/) - .map((line) => line.trim()) - .filter(Boolean); - // This waiter only performs read-only probes before snapshot state is - // applied. A bare health timeout means the proven managed gateway is still - // starting; any diagnostic or refusal beside it remains terminal. - return lines.length === 1 && lines[0] === "GATEWAY_HEALTH_TIMEOUT"; +function isExactlyRetryableManagedRecoveryFailure(result: SandboxCommandResult | null): boolean { + return isExactlyManagedControlMarker(result, "SUPERVISOR_BUSY"); } function isExactlyManagedGatewayStartupTransition( result: ManagedGatewaySupervisorActionResult | null, ): boolean { - return ( - isExactlyMissingManagedSupervisor(result) || - isExactlyPendingManagedSupervisorControl(result) || - isExactlyPendingManagedSupervisorDiscovery(result) || - isExactlyPendingManagedGatewayHealth(result) - ); + // Discovery pending is emitted only before the controller selects a + // supervisor, so it can delay recovery but cannot authorize relaunch or + // accept an identity. The health timeout is likewise a read-only startup + // observation. Any diagnostic beside an exact marker remains terminal. + return [ + "SUPERVISOR_NOT_RUNNING", + "PRIVILEGED_CONTROL_UNAVAILABLE", + "SUPERVISOR_DISCOVERY_PENDING", + "GATEWAY_HEALTH_TIMEOUT", + ].some((marker) => isExactlyManagedControlMarker(result, marker)); } function isExactlyRetryableManagedControlTransition( @@ -672,7 +640,13 @@ export function confirmRecoveredSandboxGatewayManaged( options.requestGatewaySupervisorActionImpl ?? executeGatewaySupervisorAction; const result = requestGatewaySupervisorAction(sandboxName, "probe"); if (hasGatewayRecoveryMarker(result)) return true; - if (result === null || isExactlyRetryableManagedRecoveryFailure(result)) return null; + if ( + result === null || + isExactlyRetryableManagedRecoveryFailure(result) || + isExactlyManagedControlMarker(result, "SUPERVISOR_DISCOVERY_PENDING") + ) { + return null; + } return false; } @@ -772,7 +746,7 @@ function recoverSandboxProcesses( onFailureLayer?.(failure.layer, failure.detail); if ( failure.layer === "supervisor not running" && - isExactlyMissingManagedSupervisor(execResult) + isExactlyManagedControlMarker(execResult, "SUPERVISOR_NOT_RUNNING") ) { const relaunch = relaunchManagedSupervisorSessionImpl(sandboxName, { quiet, @@ -788,8 +762,9 @@ function recoverSandboxProcesses( typeof options?.timeout === "number" ? options.timeout : OPENSHELL_PROBE_TIMEOUT_MS, }).output, confirmMissingSupervisor: (containerId) => - isExactlyMissingManagedSupervisor( + isExactlyManagedControlMarker( requestPinnedGatewaySupervisorAction(sandboxName, "probe", 210000, containerId), + "SUPERVISOR_NOT_RUNNING", ), restartRestoredManagedGateway: (containerId) => { const restarted = parseManagedGatewayControlCompletion( diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index 36f39fe2606..e0c89ef0e17 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -289,7 +289,7 @@ describe("finalizeDockerGpuPatchBackup", () => { dockerRun, runCaptureOpenshell: vi .fn() - .mockReturnValueOnce("restored-name 2026-08-23 01:40:35 Error\n") + .mockReturnValueOnce("retiring-name 2026-08-23 01:40:35 Error\n") .mockReturnValue("restored-name 2026-08-23 01:40:37 Ready\n"), runOpenshell: vi.fn(() => ({ status: 0 })), sleep: vi.fn(), @@ -302,17 +302,6 @@ describe("finalizeDockerGpuPatchBackup", () => { finalHandoffAcknowledged: true, }); expect(dockerRun.mock.calls[0]?.[0]).toEqual([ - "ps", - "-a", - "--no-trunc", - "--filter", - `id=${result.newContainerId}`, - "--filter", - "label=openshell.ai/managed-by=openshell", - "--format", - "{{.ID}}", - ]); - expect(dockerRun.mock.calls[1]?.[0]).toEqual([ "inspect", "--type", "container", @@ -320,7 +309,7 @@ describe("finalizeDockerGpuPatchBackup", () => { '{{ index .Config.Labels "openshell.ai/sandbox-namespace" }}', result.newContainerId, ]); - expect(dockerRun.mock.calls[2]?.[0]).toEqual([ + expect(dockerRun.mock.calls[1]?.[0]).toEqual([ "ps", "-a", "--no-trunc", @@ -333,6 +322,14 @@ describe("finalizeDockerGpuPatchBackup", () => { "--format", "{{.ID}}", ]); + expect(dockerRun.mock.calls[2]?.[0]).toEqual([ + "inspect", + "--type", + "container", + "--format", + "{{json .State.Running}}", + result.newContainerId, + ]); }); it("rejects multiple same-name containers within the replacement gateway namespace", () => { @@ -376,17 +373,35 @@ describe("finalizeDockerGpuPatchBackup", () => { }); }); - it("captures the retiring lifecycle row before restarting the exact replacement (#10153)", () => { + it("waits for captured name absence before restarting the exact replacement (#10153)", () => { const result = exactDeferredCreateResult(); + const events: string[] = []; const dockerRunResults = { inspect: { status: 0, stdout: "true\n" }, ps: { status: 0, stdout: `${result.newContainerId}\n` }, } as const; - const dockerStart = vi.fn(() => ({ status: 0 })); + const dockerStart = vi.fn(() => { + events.push("start replacement"); + return { status: 0 }; + }); const runCaptureOpenshell = vi .fn() - .mockReturnValueOnce("alpha 2026-08-23 10:00:00 Error\n") - .mockReturnValueOnce("alpha 2026-08-23 10:00:02 Ready\n"); + .mockImplementationOnce(() => { + events.push("observe error"); + return "alpha 2026-08-23 10:00:00 Error\n"; + }) + .mockImplementationOnce(() => { + events.push("observe deleting"); + return "alpha 2026-08-23 10:00:02 Deleting\n"; + }) + .mockImplementationOnce(() => { + events.push("observe lifecycle release"); + return "beta 2026-08-23 10:00:04 Ready\n"; + }) + .mockImplementationOnce(() => { + events.push("observe replacement ready"); + return "alpha 2026-08-23 10:00:06 Ready\n"; + }); const runOpenshell = vi.fn(() => ({ status: 0 })); const outcome = finalizeDockerGpuPatchBackup( @@ -394,7 +409,7 @@ describe("finalizeDockerGpuPatchBackup", () => { result, supervisorReady: true, sandboxName: "alpha", - finalHandoffTimeoutSecs: 1, + finalHandoffTimeoutSecs: 4, }, { dockerStop: vi.fn(() => ({ status: 0 })), @@ -420,23 +435,26 @@ describe("finalizeDockerGpuPatchBackup", () => { result.newContainerId, expect.objectContaining({ ignoreError: true }), ); + expect(events).toEqual([ + "observe error", + "observe deleting", + "observe lifecycle release", + "start replacement", + "observe replacement ready", + ]); expect(runOpenshell).not.toHaveBeenCalledWith( ["sandbox", "list"], expect.any(Object), ); }); - it.each([ - ["a failed query", { status: 1, stderr: "daemon unavailable" }], - ["no labeled replacement", { status: 0, stdout: "" }], - ["a different replacement", { status: 0, stdout: `${"c".repeat(64)}\n` }], - [ - "multiple labeled replacements", - { status: 0, stdout: `${"b".repeat(64)}\n${"c".repeat(64)}\n` }, - ], - ])("rejects retiring Error when the canonical query returns %s (#9531)", (_case, query) => { + it("does not restart while captured lifecycle rows still name the sandbox (#10153)", () => { const result = exactDeferredCreateResult(); const dockerStart = vi.fn(() => ({ status: 0 })); + const runCaptureOpenshell = vi + .fn() + .mockReturnValueOnce("alpha 2026-08-23 01:40:35 Error\n") + .mockReturnValue("alpha 2026-08-23 01:40:37 Deleting\n"); const outcome = finalizeDockerGpuPatchBackup( { @@ -448,9 +466,9 @@ describe("finalizeDockerGpuPatchBackup", () => { { dockerStop: vi.fn(() => ({ status: 0 })), dockerRm: vi.fn(() => ({ status: 0 })), - dockerRun: vi.fn(() => query), + dockerRun: vi.fn(), dockerStart, - runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 01:40:35 Error\n"), + runCaptureOpenshell, runOpenshell: vi.fn(() => ({ status: 0 })), sleep: vi.fn(), }, @@ -462,6 +480,7 @@ describe("finalizeDockerGpuPatchBackup", () => { replacementRestarted: false, }); expect(dockerStart).not.toHaveBeenCalled(); + expect(runCaptureOpenshell).toHaveBeenCalledTimes(2); }); it("rolls back to the backup container when supervisor reconnect failed", () => { diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index 2939ec6ed62..c9e3d7929e3 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -40,8 +40,6 @@ import { waitForOpenShellSandboxLifecycleRelease, } from "./docker-gpu-supervisor-reconnect"; import { - OPENSHELL_MANAGED_BY_LABEL, - OPENSHELL_MANAGED_BY_VALUE, OPENSHELL_SANDBOX_NAMESPACE_LABEL, queryOpenShellDockerSandboxContainers, } from "./openshell-docker-sandbox-containers"; @@ -77,45 +75,6 @@ export type DockerGpuPatchFinalizeOutcome = { replacementPresence?: "absent" | "present" | "unknown"; }; -function isExactOpenShellReplacement( - replacementContainerId: string, - dockerRun: NonNullable, - timeoutMs: number, -): boolean { - const expectedContainerId = fullDockerContainerId(replacementContainerId); - if (!expectedContainerId || timeoutMs <= 0) return false; - try { - const query = dockerRun( - [ - "ps", - "-a", - "--no-trunc", - "--filter", - `id=${expectedContainerId}`, - "--filter", - `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, - "--format", - "{{.ID}}", - ], - { - ignoreError: true, - suppressOutput: true, - timeout: Math.max(1, Math.min(DOCKER_GPU_PATCH_TIMEOUT_MS, Math.floor(timeoutMs))), - }, - ); - if (!hasZeroDockerExitStatus(query)) return false; - const containerIds = String(query.stdout ?? "") - .split(/\r?\n/u) - .map((line) => line.trim()) - .filter(Boolean); - return ( - containerIds.length === 1 && fullDockerContainerId(containerIds[0]) === expectedContainerId - ); - } catch { - return false; - } -} - function isExactRunningReplacement( sandboxName: string, replacementContainerId: string, @@ -247,12 +206,6 @@ export function finalizeDockerGpuPatchBackup( { runCaptureOpenshell: deps.runCaptureOpenshell, sleep: deps.sleep, - soleLabeledReplacementCorroboratesRetiringPhase: (remainingMs) => - isExactOpenShellReplacement( - options.result.newContainerId, - resolved.dockerRun, - remainingMs, - ), }, ); if (!lifecycleReleaseObserved) { diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts index 503149498f7..31a11d700ea 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts @@ -12,20 +12,19 @@ import { } from "./docker-gpu-supervisor-reconnect"; describe("Docker GPU final lifecycle release", () => { - it("requires corroboration for a retiring lifecycle row (#9962)", () => { - const corroborate = vi.fn(() => true); - const runCaptureOpenshell = vi.fn( - () => "alpha 2026-08-23 01:40:35 Deleting\n", - ); + it.each([ + ["an explicit empty list", "No sandboxes found.\n"], + ["another phase-bearing sandbox", "beta 2026-08-21 05:53:18 Ready\n"], + ])("accepts %s as a captured release receipt (#9531)", (_receipt, output) => { + const runCaptureOpenshell = vi.fn(() => output); expect( waitForOpenShellSandboxLifecycleRelease("alpha", 1, { runCaptureOpenshell, sleep: vi.fn(), - soleLabeledReplacementCorroboratesRetiringPhase: corroborate, }), ).toBe(true); - expect(corroborate).toHaveBeenCalledOnce(); + expect(runCaptureOpenshell).toHaveBeenCalledOnce(); expect(runCaptureOpenshell).toHaveBeenCalledWith( ["sandbox", "list"], expect.objectContaining({ @@ -36,8 +35,18 @@ describe("Docker GPU final lifecycle release", () => { ); }); - it("does not accept an uncorroborated retiring lifecycle row (#9962)", () => { - const runCaptureOpenshell = vi.fn(() => "alpha 2026-08-23 01:40:35 Error\n"); + it.each([ + ["a header", "NAME CREATED PHASE\n"], + ["a gateway error", "Error: gateway unavailable\n"], + ["a phase-free row", "beta 2026-08-21 05:53:18\n"], + ["an unrecognized phase", "beta 2026-08-21 05:53:18 Retiring\n"], + ["the selected sandbox in Deleting", "alpha 2026-08-21 05:53:18 Deleting\n"], + ["the selected sandbox in Ready", "alpha 2026-08-21 05:53:18 Ready\n"], + ["the selected sandbox in Provisioning", "alpha 2026-08-21 05:53:18 Provisioning\n"], + ["the selected sandbox in Error", "alpha 2026-08-21 05:53:18 Error\n"], + ["the selected sandbox in Failed", "alpha 2026-08-21 05:53:18 Failed\n"], + ])("rejects %s as a captured release receipt (#9531)", (_case, output) => { + const runCaptureOpenshell = vi.fn(() => output); expect( waitForOpenShellSandboxLifecycleRelease("alpha", 1, { @@ -45,21 +54,21 @@ describe("Docker GPU final lifecycle release", () => { sleep: vi.fn(), }), ).toBe(false); + expect(runCaptureOpenshell).toHaveBeenCalledTimes(2); }); it("does not release the lifecycle when the captured sandbox list is unavailable (#10153)", () => { - const corroborate = vi.fn(() => true); + const runCaptureOpenshell = vi.fn(() => { + throw new Error("sandbox list transport unavailable"); + }); expect( waitForOpenShellSandboxLifecycleRelease("alpha", 1, { - runCaptureOpenshell: vi.fn(() => { - throw new Error("sandbox list transport unavailable"); - }), + runCaptureOpenshell, sleep: vi.fn(), - soleLabeledReplacementCorroboratesRetiringPhase: corroborate, }), ).toBe(false); - expect(corroborate).not.toHaveBeenCalled(); + expect(runCaptureOpenshell).toHaveBeenCalledTimes(2); }); it("does not report reconnect without an OpenShell execution boundary (#9531)", () => { @@ -218,7 +227,10 @@ describe("docker-gpu-supervisor-reconnect Error-phase debounce", () => { it("short-circuits the supervisor-reconnect wait when the sandbox enters Error phase", () => { const runOpenshell = vi.fn(() => ({ status: 1, stderr: "sandbox not ready" })); - const listOutputs = ["alpha Provisioning 1s ago", "alpha Error 3s ago"]; + const listOutputs = [ + "alpha Provisioning 1s ago", + "alpha \u001b[31mError\u001b[0m 3s ago", + ]; let index = 0; const runCaptureOpenshell = vi.fn(() => listOutputs[Math.min(index++, listOutputs.length - 1)]); const sleep = vi.fn(); diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts index bad9cd26943..80ccb12ccb3 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts @@ -76,10 +76,7 @@ export type DockerGpuSupervisorReconnectDeps = { type DockerLifecycleReleaseDeps = Pick< DockerGpuSupervisorReconnectDeps, "runCaptureOpenshell" | "sleep" -> & { - /** Corroborates a retiring lifecycle row with the stopped exact replacement. */ - soleLabeledReplacementCorroboratesRetiringPhase?: (remainingMs: number) => boolean; -}; +>; type DockerFinalHandoffDeps = Required< Pick @@ -106,8 +103,9 @@ const PROCESS_TREE_BOUNDED_OPENSHELL_OPTIONS = { /** * Wait for OpenShell to retire the pre-replacement lifecycle record before - * restarting the replacement. A retiring Error or Deleting row is accepted - * only with an identity-bound Docker corroboration. + * restarting the replacement. The selected sandbox name must be absent from + * a phase-bearing list receipt because a row cannot prove which container + * owns the lifecycle, even when Docker still exposes the exact replacement. */ export function waitForOpenShellSandboxLifecycleRelease( sandboxName: string, @@ -140,20 +138,8 @@ export function waitForOpenShellSandboxLifecycleRelease( if (output) { const entries = parseLiveSandboxEntries(output); const sandboxPresent = entries.some((entry) => entry.name === sandboxName); - const retiring = entries.some( - (entry) => - entry.name === sandboxName && (entry.phase === "Error" || entry.phase === "Deleting"), - ); - const corroborated = - retiring && - deadline - Date.now() > 0 && - deps.soleLabeledReplacementCorroboratesRetiringPhase?.(deadline - Date.now()) === true; const explicitEmptyList = output === "No sandboxes found" || output === "No sandboxes found."; - if ( - explicitEmptyList || - corroborated || - (entries.some((entry) => entry.phase !== null) && !sandboxPresent) - ) { + if (explicitEmptyList || (entries.some((entry) => entry.phase !== null) && !sandboxPresent)) { return true; } } @@ -263,19 +249,6 @@ function defaultSleep(seconds: number): void { Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Math.max(0, seconds) * 1000); } -const ANSI_RE = /\x1b\[[0-9;]*m/g; - -function parseSandboxListFailurePhase(output: string, sandboxName: string): string | null { - if (typeof output !== "string" || !output.includes(sandboxName)) return null; - for (const line of output.replace(ANSI_RE, "").split(/\r?\n/)) { - const cols = line.trim().split(/\s+/); - if (cols[0] === sandboxName) { - return cols.find((col) => TERMINAL_SANDBOX_FAILURE_PHASES.has(col)) ?? null; - } - } - return null; -} - function sandboxListShowsErrorPhase( sandboxName: string, runCaptureOpenshell: RunCaptureOpenshellFn, @@ -287,7 +260,12 @@ function sandboxListShowsErrorPhase( suppressOutput: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, }); - return parseSandboxListFailurePhase(list, sandboxName) !== null; + return parseLiveSandboxEntries(list).some( + (entry) => + entry.name === sandboxName && + entry.phase !== null && + TERMINAL_SANDBOX_FAILURE_PHASES.has(entry.phase), + ); } catch { return false; } diff --git a/test/e2e/live/gateway-guard-legacy-keepalive-fixture.ts b/test/e2e/live/gateway-guard-legacy-keepalive-fixture.ts index 6198ac1cffa..dc77d766756 100644 --- a/test/e2e/live/gateway-guard-legacy-keepalive-fixture.ts +++ b/test/e2e/live/gateway-guard-legacy-keepalive-fixture.ts @@ -98,26 +98,40 @@ function requireFixtureInput(condition: boolean, message: string): asserts condi if (!condition) throw new Error(message); } -/** Read the final machine receipt without treating recreation progress as JSON. */ +function isLegacyKeepaliveHandoffReceiptCandidate( + value: unknown, +): value is Record { + return ( + typeof value === "object" && + value !== null && + !Array.isArray(value) && + ["oldContainerId", "newContainerId", "startupCommand"].some((key) => + Object.hasOwn(value, key), + ) + ); +} + +/** Read one final machine receipt without treating recreation progress as JSON. */ export function parseLegacyKeepaliveHandoffReceipt( output: string, ): LegacyKeepaliveHandoffReceipt { - const receiptLine = output + const lines = output .split(/\r?\n/u) .map((line) => line.trim()) - .filter(Boolean) - .at(-1); - let parsed: unknown; - try { - parsed = JSON.parse(receiptLine ?? ""); - } catch { - throw new Error("legacy keepalive fixture did not emit a final JSON handoff receipt"); - } + .filter(Boolean); + const receiptCandidates = lines.flatMap((line, index) => { + try { + const value: unknown = JSON.parse(line); + return isLegacyKeepaliveHandoffReceiptCandidate(value) ? [{ index, value }] : []; + } catch { + return []; + } + }); requireFixtureInput( - typeof parsed === "object" && parsed !== null && !Array.isArray(parsed), - "legacy keepalive fixture handoff receipt must be an object", + receiptCandidates.length === 1 && receiptCandidates[0]?.index === lines.length - 1, + "legacy keepalive fixture must emit exactly one final JSON handoff receipt", ); - const receipt = parsed as Record; + const receipt = receiptCandidates[0].value; requireFixtureInput( DOCKER_CONTAINER_ID_PATTERN.test(String(receipt.oldContainerId ?? "")) && DOCKER_CONTAINER_ID_PATTERN.test(String(receipt.newContainerId ?? "")) && diff --git a/test/e2e/support/gateway-guard-legacy-keepalive-fixture.test.ts b/test/e2e/support/gateway-guard-legacy-keepalive-fixture.test.ts index 7ea8f360d3e..2c223b7038f 100644 --- a/test/e2e/support/gateway-guard-legacy-keepalive-fixture.test.ts +++ b/test/e2e/support/gateway-guard-legacy-keepalive-fixture.test.ts @@ -150,6 +150,21 @@ describe("gateway guard legacy keepalive fixture", () => { it.each([ { name: "missing receipt", output: "Waiting for handoff\n" }, + { + name: "conflicting receipts", + output: [ + JSON.stringify({ + oldContainerId: OLD_CONTAINER_ID, + newContainerId: NEW_CONTAINER_ID, + startupCommand: "sleep infinity", + }), + JSON.stringify({ + oldContainerId: NEW_CONTAINER_ID, + newContainerId: "c".repeat(64), + startupCommand: "sleep infinity", + }), + ].join("\n"), + }, { name: "unchanged container identity", output: JSON.stringify({ @@ -168,7 +183,7 @@ describe("gateway guard legacy keepalive fixture", () => { }, ])("rejects $name in the handoff receipt", ({ output }) => { expect(() => parseLegacyKeepaliveHandoffReceipt(output)).toThrow( - /final JSON handoff receipt|handoff receipt is invalid/u, + /exactly one final JSON handoff receipt|handoff receipt is invalid/u, ); }); From 403b25d1104efbcb29c586d08fd6315ae851cb25 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 26 Aug 2026 16:45:56 -0700 Subject: [PATCH 07/12] test(recovery): cover lifecycle deletion sequence Signed-off-by: Prekshi Vyas --- .../docker-final-handoff-lifecycle.test.ts | 147 ++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 test/onboarding/docker-final-handoff-lifecycle.test.ts diff --git a/test/onboarding/docker-final-handoff-lifecycle.test.ts b/test/onboarding/docker-final-handoff-lifecycle.test.ts new file mode 100644 index 00000000000..232cc16a415 --- /dev/null +++ b/test/onboarding/docker-final-handoff-lifecycle.test.ts @@ -0,0 +1,147 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { createDockerGpuInspectFixture } from "../../src/lib/onboard/__test-helpers__/docker-gpu-patch-fixtures"; +import { getDockerGpuPatchFailureContext } from "../../src/lib/onboard/docker-gpu-patch"; +import { recreateOpenShellDockerSandboxWithStartupCommand } from "../../src/lib/onboard/docker-startup-command-patch"; + +const OLD_CONTAINER_ID = "a".repeat(64); +const NEW_CONTAINER_ID = "b".repeat(64); + +function dockerCaptureFixture() { + const inspect = createDockerGpuInspectFixture(); + inspect.Id = OLD_CONTAINER_ID; + return vi.fn((args: readonly string[]) => { + if (args[0] === "ps") return `${OLD_CONTAINER_ID}\n`; + if (args[0] === "inspect") return JSON.stringify([inspect]); + return ""; + }); +} + +function dockerRunFixture() { + return vi.fn((args: readonly string[]) => { + if (args[0] === "ps") return { status: 0, stdout: `${NEW_CONTAINER_ID}\n` }; + if (args[0] === "inspect" && String(args[4]).includes("sandbox-namespace")) { + return { status: 0, stdout: "current-gateway\n" }; + } + return { status: 0, stdout: "true\n" }; + }); +} + +describe("Docker final handoff lifecycle integration", () => { + it("restarts only after Error and Deleting release the selected sandbox name (#10153)", () => { + const events: string[] = []; + const dockerStart = vi.fn(() => { + events.push("start replacement"); + return { status: 0 }; + }); + const runCaptureOpenshell = vi + .fn() + .mockImplementationOnce(() => { + events.push("observe error"); + return "alpha 2026-08-23 10:00:00 Error\n"; + }) + .mockImplementationOnce(() => { + events.push("observe deleting"); + return "alpha 2026-08-23 10:00:02 Deleting\n"; + }) + .mockImplementationOnce(() => { + events.push("observe name absence"); + return "beta 2026-08-23 10:00:04 Ready\n"; + }) + .mockImplementationOnce(() => { + events.push("observe replacement ready"); + return "alpha 2026-08-23 10:00:06 Ready\n"; + }); + + const result = recreateOpenShellDockerSandboxWithStartupCommand( + { + sandboxName: "alpha", + expectedOldContainerId: OLD_CONTAINER_ID, + openshellSandboxCommand: ["sleep", "infinity"], + timeoutSecs: 4, + }, + { + dockerCapture: dockerCaptureFixture(), + dockerRun: dockerRunFixture(), + dockerRunDetached: vi.fn(() => ({ status: 0, stdout: `${NEW_CONTAINER_ID}\n` })), + dockerRename: vi.fn(() => ({ status: 0 })), + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + dockerStart, + runCaptureOpenshell, + runOpenshell: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), + now: () => new Date("2026-05-12T00:00:00Z"), + detectSandboxFallbackDns: vi.fn(() => null), + readDir: vi.fn(() => null), + readFile: vi.fn(() => null), + }, + ); + + expect(result).toMatchObject({ + backupRemoved: true, + mode: { kind: "startup-command" }, + newContainerId: NEW_CONTAINER_ID, + oldContainerId: OLD_CONTAINER_ID, + }); + expect(events).toEqual([ + "observe error", + "observe deleting", + "observe name absence", + "start replacement", + "observe replacement ready", + ]); + }); + + it("never restarts when Error advances to Deleting without a name-absence receipt (#10153)", () => { + const dockerStart = vi.fn(() => ({ status: 0 })); + const runCaptureOpenshell = vi + .fn() + .mockReturnValueOnce("alpha 2026-08-23 10:00:00 Error\n") + .mockReturnValue("alpha 2026-08-23 10:00:02 Deleting\n"); + let failure: unknown; + + try { + recreateOpenShellDockerSandboxWithStartupCommand( + { + sandboxName: "alpha", + expectedOldContainerId: OLD_CONTAINER_ID, + openshellSandboxCommand: ["sleep", "infinity"], + timeoutSecs: 1, + }, + { + dockerCapture: dockerCaptureFixture(), + dockerRun: dockerRunFixture(), + dockerRunDetached: vi.fn(() => ({ status: 0, stdout: `${NEW_CONTAINER_ID}\n` })), + dockerRename: vi.fn(() => ({ status: 0 })), + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + dockerStart, + runCaptureOpenshell, + runOpenshell: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), + now: () => new Date("2026-05-12T00:00:00Z"), + detectSandboxFallbackDns: vi.fn(() => null), + readDir: vi.fn(() => null), + readFile: vi.fn(() => null), + }, + ); + } catch (error) { + failure = error; + } + + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toContain("final replacement handoff"); + expect(getDockerGpuPatchFailureContext(failure)).toMatchObject({ + backupRemoved: true, + newContainerId: NEW_CONTAINER_ID, + oldContainerId: OLD_CONTAINER_ID, + rolledBack: false, + }); + expect(runCaptureOpenshell).toHaveBeenCalledTimes(2); + expect(dockerStart).not.toHaveBeenCalled(); + }); +}); From 92b5c5f1373fa0d417fd8516d5bedd78d9bca3bb Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 26 Aug 2026 16:45:56 -0700 Subject: [PATCH 08/12] test(recovery): cover lifecycle deletion sequence Signed-off-by: Prekshi Vyas --- .../docker-gpu-patch-fixtures.ts | 24 ++++ .../docker-final-handoff-lifecycle.test.ts | 130 ++++++++++++++++++ 2 files changed, 154 insertions(+) create mode 100644 test/onboarding/docker-final-handoff-lifecycle.test.ts diff --git a/src/lib/onboard/__test-helpers__/docker-gpu-patch-fixtures.ts b/src/lib/onboard/__test-helpers__/docker-gpu-patch-fixtures.ts index 0c0f3f0a89a..55fd09a79d6 100644 --- a/src/lib/onboard/__test-helpers__/docker-gpu-patch-fixtures.ts +++ b/src/lib/onboard/__test-helpers__/docker-gpu-patch-fixtures.ts @@ -67,6 +67,30 @@ export function createDockerGpuInspectFixture(): DockerContainerInspect { }; } +export function createDockerFinalHandoffCaptureFixture( + oldContainerId: string, +): (args: readonly string[]) => string { + const inspect = createDockerGpuInspectFixture(); + inspect.Id = oldContainerId; + return (args) => { + if (args[0] === "ps") return `${oldContainerId}\n`; + if (args[0] === "inspect") return JSON.stringify([inspect]); + return ""; + }; +} + +export function createDockerFinalHandoffRunFixture( + newContainerId: string, +): (args: readonly string[]) => { status: number; stdout: string } { + return (args) => { + if (args[0] === "ps") return { status: 0, stdout: `${newContainerId}\n` }; + if (args[0] === "inspect" && String(args[4]).includes("sandbox-namespace")) { + return { status: 0, stdout: "current-gateway\n" }; + } + return { status: 0, stdout: "true\n" }; + }; +} + export function createDockerGpuDnsInspectFixture(): DockerContainerInspect { return { Id: "old-container-id", diff --git a/test/onboarding/docker-final-handoff-lifecycle.test.ts b/test/onboarding/docker-final-handoff-lifecycle.test.ts new file mode 100644 index 00000000000..4af946f5dfc --- /dev/null +++ b/test/onboarding/docker-final-handoff-lifecycle.test.ts @@ -0,0 +1,130 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + createDockerFinalHandoffCaptureFixture, + createDockerFinalHandoffRunFixture, +} from "../../src/lib/onboard/__test-helpers__/docker-gpu-patch-fixtures"; +import { getDockerGpuPatchFailureContext } from "../../src/lib/onboard/docker-gpu-patch"; +import { recreateOpenShellDockerSandboxWithStartupCommand } from "../../src/lib/onboard/docker-startup-command-patch"; + +const OLD_CONTAINER_ID = "a".repeat(64); +const NEW_CONTAINER_ID = "b".repeat(64); + +describe("Docker final handoff lifecycle integration", () => { + it("restarts only after Error and Deleting release the selected sandbox name (#10153)", () => { + const events: string[] = []; + const dockerStart = vi.fn(() => { + events.push("start replacement"); + return { status: 0 }; + }); + const runCaptureOpenshell = vi + .fn() + .mockImplementationOnce(() => { + events.push("observe error"); + return "alpha 2026-08-23 10:00:00 Error\n"; + }) + .mockImplementationOnce(() => { + events.push("observe deleting"); + return "alpha 2026-08-23 10:00:02 Deleting\n"; + }) + .mockImplementationOnce(() => { + events.push("observe name absence"); + return "beta 2026-08-23 10:00:04 Ready\n"; + }) + .mockImplementationOnce(() => { + events.push("observe replacement ready"); + return "alpha 2026-08-23 10:00:06 Ready\n"; + }); + + const result = recreateOpenShellDockerSandboxWithStartupCommand( + { + sandboxName: "alpha", + expectedOldContainerId: OLD_CONTAINER_ID, + openshellSandboxCommand: ["sleep", "infinity"], + timeoutSecs: 4, + }, + { + dockerCapture: vi.fn(createDockerFinalHandoffCaptureFixture(OLD_CONTAINER_ID)), + dockerRun: vi.fn(createDockerFinalHandoffRunFixture(NEW_CONTAINER_ID)), + dockerRunDetached: vi.fn(() => ({ status: 0, stdout: `${NEW_CONTAINER_ID}\n` })), + dockerRename: vi.fn(() => ({ status: 0 })), + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + dockerStart, + runCaptureOpenshell, + runOpenshell: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), + now: () => new Date("2026-05-12T00:00:00Z"), + detectSandboxFallbackDns: vi.fn(() => null), + readDir: vi.fn(() => null), + readFile: vi.fn(() => null), + }, + ); + + expect(result).toMatchObject({ + backupRemoved: true, + mode: { kind: "startup-command" }, + newContainerId: NEW_CONTAINER_ID, + oldContainerId: OLD_CONTAINER_ID, + }); + expect(events).toEqual([ + "observe error", + "observe deleting", + "observe name absence", + "start replacement", + "observe replacement ready", + ]); + }); + + it("never restarts when Error advances to Deleting without a name-absence receipt (#10153)", () => { + const dockerStart = vi.fn(() => ({ status: 0 })); + const runCaptureOpenshell = vi + .fn() + .mockReturnValueOnce("alpha 2026-08-23 10:00:00 Error\n") + .mockReturnValue("alpha 2026-08-23 10:00:02 Deleting\n"); + let failure: unknown; + + try { + recreateOpenShellDockerSandboxWithStartupCommand( + { + sandboxName: "alpha", + expectedOldContainerId: OLD_CONTAINER_ID, + openshellSandboxCommand: ["sleep", "infinity"], + timeoutSecs: 1, + }, + { + dockerCapture: vi.fn(createDockerFinalHandoffCaptureFixture(OLD_CONTAINER_ID)), + dockerRun: vi.fn(createDockerFinalHandoffRunFixture(NEW_CONTAINER_ID)), + dockerRunDetached: vi.fn(() => ({ status: 0, stdout: `${NEW_CONTAINER_ID}\n` })), + dockerRename: vi.fn(() => ({ status: 0 })), + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + dockerStart, + runCaptureOpenshell, + runOpenshell: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), + now: () => new Date("2026-05-12T00:00:00Z"), + detectSandboxFallbackDns: vi.fn(() => null), + readDir: vi.fn(() => null), + readFile: vi.fn(() => null), + }, + ); + } catch (error) { + failure = error; + } + + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toContain("final replacement handoff"); + expect(getDockerGpuPatchFailureContext(failure)).toMatchObject({ + backupRemoved: true, + newContainerId: NEW_CONTAINER_ID, + oldContainerId: OLD_CONTAINER_ID, + rolledBack: false, + }); + expect(runCaptureOpenshell).toHaveBeenCalledTimes(2); + expect(dockerStart).not.toHaveBeenCalled(); + }); +}); From faa54ccc2def8a08f3778575716d9d127f4233a4 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 26 Aug 2026 20:22:34 -0700 Subject: [PATCH 09/12] test(onboard): capture lifecycle fixture output Signed-off-by: Prekshi Vyas --- test/helpers/onboard-script-mocks.cjs | 106 +++++++++++------- .../onboard-custom-dockerfile.test.ts | 2 +- ...oard-extra-provider-reconciliation.test.ts | 2 +- .../onboard-installer-restore-intent.test.ts | 2 +- test/onboarding/onboard-messaging.test.ts | 6 +- .../onboard-reservation-recreate.test.ts | 2 +- 6 files changed, 72 insertions(+), 48 deletions(-) diff --git a/test/helpers/onboard-script-mocks.cjs b/test/helpers/onboard-script-mocks.cjs index 8da382bc4dc..17d07d6dca6 100644 --- a/test/helpers/onboard-script-mocks.cjs +++ b/test/helpers/onboard-script-mocks.cjs @@ -629,59 +629,81 @@ function mockStandaloneGatewayTeardownAuthority() { function mockDockerSandboxLifecycleReleaseFromRunner() { const runner = require(path.resolve(__dirname, "../../src/lib/runner.ts")); - if (runner.run.__nemoclawDockerLifecycleFixture === true) return; - const run = runner.run; - let finalCommitReleased = false; - let lifecycleReleased = false; - const wrappedRun = (command, options) => { - const normalized = normalizeCommand(command); + const state = runner.run.__nemoclawDockerLifecycleState ?? { + finalCommitReleased: false, + lifecycleReleased: false, + replacementRestarted: false, + }; + const captureOutput = (normalized) => { if ( - finalCommitReleased && - ((normalized.startsWith("docker ps -a --no-trunc ") && - normalized.includes("label=openshell.ai/sandbox-name=my-assistant") && - normalized.endsWith("--format {{.ID}}")) || - normalized === - `docker inspect --type container --format {{ index .Config.Labels "openshell.ai/sandbox-namespace" }} ${ONBOARD_SANDBOX_NEW_CONTAINER_ID}`) + state.finalCommitReleased && + normalized.startsWith("docker ps -a --no-trunc ") && + normalized.includes("label=openshell.ai/sandbox-name=my-assistant") && + normalized.endsWith("--format {{.ID}}") ) { - return { - status: 0, - stdout: Buffer.from( - normalized.startsWith("docker inspect ") - ? "test-gateway\n" - : `${ONBOARD_SANDBOX_NEW_CONTAINER_ID}\n`, - ), - stderr: Buffer.alloc(0), - }; + return `${ONBOARD_SANDBOX_NEW_CONTAINER_ID}\n`; + } + if ( + state.finalCommitReleased && + normalized === + `docker inspect --type container --format {{ index .Config.Labels "openshell.ai/sandbox-namespace" }} ${ONBOARD_SANDBOX_NEW_CONTAINER_ID}` + ) { + return "test-gateway\n"; } if ( - finalCommitReleased && + state.finalCommitReleased && normalized === `docker inspect --type container --format {{json .State.Running}} ${ONBOARD_SANDBOX_NEW_CONTAINER_ID}` ) { - return { - status: 0, - stdout: Buffer.from("true\n"), - stderr: Buffer.alloc(0), - }; + return "true\n"; } - if (lifecycleReleased && normalized.includes("sandbox list")) { - return { - status: 0, - stdout: Buffer.from("No sandboxes found\n"), - stderr: Buffer.alloc(0), - }; + if (state.replacementRestarted && normalized.includes("sandbox list")) { + return "my-assistant 2026-08-27 Ready\n"; } - const result = run(command, options); - if (normalized.startsWith("docker rm ") && result?.status === 0) { - lifecycleReleased = true; - if (normalized === `docker rm ${ONBOARD_SANDBOX_OLD_CONTAINER_ID}`) { - finalCommitReleased = true; - } + if (state.lifecycleReleased && normalized.includes("sandbox list")) { + return "No sandboxes found\n"; } - return result; + return null; }; - wrappedRun.__nemoclawDockerLifecycleFixture = true; - runner.run = wrappedRun; + if (runner.run.__nemoclawDockerLifecycleFixture !== true) { + const run = runner.run; + const wrappedRun = (command, options) => { + const normalized = normalizeCommand(command); + const captured = captureOutput(normalized); + if (captured !== null) { + return { + status: 0, + stdout: Buffer.from(captured), + stderr: Buffer.alloc(0), + }; + } + const result = run(command, options); + if (normalized.startsWith("docker rm ") && result?.status === 0) { + state.lifecycleReleased = true; + if (normalized === `docker rm ${ONBOARD_SANDBOX_OLD_CONTAINER_ID}`) { + state.finalCommitReleased = true; + } + } + if ( + normalized === `docker start ${ONBOARD_SANDBOX_NEW_CONTAINER_ID}` && + result?.status === 0 + ) { + state.replacementRestarted = true; + } + return result; + }; + wrappedRun.__nemoclawDockerLifecycleFixture = true; + wrappedRun.__nemoclawDockerLifecycleState = state; + runner.run = wrappedRun; + } + if (runner.runCapture.__nemoclawDockerLifecycleFixture !== true) { + const runCapture = runner.runCapture; + const wrappedRunCapture = (command, options) => { + return captureOutput(normalizeCommand(command)) ?? runCapture(command, options); + }; + wrappedRunCapture.__nemoclawDockerLifecycleFixture = true; + runner.runCapture = wrappedRunCapture; + } } function mockFreshOpenClawPluginDiscovery() { diff --git a/test/onboarding/onboard-custom-dockerfile.test.ts b/test/onboarding/onboard-custom-dockerfile.test.ts index b1d970dc600..646bebfd0a9 100644 --- a/test/onboarding/onboard-custom-dockerfile.test.ts +++ b/test/onboarding/onboard-custom-dockerfile.test.ts @@ -238,7 +238,6 @@ runner.run = (command, opts = {}) => { ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; -fixtureMocks.mockDockerSandboxLifecycleReleaseFromRunner(); runner.runCapture = (command) => { const normalized = _n(command); if (normalized.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:8080"; @@ -256,6 +255,7 @@ runner.runCapture = (command) => { if (normalized.includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running"; return ""; }; +fixtureMocks.mockDockerSandboxLifecycleReleaseFromRunner(); const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { sandboxName: "my-assistant", provider: "openai-api", diff --git a/test/onboarding/onboard-extra-provider-reconciliation.test.ts b/test/onboarding/onboard-extra-provider-reconciliation.test.ts index 91a384920f3..4d9a0cc1174 100644 --- a/test/onboarding/onboard-extra-provider-reconciliation.test.ts +++ b/test/onboarding/onboard-extra-provider-reconciliation.test.ts @@ -89,7 +89,6 @@ runner.run = (command, opts = {}) => { ? { status: 0, stdout: Buffer.from("my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; -require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); runner.runCapture = (command) => { const normalized = _n(command); const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command); @@ -103,6 +102,7 @@ runner.runCapture = (command) => { } return ""; }; +require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; sandboxBaseImage.resolveSandboxBaseImage = () => ({ diff --git a/test/onboarding/onboard-installer-restore-intent.test.ts b/test/onboarding/onboard-installer-restore-intent.test.ts index 308eded935a..bf0bf2f39de 100644 --- a/test/onboarding/onboard-installer-restore-intent.test.ts +++ b/test/onboarding/onboard-installer-restore-intent.test.ts @@ -81,7 +81,6 @@ runner.run = (command) => { ? { status: 0, stdout: Buffer.from("my-assistant\nId: fixture-created-sandbox\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; -fixtureMocks.mockDockerSandboxLifecycleReleaseFromRunner(); runner.runCapture = (command) => { const cmd = _n(command); if (cmd.includes("gateway info")) return "Gateway endpoint: http://127.0.0.1:8080"; @@ -103,6 +102,7 @@ runner.runCapture = (command) => { } return ""; }; +fixtureMocks.mockDockerSandboxLifecycleReleaseFromRunner(); const sourceEntry = fixtureMocks.managedSandboxPolicyReceiptFixture({ name: "my-assistant", gpuEnabled: false, diff --git a/test/onboarding/onboard-messaging.test.ts b/test/onboarding/onboard-messaging.test.ts index 8222c02132d..7cec980e00d 100644 --- a/test/onboarding/onboard-messaging.test.ts +++ b/test/onboarding/onboard-messaging.test.ts @@ -541,7 +541,7 @@ runner.run = (command, opts = {}) => { if (refresh && gatewaySecrets.has(refresh)) { if (refresh === process.env.NEMOCLAW_TEST_FAIL_PROVIDER) return { status: 1 }; revisions.set(refresh, revisions.get(refresh) + 1); return { status: 0 }; } if (normalized.includes("provider get")) return { status: 1 }; return normalized.includes("sandbox get") && normalized.includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } : { status: 0 }; -}; require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); +}; runner.runCapture = (command) => { const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command); if (createdIdentity !== null) return createdIdentity; @@ -552,6 +552,7 @@ runner.runCapture = (command) => { if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running\nmy-assistant 127.0.0.1 8642 12346 running"; return ""; }; +require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); registry.registerSandbox = (entry) => { registered = entry; return true; }; registry.updateSandbox = () => true; registry.setDefault = () => true; registry.removeSandbox = () => true; const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { sandboxName: "my-assistant", @@ -718,7 +719,7 @@ runner.run = (command, opts = {}) => { if (normalized.includes("provider get -g nemoclaw my-assistant-telegram-bridge")) return { status: 0, stdout: "Name: my-assistant-telegram-bridge\nType: nemoclaw-mcp-v1\nCredential keys: TELEGRAM_BOT_TOKEN\nConfig keys: \n" }; if (normalized.includes("provider get")) return { status: 1 }; return normalized.includes("sandbox get") && normalized.includes("my-assistant") ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } : { status: 0 }; -}; require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); +}; runner.runCapture = (command) => { const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command); if (createdIdentity !== null) return createdIdentity; @@ -731,6 +732,7 @@ runner.runCapture = (command) => { if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running\nmy-assistant 127.0.0.1 8642 12346 running"; return ""; }; +require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); registry.registerSandbox = (entry) => { registerCalls.push(entry); return true; diff --git a/test/onboarding/onboard-reservation-recreate.test.ts b/test/onboarding/onboard-reservation-recreate.test.ts index 6ebb4d147a4..9653ac812f9 100644 --- a/test/onboarding/onboard-reservation-recreate.test.ts +++ b/test/onboarding/onboard-reservation-recreate.test.ts @@ -80,7 +80,6 @@ runner.run = (command) => { ? { status: 0, stdout: Buffer.from("Name: my-assistant\nId: sbx-4f2a91c0d7\n"), stderr: Buffer.alloc(0) } : { status: 0 }; }; -require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); runner.runCapture = (command) => { const cmd = _n(command); const createdIdentity = fixtureMocks.mockCreatedSandboxIdentityList(command); @@ -98,6 +97,7 @@ runner.runCapture = (command) => { } return ""; }; +require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); onboardSession.loadSession = () => ({ sessionId: "session-owner" }); From 3eb72e998bc1525ead7cd689460fc6bd77b5bd9f Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 26 Aug 2026 20:26:06 -0700 Subject: [PATCH 10/12] test(onboard): keep messaging fixture within budget Signed-off-by: Prekshi Vyas --- test/onboarding/onboard-messaging.test.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/onboarding/onboard-messaging.test.ts b/test/onboarding/onboard-messaging.test.ts index 7cec980e00d..9e81424fe92 100644 --- a/test/onboarding/onboard-messaging.test.ts +++ b/test/onboarding/onboard-messaging.test.ts @@ -551,8 +551,7 @@ runner.runCapture = (command) => { if (mockedCapture !== null) return mockedCapture; if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running\nmy-assistant 127.0.0.1 8642 12346 running"; return ""; -}; -require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); +}; require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); registry.registerSandbox = (entry) => { registered = entry; return true; }; registry.updateSandbox = () => true; registry.setDefault = () => true; registry.removeSandbox = () => true; const createFixture = fixtureMocks.installVerifiedSandboxCreateFixture(registry, { sandboxName: "my-assistant", @@ -731,8 +730,7 @@ runner.runCapture = (command) => { } if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running\nmy-assistant 127.0.0.1 8642 12346 running"; return ""; -}; -require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); +}; require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); registry.registerSandbox = (entry) => { registerCalls.push(entry); return true; From 6015a9386ffbdbe0735113835da58ea0ba12c51f Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 26 Aug 2026 21:05:21 -0700 Subject: [PATCH 11/12] test(onboard): cover messaging lifecycle capture Signed-off-by: Prekshi Vyas --- test/onboarding/onboard-messaging.test.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/onboarding/onboard-messaging.test.ts b/test/onboarding/onboard-messaging.test.ts index 9e81424fe92..514a881d63e 100644 --- a/test/onboarding/onboard-messaging.test.ts +++ b/test/onboarding/onboard-messaging.test.ts @@ -99,7 +99,7 @@ runner.runCapture = (command) => { if (mockedCapture !== null) return mockedCapture; } return ""; -}; +}; require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); registry.registerSandbox = () => true; registry.updateSandbox = () => true; registry.setDefault = () => true; @@ -382,7 +382,7 @@ runner.runCapture = (command) => { if (mockedCapture !== null) return mockedCapture; } return ""; -}; +}; require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); registry.registerSandbox = (entry) => { registeredSandbox = entry; return true; @@ -892,7 +892,7 @@ runner.runCapture = (command) => { } if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running\nmy-assistant 127.0.0.1 8642 12346 running"; return ""; -}; +}; require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); registry.registerSandbox = (entry) => { registerCalls.push(entry); return true; @@ -1062,7 +1062,7 @@ runner.runCapture = (command) => { } if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running\nmy-assistant 127.0.0.1 8642 12346 running"; return ""; -}; +}; require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); registry.registerSandbox = (entry) => { registerCalls.push(entry); return true; @@ -1206,7 +1206,7 @@ runner.runCapture = (command) => { if (_n(command).includes("sandbox get")) return ""; if (_n(command).includes("sandbox list")) return ""; return ""; -}; +}; require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); registry.registerSandbox = () => true; registry.updateSandbox = () => true; registry.setDefault = () => true; @@ -1398,7 +1398,7 @@ runner.runCapture = (command) => { } if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running\nmy-assistant 127.0.0.1 8642 12346 running"; return ""; -}; +}; require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); registry.registerSandbox = () => true; registry.updateSandbox = () => true; registry.setDefault = () => true; @@ -1539,7 +1539,7 @@ runner.runCapture = (command) => { } if (_n(command).includes("forward list")) return "my-assistant 127.0.0.1 18789 12345 running\nmy-assistant 127.0.0.1 8642 12346 running"; return ""; -}; +}; require(${onboardScriptMocksPath}).mockDockerSandboxLifecycleReleaseFromRunner(); registry.registerSandbox = () => true; registry.updateSandbox = () => true; registry.setDefault = () => true; From e68696f3c1f67defdec3b233b33f9b0fdddeba18 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 26 Aug 2026 22:40:44 -0700 Subject: [PATCH 12/12] fix(recovery): fence final handoff through OpenShell --- docs/reference/commands.mdx | 12 +- docs/reference/troubleshooting.mdx | 7 +- src/lib/actions/sandbox/process-recovery.ts | 8 +- .../onboard/docker-gpu-patch-finalize.test.ts | 211 +++++++++--------- src/lib/onboard/docker-gpu-patch-finalize.ts | 116 ++++++---- .../onboard/docker-gpu-patch-recreate.test.ts | 29 +-- .../onboard/docker-gpu-patch-rollback.test.ts | 7 +- .../docker-gpu-supervisor-reconnect.test.ts | 62 +---- .../docker-gpu-supervisor-reconnect.ts | 57 +---- test/e2e/mock-parity.json | 4 +- test/helpers/onboard-script-mocks.cjs | 14 +- .../docker-final-handoff-lifecycle.test.ts | 90 +++++--- .../onboard-custom-dockerfile.test.ts | 5 + test/onboarding/onboard-messaging.test.ts | 4 +- ...ocess-recovery-supervisor-relaunch.test.ts | 29 ++- 15 files changed, 303 insertions(+), 352 deletions(-) diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index f760ed78b8c..53c1ab15ac0 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -1134,8 +1134,9 @@ On Jetson/Tegra hosts, the compatibility path uses the NVIDIA runtime and adds e These include selected `/dev/nvmap`, `/dev/nvhost-*`, and `/dev/nvgpu/igpu0/*` nodes plus real `/dev/dri/renderD*` character devices. After compatibility recreation starts, onboarding keeps the pre-patch container as a rollback backup until the replacement passes the Ready, GPU, and applicable local-inference checks. If one of those checks fails before backup removal, onboarding prints failure diagnostics and attempts to restore the pre-patch container. -To commit the replacement, NemoClaw stops it, removes the rollback backup, and waits until a successful OpenShell sandbox list has no row with that sandbox name. -NemoClaw then starts the replacement as the final container lifecycle event and verifies OpenShell supervisor readiness again within the same handoff deadline. +To commit the replacement, NemoClaw first asks OpenShell to stop the sandbox so its durable lifecycle row reaches `Stopped` before any irreversible Docker mutation. +It then stops the exact transaction-owned replacement, removes the rollback backup, and asks OpenShell to start the sandbox so OpenShell owns the `Starting` lifecycle fence. +NemoClaw verifies a `Ready` row, a working sandbox exec, and that the exact replacement is the sole running labeled container within the final handoff deadline. If that final handoff cannot be confirmed, onboarding exits with the container diagnostics and cleanup guidance instead of reporting success. If rollback fails, onboarding reports that the pre-patch container was not restored and prints container-cleanup guidance. GPU-proof diagnostics are captured before rollback and can print that guidance before the final container state is known, so inspect the sandbox and its labeled Docker containers before running a deletion command. @@ -1844,9 +1845,10 @@ NemoClaw waits for the exact replacement to pass managed gateway health and Open After state restoration, it restarts the gateway in the exact replacement container and requires an authenticated `ok` result. It then runs the managed settle check. It commits only after the replacement identity, state restoration, gateway restart, and settle check pass. -At the final commit handoff, NemoClaw stops the replacement, removes the rollback container, and waits for a captured, phase-bearing `openshell sandbox list` result that omits the selected sandbox name before it restarts the replacement. -An `Error` or `Deleting` row for the selected sandbox does not prove lifecycle release, even when Docker still exposes the exact replacement container. -If that release proof does not arrive after the rollback container has been removed, NemoClaw leaves the replacement stopped and reports that automatic rollback is unavailable. +At the final commit handoff, NemoClaw asks OpenShell to stop the sandbox before it mutates either exact container. +After OpenShell acknowledges that stop, NemoClaw stops the exact replacement, removes the rollback container, and asks OpenShell to start the sandbox through its authoritative lifecycle path. +This preserves OpenShell's stopped/starting event fence while stale Docker removal snapshots settle; raw Docker stop/start events cannot strand the lifecycle row in `Error` or `Deleting`. +If the authoritative stop fails, NemoClaw leaves both containers intact. If the start or final `Ready`/exec/exact-container proof fails after rollback-container removal, NemoClaw reports that automatic rollback is unavailable. If OpenShell re-registration, state restoration, or a later gateway check fails, NemoClaw attempts to roll back the replacement. The primary dashboard or API host forward stays stopped. NemoClaw removes the temporary state backup after a successful restore or rollback. diff --git a/docs/reference/troubleshooting.mdx b/docs/reference/troubleshooting.mdx index f923e3f82fd..6f10c857734 100644 --- a/docs/reference/troubleshooting.mdx +++ b/docs/reference/troubleshooting.mdx @@ -1755,9 +1755,10 @@ NemoClaw treats `SUPERVISOR_UNAVAILABLE` as terminal because it can report unrea `SUPERVISOR_NOT_RUNNING` is a separate result that requires two zero-supervisor scans with a stable PID 1. It enters the bounded startup retry first; only an exact missing-supervisor result that remains after the bound can authorize a container-identity-pinned recreation on a supported local Docker-driver sandbox with the legacy keepalive startup. That recreation commits only after managed health and settle checks pass. -At the final commit handoff, NemoClaw stops the replacement, removes the rollback container, and waits for a captured, phase-bearing `openshell sandbox list` result that omits the selected sandbox name before it restarts the replacement. -An `Error` or `Deleting` row for the selected sandbox does not prove lifecycle release, even when Docker still exposes the exact replacement container. -If the release proof does not arrive after the rollback container has been removed, NemoClaw leaves the replacement stopped and reports that automatic rollback is unavailable. +At the final commit handoff, NemoClaw asks OpenShell to stop the sandbox before it mutates either exact container. +After OpenShell acknowledges that stop, NemoClaw stops the exact replacement, removes the rollback container, and asks OpenShell to start the sandbox through its authoritative lifecycle path. +This preserves OpenShell's stopped/starting event fence while stale Docker removal snapshots settle; raw Docker stop/start events cannot strand the lifecycle row in `Error` or `Deleting`. +If the authoritative stop fails, NemoClaw leaves both containers intact. If the start or final `Ready`/exec/exact-container proof fails after rollback-container removal, NemoClaw reports that automatic rollback is unavailable. To bypass that trusted recreation while troubleshooting, run `NEMOCLAW_DISABLE_SUPERVISOR_RELAUNCH=1 $$nemoclaw recover`; NemoClaw leaves the container unchanged and returns rebuild or re-onboard guidance. If recovery stops after 3 `SUPERVISOR_BUSY` results, or if `gateway restart` reports `SUPERVISOR_BUSY`, wait for the active request to finish and retry the command. If recovery exhausts the transition bound after status `137` or the Docker restart result, wait for the container to finish restarting and retry the command. diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 28a9fadc814..12bc677ffe2 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -484,14 +484,14 @@ function waitForFinalRelaunchManagedSupervisor( function finalRelaunchContainerFailureDetail( completion: ReturnType, ): string | null { + if (completion.lifecycleStopAcknowledged === false) { + return "OpenShell did not acknowledge the authoritative stop before the final replacement handoff. NemoClaw did not start the primary dashboard/API host forward"; + } if (completion.replacementStoppedForCommit === false) { return "Docker could not stop the replacement container for the final recovery handoff. NemoClaw did not start the primary dashboard/API host forward"; } - if (completion.lifecycleReleaseObserved === false) { - return "OpenShell did not retire the previous lifecycle record before the final replacement start. NemoClaw did not start the primary dashboard/API host forward"; - } if (completion.replacementRestarted === false) { - return "Docker could not start the replacement container to complete the final recovery handoff. NemoClaw did not start the primary dashboard/API host forward"; + return "OpenShell could not start the replacement container through the authoritative lifecycle path. NemoClaw did not start the primary dashboard/API host forward"; } if (completion.finalHandoffAcknowledged === false) { return `OpenShell did not acknowledge the final replacement container handoff${ diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index e0c89ef0e17..7f915071cfb 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -40,10 +40,7 @@ function exactDeferredCreateResult(): DockerGpuPatchResult { function readyHandoffDeps() { return { - runCaptureOpenshell: vi - .fn() - .mockReturnValueOnce("beta 2026-08-23 10:00:00 Ready\n") - .mockReturnValue("alpha 2026-08-23 10:00:02 Ready\n"), + runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 10:00:02 Ready\n"), runOpenshell: vi.fn(() => ({ status: 0 })), sleep: vi.fn(), }; @@ -101,6 +98,7 @@ describe("finalizeDockerGpuPatchBackup", () => { backupRemoved: false, rolledBack: false, replacementStoppedForCommit: false, + lifecycleStopAcknowledged: false, finalHandoffAcknowledged: false, lastSandboxPhase: null, }); @@ -109,7 +107,7 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(dockerStart).not.toHaveBeenCalled(); }); - it("uses one exact stop, remove, start, and Ready acknowledgement handoff (#9531)", () => { + it("uses authoritative OpenShell stop/start around the exact Docker commit (#9531)", () => { const result = exactDeferredCreateResult(); const events: string[] = []; const dockerStop = vi.fn(() => { @@ -120,22 +118,19 @@ describe("finalizeDockerGpuPatchBackup", () => { events.push("remove backup"); return { status: 0 }; }); - const dockerStart = vi.fn(() => { - events.push("start replacement"); - return { status: 0 }; + const dockerStart = vi.fn(() => ({ status: 0 })); + const runCaptureOpenshell = vi.fn(() => { + events.push("observe ready"); + return "alpha 2026-08-23 10:00:02 Ready\n"; }); - const runCaptureOpenshell = vi - .fn() - .mockImplementationOnce(() => { - events.push("observe lifecycle release"); - return "beta 2026-08-23 10:00:00 Ready\n"; - }) - .mockImplementation(() => { - events.push("observe ready"); - return "alpha 2026-08-23 10:00:02 Ready\n"; - }); - const runOpenshell = vi.fn(() => { - events.push("exec ready"); + const runOpenshell = vi.fn((args: string[]) => { + events.push( + args[1] === "stop" + ? "stop through OpenShell" + : args[1] === "start" + ? "start through OpenShell" + : "exec ready", + ); return { status: 0 }; }); const dockerResults = { @@ -181,15 +176,15 @@ describe("finalizeDockerGpuPatchBackup", () => { rolledBack: false, replacementStoppedForCommit: true, replacementRestarted: true, - lifecycleReleaseObserved: true, + lifecycleStopAcknowledged: true, finalHandoffAcknowledged: true, lastSandboxPhase: "Ready", }); expect(events).toEqual([ + "stop through OpenShell", "stop replacement", "remove backup", - "observe lifecycle release", - "start replacement", + "start through OpenShell", "observe ready", "exec ready", "read replacement namespace", @@ -204,9 +199,24 @@ describe("finalizeDockerGpuPatchBackup", () => { result.oldContainerId, expect.objectContaining({ ignoreError: true }), ); - expect(dockerStart).toHaveBeenCalledWith( - result.newContainerId, - expect.objectContaining({ ignoreError: true }), + expect(dockerStart).not.toHaveBeenCalled(); + expect(runOpenshell).toHaveBeenNthCalledWith( + 1, + ["sandbox", "stop", "alpha"], + expect.objectContaining({ + killProcessTreeOnTimeout: true, + killSignal: "SIGKILL", + timeout: 60_000, + }), + ); + expect(runOpenshell).toHaveBeenNthCalledWith( + 2, + ["sandbox", "start", "alpha"], + expect.objectContaining({ + killProcessTreeOnTimeout: true, + killSignal: "SIGKILL", + timeout: 60_000, + }), ); }); @@ -230,18 +240,15 @@ describe("finalizeDockerGpuPatchBackup", () => { events.push("remove backup"); return { status: 0 }; }), - dockerStart: vi.fn(() => { - events.push("start replacement"); + dockerStart: vi.fn(() => ({ status: 0 })), + runCaptureOpenshell: vi.fn(() => { + events.push("observe deleting"); + return "alpha 2026-08-23 10:00:02 Deleting\n"; + }), + runOpenshell: vi.fn((args: string[]) => { + events.push(args[1] === "stop" ? "stop through OpenShell" : "start through OpenShell"); return { status: 0 }; }), - runCaptureOpenshell: vi - .fn() - .mockReturnValueOnce("beta 2026-08-23 10:00:00 Ready\n") - .mockImplementation(() => { - events.push("observe deleting"); - return "alpha 2026-08-23 10:00:02 Deleting\n"; - }), - runOpenshell: vi.fn(() => ({ status: 1 })), dockerRun: vi.fn(() => ({ status: 0, stdout: `${result.newContainerId}\n` })), sleep, }, @@ -252,14 +259,15 @@ describe("finalizeDockerGpuPatchBackup", () => { rolledBack: false, replacementStoppedForCommit: true, replacementRestarted: true, - lifecycleReleaseObserved: true, + lifecycleStopAcknowledged: true, finalHandoffAcknowledged: false, lastSandboxPhase: "Deleting", }); expect(events).toEqual([ + "stop through OpenShell", "stop replacement", "remove backup", - "start replacement", + "start through OpenShell", "observe deleting", ]); expect(sleep).not.toHaveBeenCalled(); @@ -287,10 +295,7 @@ describe("finalizeDockerGpuPatchBackup", () => { dockerRm: vi.fn(() => ({ status: 0 })), dockerStart: vi.fn(() => ({ status: 0 })), dockerRun, - runCaptureOpenshell: vi - .fn() - .mockReturnValueOnce("retiring-name 2026-08-23 01:40:35 Error\n") - .mockReturnValue("restored-name 2026-08-23 01:40:37 Ready\n"), + runCaptureOpenshell: vi.fn(() => "restored-name 2026-08-23 01:40:37 Ready\n"), runOpenshell: vi.fn(() => ({ status: 0 })), sleep: vi.fn(), }, @@ -298,7 +303,7 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(outcome).toMatchObject({ backupRemoved: true, - lifecycleReleaseObserved: true, + lifecycleStopAcknowledged: true, finalHandoffAcknowledged: true, }); expect(dockerRun.mock.calls[0]?.[0]).toEqual([ @@ -356,10 +361,7 @@ describe("finalizeDockerGpuPatchBackup", () => { dockerRm: vi.fn(() => ({ status: 0 })), dockerStart: vi.fn(() => ({ status: 0 })), dockerRun, - runCaptureOpenshell: vi - .fn() - .mockReturnValueOnce("beta 2026-08-23 01:40:35 Ready\n") - .mockReturnValue("alpha 2026-08-23 01:40:37 Ready\n"), + runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 01:40:37 Ready\n"), runOpenshell: vi.fn(() => ({ status: 0 })), sleep: vi.fn(), }, @@ -367,42 +369,34 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(outcome).toMatchObject({ backupRemoved: true, - lifecycleReleaseObserved: true, + lifecycleStopAcknowledged: true, replacementRestarted: true, finalHandoffAcknowledged: false, }); }); - it("waits for captured name absence before restarting the exact replacement (#10153)", () => { + it("does not require name absence between authoritative stop and start (#10153)", () => { const result = exactDeferredCreateResult(); const events: string[] = []; const dockerRunResults = { inspect: { status: 0, stdout: "true\n" }, ps: { status: 0, stdout: `${result.newContainerId}\n` }, } as const; - const dockerStart = vi.fn(() => { - events.push("start replacement"); + const dockerStart = vi.fn(() => ({ status: 0 })); + const runCaptureOpenshell = vi.fn(() => { + events.push("observe replacement ready"); + return "alpha 2026-08-23 10:00:06 Ready\n"; + }); + const runOpenshell = vi.fn((args: string[]) => { + events.push( + args[1] === "stop" + ? "stop through OpenShell" + : args[1] === "start" + ? "start through OpenShell" + : "exec ready", + ); return { status: 0 }; }); - const runCaptureOpenshell = vi - .fn() - .mockImplementationOnce(() => { - events.push("observe error"); - return "alpha 2026-08-23 10:00:00 Error\n"; - }) - .mockImplementationOnce(() => { - events.push("observe deleting"); - return "alpha 2026-08-23 10:00:02 Deleting\n"; - }) - .mockImplementationOnce(() => { - events.push("observe lifecycle release"); - return "beta 2026-08-23 10:00:04 Ready\n"; - }) - .mockImplementationOnce(() => { - events.push("observe replacement ready"); - return "alpha 2026-08-23 10:00:06 Ready\n"; - }); - const runOpenshell = vi.fn(() => ({ status: 0 })); const outcome = finalizeDockerGpuPatchBackup( { @@ -427,34 +421,26 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(outcome).toMatchObject({ backupRemoved: true, - lifecycleReleaseObserved: true, + lifecycleStopAcknowledged: true, replacementRestarted: true, finalHandoffAcknowledged: true, }); - expect(dockerStart).toHaveBeenCalledWith( - result.newContainerId, - expect.objectContaining({ ignoreError: true }), - ); + expect(dockerStart).not.toHaveBeenCalled(); expect(events).toEqual([ - "observe error", - "observe deleting", - "observe lifecycle release", - "start replacement", + "stop through OpenShell", + "start through OpenShell", "observe replacement ready", + "exec ready", ]); - expect(runOpenshell).not.toHaveBeenCalledWith( - ["sandbox", "list"], - expect.any(Object), - ); + expect(runOpenshell).not.toHaveBeenCalledWith(["sandbox", "list"], expect.any(Object)); }); - it("does not restart while captured lifecycle rows still name the sandbox (#10153)", () => { + it("does not mutate containers when authoritative OpenShell stop fails (#10153)", () => { const result = exactDeferredCreateResult(); + const dockerStop = vi.fn(() => ({ status: 0 })); + const dockerRm = vi.fn(() => ({ status: 0 })); const dockerStart = vi.fn(() => ({ status: 0 })); - const runCaptureOpenshell = vi - .fn() - .mockReturnValueOnce("alpha 2026-08-23 01:40:35 Error\n") - .mockReturnValue("alpha 2026-08-23 01:40:37 Deleting\n"); + const runCaptureOpenshell = vi.fn(() => "alpha 2026-08-23 01:40:35 Ready\n"); const outcome = finalizeDockerGpuPatchBackup( { @@ -464,23 +450,25 @@ describe("finalizeDockerGpuPatchBackup", () => { finalHandoffTimeoutSecs: 1, }, { - dockerStop: vi.fn(() => ({ status: 0 })), - dockerRm: vi.fn(() => ({ status: 0 })), + dockerStop, + dockerRm, dockerRun: vi.fn(), dockerStart, runCaptureOpenshell, - runOpenshell: vi.fn(() => ({ status: 0 })), + runOpenshell: vi.fn(() => ({ status: 1 })), sleep: vi.fn(), }, ); expect(outcome).toMatchObject({ - backupRemoved: true, - lifecycleReleaseObserved: false, - replacementRestarted: false, + backupRemoved: false, + lifecycleStopAcknowledged: false, + replacementStoppedForCommit: false, }); + expect(dockerStop).not.toHaveBeenCalled(); + expect(dockerRm).not.toHaveBeenCalled(); expect(dockerStart).not.toHaveBeenCalled(); - expect(runCaptureOpenshell).toHaveBeenCalledTimes(2); + expect(runCaptureOpenshell).not.toHaveBeenCalled(); }); it("rolls back to the backup container when supervisor reconnect failed", () => { @@ -606,6 +594,7 @@ describe("finalizeDockerGpuPatchBackup", () => { rolledBack: false, replacementStoppedForCommit: true, replacementRestarted: true, + lifecycleStopAcknowledged: true, finalHandoffAcknowledged: false, lastSandboxPhase: null, }); @@ -613,10 +602,7 @@ describe("finalizeDockerGpuPatchBackup", () => { "old-container-id", expect.objectContaining({ ignoreError: true }), ); - expect(dockerStart).toHaveBeenCalledWith( - "new-container-id", - expect.objectContaining({ ignoreError: true }), - ); + expect(dockerStart).not.toHaveBeenCalled(); }); it("fails closed when backup removal has no exit status", () => { @@ -637,13 +623,11 @@ describe("finalizeDockerGpuPatchBackup", () => { rolledBack: false, replacementStoppedForCommit: true, replacementRestarted: true, + lifecycleStopAcknowledged: true, finalHandoffAcknowledged: false, lastSandboxPhase: null, }); - expect(dockerStart).toHaveBeenCalledWith( - "new-container-id", - expect.objectContaining({ ignoreError: true }), - ); + expect(dockerStart).not.toHaveBeenCalled(); }); it("retains the backup when the replacement cannot be stopped for the final handoff", () => { @@ -665,12 +649,18 @@ describe("finalizeDockerGpuPatchBackup", () => { backupRemoved: false, rolledBack: false, replacementStoppedForCommit: false, + replacementRestarted: true, + lifecycleStopAcknowledged: true, }); expect(dockerRm).not.toHaveBeenCalled(); expect(dockerStart).not.toHaveBeenCalled(); }); it("reports a failed replacement restart after the backup is removed", () => { + const runOpenshell = vi + .fn() + .mockReturnValueOnce({ status: 0 }) + .mockReturnValueOnce({ status: 1 }); const outcome = finalizeDockerGpuPatchBackup( { result: deferredCreateResult(), @@ -682,7 +672,9 @@ describe("finalizeDockerGpuPatchBackup", () => { dockerStop: vi.fn(() => ({ status: 0 })), dockerRm: vi.fn(() => ({ status: 0 })), dockerStart: vi.fn(() => ({ status: 1 })), - ...readyHandoffDeps(), + runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 10:00:02 Ready\n"), + runOpenshell, + sleep: vi.fn(), }, ); @@ -691,9 +683,20 @@ describe("finalizeDockerGpuPatchBackup", () => { rolledBack: false, replacementStoppedForCommit: true, replacementRestarted: false, + lifecycleStopAcknowledged: true, finalHandoffAcknowledged: false, lastSandboxPhase: null, }); + expect(runOpenshell).toHaveBeenNthCalledWith( + 1, + ["sandbox", "stop", "alpha"], + expect.any(Object), + ); + expect(runOpenshell).toHaveBeenNthCalledWith( + 2, + ["sandbox", "start", "alpha"], + expect.any(Object), + ); }); it("records a remaining exact-ID replacement when removal fails (#7996)", () => { diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index c9e3d7929e3..9d88828f100 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -10,10 +10,10 @@ // post-create container recreation NemoClaw performs here. Until OpenShell // supports that natively, NemoClaw recreates the container with GPU access // and uses this module to either restore the pre-patch backup before commit or -// complete the exact stop/remove/start handoff and require OpenShell's final -// Ready acknowledgement. Removing the old container is the irreversible -// commit point: later failures require a sandbox rebuild rather than automatic -// rollback. Regression coverage: +// complete the OpenShell stop / exact remove / OpenShell start handoff and +// require OpenShell's final Ready acknowledgement. Removing the old container +// is the irreversible commit point: later failures require a sandbox rebuild +// rather than automatic rollback. Regression coverage: // * src/lib/onboard/docker-gpu-patch-finalize.test.ts — direct unit tests // for exact final handoff, terminal phase, rollback, and failure outcomes. // * src/lib/onboard/docker-gpu-patch-rollback.test.ts — composed @@ -35,10 +35,7 @@ import { } from "./docker-gpu-patch-rollback"; import { fullDockerContainerId } from "./docker-gpu-patch-clone"; import type { DockerGpuPatchDeps, DockerGpuPatchResult } from "./docker-gpu-patch-types"; -import { - waitForOpenShellFinalHandoff, - waitForOpenShellSandboxLifecycleRelease, -} from "./docker-gpu-supervisor-reconnect"; +import { waitForOpenShellFinalHandoff } from "./docker-gpu-supervisor-reconnect"; import { OPENSHELL_SANDBOX_NAMESPACE_LABEL, queryOpenShellDockerSandboxContainers, @@ -67,7 +64,7 @@ export type DockerGpuPatchFinalizeOutcome = { rolledBack: boolean; replacementStoppedForCommit?: boolean; replacementRestarted?: boolean; - lifecycleReleaseObserved?: boolean; + lifecycleStopAcknowledged?: boolean; finalHandoffAcknowledged?: boolean; lastSandboxPhase?: string | null; replacementStopConfirmed?: boolean; @@ -75,6 +72,30 @@ export type DockerGpuPatchFinalizeOutcome = { replacementPresence?: "absent" | "present" | "unknown"; }; +const PROCESS_TREE_BOUNDED_OPENSHELL_OPTIONS = { + killProcessTreeOnTimeout: true, + killSignal: "SIGKILL", +} as const; + +function runOpenShellLifecycleCommand( + runOpenshell: NonNullable, + args: string[], + timeoutSecs: number, +): boolean { + try { + return hasZeroDockerExitStatus( + runOpenshell(args, { + ignoreError: true, + ...PROCESS_TREE_BOUNDED_OPENSHELL_OPTIONS, + suppressOutput: true, + timeout: Math.max(1, Math.round(timeoutSecs * 1000)), + }), + ); + } catch { + return false; + } +} + function isExactRunningReplacement( sandboxName: string, replacementContainerId: string, @@ -159,74 +180,91 @@ export function finalizeDockerGpuPatchBackup( return { backupRemoved: true, rolledBack: false }; } if (options.supervisorReady) { - // Stop the exact replacement before retiring the exact backup, then start - // the replacement afterward. The final start is the authoritative Docker - // lifecycle event. Backup removal is the irreversible commit point; - // failures after it require a sandbox rebuild. Success is withheld until - // OpenShell reports Ready and Docker still proves the exact replacement is - // the sole running labeled container (#9531). + // Move the durable OpenShell row to Stopped before touching the exact + // containers. OpenShell 0.0.106 keeps Stopped stable while the Docker + // driver's duplicate-ID snapshot catches up with removal of the rollback + // container. Starting through OpenShell then owns the lifecycle fence and + // prevents the stopped replacement snapshot from regressing the row to + // Error or sticky Deleting. Backup removal remains the irreversible commit + // point; failures after it require a sandbox rebuild. Success is withheld + // until OpenShell reports Ready and Docker still proves the exact + // replacement is the sole running labeled container (#9531, #10153). if (!deps.runOpenshell || !deps.runCaptureOpenshell) { return { backupRemoved: false, rolledBack: false, replacementStoppedForCommit: false, + lifecycleStopAcknowledged: false, + finalHandoffAcknowledged: false, + lastSandboxPhase: null, + }; + } + console.log( + ` Stopping the replacement through OpenShell for the final handoff (up to ${options.finalHandoffTimeoutSecs}s)...`, + ); + const lifecycleStopAcknowledged = runOpenShellLifecycleCommand( + deps.runOpenshell, + ["sandbox", "stop", options.sandboxName], + options.finalHandoffTimeoutSecs, + ); + if (!lifecycleStopAcknowledged) { + return { + backupRemoved: false, + rolledBack: false, + replacementStoppedForCommit: false, + lifecycleStopAcknowledged: false, finalHandoffAcknowledged: false, lastSandboxPhase: null, }; } const stopResult = resolved.dockerStop(options.result.newContainerId, containerOpts); if (!hasZeroDockerExitStatus(stopResult)) { + const replacementRestarted = runOpenShellLifecycleCommand( + deps.runOpenshell, + ["sandbox", "start", options.sandboxName], + options.finalHandoffTimeoutSecs, + ); return { backupRemoved: false, rolledBack: false, replacementStoppedForCommit: false, + replacementRestarted, + lifecycleStopAcknowledged: true, }; } const rmResult = resolved.dockerRm(options.result.oldContainerId, containerOpts); const backupRemoved = hasZeroDockerExitStatus(rmResult); if (!backupRemoved) { - const replacementRestarted = hasZeroDockerExitStatus( - resolved.dockerStart(options.result.newContainerId, containerOpts), + const replacementRestarted = runOpenShellLifecycleCommand( + deps.runOpenshell, + ["sandbox", "start", options.sandboxName], + options.finalHandoffTimeoutSecs, ); return { backupRemoved: false, rolledBack: false, replacementStoppedForCommit: true, replacementRestarted, + lifecycleStopAcknowledged: true, finalHandoffAcknowledged: false, lastSandboxPhase: null, }; } console.log( - ` Waiting for OpenShell to retire the previous lifecycle record before restarting the replacement (up to ${options.finalHandoffTimeoutSecs}s)...`, + ` Starting the exact replacement through OpenShell to complete the final handoff (up to ${options.finalHandoffTimeoutSecs}s)...`, ); - const lifecycleReleaseObserved = waitForOpenShellSandboxLifecycleRelease( - options.sandboxName, + const replacementRestarted = runOpenShellLifecycleCommand( + deps.runOpenshell, + ["sandbox", "start", options.sandboxName], options.finalHandoffTimeoutSecs, - { - runCaptureOpenshell: deps.runCaptureOpenshell, - sleep: deps.sleep, - }, ); - if (!lifecycleReleaseObserved) { - return { - backupRemoved: true, - rolledBack: false, - replacementStoppedForCommit: true, - replacementRestarted: false, - lifecycleReleaseObserved: false, - finalHandoffAcknowledged: false, - lastSandboxPhase: null, - }; - } - const startResult = resolved.dockerStart(options.result.newContainerId, containerOpts); - const replacementRestarted = hasZeroDockerExitStatus(startResult); if (!replacementRestarted) { return { backupRemoved: true, rolledBack: false, replacementStoppedForCommit: true, - replacementRestarted, + replacementRestarted: false, + lifecycleStopAcknowledged: true, finalHandoffAcknowledged: false, lastSandboxPhase: null, }; @@ -255,7 +293,7 @@ export function finalizeDockerGpuPatchBackup( rolledBack: false, replacementStoppedForCommit: true, replacementRestarted: true, - lifecycleReleaseObserved: true, + lifecycleStopAcknowledged: true, finalHandoffAcknowledged: acknowledgement.acknowledged, lastSandboxPhase: acknowledgement.lastSandboxPhase, }; diff --git a/src/lib/onboard/docker-gpu-patch-recreate.test.ts b/src/lib/onboard/docker-gpu-patch-recreate.test.ts index a58c2ef3d28..d5731c62dab 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.test.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.test.ts @@ -39,13 +39,8 @@ describe("Docker GPU recreate orchestration", () => { const dockerStop = vi.fn(() => ({ status: 0 })); const dockerRm = vi.fn(() => ({ status: 0 })); const dockerStart = vi.fn(() => ({ status: 0 })); - const runOpenshell = vi.fn((args: readonly string[]) => - args[1] === "list" ? { status: 0, stdout: "No sandboxes found.\n" } : { status: 0 }, - ); - const runCaptureOpenshell = vi - .fn() - .mockReturnValueOnce("No sandboxes found.\n") - .mockReturnValue("alpha 2026-08-23 10:00:02 Ready\n"); + const runOpenshell = vi.fn(() => ({ status: 0 })); + const runCaptureOpenshell = vi.fn(() => "alpha 2026-08-23 10:00:02 Ready\n"); const result = recreateOpenShellDockerSandboxWithGpu( { sandboxName: "alpha", timeoutSecs: 1 }, @@ -105,9 +100,14 @@ describe("Docker GPU recreate orchestration", () => { expect(dockerRm.mock.invocationCallOrder[backupRmCall]).toBeGreaterThan( runOpenshell.mock.invocationCallOrder[0], ); - expect(dockerStart).toHaveBeenCalledWith( - NEW_CONTAINER_ID, - expect.objectContaining({ ignoreError: true }), + expect(dockerStart).not.toHaveBeenCalled(); + expect(runOpenshell).toHaveBeenCalledWith( + ["sandbox", "stop", "alpha"], + expect.objectContaining({ ignoreError: true, timeout: 1000 }), + ); + expect(runOpenshell).toHaveBeenCalledWith( + ["sandbox", "start", "alpha"], + expect.objectContaining({ ignoreError: true, timeout: 1000 }), ); expect(runCaptureOpenshell).toHaveBeenCalledWith( ["sandbox", "list"], @@ -128,13 +128,8 @@ describe("Docker GPU recreate orchestration", () => { dockerStop: vi.fn(() => ({ status: 0 })), dockerRm: vi.fn(() => ({ status: 0 })), dockerStart: vi.fn(() => ({ status: 0 })), - runCaptureOpenshell: vi - .fn() - .mockReturnValueOnce("No sandboxes found.\n") - .mockReturnValue("alpha 2026-08-23 10:00:02 Deleting\n"), - runOpenshell: vi.fn((args: readonly string[]) => - args[1] === "list" ? { status: 0, stdout: "No sandboxes found.\n" } : { status: 0 }, - ), + runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 10:00:02 Deleting\n"), + runOpenshell: vi.fn(() => ({ status: 0 })), sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), detectSandboxFallbackDns: vi.fn(() => null), diff --git a/src/lib/onboard/docker-gpu-patch-rollback.test.ts b/src/lib/onboard/docker-gpu-patch-rollback.test.ts index d541200ae48..14ecb44988d 100644 --- a/src/lib/onboard/docker-gpu-patch-rollback.test.ts +++ b/src/lib/onboard/docker-gpu-patch-rollback.test.ts @@ -273,9 +273,10 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { dockerRename, dockerStart, dockerLogs: vi.fn(() => ""), - runOpenshell: vi.fn((args: readonly string[]) => - args[1] === "list" ? { status: 0, stdout: "No sandboxes found.\n" } : { status: 0 }, - ), + runOpenshell: vi.fn((args: readonly string[]) => { + startTargets.push(...(args[1] === "start" ? [retryId] : [])); + return { status: 0 }; + }), runCaptureOpenshell: vi.fn(() => !restoredPresent && !startTargets.includes(retryId) ? "No sandboxes found.\n" diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts index 31a11d700ea..67bbfdbfa07 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts @@ -7,70 +7,10 @@ import { getDockerGpuSupervisorReconnectErrorDebouncePolls, getDockerGpuSupervisorReconnectTimeoutSecs, waitForOpenShellFinalHandoff, - waitForOpenShellSandboxLifecycleRelease, waitForOpenShellSupervisorReconnect, } from "./docker-gpu-supervisor-reconnect"; -describe("Docker GPU final lifecycle release", () => { - it.each([ - ["an explicit empty list", "No sandboxes found.\n"], - ["another phase-bearing sandbox", "beta 2026-08-21 05:53:18 Ready\n"], - ])("accepts %s as a captured release receipt (#9531)", (_receipt, output) => { - const runCaptureOpenshell = vi.fn(() => output); - - expect( - waitForOpenShellSandboxLifecycleRelease("alpha", 1, { - runCaptureOpenshell, - sleep: vi.fn(), - }), - ).toBe(true); - expect(runCaptureOpenshell).toHaveBeenCalledOnce(); - expect(runCaptureOpenshell).toHaveBeenCalledWith( - ["sandbox", "list"], - expect.objectContaining({ - killProcessTreeOnTimeout: true, - killSignal: "SIGKILL", - timeout: expect.any(Number), - }), - ); - }); - - it.each([ - ["a header", "NAME CREATED PHASE\n"], - ["a gateway error", "Error: gateway unavailable\n"], - ["a phase-free row", "beta 2026-08-21 05:53:18\n"], - ["an unrecognized phase", "beta 2026-08-21 05:53:18 Retiring\n"], - ["the selected sandbox in Deleting", "alpha 2026-08-21 05:53:18 Deleting\n"], - ["the selected sandbox in Ready", "alpha 2026-08-21 05:53:18 Ready\n"], - ["the selected sandbox in Provisioning", "alpha 2026-08-21 05:53:18 Provisioning\n"], - ["the selected sandbox in Error", "alpha 2026-08-21 05:53:18 Error\n"], - ["the selected sandbox in Failed", "alpha 2026-08-21 05:53:18 Failed\n"], - ])("rejects %s as a captured release receipt (#9531)", (_case, output) => { - const runCaptureOpenshell = vi.fn(() => output); - - expect( - waitForOpenShellSandboxLifecycleRelease("alpha", 1, { - runCaptureOpenshell, - sleep: vi.fn(), - }), - ).toBe(false); - expect(runCaptureOpenshell).toHaveBeenCalledTimes(2); - }); - - it("does not release the lifecycle when the captured sandbox list is unavailable (#10153)", () => { - const runCaptureOpenshell = vi.fn(() => { - throw new Error("sandbox list transport unavailable"); - }); - - expect( - waitForOpenShellSandboxLifecycleRelease("alpha", 1, { - runCaptureOpenshell, - sleep: vi.fn(), - }), - ).toBe(false); - expect(runCaptureOpenshell).toHaveBeenCalledTimes(2); - }); - +describe("Docker GPU supervisor reconnect", () => { it("does not report reconnect without an OpenShell execution boundary (#9531)", () => { expect(waitForOpenShellSupervisorReconnect("alpha", 1, { sleep: vi.fn() })).toBe(false); }); diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts index 80ccb12ccb3..b56462a6cff 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts @@ -73,11 +73,6 @@ export type DockerGpuSupervisorReconnectDeps = { errorPhaseDebouncePolls?: number; }; -type DockerLifecycleReleaseDeps = Pick< - DockerGpuSupervisorReconnectDeps, - "runCaptureOpenshell" | "sleep" ->; - type DockerFinalHandoffDeps = Required< Pick > & @@ -101,56 +96,6 @@ const PROCESS_TREE_BOUNDED_OPENSHELL_OPTIONS = { killSignal: "SIGKILL", } as const; -/** - * Wait for OpenShell to retire the pre-replacement lifecycle record before - * restarting the replacement. The selected sandbox name must be absent from - * a phase-bearing list receipt because a row cannot prove which container - * owns the lifecycle, even when Docker still exposes the exact replacement. - */ -export function waitForOpenShellSandboxLifecycleRelease( - sandboxName: string, - timeoutSecs: number, - deps: DockerLifecycleReleaseDeps, -): boolean { - if (!deps.runCaptureOpenshell) return false; - const sleep = deps.sleep ?? defaultSleep; - const deadline = Date.now() + Math.max(1, Math.round(timeoutSecs)) * 1000; - const maxAttempts = Math.max(1, Math.ceil(Math.max(1, Math.round(timeoutSecs)) / 2) + 1); - - for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { - const remainingMs = deadline - Date.now(); - if (remainingMs <= 0) break; - let output = ""; - try { - // The streaming runner does not return sandbox-list stdout. Capture the - // row so a successful command cannot hide a retiring lifecycle phase. - output = deps - .runCaptureOpenshell(["sandbox", "list"], { - ignoreError: true, - ...PROCESS_TREE_BOUNDED_OPENSHELL_OPTIONS, - suppressOutput: true, - timeout: Math.min(DOCKER_GPU_PATCH_TIMEOUT_MS, remainingMs), - }) - .trim(); - } catch { - output = ""; - } - if (output) { - const entries = parseLiveSandboxEntries(output); - const sandboxPresent = entries.some((entry) => entry.name === sandboxName); - const explicitEmptyList = output === "No sandboxes found" || output === "No sandboxes found."; - if (explicitEmptyList || (entries.some((entry) => entry.phase !== null) && !sandboxPresent)) { - return true; - } - } - const remainingBeforeSleepMs = deadline - Date.now(); - if (attempt < maxAttempts && remainingBeforeSleepMs > 0) { - sleep(Math.min(2, remainingBeforeSleepMs / 1000)); - } - } - return false; -} - function exactReplacementIsRunning( callback: DockerFinalHandoffDeps["replacementIsExactAndRunning"], remainingMs: number, @@ -166,7 +111,7 @@ function exactReplacementIsRunning( /** * Confirm the final Docker replacement handoff through OpenShell and Docker. * - * The final replacement start is the authoritative lifecycle event. Success + * The preceding OpenShell start is the authoritative lifecycle event. Success * requires both an OpenShell Ready row with a working sandbox exec and a * bounded Docker proof that the exact transaction-owned replacement is the * sole running labeled container. Deleting is terminal after that start. diff --git a/test/e2e/mock-parity.json b/test/e2e/mock-parity.json index ed4b34c0936..1f82b65d2ba 100644 --- a/test/e2e/mock-parity.json +++ b/test/e2e/mock-parity.json @@ -186,12 +186,14 @@ "live": "test/e2e/live/gateway-guard-recovery.test.ts", "fast": [ "src/lib/actions/sandbox/supervisor-relaunch.test.ts", + "src/lib/onboard/docker-gpu-patch-finalize.test.ts", "src/lib/onboard/docker-startup-command-patch.test.ts", "src/lib/sandbox/privileged-exec.test.ts", "test/inference/managed/managed-gateway-control.test.ts", "test/agents/openclaw/runtime/nemoclaw-start-guard-recovery.test.ts", "test/e2e/support/gateway-guard-legacy-keepalive-fixture.test.ts", - "test/process-recovery/process-recovery-supervisor-relaunch.test.ts" + "test/process-recovery/process-recovery-supervisor-relaunch.test.ts", + "test/onboarding/docker-final-handoff-lifecycle.test.ts" ] }, { diff --git a/test/helpers/onboard-script-mocks.cjs b/test/helpers/onboard-script-mocks.cjs index 17d07d6dca6..1d30af8ca2f 100644 --- a/test/helpers/onboard-script-mocks.cjs +++ b/test/helpers/onboard-script-mocks.cjs @@ -631,7 +631,7 @@ function mockDockerSandboxLifecycleReleaseFromRunner() { const runner = require(path.resolve(__dirname, "../../src/lib/runner.ts")); const state = runner.run.__nemoclawDockerLifecycleState ?? { finalCommitReleased: false, - lifecycleReleased: false, + lifecycleStopped: false, replacementRestarted: false, }; const captureOutput = (normalized) => { @@ -660,8 +660,8 @@ function mockDockerSandboxLifecycleReleaseFromRunner() { if (state.replacementRestarted && normalized.includes("sandbox list")) { return "my-assistant 2026-08-27 Ready\n"; } - if (state.lifecycleReleased && normalized.includes("sandbox list")) { - return "No sandboxes found\n"; + if (state.lifecycleStopped && normalized.includes("sandbox list")) { + return "my-assistant 2026-08-27 Stopped\n"; } return null; }; @@ -679,13 +679,17 @@ function mockDockerSandboxLifecycleReleaseFromRunner() { } const result = run(command, options); if (normalized.startsWith("docker rm ") && result?.status === 0) { - state.lifecycleReleased = true; if (normalized === `docker rm ${ONBOARD_SANDBOX_OLD_CONTAINER_ID}`) { state.finalCommitReleased = true; } } + if (normalized.includes("sandbox stop my-assistant") && result?.status === 0) { + state.lifecycleStopped = true; + state.replacementRestarted = false; + } if ( - normalized === `docker start ${ONBOARD_SANDBOX_NEW_CONTAINER_ID}` && + state.finalCommitReleased && + normalized.includes("sandbox start my-assistant") && result?.status === 0 ) { state.replacementRestarted = true; diff --git a/test/onboarding/docker-final-handoff-lifecycle.test.ts b/test/onboarding/docker-final-handoff-lifecycle.test.ts index 4af946f5dfc..b80e4485f15 100644 --- a/test/onboarding/docker-final-handoff-lifecycle.test.ts +++ b/test/onboarding/docker-final-handoff-lifecycle.test.ts @@ -14,30 +14,35 @@ const OLD_CONTAINER_ID = "a".repeat(64); const NEW_CONTAINER_ID = "b".repeat(64); describe("Docker final handoff lifecycle integration", () => { - it("restarts only after Error and Deleting release the selected sandbox name (#10153)", () => { + it("commits only between authoritative OpenShell stop and start receipts (#10153)", () => { const events: string[] = []; - const dockerStart = vi.fn(() => { - events.push("start replacement"); + const dockerStop = vi.fn((containerId: string) => { + events.push( + containerId === OLD_CONTAINER_ID ? "stop original for recreate" : "stop exact replacement", + ); + return { status: 0 }; + }); + const dockerRm = vi.fn(() => { + events.push("remove exact backup"); + return { status: 0 }; + }); + const dockerStart = vi.fn(() => ({ status: 0 })); + const runCaptureOpenshell = vi.fn(() => { + events.push("observe replacement ready"); + return "alpha 2026-08-23 10:00:06 Ready\n"; + }); + const runOpenshell = vi.fn((args: string[]) => { + events.push( + args[1] === "stop" + ? "stop through OpenShell" + : args[1] === "start" + ? "start through OpenShell" + : args[1] === "exec" && events.includes("stop through OpenShell") + ? "exec final ready" + : "exec supervisor ready", + ); return { status: 0 }; }); - const runCaptureOpenshell = vi - .fn() - .mockImplementationOnce(() => { - events.push("observe error"); - return "alpha 2026-08-23 10:00:00 Error\n"; - }) - .mockImplementationOnce(() => { - events.push("observe deleting"); - return "alpha 2026-08-23 10:00:02 Deleting\n"; - }) - .mockImplementationOnce(() => { - events.push("observe name absence"); - return "beta 2026-08-23 10:00:04 Ready\n"; - }) - .mockImplementationOnce(() => { - events.push("observe replacement ready"); - return "alpha 2026-08-23 10:00:06 Ready\n"; - }); const result = recreateOpenShellDockerSandboxWithStartupCommand( { @@ -51,11 +56,11 @@ describe("Docker final handoff lifecycle integration", () => { dockerRun: vi.fn(createDockerFinalHandoffRunFixture(NEW_CONTAINER_ID)), dockerRunDetached: vi.fn(() => ({ status: 0, stdout: `${NEW_CONTAINER_ID}\n` })), dockerRename: vi.fn(() => ({ status: 0 })), - dockerStop: vi.fn(() => ({ status: 0 })), - dockerRm: vi.fn(() => ({ status: 0 })), + dockerStop, + dockerRm, dockerStart, runCaptureOpenshell, - runOpenshell: vi.fn(() => ({ status: 0 })), + runOpenshell, sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), detectSandboxFallbackDns: vi.fn(() => null), @@ -71,20 +76,28 @@ describe("Docker final handoff lifecycle integration", () => { oldContainerId: OLD_CONTAINER_ID, }); expect(events).toEqual([ - "observe error", - "observe deleting", - "observe name absence", - "start replacement", + "stop original for recreate", + "exec supervisor ready", + "stop through OpenShell", + "stop exact replacement", + "remove exact backup", + "start through OpenShell", "observe replacement ready", + "exec final ready", ]); + expect(dockerRm).toHaveBeenCalledWith(OLD_CONTAINER_ID, expect.any(Object)); + expect(dockerStart).not.toHaveBeenCalled(); }); - it("never restarts when Error advances to Deleting without a name-absence receipt (#10153)", () => { + it("does not cross the final Docker commit when authoritative OpenShell stop fails (#10153)", () => { + const dockerStop = vi.fn(() => ({ status: 0 })); + const dockerRm = vi.fn(() => ({ status: 0 })); const dockerStart = vi.fn(() => ({ status: 0 })); - const runCaptureOpenshell = vi + const runCaptureOpenshell = vi.fn(() => "alpha 2026-08-23 10:00:00 Ready\n"); + const runOpenshell = vi .fn() - .mockReturnValueOnce("alpha 2026-08-23 10:00:00 Error\n") - .mockReturnValue("alpha 2026-08-23 10:00:02 Deleting\n"); + .mockReturnValueOnce({ status: 0 }) + .mockReturnValueOnce({ status: 1 }); let failure: unknown; try { @@ -100,11 +113,11 @@ describe("Docker final handoff lifecycle integration", () => { dockerRun: vi.fn(createDockerFinalHandoffRunFixture(NEW_CONTAINER_ID)), dockerRunDetached: vi.fn(() => ({ status: 0, stdout: `${NEW_CONTAINER_ID}\n` })), dockerRename: vi.fn(() => ({ status: 0 })), - dockerStop: vi.fn(() => ({ status: 0 })), - dockerRm: vi.fn(() => ({ status: 0 })), + dockerStop, + dockerRm, dockerStart, runCaptureOpenshell, - runOpenshell: vi.fn(() => ({ status: 0 })), + runOpenshell, sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), detectSandboxFallbackDns: vi.fn(() => null), @@ -119,12 +132,15 @@ describe("Docker final handoff lifecycle integration", () => { expect(failure).toBeInstanceOf(Error); expect((failure as Error).message).toContain("final replacement handoff"); expect(getDockerGpuPatchFailureContext(failure)).toMatchObject({ - backupRemoved: true, + backupRemoved: false, newContainerId: NEW_CONTAINER_ID, oldContainerId: OLD_CONTAINER_ID, rolledBack: false, }); - expect(runCaptureOpenshell).toHaveBeenCalledTimes(2); + expect(runCaptureOpenshell).not.toHaveBeenCalled(); + expect(dockerStop).toHaveBeenCalledTimes(1); + expect(dockerStop).toHaveBeenCalledWith(OLD_CONTAINER_ID, expect.any(Object)); + expect(dockerRm).not.toHaveBeenCalled(); expect(dockerStart).not.toHaveBeenCalled(); }); }); diff --git a/test/onboarding/onboard-custom-dockerfile.test.ts b/test/onboarding/onboard-custom-dockerfile.test.ts index 646bebfd0a9..5171a1dc40a 100644 --- a/test/onboarding/onboard-custom-dockerfile.test.ts +++ b/test/onboarding/onboard-custom-dockerfile.test.ts @@ -464,6 +464,7 @@ createSandbox( NEMOCLAW_RECREATE_SANDBOX: "1", SANDBOX_LIVE: sandboxLive, }, + timeout: 30_000, }); assert.equal(result.status, 1, result.stderr); @@ -533,6 +534,7 @@ const { createSandbox } = require(${onboardPath}); PATH: `${fakeBin}:${process.env.PATH || ""}`, NEMOCLAW_NON_INTERACTIVE: "1", }, + timeout: 30_000, }); assert.equal(result.status, 1, "should exit 1 when fromDockerfile path is missing"); @@ -597,6 +599,7 @@ const { createSandbox } = require(${onboardPath}); PATH: `${fakeBin}:${process.env.PATH || ""}`, NEMOCLAW_NON_INTERACTIVE: "1", }, + timeout: 30_000, }); assert.equal(result.status, 1, "should exit 1 when fromDockerfile path is a directory"); @@ -671,6 +674,7 @@ const { createSandbox } = require(${onboardPath}); PATH: `${fakeBin}:${process.env.PATH || ""}`, NEMOCLAW_NON_INTERACTIVE: "1", }, + timeout: 30_000, }); assert.equal(result.status, 1, "should exit 1 when fromDockerfile is ignored"); @@ -773,6 +777,7 @@ const { createSandbox } = require(${onboardPath}); PATH: `${fakeBin}:${process.env.PATH || ""}`, NEMOCLAW_NON_INTERACTIVE: "1", }, + timeout: 30_000, }); assert.equal(result.status, 0, result.stderr); diff --git a/test/onboarding/onboard-messaging.test.ts b/test/onboarding/onboard-messaging.test.ts index 514a881d63e..a1bcd67f2cb 100644 --- a/test/onboarding/onboard-messaging.test.ts +++ b/test/onboarding/onboard-messaging.test.ts @@ -790,7 +790,7 @@ const { createSandbox } = require(${onboardPath}); const result = spawnSync(process.execPath, [scriptPath], { cwd: repoRoot, - encoding: "utf-8", + encoding: "utf-8", timeout: 30_000, env: { ...process.env, HOME: tmpDir, @@ -801,7 +801,7 @@ const { createSandbox } = require(${onboardPath}); }, }); - assert.equal(result.status, 0, result.stderr); + assert.equal(result.status, 0, result.stderr || result.error?.message); const payload = parseStdoutJson(result.stdout); const createCommand = payload.commands.find((entry: CommandEntry) => diff --git a/test/process-recovery/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery/process-recovery-supervisor-relaunch.test.ts index f944cdfced6..c2f94ae1f9e 100644 --- a/test/process-recovery/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery/process-recovery-supervisor-relaunch.test.ts @@ -81,7 +81,10 @@ function composedRelaunchTransaction( .fn() .mockReturnValueOnce(containerIds.old) .mockReturnValue(containerIds.replacement); - const runOpenshell = vi.fn(() => ({ status: 0, stdout: "No sandboxes found.\n" })); + const runOpenshell = vi.fn((args: readonly string[]) => { + order.push(`openshell-${args[1]}`); + return { status: 0, stdout: "No sandboxes found.\n" }; + }); let activeSandboxName = ""; const runCaptureOpenshell = vi.fn(() => runCaptureOpenshell.mock.calls.length === 1 @@ -586,14 +589,10 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { oldContainerId, expect.objectContaining({ ignoreError: true }), ); - expect(dockerStart).toHaveBeenCalledWith( - replacementContainerId, - expect.objectContaining({ ignoreError: true }), - ); + expect(dockerStart).not.toHaveBeenCalled(); + expect(order).toContain("openshell-stop"); + expect(order).toContain("openshell-start"); expect(waitForRecreatedSandboxOpenShellReadyImpl).toHaveBeenCalledTimes(2); - expect(dockerStart.mock.invocationCallOrder[0]).toBeLessThan( - waitForRecreatedSandboxOpenShellReadyImpl.mock.invocationCallOrder[1], - ); expect(waitForRecreatedSandboxOpenShellReadyImpl.mock.invocationCallOrder[1]).toBeLessThan( runOpenshell.mock.invocationCallOrder[0], ); @@ -625,22 +624,22 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { finalReadinessReady: true, }, { - condition: "OpenShell does not retire the previous lifecycle record", + condition: "OpenShell does not acknowledge the authoritative stop", finalizeOutcome: () => ({ - backupRemoved: true, - lifecycleReleaseObserved: false, + backupRemoved: false, + lifecycleStopAcknowledged: false, replacementRestarted: false, - replacementStoppedForCommit: true, + replacementStoppedForCommit: false, rolledBack: false, stateRestored: true, }), - expectedDetail: "OpenShell did not retire the previous lifecycle record", + expectedDetail: "OpenShell did not acknowledge the authoritative stop", expectedReadinessCalls: 1, finalPinnedAction: () => ACCEPTED_MANAGED_PROBE, finalReadinessReady: true, }, { - condition: "Docker cannot start the replacement container", + condition: "OpenShell cannot start the replacement container", finalizeOutcome: () => ({ backupRemoved: true, replacementRestarted: false, @@ -648,7 +647,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { rolledBack: false, stateRestored: true, }), - expectedDetail: "Docker could not start the replacement container", + expectedDetail: "OpenShell could not start the replacement container", expectedReadinessCalls: 1, finalPinnedAction: () => ACCEPTED_MANAGED_PROBE, finalReadinessReady: true,