From 1ceef4dc5bb48390866129ba600a58a6fe3e6768 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sun, 23 Aug 2026 16:16:24 -0700 Subject: [PATCH 01/12] fix(onboard): finalize exact Docker sandbox handoff Signed-off-by: Apurv Kumaria --- src/lib/actions/sandbox/process-recovery.ts | 17 +- .../sandbox/supervisor-relaunch.test.ts | 16 +- .../actions/sandbox/supervisor-relaunch.ts | 9 +- .../onboard/docker-gpu-patch-diagnostics.ts | 1 + .../onboard/docker-gpu-patch-finalize.test.ts | 439 ++++++------------ src/lib/onboard/docker-gpu-patch-finalize.ts | 175 +++---- .../onboard/docker-gpu-patch-jetson.test.ts | 4 +- .../docker-gpu-patch-mode-selection.test.ts | 2 +- .../docker-gpu-patch-recreate-dns.test.ts | 25 +- .../onboard/docker-gpu-patch-recreate.test.ts | 85 +++- src/lib/onboard/docker-gpu-patch-recreate.ts | 43 +- .../onboard/docker-gpu-patch-rollback.test.ts | 4 +- src/lib/onboard/docker-gpu-patch-types.ts | 11 +- ...ocker-gpu-sandbox-create-lifecycle.test.ts | 14 +- src/lib/onboard/docker-gpu-sandbox-create.ts | 34 +- .../docker-gpu-supervisor-reconnect.test.ts | 149 +++--- .../docker-gpu-supervisor-reconnect.ts | 152 +++--- src/lib/runtime-recovery.ts | 1 + .../gateway-guard-legacy-keepalive-fixture.ts | 19 + test/e2e/live/gateway-guard-recovery.test.ts | 40 +- ...way-guard-legacy-keepalive-fixture.test.ts | 6 +- ...ocess-recovery-supervisor-relaunch.test.ts | 147 +++--- 22 files changed, 733 insertions(+), 660 deletions(-) diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 1eca94f8727..80e9cdf51ca 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -505,12 +505,14 @@ function finalRelaunchContainerFailureDetail( if (completion.replacementStoppedForCommit === false) { return "Docker could not stop the replacement container for the final recovery handoff. NemoClaw did not start the primary dashboard/API host forward"; } - if (completion.lifecycleReleaseObserved === false) { - return "OpenShell did not release the sandbox name before the final recovery handoff. NemoClaw did not restart the replacement container or start the primary dashboard/API host forward"; - } if (completion.replacementRestarted === false) { return "Docker could not start the replacement container to complete the final recovery handoff. NemoClaw did not start the primary dashboard/API host forward"; } + if (completion.finalHandoffAcknowledged === false) { + return `OpenShell did not acknowledge the final replacement container handoff${ + completion.lastSandboxPhase ? `; last sandbox phase was ${completion.lastSandboxPhase}` : "" + }. NemoClaw did not start the primary dashboard/API host forward`; + } return null; } @@ -759,6 +761,13 @@ function recoverSandboxProcesses( quiet, deps: { runOpenshell, + runCaptureOpenshell: (args, options) => + captureOpenshell(args, { + ignoreError: true, + includeStderr: true, + timeout: + typeof options?.timeout === "number" ? options.timeout : OPENSHELL_PROBE_TIMEOUT_MS, + }).output, confirmMissingSupervisor: (containerId) => isExactlyMissingManagedSupervisor( requestPinnedGatewaySupervisorAction(sandboxName, "probe", 210000, containerId), @@ -847,7 +856,7 @@ export function restartSandboxGateway( requestGatewaySupervisorActionImpl: deps.requestGatewaySupervisorAction ?? executeGatewaySupervisorAction, }), - }), + }), ensureSandboxPortForward, ensureHermesDashboardPortForwardIfEnabled, recoverMessagingHostForward, diff --git a/src/lib/actions/sandbox/supervisor-relaunch.test.ts b/src/lib/actions/sandbox/supervisor-relaunch.test.ts index 08ea2622049..62a8d4ce2c7 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.test.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.test.ts @@ -75,6 +75,7 @@ function baseDeps(overrides: ManagedSupervisorRelaunchDeps = {}) { })), removeBackup: vi.fn(() => true), runOpenshell: vi.fn(() => ({ status: 0, stdout: "No sandboxes found.\n" })), + runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 10:00:00 Ready\n"), recreate: vi.fn(() => patchResult()), finalize: vi.fn(({ supervisorReady }) => supervisorReady @@ -156,12 +157,15 @@ describe("relaunchManagedSupervisorSession", () => { expect(deps.removeBackup).toHaveBeenCalledWith("alpha", "/tmp/rebuild-backups/alpha/recovery"); expect(deps.finalize).toHaveBeenCalledWith( { - lifecycleReleaseTimeoutSecs: 900, + finalHandoffTimeoutSecs: 900, result: expect.objectContaining({ newContainerId: "new-container-id" }), sandboxName: "alpha", supervisorReady: true, }, - { runOpenshell: deps.runOpenshell }, + { + runCaptureOpenshell: deps.runCaptureOpenshell, + runOpenshell: deps.runOpenshell, + }, ); }); @@ -389,12 +393,15 @@ describe("relaunchManagedSupervisorSession", () => { expect(deps.restartRestoredManagedGateway).toHaveBeenCalledWith("new-container-id"); expect(deps.finalize).toHaveBeenCalledWith( { - lifecycleReleaseTimeoutSecs: 900, + finalHandoffTimeoutSecs: 900, result: expect.objectContaining({ newContainerId: "new-container-id" }), sandboxName: "alpha", supervisorReady: true, }, - { runOpenshell: deps.runOpenshell }, + { + runCaptureOpenshell: deps.runCaptureOpenshell, + runOpenshell: deps.runOpenshell, + }, ); }); @@ -405,6 +412,7 @@ describe("relaunchManagedSupervisorSession", () => { expect(relaunch?.finalize(true)).toMatchObject({ backupRemoved: true, rolledBack: false }); expect(deps.finalize).toHaveBeenCalledWith(expect.objectContaining({ supervisorReady: true }), { + runCaptureOpenshell: deps.runCaptureOpenshell, runOpenshell: deps.runOpenshell, sleep, }); diff --git a/src/lib/actions/sandbox/supervisor-relaunch.ts b/src/lib/actions/sandbox/supervisor-relaunch.ts index cf71bf3bf68..c892ce6e522 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.ts @@ -59,6 +59,9 @@ export type ManagedSupervisorRelaunchDeps = { recreate?: typeof recreateOpenShellDockerSandboxWithStartupCommand; finalize?: typeof finalizeDockerGpuPatchBackup; runOpenshell?: NonNullable[1]>["runOpenshell"]; + runCaptureOpenshell?: NonNullable< + Parameters[1] + >["runCaptureOpenshell"]; }; function inspectContainer(containerId: string): DockerContainerInspect { @@ -287,8 +290,10 @@ export function relaunchManagedSupervisorSession( return finalizeFailure(); } const runLifecycleProbe = deps.runOpenshell; - if (!runLifecycleProbe) return finalizeFailure(); + const captureLifecycleProbe = deps.runCaptureOpenshell; + if (!runLifecycleProbe || !captureLifecycleProbe) return finalizeFailure(); const lifecycleDeps = { + runCaptureOpenshell: captureLifecycleProbe, runOpenshell: runLifecycleProbe, ...(deps.sleep ? { sleep: deps.sleep } : {}), }; @@ -298,7 +303,7 @@ export function relaunchManagedSupervisorSession( result, supervisorReady: true, sandboxName, - lifecycleReleaseTimeoutSecs: getDockerGpuSupervisorReconnectTimeoutSecs(1), + finalHandoffTimeoutSecs: getDockerGpuSupervisorReconnectTimeoutSecs(1), }, lifecycleDeps, ), diff --git a/src/lib/onboard/docker-gpu-patch-diagnostics.ts b/src/lib/onboard/docker-gpu-patch-diagnostics.ts index ef9fda3317e..1ad91880cbd 100644 --- a/src/lib/onboard/docker-gpu-patch-diagnostics.ts +++ b/src/lib/onboard/docker-gpu-patch-diagnostics.ts @@ -243,6 +243,7 @@ export function collectDockerGpuPatchDiagnostics( `old_container_id=${redactor.redactText(context?.oldContainerId ?? "unknown")}`, `new_container_id=${redactor.redactText(context?.newContainerId ?? "unknown")}`, `backup_container_name=${redactor.redactText(context?.backupContainerName ?? "none")}`, + `last_openshell_phase=${redactor.redactText(context?.lastSandboxPhase ?? "unknown")}`, `rolled_back=${cleanupPendingRollback ? "pending" : context?.rolledBack === true ? "yes" : context?.rolledBack === false ? "failed" : "no"}`, ...(context?.replacementStopConfirmed !== undefined ? [`replacement_stop_confirmed=${confirmationValue(context.replacementStopConfirmed)}`] diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index 3ba19ac1e88..7f83db94d1e 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -30,6 +30,21 @@ function deferredCreateResult(): DockerGpuPatchResult { }; } +function exactDeferredCreateResult(): DockerGpuPatchResult { + return { + ...deferredCreateResult(), + oldContainerId: "a".repeat(64), + newContainerId: "b".repeat(64), + }; +} + +function readyHandoffDeps() { + return { + runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 10:00:00 Ready\n"), + runOpenshell: vi.fn(() => ({ status: 0 })), + }; +} + function collectRollbackDiagnostics( newContainerId: string, outcome: DockerGpuPatchFinalizeOutcome, @@ -63,52 +78,35 @@ function collectRollbackDiagnostics( } describe("finalizeDockerGpuPatchBackup", () => { - it("makes the replacement restart the final lifecycle event after removing the backup", () => { + it("retains both containers when final acknowledgement probes are unavailable (#9531)", () => { const dockerStop = vi.fn(() => ({ status: 0 })); - const dockerRm = vi.fn((_name: string) => ({ status: 0 })); + const dockerRm = vi.fn(() => ({ status: 0 })); const dockerStart = vi.fn(() => ({ status: 0 })); - const outcome = finalizeDockerGpuPatchBackup( - { - result: deferredCreateResult(), - supervisorReady: true, - sandboxName: "alpha", - lifecycleReleaseTimeoutSecs: 60, - }, - { - dockerStop, - dockerRm, - dockerStart, - runOpenshell: vi.fn(() => ({ status: 0, stdout: "No sandboxes found.\n" })), - }, - ); - expect(outcome).toEqual({ - backupRemoved: true, + + expect( + finalizeDockerGpuPatchBackup( + { + result: exactDeferredCreateResult(), + supervisorReady: true, + sandboxName: "alpha", + finalHandoffTimeoutSecs: 60, + }, + { dockerRm, dockerStart, dockerStop }, + ), + ).toEqual({ + backupRemoved: false, rolledBack: false, - replacementStoppedForCommit: true, - replacementRestarted: true, - lifecycleReleaseObserved: true, + replacementStoppedForCommit: false, + finalHandoffAcknowledged: false, + lastSandboxPhase: null, }); - expect(dockerStop).toHaveBeenCalledWith( - "new-container-id", - expect.objectContaining({ ignoreError: true }), - ); - expect(dockerRm).toHaveBeenCalledWith( - "openshell-alpha-nemoclaw-gpu-backup-1780491860342", - expect.objectContaining({ ignoreError: true }), - ); - expect(dockerStart).toHaveBeenCalledWith( - "new-container-id", - expect.objectContaining({ ignoreError: true }), - ); - expect(dockerStop.mock.invocationCallOrder[0]).toBeLessThan( - dockerRm.mock.invocationCallOrder[0], - ); - expect(dockerRm.mock.invocationCallOrder[0]).toBeLessThan( - dockerStart.mock.invocationCallOrder[0], - ); + expect(dockerStop).not.toHaveBeenCalled(); + expect(dockerRm).not.toHaveBeenCalled(); + expect(dockerStart).not.toHaveBeenCalled(); }); - it("waits for the sandbox name to disappear before restarting the replacement (#9531)", () => { + it("uses one exact stop, remove, start, and Ready acknowledgement handoff (#9531)", () => { + const result = exactDeferredCreateResult(); const events: string[] = []; const dockerStop = vi.fn(() => { events.push("stop replacement"); @@ -122,281 +120,125 @@ describe("finalizeDockerGpuPatchBackup", () => { events.push("start replacement"); return { status: 0 }; }); - const runOpenshell = vi - .fn() - .mockImplementationOnce(() => { - events.push("observe deleting"); - return { status: 0, stdout: "alpha 2026-08-21 05:53:16 Deleting\n" }; - }) - .mockImplementationOnce(() => { - events.push("observe error"); - return { status: 0, stdout: "alpha 2026-08-21 05:53:18 Error\n" }; - }) - .mockImplementationOnce(() => { - events.push("observe name absence"); - return { status: 0, stdout: "beta 2026-08-21 05:53:20 Ready\n" }; - }); - - const outcome = finalizeDockerGpuPatchBackup( - { - result: deferredCreateResult(), - supervisorReady: true, - sandboxName: "alpha", - lifecycleReleaseTimeoutSecs: 60, - }, - { dockerStop, dockerRm, dockerStart, runOpenshell, sleep: vi.fn() }, - ); - - expect(outcome).toMatchObject({ - backupRemoved: true, - lifecycleReleaseObserved: true, - replacementRestarted: true, - }); - expect(events).toEqual([ - "stop replacement", - "remove backup", - "observe deleting", - "observe error", - "observe name absence", - "start replacement", - ]); - }); - - it("accepts Error only when the stopped replacement is the sole labeled container (#9962)", () => { - const replacementContainerId = "a".repeat(64); - const events: string[] = []; - const dockerStop = vi.fn(() => { - events.push("stop replacement"); - return { status: 0 }; - }); - const dockerRm = vi.fn(() => { - events.push("remove backup"); - return { status: 0 }; - }); - const dockerRun = vi.fn(() => { - events.push("confirm exact replacement"); - return { status: 0, stdout: `${replacementContainerId}\n` }; + const runCaptureOpenshell = vi.fn(() => { + events.push("observe ready"); + return "alpha 2026-08-23 10:00:00 Ready\n"; }); - const dockerStart = vi.fn(() => { - events.push("start replacement"); + const runOpenshell = vi.fn(() => { + events.push("exec ready"); return { status: 0 }; }); - const runOpenshell = vi.fn(() => { - events.push("observe error"); - return { status: 0, stdout: "alpha 2026-08-23 01:40:35 Error\n" }; + const dockerResults = { + ps: { + event: "confirm sole replacement", + result: { status: 0, stdout: `${result.newContainerId}\n` }, + }, + inspect: { event: "confirm running replacement", result: { status: 0, stdout: "true\n" } }, + } as const; + const dockerRun = vi.fn((args: readonly string[]) => { + const response = dockerResults[String(args[0]) as keyof typeof dockerResults]; + events.push(response.event); + return response.result; }); const outcome = finalizeDockerGpuPatchBackup( { - result: { ...deferredCreateResult(), newContainerId: replacementContainerId }, + result, supervisorReady: true, sandboxName: "alpha", - lifecycleReleaseTimeoutSecs: 60, + finalHandoffTimeoutSecs: 60, + }, + { + dockerStop, + dockerRm, + dockerRun, + dockerStart, + runCaptureOpenshell, + runOpenshell, + sleep: vi.fn(), }, - { dockerStop, dockerRm, dockerRun, dockerStart, runOpenshell, sleep: vi.fn() }, ); - expect(outcome).toMatchObject({ + expect(outcome).toEqual({ backupRemoved: true, - lifecycleReleaseObserved: true, + rolledBack: false, + replacementStoppedForCommit: true, replacementRestarted: true, + finalHandoffAcknowledged: true, + lastSandboxPhase: "Ready", }); expect(events).toEqual([ "stop replacement", "remove backup", - "observe error", - "confirm exact replacement", "start replacement", + "observe ready", + "exec ready", + "confirm sole replacement", + "confirm running replacement", ]); - expect(dockerRun).toHaveBeenCalledWith( - expect.arrayContaining([ - "--no-trunc", - "label=openshell.ai/managed-by=openshell", - "label=openshell.ai/sandbox-name=alpha", - ]), + expect(dockerStop).toHaveBeenCalledWith( + result.newContainerId, expect.objectContaining({ ignoreError: true }), ); - }); - - it("caps Error corroboration to the remaining lifecycle-release budget (#9962)", () => { - const replacementContainerId = "a".repeat(64); - vi.useFakeTimers(); - vi.setSystemTime(0); - const dockerRun = vi.fn(() => ({ - status: 0, - stdout: `${replacementContainerId}\n`, - })); - - let outcome: DockerGpuPatchFinalizeOutcome; - try { - outcome = finalizeDockerGpuPatchBackup( - { - result: { ...deferredCreateResult(), newContainerId: replacementContainerId }, - supervisorReady: true, - sandboxName: "alpha", - lifecycleReleaseTimeoutSecs: 1, - }, - { - dockerStop: vi.fn(() => ({ status: 0 })), - dockerRm: vi.fn(() => ({ status: 0 })), - dockerRun, - dockerStart: vi.fn(() => ({ status: 0 })), - runOpenshell: vi.fn(() => { - vi.setSystemTime(750); - return { - status: 0, - stdout: "alpha 2026-08-23 01:40:35 Error\n", - }; - }), - sleep: vi.fn(), - }, - ); - } finally { - vi.useRealTimers(); - } - - expect(outcome).toMatchObject({ - lifecycleReleaseObserved: true, - replacementRestarted: true, - }); - expect(dockerRun).toHaveBeenCalledWith( - expect.any(Array), - expect.objectContaining({ timeout: 250 }), + expect(dockerRm).toHaveBeenCalledWith( + result.oldContainerId, + expect.objectContaining({ ignoreError: true }), ); - }); - - it.each([ - ["a failed Docker query", { status: 1, stderr: "daemon unavailable" }], - ["no labeled container", { status: 0, stdout: "" }], - ["another labeled container", { status: 0, stdout: `${"b".repeat(64)}\n` }], - [ - "multiple labeled containers", - { status: 0, stdout: `${"a".repeat(64)}\n${"b".repeat(64)}\n` }, - ], - ["a truncated replacement ID", { status: 0, stdout: `${"a".repeat(12)}\n` }], - ])("does not accept Error with %s (#9962)", (_case, dockerResult) => { - const replacementContainerId = "a".repeat(64); - const dockerStart = vi.fn(() => ({ status: 0 })); - const outcome = finalizeDockerGpuPatchBackup( - { - result: { ...deferredCreateResult(), newContainerId: replacementContainerId }, - supervisorReady: true, - sandboxName: "alpha", - lifecycleReleaseTimeoutSecs: 1, - }, - { - dockerStop: vi.fn(() => ({ status: 0 })), - dockerRm: vi.fn(() => ({ status: 0 })), - dockerRun: vi.fn(() => dockerResult), - dockerStart, - runOpenshell: vi.fn(() => ({ - status: 0, - stdout: "alpha 2026-08-23 01:40:35 Error\n", - })), - sleep: vi.fn(), - }, + expect(dockerStart).toHaveBeenCalledWith( + result.newContainerId, + expect.objectContaining({ ignoreError: true }), ); - - expect(outcome).toMatchObject({ - lifecycleReleaseObserved: false, - replacementRestarted: false, - }); - expect(dockerStart).not.toHaveBeenCalled(); }); - it("does not accept Error when the exact Docker query throws (#9962)", () => { - const dockerStart = vi.fn(() => ({ status: 0 })); + it("fails immediately when OpenShell reports Deleting after the final start (#9531)", () => { + const result = exactDeferredCreateResult(); + const events: string[] = []; + const sleep = vi.fn(); const outcome = finalizeDockerGpuPatchBackup( { - result: { ...deferredCreateResult(), newContainerId: "a".repeat(64) }, + result, supervisorReady: true, sandboxName: "alpha", - lifecycleReleaseTimeoutSecs: 1, + finalHandoffTimeoutSecs: 60, }, { - dockerStop: vi.fn(() => ({ status: 0 })), - dockerRm: vi.fn(() => ({ status: 0 })), - dockerRun: vi.fn(() => { - throw new Error("daemon unavailable"); + dockerStop: vi.fn(() => { + events.push("stop replacement"); + return { status: 0 }; }), - dockerStart, - runOpenshell: vi.fn(() => ({ - status: 0, - stdout: "alpha 2026-08-23 01:40:35 Error\n", - })), - sleep: vi.fn(), - }, - ); - - expect(outcome).toMatchObject({ - lifecycleReleaseObserved: false, - replacementRestarted: false, - }); - expect(dockerStart).not.toHaveBeenCalled(); - }); - - it("does not treat failed lifecycle probes as a release receipt (#9531)", () => { - const runOpenshell = vi - .fn() - .mockReturnValueOnce({ status: 0, stdout: "Error: gateway unavailable" }) - .mockReturnValueOnce({ status: 1, stderr: "gateway unavailable" }); - const dockerStart = vi.fn(() => ({ status: 0 })); - - const outcome = finalizeDockerGpuPatchBackup( - { - result: deferredCreateResult(), - supervisorReady: true, - sandboxName: "alpha", - lifecycleReleaseTimeoutSecs: 1, - }, - { - dockerStop: vi.fn(() => ({ status: 0 })), - dockerRm: vi.fn(() => ({ status: 0 })), - dockerStart, - runOpenshell, - sleep: vi.fn(), + dockerRm: vi.fn(() => { + events.push("remove backup"); + return { status: 0 }; + }), + dockerStart: vi.fn(() => { + events.push("start replacement"); + return { status: 0 }; + }), + runCaptureOpenshell: vi.fn(() => { + events.push("observe deleting"); + return "alpha 2026-08-23 10:00:00 Deleting\n"; + }), + runOpenshell: vi.fn(() => ({ status: 1 })), + dockerRun: vi.fn(() => ({ status: 0, stdout: `${result.newContainerId}\n` })), + sleep, }, ); - expect(outcome).toMatchObject({ + expect(outcome).toEqual({ backupRemoved: true, - lifecycleReleaseObserved: false, - replacementRestarted: false, + rolledBack: false, + replacementStoppedForCommit: true, + replacementRestarted: true, + finalHandoffAcknowledged: false, + lastSandboxPhase: "Deleting", }); - expect(runOpenshell).toHaveBeenCalledTimes(2); - expect(runOpenshell.mock.calls[0]?.[1]?.timeout).toBeGreaterThan(0); - expect(runOpenshell.mock.calls[0]?.[1]?.timeout).toBeLessThanOrEqual(1000); - expect(runOpenshell.mock.calls[1]?.[1]?.timeout).toBeGreaterThan(0); - expect(runOpenshell.mock.calls[1]?.[1]?.timeout).toBeLessThanOrEqual(1000); - expect(dockerStart).not.toHaveBeenCalled(); - }); - - it("does not treat an unrelated terminal lifecycle phase as the stopped replacement (#9531)", () => { - const runOpenshell = vi.fn(() => ({ - status: 0, - stdout: "alpha 2026-08-21 05:53:18 Failed\n", - })); - - const dockerStart = vi.fn(() => ({ status: 0 })); - const outcome = finalizeDockerGpuPatchBackup( - { - result: deferredCreateResult(), - supervisorReady: true, - sandboxName: "alpha", - lifecycleReleaseTimeoutSecs: 1, - }, - { - dockerStop: vi.fn(() => ({ status: 0 })), - dockerRm: vi.fn(() => ({ status: 0 })), - dockerStart, - runOpenshell, - sleep: vi.fn(), - }, - ); - - expect(outcome.lifecycleReleaseObserved).toBe(false); - expect(runOpenshell).toHaveBeenCalledTimes(2); - expect(dockerStart).not.toHaveBeenCalled(); + expect(events).toEqual([ + "stop replacement", + "remove backup", + "start replacement", + "observe deleting", + ]); + expect(sleep).not.toHaveBeenCalled(); }); it("rolls back to the backup container when supervisor reconnect failed", () => { @@ -493,7 +335,7 @@ describe("finalizeDockerGpuPatchBackup", () => { result, supervisorReady: true, sandboxName: "alpha", - lifecycleReleaseTimeoutSecs: 60, + finalHandoffTimeoutSecs: 60, }, { dockerRm }, ); @@ -513,22 +355,26 @@ describe("finalizeDockerGpuPatchBackup", () => { result: deferredCreateResult(), supervisorReady: true, sandboxName: "alpha", - lifecycleReleaseTimeoutSecs: 60, + finalHandoffTimeoutSecs: 60, }, - { dockerStop, dockerRm, dockerStart }, + { dockerStop, dockerRm, dockerStart, ...readyHandoffDeps() }, ); expect(outcome).toEqual({ backupRemoved: false, rolledBack: false, replacementStoppedForCommit: true, - replacementRestarted: false, - lifecycleReleaseObserved: false, + replacementRestarted: true, + finalHandoffAcknowledged: false, + lastSandboxPhase: null, }); expect(dockerRm).toHaveBeenCalledWith( - "openshell-alpha-nemoclaw-gpu-backup-1780491860342", + "old-container-id", + expect.objectContaining({ ignoreError: true }), + ); + expect(dockerStart).toHaveBeenCalledWith( + "new-container-id", expect.objectContaining({ ignoreError: true }), ); - expect(dockerStart).not.toHaveBeenCalled(); }); it("fails closed when backup removal has no exit status", () => { @@ -540,18 +386,22 @@ describe("finalizeDockerGpuPatchBackup", () => { result: deferredCreateResult(), supervisorReady: true, sandboxName: "alpha", - lifecycleReleaseTimeoutSecs: 60, + finalHandoffTimeoutSecs: 60, }, - { dockerStop, dockerRm, dockerStart }, + { dockerStop, dockerRm, dockerStart, ...readyHandoffDeps() }, ); expect(outcome).toEqual({ backupRemoved: false, rolledBack: false, replacementStoppedForCommit: true, - replacementRestarted: false, - lifecycleReleaseObserved: false, + replacementRestarted: true, + finalHandoffAcknowledged: false, + lastSandboxPhase: null, }); - expect(dockerStart).not.toHaveBeenCalled(); + expect(dockerStart).toHaveBeenCalledWith( + "new-container-id", + expect.objectContaining({ ignoreError: true }), + ); }); it("retains the backup when the replacement cannot be stopped for the final handoff", () => { @@ -564,9 +414,9 @@ describe("finalizeDockerGpuPatchBackup", () => { result: deferredCreateResult(), supervisorReady: true, sandboxName: "alpha", - lifecycleReleaseTimeoutSecs: 60, + finalHandoffTimeoutSecs: 60, }, - { dockerStop, dockerRm, dockerStart }, + { dockerStop, dockerRm, dockerStart, ...readyHandoffDeps() }, ); expect(outcome).toEqual({ @@ -584,13 +434,13 @@ describe("finalizeDockerGpuPatchBackup", () => { result: deferredCreateResult(), supervisorReady: true, sandboxName: "alpha", - lifecycleReleaseTimeoutSecs: 60, + finalHandoffTimeoutSecs: 60, }, { dockerStop: vi.fn(() => ({ status: 0 })), dockerRm: vi.fn(() => ({ status: 0 })), dockerStart: vi.fn(() => ({ status: 1 })), - runOpenshell: vi.fn(() => ({ status: 0, stdout: "No sandboxes found.\n" })), + ...readyHandoffDeps(), }, ); @@ -599,7 +449,8 @@ describe("finalizeDockerGpuPatchBackup", () => { rolledBack: false, replacementStoppedForCommit: true, replacementRestarted: false, - lifecycleReleaseObserved: true, + finalHandoffAcknowledged: false, + lastSandboxPhase: null, }); }); diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index 20fa338cc5a..b64c2a2e914 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -9,10 +9,11 @@ // that natively accepts NVIDIA GPU access would remove the need for the // post-create container recreation NemoClaw performs here. Until OpenShell // supports that natively, NemoClaw recreates the container with GPU access -// and uses this module to either confirm the new container or restore the -// pre-patch backup. Regression coverage: +// and uses this module to either restore the pre-patch backup before commit or +// complete the exact stop/remove/start handoff and require OpenShell's final +// Ready acknowledgement. Regression coverage: // * src/lib/onboard/docker-gpu-patch-finalize.test.ts — direct unit tests -// for finalize success / rollback / no-op / rollback failure outcomes. +// for exact final handoff, terminal phase, rollback, and failure outcomes. // * src/lib/onboard/docker-gpu-patch-rollback.test.ts — composed // recreate-with-rollback scenarios. // * src/lib/onboard/docker-gpu-sandbox-create.test.ts — composed create @@ -31,7 +32,7 @@ import { } from "./docker-gpu-patch-rollback"; import { fullDockerContainerId } from "./docker-gpu-patch-clone"; import type { DockerGpuPatchDeps, DockerGpuPatchResult } from "./docker-gpu-patch-types"; -import { waitForOpenShellSandboxLifecycleRelease } from "./docker-gpu-supervisor-reconnect"; +import { waitForOpenShellFinalHandoff } from "./docker-gpu-supervisor-reconnect"; import { queryOpenShellDockerSandboxContainers } from "./openshell-docker-sandbox-containers"; export { @@ -48,7 +49,7 @@ export type DockerGpuPatchFinalizeOptions = result: DockerGpuPatchResult; supervisorReady: true; sandboxName: string; - lifecycleReleaseTimeoutSecs: number; + finalHandoffTimeoutSecs: number; }; export type DockerGpuPatchFinalizeOutcome = { @@ -56,13 +57,14 @@ export type DockerGpuPatchFinalizeOutcome = { rolledBack: boolean; replacementStoppedForCommit?: boolean; replacementRestarted?: boolean; - lifecycleReleaseObserved?: boolean; + finalHandoffAcknowledged?: boolean; + lastSandboxPhase?: string | null; replacementStopConfirmed?: boolean; replacementRemovalConfirmed?: boolean; replacementPresence?: "absent" | "present" | "unknown"; }; -function isSoleLabeledReplacement( +function isExactRunningReplacement( sandboxName: string, replacementContainerId: string, dockerRun: NonNullable, @@ -71,12 +73,33 @@ function isSoleLabeledReplacement( const expectedContainerId = fullDockerContainerId(replacementContainerId); if (!expectedContainerId || timeoutMs <= 0) return false; try { + const deadline = Date.now() + timeoutMs; const containers = queryOpenShellDockerSandboxContainers(sandboxName, { dockerRun }, timeoutMs); - return ( - containers.ok && - containers.ids.length === 1 && - fullDockerContainerId(containers.ids[0]) === expectedContainerId + if ( + !containers.ok || + containers.ids.length !== 1 || + fullDockerContainerId(containers.ids[0]) !== expectedContainerId + ) { + return false; + } + const remainingMs = deadline - Date.now(); + if (remainingMs <= 0) return false; + const inspect = dockerRun( + [ + "inspect", + "--type", + "container", + "--format", + "{{json .State.Running}}", + expectedContainerId, + ], + { + ignoreError: true, + suppressOutput: true, + timeout: Math.min(DOCKER_GPU_PATCH_TIMEOUT_MS, remainingMs), + }, ); + return hasZeroDockerExitStatus(inspect) && String(inspect.stdout ?? "").trim() === "true"; } catch { return false; } @@ -96,12 +119,20 @@ export function finalizeDockerGpuPatchBackup( return { backupRemoved: true, rolledBack: false }; } if (options.supervisorReady) { - // Stop the replacement before retiring the labelled backup, then start it - // afterward. OpenShell observes Docker lifecycle events for both containers; - // leaving the backup's removal as the final event can demote the already - // reconnected replacement back to not-ready. The final start makes the live - // replacement's registration authoritative while the rollback container is - // still retained until the destructive removal succeeds. + // Stop the exact replacement before retiring the exact backup, then start + // the replacement afterward. The final start is the authoritative Docker + // lifecycle event. Success is withheld until OpenShell reports Ready and + // Docker still proves the exact replacement is the sole running labeled + // container (#9531). + if (!deps.runOpenshell || !deps.runCaptureOpenshell) { + return { + backupRemoved: false, + rolledBack: false, + replacementStoppedForCommit: false, + finalHandoffAcknowledged: false, + lastSandboxPhase: null, + }; + } const stopResult = resolved.dockerStop(options.result.newContainerId, containerOpts); if (!hasZeroDockerExitStatus(stopResult)) { return { @@ -110,49 +141,59 @@ export function finalizeDockerGpuPatchBackup( replacementStoppedForCommit: false, }; } - const rmResult = resolved.dockerRm(options.result.backupContainerName, containerOpts); + const rmResult = resolved.dockerRm(options.result.oldContainerId, containerOpts); const backupRemoved = hasZeroDockerExitStatus(rmResult); - const sandboxName = options.sandboxName; - const lifecycleReleaseTimeoutSecs = options.lifecycleReleaseTimeoutSecs; - const hasLifecycleContext = - sandboxName.length > 0 && - Number.isFinite(lifecycleReleaseTimeoutSecs) && - lifecycleReleaseTimeoutSecs > 0; - if (backupRemoved && hasLifecycleContext) { - console.log( - ` Waiting for OpenShell to retire the previous lifecycle record before restarting the replacement (up to ${lifecycleReleaseTimeoutSecs}s)...`, + if (!backupRemoved) { + const replacementRestarted = hasZeroDockerExitStatus( + resolved.dockerStart(options.result.newContainerId, containerOpts), ); - } - const lifecycleReleaseObserved = - backupRemoved && hasLifecycleContext - ? waitForOpenShellSandboxLifecycleRelease(sandboxName, lifecycleReleaseTimeoutSecs, { - runOpenshell: deps.runOpenshell, - sleep: deps.sleep, - soleLabeledReplacementCorroboratesError: (remainingMs) => - isSoleLabeledReplacement( - sandboxName, - options.result.newContainerId, - resolved.dockerRun, - remainingMs, - ), - }) - : false; - if (!lifecycleReleaseObserved) { return { - backupRemoved, + backupRemoved: false, rolledBack: false, replacementStoppedForCommit: true, - replacementRestarted: false, - lifecycleReleaseObserved: false, + replacementRestarted, + finalHandoffAcknowledged: false, + lastSandboxPhase: null, }; } const startResult = resolved.dockerStart(options.result.newContainerId, containerOpts); + const replacementRestarted = hasZeroDockerExitStatus(startResult); + if (!replacementRestarted) { + return { + backupRemoved: true, + rolledBack: false, + replacementStoppedForCommit: true, + replacementRestarted, + finalHandoffAcknowledged: false, + lastSandboxPhase: null, + }; + } + console.log( + ` Waiting for OpenShell to confirm the final replacement handoff (up to ${options.finalHandoffTimeoutSecs}s)...`, + ); + const acknowledgement = waitForOpenShellFinalHandoff( + options.sandboxName, + options.finalHandoffTimeoutSecs, + { + runCaptureOpenshell: deps.runCaptureOpenshell, + runOpenshell: deps.runOpenshell, + sleep: deps.sleep, + replacementIsExactAndRunning: (remainingMs) => + isExactRunningReplacement( + options.sandboxName, + options.result.newContainerId, + resolved.dockerRun, + remainingMs, + ), + }, + ); return { - backupRemoved, + backupRemoved: true, rolledBack: false, replacementStoppedForCommit: true, - replacementRestarted: hasZeroDockerExitStatus(startResult), - lifecycleReleaseObserved: true, + replacementRestarted: true, + finalHandoffAcknowledged: acknowledgement.acknowledged, + lastSandboxPhase: acknowledgement.lastSandboxPhase, }; } const rollback = rollbackToBackupContainer( @@ -165,39 +206,3 @@ export function finalizeDockerGpuPatchBackup( ); return { backupRemoved: false, ...rollback }; } - -export type SupervisorReconnectOutcome = - | { execReady: true; backupRemoved: boolean } - | ({ execReady: false; error: Error } & Omit); - -export function reconcileSupervisorReconnect( - execReady: boolean, - refs: { newContainerId: string; backupContainerName: string; originalName: string }, - deps: DockerGpuPatchDeps, -): SupervisorReconnectOutcome { - const resolved = resolveDockerGpuPatchRollbackDeps(deps); - const containerOpts = { - ignoreError: true, - suppressOutput: true, - timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, - }; - if (execReady) { - // Backup removal is best-effort here too: the supervisor probe already - // confirmed the new container is reachable, so a failed rm leaves a - // leaked backup container but the user-visible sandbox is healthy. - // Surface the actual rm status so callers can fold it into diagnostics - // alongside the deferred-finalize path in `finalizeDockerGpuPatchBackup`. - const rmResult = resolved.dockerRm(refs.backupContainerName, containerOpts); - return { execReady: true, backupRemoved: hasZeroDockerExitStatus(rmResult) }; - } - const rollback = rollbackToBackupContainer(refs, resolved); - return { - execReady: false, - ...rollback, - error: new Error( - rollback.rolledBack - ? "OpenShell supervisor did not reconnect to the GPU-enabled container; pre-patch sandbox restored." - : "OpenShell supervisor did not reconnect to the GPU-enabled container and rollback failed; pre-patch sandbox was NOT restored.", - ), - }; -} diff --git a/src/lib/onboard/docker-gpu-patch-jetson.test.ts b/src/lib/onboard/docker-gpu-patch-jetson.test.ts index e091c45a137..fc6c769e70b 100644 --- a/src/lib/onboard/docker-gpu-patch-jetson.test.ts +++ b/src/lib/onboard/docker-gpu-patch-jetson.test.ts @@ -59,7 +59,7 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { ); recreateOpenShellDockerSandboxWithGpu( - { sandboxName: "alpha", timeoutSecs: 1, backend: "jetson" }, + { sandboxName: "alpha", timeoutSecs: 1, backend: "jetson", waitForSupervisor: false }, { dockerCapture: dockerCaptureFixture(), dockerRun: vi.fn(() => ({ status: 0, stdout: "probe-id\n" })), @@ -90,7 +90,7 @@ describe("Jetson device-node group propagation (#4231, #7610)", () => { const detectTegraDeviceGroupGidsStub = vi.fn(() => ["44"]); recreateOpenShellDockerSandboxWithGpu( - { sandboxName: "alpha", timeoutSecs: 1, backend: "generic" }, + { sandboxName: "alpha", timeoutSecs: 1, backend: "generic", waitForSupervisor: false }, { dockerCapture: dockerCaptureFixture(), dockerRun: vi.fn(() => ({ status: 0, stdout: "probe-id\n" })), diff --git a/src/lib/onboard/docker-gpu-patch-mode-selection.test.ts b/src/lib/onboard/docker-gpu-patch-mode-selection.test.ts index ad158d50191..1a5c852fef9 100644 --- a/src/lib/onboard/docker-gpu-patch-mode-selection.test.ts +++ b/src/lib/onboard/docker-gpu-patch-mode-selection.test.ts @@ -302,7 +302,7 @@ describe("docker-gpu-patch CDI-first mode selection (#4948)", () => { const detectSandboxFallbackDns = vi.fn(() => null); const result = recreateOpenShellDockerSandboxWithGpu( - { sandboxName: "alpha", timeoutSecs: 1 }, + { sandboxName: "alpha", timeoutSecs: 1, waitForSupervisor: false }, { dockerCapture, readDir: host.readDir, diff --git a/src/lib/onboard/docker-gpu-patch-recreate-dns.test.ts b/src/lib/onboard/docker-gpu-patch-recreate-dns.test.ts index 42f33fb00b5..55be092092c 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate-dns.test.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate-dns.test.ts @@ -92,7 +92,10 @@ describe("Docker GPU recreate DNS fallback (#3579)", () => { it("plumbs the discovered upstream through recreate into clone args", () => { const deps = recreateDeps("9.9.9.9"); - recreateOpenShellDockerSandboxWithGpu({ sandboxName: "alpha", timeoutSecs: 1 }, deps); + recreateOpenShellDockerSandboxWithGpu( + { sandboxName: "alpha", timeoutSecs: 1, waitForSupervisor: false }, + deps, + ); expect(deps.detectSandboxFallbackDns).toHaveBeenCalled(); expect(deps.probeContainerDns).toHaveBeenCalledWith({ dnsServer: "9.9.9.9" }); @@ -104,7 +107,10 @@ describe("Docker GPU recreate DNS fallback (#3579)", () => { it("does not add DNS through recreate when no upstream is discovered", () => { const deps = recreateDeps(null); - recreateOpenShellDockerSandboxWithGpu({ sandboxName: "alpha", timeoutSecs: 1 }, deps); + recreateOpenShellDockerSandboxWithGpu( + { sandboxName: "alpha", timeoutSecs: 1, waitForSupervisor: false }, + deps, + ); expect(deps.dockerRunDetached).not.toHaveBeenCalledWith( expect.arrayContaining(["--dns"]), @@ -123,7 +129,10 @@ describe("Docker GPU recreate DNS fallback (#3579)", () => { const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); try { - recreateOpenShellDockerSandboxWithGpu({ sandboxName: "alpha", timeoutSecs: 1 }, deps); + recreateOpenShellDockerSandboxWithGpu( + { sandboxName: "alpha", timeoutSecs: 1, waitForSupervisor: false }, + deps, + ); expect(warn).toHaveBeenCalledWith( expect.stringContaining( @@ -152,7 +161,10 @@ describe("Docker GPU recreate DNS fallback (#3579)", () => { }; const deps = recreateDeps("9.9.9.9", inspect); - recreateOpenShellDockerSandboxWithGpu({ sandboxName: "alpha", timeoutSecs: 1 }, deps); + recreateOpenShellDockerSandboxWithGpu( + { sandboxName: "alpha", timeoutSecs: 1, waitForSupervisor: false }, + deps, + ); expect(deps.probeContainerDns).not.toHaveBeenCalled(); expect(deps.dockerRunDetached).not.toHaveBeenCalledWith( @@ -170,7 +182,10 @@ describe("Docker GPU recreate DNS fallback (#3579)", () => { }); expect(() => - recreateOpenShellDockerSandboxWithGpu({ sandboxName: "alpha", timeoutSecs: 1 }, deps), + recreateOpenShellDockerSandboxWithGpu( + { sandboxName: "alpha", timeoutSecs: 1, waitForSupervisor: false }, + deps, + ), ).toThrow(/using --dns 9\.9\.9\.9.*servers_unreachable.*before container recreation/s); expect(deps.dockerStop).not.toHaveBeenCalled(); }); diff --git a/src/lib/onboard/docker-gpu-patch-recreate.test.ts b/src/lib/onboard/docker-gpu-patch-recreate.test.ts index bd4f3889618..f9c708638b0 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.test.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.test.ts @@ -3,11 +3,17 @@ import { describe, expect, it, vi } from "vitest"; import { createDockerGpuInspectFixture as inspectFixture } from "./__test-helpers__/docker-gpu-patch-fixtures"; -import { recreateOpenShellDockerSandboxWithGpu } from "./docker-gpu-patch"; +import { + getDockerGpuPatchFailureContext, + recreateOpenShellDockerSandboxWithGpu, +} from "./docker-gpu-patch"; + +const OLD_CONTAINER_ID = "a".repeat(64); +const NEW_CONTAINER_ID = "b".repeat(64); function dockerCaptureFixture() { const responses: Record = { - ps: "old-container-id\n", + ps: `${OLD_CONTAINER_ID}\n`, inspect: JSON.stringify([inspectFixture()]), info: "", }; @@ -17,12 +23,24 @@ function dockerCaptureFixture() { describe("Docker GPU recreate orchestration", () => { it("recreates the OpenShell-managed container and waits for supervisor exec", () => { const dockerCapture = dockerCaptureFixture(); - const dockerRun = vi.fn(() => ({ status: 0, stdout: "probe-id\n" })); - const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: "new-container-id\n" })); + const dockerRunResults = { + ps: { status: 0, stdout: `${NEW_CONTAINER_ID}\n` }, + inspect: { status: 0, stdout: "true\n" }, + }; + const dockerRun = vi.fn( + (args: readonly string[]) => + dockerRunResults[String(args[0]) as keyof typeof dockerRunResults] ?? { + status: 0, + stdout: "probe-id\n", + }, + ); + const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: `${NEW_CONTAINER_ID}\n` })); const dockerRename = vi.fn(() => ({ status: 0 })); const dockerStop = vi.fn(() => ({ status: 0 })); const dockerRm = vi.fn(() => ({ status: 0 })); + const dockerStart = vi.fn(() => ({ status: 0 })); const runOpenshell = vi.fn(() => ({ status: 0 })); + const runCaptureOpenshell = vi.fn(() => "alpha 2026-08-23 10:00:00 Ready\n"); const result = recreateOpenShellDockerSandboxWithGpu( { sandboxName: "alpha", timeoutSecs: 1 }, @@ -33,6 +51,8 @@ describe("Docker GPU recreate orchestration", () => { dockerRename, dockerStop, dockerRm, + dockerStart, + runCaptureOpenshell, runOpenshell, sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), @@ -42,10 +62,11 @@ describe("Docker GPU recreate orchestration", () => { }, ); - expect(result.newContainerId).toBe("new-container-id"); + expect(result.newContainerId).toBe(NEW_CONTAINER_ID); + expect(result.backupRemoved).toBe(true); expect(result.mode.kind).toBe("gpus"); expect(dockerStop).toHaveBeenCalledWith( - "old-container-id", + OLD_CONTAINER_ID, expect.objectContaining({ timeout: 90_000 }), ); expect(dockerRunDetached).toHaveBeenCalledWith( @@ -74,13 +95,59 @@ describe("Docker GPU recreate orchestration", () => { expect.objectContaining({ ignoreError: true, suppressOutput: true }), ); const dockerRmCalls = dockerRm.mock.calls as unknown[][]; - const backupRmCall = dockerRmCalls.findIndex((call) => - String(call[0]).includes("nemoclaw-gpu-backup"), - ); + const backupRmCall = dockerRmCalls.findIndex((call) => call[0] === OLD_CONTAINER_ID); expect(backupRmCall).toBeGreaterThanOrEqual(0); expect(dockerRm.mock.invocationCallOrder[backupRmCall]).toBeGreaterThan( runOpenshell.mock.invocationCallOrder[0], ); + expect(dockerStart).toHaveBeenCalledWith( + NEW_CONTAINER_ID, + expect.objectContaining({ ignoreError: true }), + ); + expect(runCaptureOpenshell).toHaveBeenCalledWith( + ["sandbox", "list"], + expect.objectContaining({ ignoreError: true, suppressOutput: true }), + ); + expect(runOpenshell).toHaveBeenCalledTimes(2); + expect(dockerStart.mock.invocationCallOrder[0]).toBeLessThan( + runOpenshell.mock.invocationCallOrder[1], + ); + }); + + it("does not report success when the final OpenShell phase is Deleting (#9531)", () => { + let failure: unknown; + try { + recreateOpenShellDockerSandboxWithGpu( + { sandboxName: "alpha", timeoutSecs: 1 }, + { + dockerCapture: dockerCaptureFixture(), + dockerRun: vi.fn(() => ({ status: 0, stdout: `${NEW_CONTAINER_ID}\n` })), + dockerRunDetached: vi.fn(() => ({ status: 0, stdout: `${NEW_CONTAINER_ID}\n` })), + dockerRename: vi.fn(() => ({ status: 0 })), + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + dockerStart: vi.fn(() => ({ status: 0 })), + runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 10:00:00 Deleting\n"), + runOpenshell: vi.fn(() => ({ status: 0 })), + sleep: vi.fn(), + now: () => new Date("2026-05-12T00:00:00Z"), + detectSandboxFallbackDns: vi.fn(() => null), + readDir: vi.fn(() => null), + readFile: vi.fn(() => null), + }, + ); + } catch (error) { + failure = error; + } + + expect(failure).toBeInstanceOf(Error); + expect((failure as Error).message).toContain("final replacement handoff"); + expect(getDockerGpuPatchFailureContext(failure)).toMatchObject({ + oldContainerId: OLD_CONTAINER_ID, + newContainerId: NEW_CONTAINER_ID, + lastSandboxPhase: "Deleting", + rolledBack: false, + }); }); it("can recreate during sandbox create before supervisor exec is allowed", () => { diff --git a/src/lib/onboard/docker-gpu-patch-recreate.ts b/src/lib/onboard/docker-gpu-patch-recreate.ts index 4eb1c0ad8d4..d3618ac46b2 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.ts @@ -26,7 +26,7 @@ import { DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, DOCKER_GPU_PATCH_TIMEOUT_MS, } from "./docker-gpu-patch-constants"; -import { reconcileSupervisorReconnect } from "./docker-gpu-patch-finalize"; +import { finalizeDockerGpuPatchBackup } from "./docker-gpu-patch-finalize"; import { selectDockerGpuPatchMode } from "./docker-gpu-patch-mode"; import { restoreDockerGpuPatchBackupAfterRecreateFailure } from "./docker-gpu-patch-rollback"; import type { @@ -393,19 +393,36 @@ export function recreateOpenShellDockerSandboxContainer( options.timeoutSecs ?? DOCKER_GPU_PATCH_WAIT_SECS, deps, ); - const reconcile = reconcileSupervisorReconnect( - execReady, - { newContainerId, backupContainerName, originalName }, - deps, - ); - if (!reconcile.execReady) { - context.rolledBack = reconcile.rolledBack; - context.replacementStopConfirmed = reconcile.replacementStopConfirmed; - context.replacementRemovalConfirmed = reconcile.replacementRemovalConfirmed; - context.replacementPresence = reconcile.replacementPresence; - throw reconcile.error; + const patchResult = result(false); + const finalization = execReady + ? finalizeDockerGpuPatchBackup( + { + result: patchResult, + supervisorReady: true, + sandboxName: options.sandboxName, + finalHandoffTimeoutSecs: options.timeoutSecs ?? DOCKER_GPU_PATCH_WAIT_SECS, + }, + deps, + ) + : finalizeDockerGpuPatchBackup({ result: patchResult, supervisorReady: false }, deps); + context.rolledBack = finalization.rolledBack; + context.replacementStopConfirmed = finalization.replacementStopConfirmed; + context.replacementRemovalConfirmed = finalization.replacementRemovalConfirmed; + context.replacementPresence = finalization.replacementPresence; + context.lastSandboxPhase = finalization.lastSandboxPhase; + if (!execReady) { + throw new Error( + finalization.rolledBack + ? "OpenShell supervisor did not reconnect to the GPU-enabled container; pre-patch sandbox restored." + : "OpenShell supervisor did not reconnect to the GPU-enabled container and rollback failed; pre-patch sandbox was NOT restored.", + ); } - return result(reconcile.backupRemoved); + if (finalization.finalHandoffAcknowledged) return result(true); + throw new Error( + finalization.lastSandboxPhase + ? `OpenShell did not acknowledge the final replacement handoff; last sandbox phase was ${finalization.lastSandboxPhase}.` + : "OpenShell did not acknowledge the final replacement handoff.", + ); } catch (error) { throw decoratePatchError(error instanceof Error ? error : new Error(String(error)), context); } diff --git a/src/lib/onboard/docker-gpu-patch-rollback.test.ts b/src/lib/onboard/docker-gpu-patch-rollback.test.ts index 094e4ae8bf1..d126f50875d 100644 --- a/src/lib/onboard/docker-gpu-patch-rollback.test.ts +++ b/src/lib/onboard/docker-gpu-patch-rollback.test.ts @@ -177,8 +177,9 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { const runByCommand: Record { status: number; stdout: string }> = { ps: () => ({ status: 0, - stdout: replacementPresent ? `${replacementId}\n` : "", + stdout: `${alphaContainerIds().join("\n")}\n`, }), + inspect: () => ({ status: 0, stdout: "true\n" }), }; const dockerRun = vi.fn( (args: readonly string[]) => @@ -227,6 +228,7 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { return successfulRemoval(); }; removalHandlers.set(restoredName, removeRestored); + removalHandlers.set(restoredId, removeRestored); const dockerRm = vi.fn((target: string) => { removeTargets.push(target); return removalHandlers.get(target)?.() ?? successfulRemoval(); diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index 02be3c624c7..8dd7765cfa9 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -85,6 +85,7 @@ export type DockerGpuPatchFailureContext = { replacementStopConfirmed?: boolean; replacementRemovalConfirmed?: boolean; replacementPresence?: "absent" | "present" | "unknown"; + lastSandboxPhase?: string | null; }; export type DockerGpuPatchResult = { @@ -94,11 +95,11 @@ export type DockerGpuPatchResult = { originalName: string; backupContainerName: string; mode: DockerGpuPatchMode; - // True when the patch path also confirmed supervisor reconnect AND removed - // the backup container. False when the caller deferred the reconnect wait - // (via `waitForSupervisor: false`); the backup is still in place and the - // caller is responsible for calling `finalizeDockerGpuPatchBackup` after - // its own supervisor wait completes. + // True when the patch path confirmed the initial supervisor reconnect, + // removed the exact backup, restarted the exact replacement, and received + // the final OpenShell handoff acknowledgement. False means the caller used + // `waitForSupervisor: false`; the backup remains until the caller invokes + // `finalizeDockerGpuPatchBackup` after its own initial readiness check. backupRemoved: boolean; }; diff --git a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index 36bc94980ca..df318c1c0d2 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -50,8 +50,9 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { const finalizeBackup = vi.fn(() => ({ backupRemoved: true, rolledBack: false, - lifecycleReleaseObserved: true, replacementRestarted: true, + finalHandoffAcknowledged: true, + lastSandboxPhase: "Ready", })); const capturePreRollbackDiagnostics = vi.fn(() => null); const onPatchFailureExit = vi.fn(); @@ -97,11 +98,11 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { result, supervisorReady: true, sandboxName: "alpha", - lifecycleReleaseTimeoutSecs: 900, + finalHandoffTimeoutSecs: 900, }, deps, ); - expect(waitForSupervisor).toHaveBeenCalledTimes(2); + expect(waitForSupervisor).toHaveBeenCalledOnce(); expect(capturePreRollbackDiagnostics).not.toHaveBeenCalled(); expect(onPatchFailureExit).not.toHaveBeenCalled(); }); @@ -138,7 +139,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { result, supervisorReady: true, sandboxName: "alpha", - lifecycleReleaseTimeoutSecs: 900, + finalHandoffTimeoutSecs: 900, }, deps, ); @@ -174,7 +175,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { expect(onPatchFailureExit).toHaveBeenCalledOnce(); }); - it("rejects final handoff when OpenShell never releases the deleting lifecycle record (#9531)", async () => { + it("rejects final handoff when OpenShell reports Deleting after restart (#9531)", async () => { const deps = makeDeps(); const result = deferredCreateResult(); const waitForSupervisor = vi.fn(() => true); @@ -191,8 +192,9 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { finalizeBackup: vi.fn(() => ({ backupRemoved: true, rolledBack: false, - lifecycleReleaseObserved: false, replacementRestarted: true, + finalHandoffAcknowledged: false, + lastSandboxPhase: "Deleting", })), onPatchFailureExit, }, diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index 9e2eacdd07c..d7b0ff08b4b 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -493,14 +493,13 @@ export function createDockerGpuSandboxCreatePatch( const supervisorReconnectTimeoutSecs = getDockerGpuSupervisorReconnectTimeoutSecs( options.timeoutSecs, ); - const finalHandoffDeadlineMs = Date.now() + supervisorReconnectTimeoutSecs * 1000; const finalizeOutcome = result ? finalizeBackup( { result, supervisorReady: true, sandboxName: options.sandboxName, - lifecycleReleaseTimeoutSecs: supervisorReconnectTimeoutSecs, + finalHandoffTimeoutSecs: supervisorReconnectTimeoutSecs, }, options.deps, ) @@ -513,35 +512,26 @@ export function createDockerGpuSandboxCreatePatch( if ( finalizeOutcome.backupRemoved && finalizeOutcome.replacementRestarted && - finalizeOutcome.lifecycleReleaseObserved === true + finalizeOutcome.finalHandoffAcknowledged === true ) { - const remainingReconnectTimeoutSecs = Math.max( - 0, - Math.ceil((finalHandoffDeadlineMs - Date.now()) / 1000), - ); - console.log( - ` Waiting for OpenShell supervisor to confirm the final container handoff (up to ${remainingReconnectTimeoutSecs}s)...`, - ); - if ( - remainingReconnectTimeoutSecs > 0 && - waitForSupervisor(options.sandboxName, remainingReconnectTimeoutSecs, { - runOpenshell: options.deps.runOpenshell, - runCaptureOpenshell: options.deps.runCaptureOpenshell, - sleep: options.deps.sleep, - }) - ) { - return; - } + return; } const failure = new Error( - "Managed startup passed Ready, but its final runtime handoff did not converge.", + `Managed startup passed Ready, but its final runtime handoff did not converge.${ + finalizeOutcome.lastSandboxPhase + ? ` Last OpenShell phase: ${finalizeOutcome.lastSandboxPhase}.` + : "" + }`, ); cutoverFinalizationFailure = failure; onPatchFailureExit(options.sandboxName, failure, { runCaptureOpenshell: options.deps.runCaptureOpenshell, dockerCapture: options.deps.dockerCapture, additionalSummaryLines: routeAdapter.additionalSummaryLines, - context: failureContext(), + context: { + ...failureContext(), + lastSandboxPhase: finalizeOutcome.lastSandboxPhase, + }, }); throw failure; })(); diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts index ef4a16aafc7..64a7df2b854 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts @@ -6,88 +6,121 @@ import { describe, expect, it, vi } from "vitest"; import { getDockerGpuSupervisorReconnectErrorDebouncePolls, getDockerGpuSupervisorReconnectTimeoutSecs, - waitForOpenShellSandboxLifecycleRelease, + waitForOpenShellFinalHandoff, waitForOpenShellSupervisorReconnect, } from "./docker-gpu-supervisor-reconnect"; -describe("Docker GPU final lifecycle release", () => { - it.each([ - ["an explicit empty list", "No sandboxes found.\n"], - ["another phase-bearing sandbox", "beta 2026-08-21 05:53:18 Ready\n"], - ])("accepts %s as a release receipt (#9531)", (_receipt, stdout) => { - const runOpenshell = vi.fn(() => ({ status: 0, stdout })); +describe("Docker GPU final handoff acknowledgement", () => { + it("accepts the exact running replacement only after OpenShell reports Ready (#9531)", () => { + const events: string[] = []; + const runCaptureOpenshell = vi + .fn() + .mockImplementationOnce(() => { + events.push("observe provisioning"); + return "alpha 2026-08-23 10:00:00 Provisioning\n"; + }) + .mockImplementationOnce(() => { + events.push("observe ready"); + return "alpha 2026-08-23 10:00:02 Ready\n"; + }); + const runOpenshell = vi.fn(() => { + events.push("exec ready"); + return { status: 0 }; + }); + const replacementIsExactAndRunning = vi.fn(() => { + events.push("confirm exact replacement"); + return true; + }); + + const acknowledgement = waitForOpenShellFinalHandoff("alpha", 60, { + runCaptureOpenshell, + runOpenshell, + replacementIsExactAndRunning, + sleep: vi.fn(), + }); + + expect(acknowledgement).toEqual({ acknowledged: true, lastSandboxPhase: "Ready" }); + expect(events).toEqual([ + "observe provisioning", + "confirm exact replacement", + "observe ready", + "exec ready", + "confirm exact replacement", + ]); + }); + + it("treats Deleting after the replacement start as terminal (#9531)", () => { + const runCaptureOpenshell = vi.fn(() => "alpha 2026-08-23 10:00:00 Deleting\n"); + const runOpenshell = vi.fn(() => ({ status: 1 })); + const replacementIsExactAndRunning = vi.fn(() => true); + const sleep = vi.fn(); expect( - waitForOpenShellSandboxLifecycleRelease("alpha", 1, { + waitForOpenShellFinalHandoff("alpha", 60, { + runCaptureOpenshell, runOpenshell, - sleep: vi.fn(), + replacementIsExactAndRunning, + sleep, }), - ).toBe(true); - expect(runOpenshell).toHaveBeenCalledOnce(); + ).toEqual({ acknowledged: false, lastSandboxPhase: "Deleting" }); + expect(runCaptureOpenshell).toHaveBeenCalledOnce(); + expect(runOpenshell).not.toHaveBeenCalled(); + expect(replacementIsExactAndRunning).not.toHaveBeenCalled(); + expect(sleep).not.toHaveBeenCalled(); }); - it.each([ - ["a header", "NAME CREATED PHASE\n"], - ["a gateway error", "Error: gateway unavailable\n"], - ["a phase-free row", "beta 2026-08-21 05:53:18\n"], - ["an unrecognized phase", "beta 2026-08-21 05:53:18 Retiring\n"], - ["the selected sandbox in Deleting", "alpha 2026-08-21 05:53:18 Deleting\n"], - ["the selected sandbox in Ready", "alpha 2026-08-21 05:53:18 Ready\n"], - ["the selected sandbox in Provisioning", "alpha 2026-08-21 05:53:18 Provisioning\n"], - ["the selected sandbox in Error", "alpha 2026-08-21 05:53:18 Error\n"], - ["the selected sandbox in Failed", "alpha 2026-08-21 05:53:18 Failed\n"], - ])("rejects %s as a release receipt (#9531)", (_case, stdout) => { - const runOpenshell = vi.fn(() => ({ status: 0, stdout })); + it("continues through Error only while the exact replacement is running (#9531)", () => { + const runCaptureOpenshell = vi.fn(() => "alpha 2026-08-23 10:00:00 Error\n"); + const replacementIsExactAndRunning = vi.fn(() => false); expect( - waitForOpenShellSandboxLifecycleRelease("alpha", 1, { - runOpenshell, + waitForOpenShellFinalHandoff("alpha", 60, { + runCaptureOpenshell, + runOpenshell: vi.fn(() => ({ status: 1 })), + replacementIsExactAndRunning, sleep: vi.fn(), }), - ).toBe(false); - expect(runOpenshell).toHaveBeenCalledTimes(2); + ).toEqual({ acknowledged: false, lastSandboxPhase: "Error" }); + expect(replacementIsExactAndRunning).toHaveBeenCalledOnce(); }); - it.each([ - ["a failed probe", { status: 1, stderr: "gateway unavailable" }], - ["a probe without an exit status", { status: null, stderr: "timed out" }], - ])("rejects %s as a release receipt (#9531)", (_case, result) => { - const runOpenshell = vi.fn(() => result); + it("allows a running replacement to progress from Error to Ready (#9531)", () => { + const runCaptureOpenshell = vi + .fn() + .mockReturnValueOnce("alpha 2026-08-23 10:00:00 Error\n") + .mockReturnValueOnce("alpha 2026-08-23 10:00:02 Ready\n"); + const runOpenshell = vi.fn(() => ({ status: 0 })); + const replacementIsExactAndRunning = vi.fn(() => true); expect( - waitForOpenShellSandboxLifecycleRelease("alpha", 1, { + waitForOpenShellFinalHandoff("alpha", 60, { + runCaptureOpenshell, runOpenshell, + replacementIsExactAndRunning, sleep: vi.fn(), }), - ).toBe(false); - expect(runOpenshell).toHaveBeenCalledTimes(2); + ).toEqual({ acknowledged: true, lastSandboxPhase: "Ready" }); + expect(replacementIsExactAndRunning).toHaveBeenCalledTimes(2); + expect(runOpenshell).toHaveBeenCalledOnce(); }); - it("does not start Error corroboration after the lifecycle-release deadline (#9962)", () => { - vi.useFakeTimers(); - vi.setSystemTime(0); - const corroborate = vi.fn(() => true); - const runOpenshell = vi.fn(() => { - vi.setSystemTime(1000); - return { - status: 0, - stdout: "alpha 2026-08-23 01:40:35 Error\n", - }; - }); + it("does not reuse a stale Ready phase after the sandbox row disappears (#9531)", () => { + const runCaptureOpenshell = vi + .fn() + .mockReturnValueOnce("alpha 2026-08-23 10:00:00 Ready\n") + .mockReturnValue(""); + const runOpenshell = vi.fn(() => ({ status: 1 })); + const replacementIsExactAndRunning = vi.fn().mockReturnValueOnce(true).mockReturnValue(false); - try { - expect( - waitForOpenShellSandboxLifecycleRelease("alpha", 1, { - runOpenshell, - sleep: vi.fn(), - soleLabeledReplacementCorroboratesError: corroborate, - }), - ).toBe(false); - } finally { - vi.useRealTimers(); - } + expect( + waitForOpenShellFinalHandoff("alpha", 60, { + runCaptureOpenshell, + runOpenshell, + replacementIsExactAndRunning, + sleep: vi.fn(), + }), + ).toEqual({ acknowledged: false, lastSandboxPhase: "Ready" }); expect(runOpenshell).toHaveBeenCalledOnce(); - expect(corroborate).not.toHaveBeenCalled(); }); }); diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts index 29884386619..2e5ff44ec06 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts @@ -73,89 +73,115 @@ export type DockerGpuSupervisorReconnectDeps = { errorPhaseDebouncePolls?: number; }; -type DockerLifecycleReleaseDeps = Pick< - DockerGpuSupervisorReconnectDeps, - "runOpenshell" | "sleep" -> & { - /** - * Corroborating evidence for an Error row from a Docker query that confirms - * the transaction-owned replacement is the sole labeled sandbox container. - * The callback must fail closed and keep its child within the supplied - * remaining lifecycle-release budget. - */ - soleLabeledReplacementCorroboratesError?: (remainingMs: number) => boolean; +type DockerFinalHandoffDeps = Required< + Pick +> & + Pick & { + /** + * Prove that the transaction-owned replacement is the sole labeled + * container and is still running. The callback must fail closed and keep + * its Docker queries within the supplied remaining handoff budget. + */ + replacementIsExactAndRunning: (remainingMs: number) => boolean; + }; + +export type DockerFinalHandoffAcknowledgement = { + acknowledged: boolean; + lastSandboxPhase: string | null; }; +const FINAL_HANDOFF_TERMINAL_PHASES = new Set(["Deleting", "Failed", "CrashLoopBackOff"]); + +function exactReplacementIsRunning( + callback: DockerFinalHandoffDeps["replacementIsExactAndRunning"], + remainingMs: number, +): boolean { + if (remainingMs <= 0) return false; + try { + return callback(remainingMs); + } catch { + return false; + } +} + /** - * Workaround contract for the OpenShell lifecycle race in #9531: - * - * - Removing the rollback backup can strand the exact sandbox in `Deleting` - * while its replacement container is healthy. - * - `openshell sandbox list` owns lifecycle authority. Docker health cannot - * prove that OpenShell retired the previous record. - * - This layer waits after backup removal and before replacement restart so - * OpenShell processes the stale deletion before the new registration. - * - The caller enters this wait only after the replacement reached Ready and - * was deliberately stopped. A successful list normally omits the sandbox - * name. An Error row is also sufficient only when a separate bounded Docker - * query confirms that exact stopped replacement is the sole remaining - * labeled container. This corroborates the release condition; the OpenShell - * row alone is not an identity-bound ownership receipt. - * - `waits for the sandbox name to disappear before restarting the - * replacement (#9531)` protects the event order. `rejects final handoff when - * OpenShell never releases the deleting lifecycle record (#9531)` protects - * the composed failure path. + * Confirm the final Docker replacement handoff through OpenShell and Docker. * - * Remove this wait only when OpenShell binds deletion to the removed container - * identity or provides an identity-bound lifecycle-release receipt. + * The final replacement start is the authoritative lifecycle event. Success + * requires both an OpenShell Ready row with a working sandbox exec and a + * bounded Docker proof that the exact transaction-owned replacement is the + * sole running labeled container. Deleting is terminal after that start. + * Error remains transient only while the exact replacement stays running. */ -export function waitForOpenShellSandboxLifecycleRelease( +export function waitForOpenShellFinalHandoff( sandboxName: string, timeoutSecs: number, - deps: DockerLifecycleReleaseDeps, -): boolean { - if (!deps.runOpenshell) return false; + deps: DockerFinalHandoffDeps, +): DockerFinalHandoffAcknowledgement { const sleep = deps.sleep ?? defaultSleep; const boundedTimeoutSecs = Math.max(1, Math.round(timeoutSecs)); const deadline = Date.now() + boundedTimeoutSecs * 1000; const maxAttempts = Math.max(1, Math.ceil(boundedTimeoutSecs / 2) + 1); + let lastSandboxPhase: string | null = null; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { - const remainingBeforeProbeMs = deadline - Date.now(); - if (remainingBeforeProbeMs <= 0) break; - const result = deps.runOpenshell(["sandbox", "list"], { - ignoreError: true, - suppressOutput: true, - timeout: Math.min(DOCKER_GPU_PATCH_TIMEOUT_MS, remainingBeforeProbeMs), - }); - if (hasZeroDockerExitStatus(result)) { - const output = String(result.stdout ?? "").trim(); - const entries = parseLiveSandboxEntries(output); - const sandboxPresent = entries.some((entry) => entry.name === sandboxName); - const stoppedReplacementError = entries.some( - (entry) => entry.name === sandboxName && entry.phase === "Error", + const remainingBeforeListMs = deadline - Date.now(); + if (remainingBeforeListMs <= 0) break; + let listOutput = ""; + try { + listOutput = deps.runCaptureOpenshell(["sandbox", "list"], { + ignoreError: true, + suppressOutput: true, + timeout: Math.min(DOCKER_GPU_PATCH_TIMEOUT_MS, remainingBeforeListMs), + }); + } catch { + listOutput = ""; + } + const entry = parseLiveSandboxEntries(listOutput).find((item) => item.name === sandboxName); + const currentPhase = entry?.phase ?? null; + if (currentPhase) lastSandboxPhase = currentPhase; + if (currentPhase && FINAL_HANDOFF_TERMINAL_PHASES.has(currentPhase)) { + return { acknowledged: false, lastSandboxPhase }; + } + + const remainingBeforeRuntimeProofMs = deadline - Date.now(); + if ( + currentPhase === "Error" && + !exactReplacementIsRunning(deps.replacementIsExactAndRunning, remainingBeforeRuntimeProofMs) + ) { + return { acknowledged: false, lastSandboxPhase }; + } + + if (currentPhase === "Ready") { + const remainingBeforeExecMs = deadline - Date.now(); + if (remainingBeforeExecMs <= 0) break; + const execResult = deps.runOpenshell(["sandbox", "exec", "-n", sandboxName, "--", "true"], { + ignoreError: true, + suppressOutput: true, + timeout: Math.min(DOCKER_GPU_PATCH_TIMEOUT_MS, remainingBeforeExecMs), + }); + const remainingAfterExecMs = deadline - Date.now(); + const replacementIsRunning = exactReplacementIsRunning( + deps.replacementIsExactAndRunning, + remainingAfterExecMs, ); - const hasPhaseBearingEntry = entries.some((entry) => entry.phase !== null); - const explicitEmptyList = output === "No sandboxes found" || output === "No sandboxes found."; - const remainingBeforeCorroborationMs = deadline - Date.now(); - const soleLabeledReplacementCorroboratesError = - stoppedReplacementError && - remainingBeforeCorroborationMs > 0 && - deps.soleLabeledReplacementCorroboratesError?.(remainingBeforeCorroborationMs) === true; - if ( - explicitEmptyList || - soleLabeledReplacementCorroboratesError || - (hasPhaseBearingEntry && !sandboxPresent) - ) { - return true; + if (hasZeroDockerExitStatus(execResult) && replacementIsRunning) { + return { acknowledged: true, lastSandboxPhase }; } + if (!replacementIsRunning) return { acknowledged: false, lastSandboxPhase }; + } else if ( + currentPhase !== "Error" && + !exactReplacementIsRunning(deps.replacementIsExactAndRunning, deadline - Date.now()) + ) { + return { acknowledged: false, lastSandboxPhase }; } + const remainingBeforeSleepMs = deadline - Date.now(); if (attempt < maxAttempts && remainingBeforeSleepMs > 0) { sleep(Math.min(2, remainingBeforeSleepMs / 1000)); } } - return false; + return { acknowledged: false, lastSandboxPhase }; } function defaultSleep(seconds: number): void { @@ -196,7 +222,7 @@ export function waitForOpenShellSupervisorReconnect( timeoutSecs: number, deps: DockerGpuSupervisorReconnectDeps, ): boolean { - if (!deps.runOpenshell) return true; + if (!deps.runOpenshell) return false; const sleep = deps.sleep ?? defaultSleep; const deadline = Date.now() + Math.max(1, timeoutSecs) * 1000; const errorPhaseDebouncePolls = diff --git a/src/lib/runtime-recovery.ts b/src/lib/runtime-recovery.ts index 179d9de5198..0361fc7a24c 100644 --- a/src/lib/runtime-recovery.ts +++ b/src/lib/runtime-recovery.ts @@ -21,6 +21,7 @@ const LIVE_SANDBOX_DISPLAY_PHASES = new Set([ "Provisioning", "Creating", "Pending", + "Deleting", "Terminating", "Error", "Failed", diff --git a/test/e2e/live/gateway-guard-legacy-keepalive-fixture.ts b/test/e2e/live/gateway-guard-legacy-keepalive-fixture.ts index 6007aba571f..c96798c7eb4 100644 --- a/test/e2e/live/gateway-guard-legacy-keepalive-fixture.ts +++ b/test/e2e/live/gateway-guard-legacy-keepalive-fixture.ts @@ -5,6 +5,7 @@ import path from "node:path"; import { pathToFileURL } from "node:url"; import * as dockerRunNamespace from "../../../src/lib/adapters/docker/run.ts"; +import * as openshellRuntimeNamespace from "../../../src/lib/adapters/openshell/runtime.ts"; import * as managedBootstrapAdapterNamespace from "../../../src/lib/onboard/managed-bootstrap/adapter.ts"; import * as dockerGpuPatchCloneNamespace from "../../../src/lib/onboard/docker-gpu-patch-clone.ts"; import type { @@ -53,6 +54,11 @@ const dockerRun = ( "default" in dockerRunNamespace ? dockerRunNamespace.default : dockerRunNamespace ) as typeof import("../../../src/lib/adapters/docker/run.ts"); const { dockerCapture: defaultDockerCapture } = dockerRun; +const openshellRuntime = ( + "default" in openshellRuntimeNamespace + ? openshellRuntimeNamespace.default + : openshellRuntimeNamespace +) as typeof import("../../../src/lib/adapters/openshell/runtime.ts"); type StartupCommandRecreate = typeof recreateOpenShellDockerSandboxWithStartupCommand; type DockerCapture = NonNullable; @@ -66,11 +72,20 @@ export type LegacyKeepaliveFixtureOptions = { export type LegacyKeepaliveFixtureDeps = { recreate: StartupCommandRecreate; dockerCapture: DockerCapture; + runOpenshell: NonNullable; + runCaptureOpenshell: NonNullable; }; const defaultDeps: LegacyKeepaliveFixtureDeps = { recreate: recreateOpenShellDockerSandboxWithStartupCommand, dockerCapture: defaultDockerCapture, + runOpenshell: openshellRuntime.runOpenshell, + runCaptureOpenshell: (args, options) => + openshellRuntime.captureOpenshell(args, { + ignoreError: true, + includeStderr: true, + timeout: typeof options?.timeout === "number" ? options.timeout : undefined, + }).output, }; function requireFixtureInput(condition: boolean, message: string): asserts condition { @@ -253,6 +268,8 @@ export function createLegacyKeepaliveFixture( const recreate = deps.recreate ?? defaultDeps.recreate; const dockerCapture = deps.dockerCapture ?? defaultDeps.dockerCapture; + const runOpenshell = deps.runOpenshell ?? defaultDeps.runOpenshell; + const runCaptureOpenshell = deps.runCaptureOpenshell ?? defaultDeps.runCaptureOpenshell; const result = recreate( { sandboxName: options.sandboxName, @@ -262,6 +279,8 @@ export function createLegacyKeepaliveFixture( }, { dockerCapture: legacyKeepaliveDockerCapture(options.expectedContainerId, dockerCapture), + runCaptureOpenshell, + runOpenshell, }, ); diff --git a/test/e2e/live/gateway-guard-recovery.test.ts b/test/e2e/live/gateway-guard-recovery.test.ts index a84d3a414ae..266937e6bc6 100644 --- a/test/e2e/live/gateway-guard-recovery.test.ts +++ b/test/e2e/live/gateway-guard-recovery.test.ts @@ -202,15 +202,15 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701) ], }, }, async ({ - artifacts, - environment, - onboard, - host, - gateway, - progress, - sandbox, - secrets, - cleanup, + artifacts, + environment, + onboard, + host, + gateway, + progress, + sandbox, + secrets, + cleanup, }) => { secrets.required("NVIDIA_INFERENCE_API_KEY"); @@ -452,17 +452,23 @@ test("gateway recovery restores /tmp guard chain after pod-recreate wipe (#2701) }, ); expect(createLegacyKeepalive.exitCode, resultText(createLegacyKeepalive)).toBe(0); - // Do not overlap the fixture's recreation with the restart below. The - // fixture runs in its own process, so the host must observe the replacement - // through OpenShell before starting the next container lifecycle transition. - await waitForSandboxExecReady( - host, - instance.sandboxName, - progress, - "legacy-recreate-openshell-ready", + const handoffReceipt = JSON.parse(createLegacyKeepalive.stdout) as { + newContainerId?: unknown; + }; + expect(handoffReceipt.newContainerId).toMatch(/^[0-9a-f]{64}$/iu); + const finalHandoffExec = await host.command( + host.openshellCommandPath, + ["sandbox", "exec", "-n", instance.sandboxName, "--", "true"], + { + artifactName: "legacy-recreate-final-handoff", + env: buildAvailabilityProbeEnv(), + timeoutMs: 30_000, + }, ); + expect(finalHandoffExec.exitCode, resultText(finalHandoffExec)).toBe(0); const legacyContainerId = await findSandboxContainer(host, "legacy-restart-container-before"); + expect(legacyContainerId).toBe(handoffReceipt.newContainerId); expect(legacyContainerId).not.toBe(recoveredContainerId); expect( await inspectStartupCommand(host, legacyContainerId, "legacy-restart-command-before"), diff --git a/test/e2e/support/gateway-guard-legacy-keepalive-fixture.test.ts b/test/e2e/support/gateway-guard-legacy-keepalive-fixture.test.ts index 98b7d89e692..b14ed1124b2 100644 --- a/test/e2e/support/gateway-guard-legacy-keepalive-fixture.test.ts +++ b/test/e2e/support/gateway-guard-legacy-keepalive-fixture.test.ts @@ -159,7 +159,11 @@ describe("gateway guard legacy keepalive fixture", () => { openshellSandboxCommand: ["sleep", "infinity"], timeoutSecs: 180, }, - { dockerCapture: expect.any(Function) }, + { + dockerCapture: expect.any(Function), + runCaptureOpenshell: expect.any(Function), + runOpenshell: expect.any(Function), + }, ); }); diff --git a/test/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery-supervisor-relaunch.test.ts index 07b6697fc5b..1bce16d5aa2 100644 --- a/test/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery-supervisor-relaunch.test.ts @@ -72,11 +72,15 @@ function composedRelaunchTransaction( : { backupRemoved: false, rolledBack: true }; }, ), + containerIds: { old: string; replacement: string } = { + old: "old-container-id", + replacement: "replacement-container-id", + }, ) { const resolveContainer = vi .fn() - .mockReturnValueOnce("old-container-id") - .mockReturnValue("replacement-container-id"); + .mockReturnValueOnce(containerIds.old) + .mockReturnValue(containerIds.replacement); const runOpenshell = vi.fn(() => ({ status: 0, stdout: "No sandboxes found.\n" })); const relaunchManagedSupervisorSessionImpl = vi.fn( (sandboxName: string, options: Parameters[1]) => @@ -113,8 +117,8 @@ function composedRelaunchTransaction( runOpenshell, recreate: vi.fn(() => ({ applied: true as const, - oldContainerId: "old-container-id", - newContainerId: "replacement-container-id", + oldContainerId: containerIds.old, + newContainerId: containerIds.replacement, originalName: "openshell-recovery-box", backupContainerName: "openshell-recovery-box-nemoclaw-backup", mode: { @@ -454,11 +458,14 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { expect(finalizeTransaction).toHaveBeenCalledOnce(); expect(finalizeTransaction).toHaveBeenCalledWith( expect.objectContaining({ - lifecycleReleaseTimeoutSecs: 900, + finalHandoffTimeoutSecs: 900, sandboxName: "recovered-box", supervisorReady: true, }), - { runOpenshell }, + { + runCaptureOpenshell: expect.any(Function), + runOpenshell, + }, ); }); @@ -470,15 +477,30 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { const dockerStop = vi.fn(() => ({ status: 0 })); const dockerRm = vi.fn(() => ({ status: 0 })); const dockerStart = vi.fn(() => ({ status: 0 })); + const oldContainerId = "a".repeat(64); + const replacementContainerId = "b".repeat(64); + const dockerRun = vi.fn((args: readonly string[]) => + args[0] === "ps" + ? { status: 0, stdout: `${replacementContainerId}\n` } + : { status: 0, stdout: "true\n" }, + ); const finalizeTransaction = vi.fn( ( options: Parameters[0], deps: Parameters[1], - ) => finalizeDockerGpuPatchBackup(options, { ...deps, dockerStop, dockerRm, dockerStart }), + ) => + finalizeDockerGpuPatchBackup(options, { + ...deps, + dockerRm, + dockerRun, + dockerStart, + dockerStop, + }), ); const { relaunchManagedSupervisorSessionImpl } = composedRelaunchTransaction( order, finalizeTransaction, + { old: oldContainerId, replacement: replacementContainerId }, ); const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => action === "recover" ? { status: 1, stdout: "", stderr: "SUPERVISOR_NOT_RUNNING" } : null, @@ -509,6 +531,10 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { ? "SANDBOX BIND PORT PID STATUS\nlegacy-handoff-box 127.0.0.1 18789 12345 running" : "SANDBOX BIND PORT PID STATUS", }), + "sandbox list": () => ({ + status: 0, + output: "legacy-handoff-box 2026-08-23 10:00:00 Ready\n", + }), }; return ( responses[args.join(" ") as keyof typeof responses]?.() ?? { @@ -540,15 +566,15 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { forwardRecovered: true, }); expect(dockerStop).toHaveBeenCalledWith( - "replacement-container-id", + replacementContainerId, expect.objectContaining({ ignoreError: true }), ); expect(dockerRm).toHaveBeenCalledWith( - "openshell-recovery-box-nemoclaw-backup", + oldContainerId, expect.objectContaining({ ignoreError: true }), ); expect(dockerStart).toHaveBeenCalledWith( - "replacement-container-id", + replacementContainerId, expect.objectContaining({ ignoreError: true }), ); expect(waitForRecreatedSandboxOpenShellReadyImpl).toHaveBeenCalledTimes(2); @@ -578,21 +604,6 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { finalPinnedAction: () => ACCEPTED_MANAGED_PROBE, finalReadinessReady: true, }, - { - condition: "OpenShell does not release the sandbox name", - finalizeOutcome: () => ({ - backupRemoved: true, - lifecycleReleaseObserved: false, - replacementRestarted: false, - replacementStoppedForCommit: true, - rolledBack: false, - stateRestored: true, - }), - expectedDetail: "OpenShell did not release the sandbox name", - expectedReadinessCalls: 1, - finalPinnedAction: () => ACCEPTED_MANAGED_PROBE, - finalReadinessReady: true, - }, { condition: "Docker cannot start the replacement container", finalizeOutcome: () => ({ @@ -1076,51 +1087,51 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { ])("reports the readiness failure and unconfirmed rollback when $condition (#9364)", ({ finalizeOutcome, }) => { - mockOpenClawSandbox("rollback-box"); - setImmediateRecoveryPolling(); - const finalize = vi.fn((_supervisorReady: boolean) => finalizeOutcome()); - const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ - containerId: "replacement-container-id", - finalize, - })); - const requestGatewaySupervisorAction = vi.fn(() => MISSING_MANAGED_SUPERVISOR); - const requestPinnedGatewaySupervisorAction = vi.fn(() => ACCEPTED_MANAGED_PROBE); - const waitForRecreatedSandboxOpenShellReadyImpl = vi.fn(() => false); - const captureOpenshell = vi - .spyOn(openshellRuntime, "captureOpenshell") - .mockReturnValue({ status: 0, output: "" }); - const runOpenshell = vi - .spyOn(openshellRuntime, "runOpenshell") - .mockReturnValue({ status: 0 } as never); + mockOpenClawSandbox("rollback-box"); + setImmediateRecoveryPolling(); + const finalize = vi.fn((_supervisorReady: boolean) => finalizeOutcome()); + const relaunchManagedSupervisorSessionImpl = vi.fn(() => ({ + containerId: "replacement-container-id", + finalize, + })); + const requestGatewaySupervisorAction = vi.fn(() => MISSING_MANAGED_SUPERVISOR); + const requestPinnedGatewaySupervisorAction = vi.fn(() => ACCEPTED_MANAGED_PROBE); + const waitForRecreatedSandboxOpenShellReadyImpl = vi.fn(() => false); + const captureOpenshell = vi + .spyOn(openshellRuntime, "captureOpenshell") + .mockReturnValue({ status: 0, output: "" }); + const runOpenshell = vi + .spyOn(openshellRuntime, "runOpenshell") + .mockReturnValue({ status: 0 } as never); - const result = checkAndRecoverSandboxProcesses("rollback-box", { - quiet: true, - isSandboxGatewayRunningImpl: () => false, - requestGatewaySupervisorAction, - requestPinnedGatewaySupervisorAction, - relaunchManagedSupervisorSessionImpl, - waitForRecreatedSandboxOpenShellReadyImpl, - }); + const result = checkAndRecoverSandboxProcesses("rollback-box", { + quiet: true, + isSandboxGatewayRunningImpl: () => false, + requestGatewaySupervisorAction, + requestPinnedGatewaySupervisorAction, + relaunchManagedSupervisorSessionImpl, + waitForRecreatedSandboxOpenShellReadyImpl, + }); - expect(result).toMatchObject({ - checked: true, - wasRunning: false, - recovered: false, - forwardRecovered: false, - recoveryFailureDetail: expect.stringContaining( - "NemoClaw could not confirm rollback to the previous sandbox container", - ), - }); - expect("recoveryFailureDetail" in result ? result.recoveryFailureDetail : "").toContain( - "did not become ready in OpenShell", - ); - expect("recoveryFailureDetail" in result ? result.recoveryFailureDetail : "").not.toContain( - "opaque-finalizer-sentinel", - ); - expect(finalize).toHaveBeenCalledOnce(); - expect(finalize).toHaveBeenCalledWith(false); - expect(captureOpenshell).not.toHaveBeenCalled(); - expect(runOpenshell).not.toHaveBeenCalled(); + expect(result).toMatchObject({ + checked: true, + wasRunning: false, + recovered: false, + forwardRecovered: false, + recoveryFailureDetail: expect.stringContaining( + "NemoClaw could not confirm rollback to the previous sandbox container", + ), + }); + expect("recoveryFailureDetail" in result ? result.recoveryFailureDetail : "").toContain( + "did not become ready in OpenShell", + ); + expect("recoveryFailureDetail" in result ? result.recoveryFailureDetail : "").not.toContain( + "opaque-finalizer-sentinel", + ); + expect(finalize).toHaveBeenCalledOnce(); + expect(finalize).toHaveBeenCalledWith(false); + expect(captureOpenshell).not.toHaveBeenCalled(); + expect(runOpenshell).not.toHaveBeenCalled(); }); it("reports the last structured OpenShell error when readiness times out", () => { From 11726633d305d6c7a8edc63f81bd075a49e19c71 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sun, 23 Aug 2026 16:53:05 -0700 Subject: [PATCH 02/12] fix(onboard): retain failed handoff recovery Signed-off-by: Apurv Kumaria --- .../sandbox/supervisor-relaunch.test.ts | 25 ++++++++++- .../actions/sandbox/supervisor-relaunch.ts | 23 +++++----- test/helpers/onboard-script-mocks.cjs | 42 +++++++++++++++++-- test/onboard-sandbox-build.test.ts | 6 ++- 4 files changed, 81 insertions(+), 15 deletions(-) diff --git a/src/lib/actions/sandbox/supervisor-relaunch.test.ts b/src/lib/actions/sandbox/supervisor-relaunch.test.ts index 62a8d4ce2c7..8a3d48bcedb 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.test.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.test.ts @@ -79,7 +79,7 @@ function baseDeps(overrides: ManagedSupervisorRelaunchDeps = {}) { recreate: vi.fn(() => patchResult()), finalize: vi.fn(({ supervisorReady }) => supervisorReady - ? { backupRemoved: true, rolledBack: false } + ? { backupRemoved: true, finalHandoffAcknowledged: true, rolledBack: false } : { backupRemoved: false, rolledBack: true }, ), ...overrides, @@ -149,6 +149,7 @@ describe("relaunchManagedSupervisorSession", () => { expect(relaunch?.finalize(true)).toEqual({ backupRemoved: true, + finalHandoffAcknowledged: true, rolledBack: false, stateRestored: true, stateBackupRemoved: true, @@ -500,12 +501,34 @@ describe("relaunchManagedSupervisorSession", () => { expect(relaunch?.finalize(true)).toEqual({ backupRemoved: true, + finalHandoffAcknowledged: true, rolledBack: false, stateRestored: true, stateBackupRemoved: false, }); }); + it("retains the restored state backup when final handoff is not acknowledged (#9531)", () => { + const deps = baseDeps({ + finalize: vi.fn(() => ({ + backupRemoved: true, + finalHandoffAcknowledged: false, + lastSandboxPhase: "Deleting", + rolledBack: false, + })), + }); + const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); + + expect(relaunch?.finalize(true)).toEqual({ + backupRemoved: true, + finalHandoffAcknowledged: false, + lastSandboxPhase: "Deleting", + rolledBack: false, + stateRestored: true, + }); + expect(deps.removeBackup).not.toHaveBeenCalled(); + }); + it("returns null when the pinned recreation fails", () => { const deps = baseDeps({ recreate: vi.fn(() => { diff --git a/src/lib/actions/sandbox/supervisor-relaunch.ts b/src/lib/actions/sandbox/supervisor-relaunch.ts index c892ce6e522..14d403e2316 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.ts @@ -297,18 +297,21 @@ export function relaunchManagedSupervisorSession( runOpenshell: runLifecycleProbe, ...(deps.sleep ? { sleep: deps.sleep } : {}), }; + const finalized = finalize( + { + result, + supervisorReady: true, + sandboxName, + finalHandoffTimeoutSecs: getDockerGpuSupervisorReconnectTimeoutSecs(1), + }, + lifecycleDeps, + ); const outcome = { - ...finalize( - { - result, - supervisorReady: true, - sandboxName, - finalHandoffTimeoutSecs: getDockerGpuSupervisorReconnectTimeoutSecs(1), - }, - lifecycleDeps, - ), + ...finalized, stateRestored: true, - stateBackupRemoved: removeSettledStateBackup(), + ...(finalized.finalHandoffAcknowledged === true + ? { stateBackupRemoved: removeSettledStateBackup() } + : {}), }; completed = { supervisorReady, outcome }; return outcome; diff --git a/test/helpers/onboard-script-mocks.cjs b/test/helpers/onboard-script-mocks.cjs index bdbae155aba..10a07c4e115 100644 --- a/test/helpers/onboard-script-mocks.cjs +++ b/test/helpers/onboard-script-mocks.cjs @@ -252,6 +252,9 @@ function mockSandboxExecCurl(command, options = {}) { } function mockOnboardRunCapture(command, options = {}) { + // The companion runner seam models the exact post-commit Docker proof. Install + // it lazily after each scenario has replaced runner.run with its local recorder. + mockDockerSandboxLifecycleReleaseFromRunner(); const normalized = normalizeCommand(command); if ( normalized.startsWith("docker ps -a --no-trunc ") && @@ -332,11 +335,35 @@ function mockStandaloneGatewayTeardownAuthority() { function mockDockerSandboxLifecycleReleaseFromRunner() { const runner = require(path.resolve(__dirname, "../../src/lib/runner.ts")); + if (runner.run.__nemoclawDockerLifecycleFixture === true) return; const run = runner.run; + let finalCommitReleased = false; let lifecycleReleased = false; - runner.run = (command, options) => { + const wrappedRun = (command, options) => { const normalized = normalizeCommand(command); - if (normalized.startsWith("docker rm ")) lifecycleReleased = true; + if ( + finalCommitReleased && + normalized.startsWith("docker ps -a --no-trunc ") && + normalized.includes("label=openshell.ai/sandbox-name=my-assistant") && + normalized.endsWith("--format {{.ID}}") + ) { + return { + status: 0, + stdout: Buffer.from(`${ONBOARD_SANDBOX_NEW_CONTAINER_ID}\n`), + stderr: Buffer.alloc(0), + }; + } + if ( + finalCommitReleased && + normalized === + `docker inspect --type container --format {{json .State.Running}} ${ONBOARD_SANDBOX_NEW_CONTAINER_ID}` + ) { + return { + status: 0, + stdout: Buffer.from("true\n"), + stderr: Buffer.alloc(0), + }; + } if (lifecycleReleased && normalized.includes("sandbox list")) { return { status: 0, @@ -344,8 +371,17 @@ function mockDockerSandboxLifecycleReleaseFromRunner() { stderr: Buffer.alloc(0), }; } - return run(command, options); + const result = run(command, options); + if (normalized.startsWith("docker rm ") && Number(result?.status ?? 0) === 0) { + lifecycleReleased = true; + if (normalized === `docker rm ${ONBOARD_SANDBOX_OLD_CONTAINER_ID}`) { + finalCommitReleased = true; + } + } + return result; }; + wrappedRun.__nemoclawDockerLifecycleFixture = true; + runner.run = wrappedRun; } function mockManagedImageFallback() { diff --git a/test/onboard-sandbox-build.test.ts b/test/onboard-sandbox-build.test.ts index 65a294a17b7..c1a0e734f32 100644 --- a/test/onboard-sandbox-build.test.ts +++ b/test/onboard-sandbox-build.test.ts @@ -443,6 +443,7 @@ platform.isWsl = () => false; const commands = []; const logs = []; const baseResolutionCalls = []; +let sandboxCreated = false; const originalLog = console.log; console.log = (...args) => { logs.push(args.join(" ")); @@ -497,7 +498,9 @@ runner.runFile = (file, args = [], opts = {}) => { return { status: 0 }; }; runner.runCapture = (command) => { - if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) return ""; + if (_n(command).includes("sandbox get") && _n(command).includes("my-assistant")) { + return sandboxCreated ? "Name: my-assistant\\nPhase: Ready\\n" : ""; + } if (_n(command).includes("sandbox list")) return "my-assistant Ready"; { const mockedCapture = require(${onboardScriptMocksPath}).mockOnboardRunCapture(command); @@ -515,6 +518,7 @@ preflight.checkPortAvailable = async () => ({ ok: true }); credentials.prompt = async () => ""; childProcess.spawn = (...args) => { + sandboxCreated = true; const child = new EventEmitter(); child.stdout = new EventEmitter(); child.stderr = new EventEmitter(); From 006feaf04468ac00724534459aefd9f09b839818 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sun, 23 Aug 2026 17:23:44 -0700 Subject: [PATCH 03/12] test(onboard): require explicit fixture removal Signed-off-by: Apurv Kumaria --- test/helpers/onboard-script-mocks.cjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/helpers/onboard-script-mocks.cjs b/test/helpers/onboard-script-mocks.cjs index 10a07c4e115..5caa3e2234b 100644 --- a/test/helpers/onboard-script-mocks.cjs +++ b/test/helpers/onboard-script-mocks.cjs @@ -372,7 +372,7 @@ function mockDockerSandboxLifecycleReleaseFromRunner() { }; } const result = run(command, options); - if (normalized.startsWith("docker rm ") && Number(result?.status ?? 0) === 0) { + if (normalized.startsWith("docker rm ") && result?.status === 0) { lifecycleReleased = true; if (normalized === `docker rm ${ONBOARD_SANDBOX_OLD_CONTAINER_ID}`) { finalCommitReleased = true; From 6e6d0d1264501fe0bc6a56b84ec21a5218c57108 Mon Sep 17 00:00:00 2001 From: Apurv Kumaria Date: Sun, 23 Aug 2026 23:44:32 -0700 Subject: [PATCH 04/12] fix(onboard): define final handoff commit point Signed-off-by: Apurv Kumaria --- .../onboard/docker-gpu-patch-diagnostics.ts | 1 + .../onboard/docker-gpu-patch-finalize.test.ts | 28 ++++++---- src/lib/onboard/docker-gpu-patch-finalize.ts | 52 ++++--------------- .../onboard/docker-gpu-patch-recreate.test.ts | 15 +++--- src/lib/onboard/docker-gpu-patch-recreate.ts | 8 ++- .../onboard/docker-gpu-patch-rollback.test.ts | 6 ++- src/lib/onboard/docker-gpu-patch-types.ts | 1 + ...ocker-gpu-sandbox-create-lifecycle.test.ts | 7 ++- src/lib/onboard/docker-gpu-sandbox-create.ts | 5 ++ .../docker-gpu-supervisor-reconnect.test.ts | 5 +- .../docker-gpu-supervisor-reconnect.ts | 5 +- 11 files changed, 66 insertions(+), 67 deletions(-) diff --git a/src/lib/onboard/docker-gpu-patch-diagnostics.ts b/src/lib/onboard/docker-gpu-patch-diagnostics.ts index 1ad91880cbd..cfb61b78ef2 100644 --- a/src/lib/onboard/docker-gpu-patch-diagnostics.ts +++ b/src/lib/onboard/docker-gpu-patch-diagnostics.ts @@ -243,6 +243,7 @@ export function collectDockerGpuPatchDiagnostics( `old_container_id=${redactor.redactText(context?.oldContainerId ?? "unknown")}`, `new_container_id=${redactor.redactText(context?.newContainerId ?? "unknown")}`, `backup_container_name=${redactor.redactText(context?.backupContainerName ?? "none")}`, + `backup_removed=${confirmationValue(context?.backupRemoved)}`, `last_openshell_phase=${redactor.redactText(context?.lastSandboxPhase ?? "unknown")}`, `rolled_back=${cleanupPendingRollback ? "pending" : context?.rolledBack === true ? "yes" : context?.rolledBack === false ? "failed" : "no"}`, ...(context?.replacementStopConfirmed !== undefined diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index 95bcfd992f3..938d3d5585a 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -128,13 +128,17 @@ describe("finalizeDockerGpuPatchBackup", () => { events.push("observe ready"); return "alpha 2026-08-23 10:00:00 Ready\n"; }); + const runOpenshellResults = { + exec: { event: "exec ready", result: { status: 0 } }, + list: { + event: "observe lifecycle release", + result: { status: 0, stdout: "beta 2026-08-23 10:00:00 Ready\n" }, + }, + } as const; const runOpenshell = vi.fn((args: readonly string[]) => { - if (args[1] === "list") { - events.push("observe lifecycle release"); - return { status: 0, stdout: "beta 2026-08-23 10:00:00 Ready\n" }; - } - events.push("exec ready"); - return { status: 0 }; + const response = runOpenshellResults[args[1] === "list" ? "list" : "exec"]; + events.push(response.event); + return response.result; }); const dockerResults = { ps: { @@ -258,10 +262,14 @@ describe("finalizeDockerGpuPatchBackup", () => { it("accepts a retiring Error row only when the exact replacement has the OpenShell label (#9962)", () => { const result = exactDeferredCreateResult(); - const dockerRun = vi.fn((args: readonly string[]) => { - if (args[0] === "inspect") return { status: 0, stdout: "true\n" }; - return { status: 0, stdout: `${result.newContainerId}\n` }; - }); + const dockerRunResults = { + inspect: { status: 0, stdout: "true\n" }, + ps: { status: 0, stdout: `${result.newContainerId}\n` }, + } as const; + const dockerRun = vi.fn( + (args: readonly string[]) => + dockerRunResults[String(args[0]) as keyof typeof dockerRunResults], + ); const outcome = finalizeDockerGpuPatchBackup( { diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index c058fc1603a..29f822fc998 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -11,7 +11,9 @@ // supports that natively, NemoClaw recreates the container with GPU access // and uses this module to either restore the pre-patch backup before commit or // complete the exact stop/remove/start handoff and require OpenShell's final -// Ready acknowledgement. Regression coverage: +// Ready acknowledgement. Removing the old container is the irreversible +// commit point: later failures require a sandbox rebuild rather than automatic +// rollback. Regression coverage: // * src/lib/onboard/docker-gpu-patch-finalize.test.ts — direct unit tests // for exact final handoff, terminal phase, rollback, and failure outcomes. // * src/lib/onboard/docker-gpu-patch-rollback.test.ts — composed @@ -60,6 +62,7 @@ export type DockerGpuPatchFinalizeOptions = }; export type DockerGpuPatchFinalizeOutcome = { + /** True once the old container has crossed the irreversible removal boundary. */ backupRemoved: boolean; rolledBack: boolean; replacementStoppedForCommit?: boolean; @@ -104,8 +107,7 @@ function isSoleLabeledReplacement( .map((line) => line.trim()) .filter(Boolean); return ( - containerIds.length === 1 && - fullDockerContainerId(containerIds[0]) === expectedContainerId + containerIds.length === 1 && fullDockerContainerId(containerIds[0]) === expectedContainerId ); } catch { return false; @@ -169,9 +171,10 @@ export function finalizeDockerGpuPatchBackup( if (options.supervisorReady) { // Stop the exact replacement before retiring the exact backup, then start // the replacement afterward. The final start is the authoritative Docker - // lifecycle event. Success is withheld until OpenShell reports Ready and - // Docker still proves the exact replacement is the sole running labeled - // container (#9531). + // lifecycle event. Backup removal is the irreversible commit point; + // failures after it require a sandbox rebuild. Success is withheld until + // OpenShell reports Ready and Docker still proves the exact replacement is + // the sole running labeled container (#9531). if (!deps.runOpenshell || !deps.runCaptureOpenshell) { return { backupRemoved: false, @@ -214,11 +217,7 @@ export function finalizeDockerGpuPatchBackup( runOpenshell: deps.runOpenshell, sleep: deps.sleep, soleLabeledReplacementCorroboratesRetiringPhase: (remainingMs) => - isSoleLabeledReplacement( - options.result.newContainerId, - resolved.dockerRun, - remainingMs, - ), + isSoleLabeledReplacement(options.result.newContainerId, resolved.dockerRun, remainingMs), }, ); if (!lifecycleReleaseObserved) { @@ -283,34 +282,3 @@ export function finalizeDockerGpuPatchBackup( ); return { backupRemoved: false, ...rollback }; } - -export type SupervisorReconnectOutcome = - | { execReady: true; backupRemoved: boolean } - | ({ execReady: false; error: Error } & Omit); - -export function reconcileSupervisorReconnect( - execReady: boolean, - refs: { newContainerId: string; backupContainerName: string; originalName: string }, - deps: DockerGpuPatchDeps, -): SupervisorReconnectOutcome { - const resolved = resolveDockerGpuPatchRollbackDeps(deps); - const containerOpts = { - ignoreError: true, - suppressOutput: true, - timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, - }; - if (execReady) { - const rmResult = resolved.dockerRm(refs.backupContainerName, containerOpts); - return { execReady: true, backupRemoved: hasZeroDockerExitStatus(rmResult) }; - } - const rollback = rollbackToBackupContainer(refs, resolved); - return { - execReady: false, - ...rollback, - error: new Error( - rollback.rolledBack - ? "OpenShell supervisor did not reconnect to the GPU-enabled container; pre-patch sandbox restored." - : "OpenShell supervisor did not reconnect to the GPU-enabled container and rollback failed; pre-patch sandbox was NOT restored.", - ), - }; -} diff --git a/src/lib/onboard/docker-gpu-patch-recreate.test.ts b/src/lib/onboard/docker-gpu-patch-recreate.test.ts index f9c708638b0..98867a8a3d4 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.test.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.test.ts @@ -39,7 +39,9 @@ describe("Docker GPU recreate orchestration", () => { const dockerStop = vi.fn(() => ({ status: 0 })); const dockerRm = vi.fn(() => ({ status: 0 })); const dockerStart = vi.fn(() => ({ status: 0 })); - const runOpenshell = vi.fn(() => ({ status: 0 })); + const runOpenshell = vi.fn((args: readonly string[]) => + args[1] === "list" ? { status: 0, stdout: "No sandboxes found.\n" } : { status: 0 }, + ); const runCaptureOpenshell = vi.fn(() => "alpha 2026-08-23 10:00:00 Ready\n"); const result = recreateOpenShellDockerSandboxWithGpu( @@ -108,10 +110,6 @@ describe("Docker GPU recreate orchestration", () => { ["sandbox", "list"], expect.objectContaining({ ignoreError: true, suppressOutput: true }), ); - expect(runOpenshell).toHaveBeenCalledTimes(2); - expect(dockerStart.mock.invocationCallOrder[0]).toBeLessThan( - runOpenshell.mock.invocationCallOrder[1], - ); }); it("does not report success when the final OpenShell phase is Deleting (#9531)", () => { @@ -128,7 +126,9 @@ describe("Docker GPU recreate orchestration", () => { dockerRm: vi.fn(() => ({ status: 0 })), dockerStart: vi.fn(() => ({ status: 0 })), runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 10:00:00 Deleting\n"), - runOpenshell: vi.fn(() => ({ status: 0 })), + runOpenshell: vi.fn((args: readonly string[]) => + args[1] === "list" ? { status: 0, stdout: "No sandboxes found.\n" } : { status: 0 }, + ), sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), detectSandboxFallbackDns: vi.fn(() => null), @@ -142,7 +142,10 @@ describe("Docker GPU recreate orchestration", () => { expect(failure).toBeInstanceOf(Error); expect((failure as Error).message).toContain("final replacement handoff"); + expect((failure as Error).message).toContain("automatic rollback is unavailable"); + expect((failure as Error).message).toContain("Rebuild the sandbox before retrying"); expect(getDockerGpuPatchFailureContext(failure)).toMatchObject({ + backupRemoved: true, oldContainerId: OLD_CONTAINER_ID, newContainerId: NEW_CONTAINER_ID, lastSandboxPhase: "Deleting", diff --git a/src/lib/onboard/docker-gpu-patch-recreate.ts b/src/lib/onboard/docker-gpu-patch-recreate.ts index d3618ac46b2..7d2310ff366 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.ts @@ -406,6 +406,7 @@ export function recreateOpenShellDockerSandboxContainer( ) : finalizeDockerGpuPatchBackup({ result: patchResult, supervisorReady: false }, deps); context.rolledBack = finalization.rolledBack; + context.backupRemoved = finalization.backupRemoved; context.replacementStopConfirmed = finalization.replacementStopConfirmed; context.replacementRemovalConfirmed = finalization.replacementRemovalConfirmed; context.replacementPresence = finalization.replacementPresence; @@ -418,10 +419,13 @@ export function recreateOpenShellDockerSandboxContainer( ); } if (finalization.finalHandoffAcknowledged) return result(true); + const rebuildRequired = finalization.backupRemoved + ? " The previous container was removed at the final handoff commit point; automatic rollback is unavailable. Rebuild the sandbox before retrying." + : ""; throw new Error( finalization.lastSandboxPhase - ? `OpenShell did not acknowledge the final replacement handoff; last sandbox phase was ${finalization.lastSandboxPhase}.` - : "OpenShell did not acknowledge the final replacement handoff.", + ? `OpenShell did not acknowledge the final replacement handoff; last sandbox phase was ${finalization.lastSandboxPhase}.${rebuildRequired}` + : `OpenShell did not acknowledge the final replacement handoff.${rebuildRequired}`, ); } catch (error) { throw decoratePatchError(error instanceof Error ? error : new Error(String(error)), context); diff --git a/src/lib/onboard/docker-gpu-patch-rollback.test.ts b/src/lib/onboard/docker-gpu-patch-rollback.test.ts index d126f50875d..6c6af7c26ac 100644 --- a/src/lib/onboard/docker-gpu-patch-rollback.test.ts +++ b/src/lib/onboard/docker-gpu-patch-rollback.test.ts @@ -273,8 +273,10 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { dockerRename, dockerStart, dockerLogs: vi.fn(() => ""), - runOpenshell: vi.fn(() => ({ status: 0 })), - runCaptureOpenshell: vi.fn(() => "alpha Ready\n"), + runOpenshell: vi.fn((args: readonly string[]) => + args[1] === "list" ? { status: 0, stdout: "No sandboxes found.\n" } : { status: 0 }, + ), + runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 10:00:00 Ready\n"), sleep: vi.fn(), homedir: () => tmpDir, now: () => new Date("2026-07-03T00:00:00Z"), diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index 8dd7765cfa9..972cd858f54 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -79,6 +79,7 @@ export type DockerGpuPatchFailureContext = { oldContainerId?: string | null; newContainerId?: string | null; backupContainerName?: string | null; + backupRemoved?: boolean; selectedMode?: DockerGpuPatchMode | null; modeAttempts?: DockerGpuPatchModeAttempt[]; rolledBack?: boolean; diff --git a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index df318c1c0d2..fde42865d83 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -171,8 +171,11 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); patch.waitForSupervisorReconnectIfNeeded(); - await expect(patch.commitAfterReady()).rejects.toThrow("final runtime handoff"); + await expect(patch.commitAfterReady()).rejects.toThrow("automatic rollback is unavailable"); expect(onPatchFailureExit).toHaveBeenCalledOnce(); + expect(onPatchFailureExit.mock.calls[0]?.[2]?.context).toMatchObject({ + backupRemoved: true, + }); }); it("rejects final handoff when OpenShell reports Deleting after restart (#9531)", async () => { @@ -202,7 +205,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); patch.waitForSupervisorReconnectIfNeeded(); - await expect(patch.commitAfterReady()).rejects.toThrow("final runtime handoff"); + await expect(patch.commitAfterReady()).rejects.toThrow("automatic rollback is unavailable"); expect(waitForSupervisor).toHaveBeenCalledOnce(); expect(onPatchFailureExit).toHaveBeenCalledOnce(); diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index d7b0ff08b4b..e4ed566917b 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -521,6 +521,10 @@ export function createDockerGpuSandboxCreatePatch( finalizeOutcome.lastSandboxPhase ? ` Last OpenShell phase: ${finalizeOutcome.lastSandboxPhase}.` : "" + }${ + finalizeOutcome.backupRemoved + ? " The previous container was removed at the final handoff commit point; automatic rollback is unavailable. Rebuild the sandbox before retrying." + : "" }`, ); cutoverFinalizationFailure = failure; @@ -530,6 +534,7 @@ export function createDockerGpuSandboxCreatePatch( additionalSummaryLines: routeAdapter.additionalSummaryLines, context: { ...failureContext(), + backupRemoved: finalizeOutcome.backupRemoved, lastSandboxPhase: finalizeOutcome.lastSandboxPhase, }, }); diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts index 05f93dfd22c..a197f82343a 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts @@ -40,7 +40,10 @@ describe("Docker GPU final lifecycle release", () => { sleep: vi.fn(), }), ).toBe(false); - expect(runOpenshell).toHaveBeenCalledTimes(2); + }); + + it("does not report reconnect without an OpenShell execution boundary (#9531)", () => { + expect(waitForOpenShellSupervisorReconnect("alpha", 1, { sleep: vi.fn() })).toBe(false); }); }); diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts index 26f2a3f7339..76215a08973 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts @@ -128,7 +128,8 @@ export function waitForOpenShellSandboxLifecycleRelease( const entries = parseLiveSandboxEntries(output); const sandboxPresent = entries.some((entry) => entry.name === sandboxName); const retiring = entries.some( - (entry) => entry.name === sandboxName && (entry.phase === "Error" || entry.phase === "Deleting"), + (entry) => + entry.name === sandboxName && (entry.phase === "Error" || entry.phase === "Deleting"), ); const corroborated = retiring && @@ -281,7 +282,7 @@ export function waitForOpenShellSupervisorReconnect( timeoutSecs: number, deps: DockerGpuSupervisorReconnectDeps, ): boolean { - if (!deps.runOpenshell) return true; + if (!deps.runOpenshell) return false; const sleep = deps.sleep ?? defaultSleep; const deadline = Date.now() + Math.max(1, timeoutSecs) * 1000; const errorPhaseDebouncePolls = From c7d1e9417a5b34618cf47c03e953f92fb6ed6d65 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 25 Aug 2026 06:45:58 -0700 Subject: [PATCH 05/12] fix(onboard): use canonical sandbox container query Signed-off-by: Aaron Erickson --- .../onboard/docker-gpu-patch-finalize.test.ts | 46 ++++++++++++-- src/lib/onboard/docker-gpu-patch-finalize.ts | 61 +++++-------------- 2 files changed, 57 insertions(+), 50 deletions(-) diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index 938d3d5585a..5c917ebe1cb 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -303,15 +303,53 @@ describe("finalizeDockerGpuPatchBackup", () => { "-a", "--no-trunc", "--filter", - `id=${result.newContainerId}`, - "--filter", "label=openshell.ai/managed-by=openshell", + "--filter", + "label=openshell.ai/sandbox-name=restored-name", "--format", "{{.ID}}", ]); - expect(dockerRun.mock.calls[0]?.[0]).not.toContain( - "label=openshell.ai/sandbox-name=restored-name", + }); + + it.each([ + ["a failed query", { status: 1, stderr: "daemon unavailable" }], + ["no labeled replacement", { status: 0, stdout: "" }], + ["a different replacement", { status: 0, stdout: `${"c".repeat(64)}\n` }], + [ + "multiple labeled replacements", + { status: 0, stdout: `${"b".repeat(64)}\n${"c".repeat(64)}\n` }, + ], + ])("rejects retiring Error when the canonical query returns %s (#9531)", (_case, query) => { + const result = exactDeferredCreateResult(); + const dockerStart = vi.fn(() => ({ status: 0 })); + + const outcome = finalizeDockerGpuPatchBackup( + { + result, + supervisorReady: true, + sandboxName: "alpha", + finalHandoffTimeoutSecs: 1, + }, + { + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + dockerRun: vi.fn(() => query), + dockerStart, + runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 01:40:35 Ready\n"), + runOpenshell: vi.fn(() => ({ + status: 0, + stdout: "alpha 2026-08-23 01:40:35 Error\n", + })), + sleep: vi.fn(), + }, ); + + expect(outcome).toMatchObject({ + backupRemoved: true, + lifecycleReleaseObserved: false, + replacementRestarted: false, + }); + expect(dockerStart).not.toHaveBeenCalled(); }); it("rolls back to the backup container when supervisor reconnect failed", () => { diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index 29f822fc998..c874626d0db 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -38,11 +38,7 @@ import { waitForOpenShellFinalHandoff, waitForOpenShellSandboxLifecycleRelease, } from "./docker-gpu-supervisor-reconnect"; -import { - OPENSHELL_MANAGED_BY_LABEL, - OPENSHELL_MANAGED_BY_VALUE, - queryOpenShellDockerSandboxContainers, -} from "./openshell-docker-sandbox-containers"; +import { queryOpenShellDockerSandboxContainers } from "./openshell-docker-sandbox-containers"; export { restoreDockerGpuPatchBackupAfterRecreateFailure as rollbackDockerGpuPatchOnRecreateFailure, @@ -75,45 +71,6 @@ export type DockerGpuPatchFinalizeOutcome = { replacementPresence?: "absent" | "present" | "unknown"; }; -function isSoleLabeledReplacement( - replacementContainerId: string, - dockerRun: NonNullable, - timeoutMs: number, -): boolean { - const expectedContainerId = fullDockerContainerId(replacementContainerId); - if (!expectedContainerId || timeoutMs <= 0) return false; - try { - const query = dockerRun( - [ - "ps", - "-a", - "--no-trunc", - "--filter", - `id=${expectedContainerId}`, - "--filter", - `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, - "--format", - "{{.ID}}", - ], - { - ignoreError: true, - suppressOutput: true, - timeout: Math.max(1, Math.min(DOCKER_GPU_PATCH_TIMEOUT_MS, Math.floor(timeoutMs))), - }, - ); - if (!hasZeroDockerExitStatus(query)) return false; - const containerIds = String(query.stdout ?? "") - .split(/\r?\n/u) - .map((line) => line.trim()) - .filter(Boolean); - return ( - containerIds.length === 1 && fullDockerContainerId(containerIds[0]) === expectedContainerId - ); - } catch { - return false; - } -} - function isExactRunningReplacement( sandboxName: string, replacementContainerId: string, @@ -216,8 +173,20 @@ export function finalizeDockerGpuPatchBackup( { runOpenshell: deps.runOpenshell, sleep: deps.sleep, - soleLabeledReplacementCorroboratesRetiringPhase: (remainingMs) => - isSoleLabeledReplacement(options.result.newContainerId, resolved.dockerRun, remainingMs), + soleLabeledReplacementCorroboratesRetiringPhase: (remainingMs) => { + const expectedContainerId = fullDockerContainerId(options.result.newContainerId); + if (!expectedContainerId || remainingMs <= 0) return false; + const containers = queryOpenShellDockerSandboxContainers( + options.sandboxName, + { dockerRun: resolved.dockerRun }, + remainingMs, + ); + return ( + containers.ok && + containers.ids.length === 1 && + fullDockerContainerId(containers.ids[0]) === expectedContainerId + ); + }, }, ); if (!lifecycleReleaseObserved) { From 55487313fdc6c214a4159b5eca0467e3af7d113f Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 25 Aug 2026 08:11:44 -0700 Subject: [PATCH 06/12] fix(onboard): restore prior container after handoff failure Signed-off-by: Aaron Erickson --- .../sandbox/supervisor-relaunch.test.ts | 6 +- .../onboard/docker-gpu-patch-diagnostics.ts | 2 + .../onboard/docker-gpu-patch-finalize.test.ts | 256 ++++++++++++++++-- src/lib/onboard/docker-gpu-patch-finalize.ts | 201 +++++++++++++- .../onboard/docker-gpu-patch-recreate.test.ts | 56 +++- src/lib/onboard/docker-gpu-patch-recreate.ts | 18 +- .../onboard/docker-gpu-patch-rollback.test.ts | 2 + src/lib/onboard/docker-gpu-patch-rollback.ts | 37 ++- src/lib/onboard/docker-gpu-patch-types.ts | 2 + ...ocker-gpu-sandbox-create-lifecycle.test.ts | 17 +- src/lib/onboard/docker-gpu-sandbox-create.ts | 18 +- ...ocess-recovery-supervisor-relaunch.test.ts | 20 +- 12 files changed, 556 insertions(+), 79 deletions(-) diff --git a/src/lib/actions/sandbox/supervisor-relaunch.test.ts b/src/lib/actions/sandbox/supervisor-relaunch.test.ts index 8a3d48bcedb..438b72af361 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.test.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.test.ts @@ -508,13 +508,13 @@ describe("relaunchManagedSupervisorSession", () => { }); }); - it("retains the restored state backup when final handoff is not acknowledged (#9531)", () => { + it("restores the prior container and retains state backup when final handoff fails (#9531)", () => { const deps = baseDeps({ finalize: vi.fn(() => ({ backupRemoved: true, finalHandoffAcknowledged: false, lastSandboxPhase: "Deleting", - rolledBack: false, + rolledBack: true, })), }); const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); @@ -523,7 +523,7 @@ describe("relaunchManagedSupervisorSession", () => { backupRemoved: true, finalHandoffAcknowledged: false, lastSandboxPhase: "Deleting", - rolledBack: false, + rolledBack: true, stateRestored: true, }); expect(deps.removeBackup).not.toHaveBeenCalled(); diff --git a/src/lib/onboard/docker-gpu-patch-diagnostics.ts b/src/lib/onboard/docker-gpu-patch-diagnostics.ts index cfb61b78ef2..d832591540e 100644 --- a/src/lib/onboard/docker-gpu-patch-diagnostics.ts +++ b/src/lib/onboard/docker-gpu-patch-diagnostics.ts @@ -244,6 +244,8 @@ export function collectDockerGpuPatchDiagnostics( `new_container_id=${redactor.redactText(context?.newContainerId ?? "unknown")}`, `backup_container_name=${redactor.redactText(context?.backupContainerName ?? "none")}`, `backup_removed=${confirmationValue(context?.backupRemoved)}`, + `rollback_image_id=${redactor.redactText(context?.rollbackImageId ?? "none")}`, + `rollback_image_removed=${confirmationValue(context?.rollbackImageRemoved)}`, `last_openshell_phase=${redactor.redactText(context?.lastSandboxPhase ?? "unknown")}`, `rolled_back=${cleanupPendingRollback ? "pending" : context?.rolledBack === true ? "yes" : context?.rolledBack === false ? "failed" : "no"}`, ...(context?.replacementStopConfirmed !== undefined diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index 5c917ebe1cb..bc0f2701908 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -7,12 +7,16 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; +import { createDockerGpuInspectFixture as inspectFixture } from "./__test-helpers__/docker-gpu-patch-fixtures"; import { collectDockerGpuPatchDiagnostics, type DockerGpuPatchResult } from "./docker-gpu-patch"; import { type DockerGpuPatchFinalizeOutcome, finalizeDockerGpuPatchBackup, } from "./docker-gpu-patch-finalize"; +const ROLLBACK_IMAGE_ID = `sha256:${"d".repeat(64)}`; +const RESTORED_CONTAINER_ID = "e".repeat(64); + function deferredCreateResult(): DockerGpuPatchResult { return { applied: true, @@ -38,8 +42,36 @@ function exactDeferredCreateResult(): DockerGpuPatchResult { }; } +function rollbackPlanDeps(result: DockerGpuPatchResult = exactDeferredCreateResult()) { + return { + dockerCapture: vi.fn(() => + JSON.stringify([{ ...inspectFixture(), Id: result.oldContainerId }]), + ), + dockerRun: vi.fn((args: readonly string[]) => { + switch (args[0]) { + case "commit": + return { status: 0, stdout: `${ROLLBACK_IMAGE_ID}\n` }; + case "image": + return { status: 0 }; + case "ps": + return { status: 0, stdout: `${result.newContainerId}\n` }; + case "inspect": + return { status: 0, stdout: "true\n" }; + default: + return { status: 1, stderr: "unexpected Docker command" }; + } + }), + dockerRunDetached: vi.fn((_args: readonly string[]) => ({ + status: 0, + stdout: `${RESTORED_CONTAINER_ID}\n`, + stderr: "", + })), + }; +} + function readyHandoffDeps() { return { + ...rollbackPlanDeps(), runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 10:00:00 Ready\n"), runOpenshell: vi.fn((args: readonly string[]) => args[1] === "list" @@ -111,6 +143,7 @@ describe("finalizeDockerGpuPatchBackup", () => { it("uses one exact stop, remove, start, and Ready acknowledgement handoff (#9531)", () => { const result = exactDeferredCreateResult(); + const rollbackDeps = rollbackPlanDeps(result); const events: string[] = []; const dockerStop = vi.fn(() => { events.push("stop replacement"); @@ -148,9 +181,19 @@ describe("finalizeDockerGpuPatchBackup", () => { inspect: { event: "confirm running replacement", result: { status: 0, stdout: "true\n" } }, } as const; const dockerRun = vi.fn((args: readonly string[]) => { - const response = dockerResults[String(args[0]) as keyof typeof dockerResults]; - events.push(response.event); - return response.result; + switch (args[0]) { + case "commit": + events.push("commit rollback image"); + return rollbackDeps.dockerRun(args); + case "image": + events.push("remove rollback image"); + return rollbackDeps.dockerRun(args); + default: { + const response = dockerResults[String(args[0]) as keyof typeof dockerResults]; + events.push(response.event); + return response.result; + } + } }); const outcome = finalizeDockerGpuPatchBackup( @@ -161,9 +204,11 @@ describe("finalizeDockerGpuPatchBackup", () => { finalHandoffTimeoutSecs: 60, }, { + dockerCapture: rollbackDeps.dockerCapture, dockerStop, dockerRm, dockerRun, + dockerRunDetached: rollbackDeps.dockerRunDetached, dockerStart, runCaptureOpenshell, runOpenshell, @@ -174,6 +219,7 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(outcome).toEqual({ backupRemoved: true, rolledBack: false, + rollbackImageRemoved: true, replacementStoppedForCommit: true, replacementRestarted: true, lifecycleReleaseObserved: true, @@ -182,6 +228,7 @@ describe("finalizeDockerGpuPatchBackup", () => { }); expect(events).toEqual([ "stop replacement", + "commit rollback image", "remove backup", "observe lifecycle release", "start replacement", @@ -189,6 +236,7 @@ describe("finalizeDockerGpuPatchBackup", () => { "exec ready", "confirm sole replacement", "confirm running replacement", + "remove rollback image", ]); expect(dockerStop).toHaveBeenCalledWith( result.newContainerId, @@ -198,14 +246,20 @@ describe("finalizeDockerGpuPatchBackup", () => { result.oldContainerId, expect.objectContaining({ ignoreError: true }), ); + const commitCall = dockerRun.mock.calls.findIndex((call) => call[0][0] === "commit"); + expect(commitCall).toBeGreaterThanOrEqual(0); + expect(dockerRun.mock.invocationCallOrder[commitCall]).toBeLessThan( + dockerRm.mock.invocationCallOrder[0], + ); expect(dockerStart).toHaveBeenCalledWith( result.newContainerId, expect.objectContaining({ ignoreError: true }), ); }); - it("fails immediately when OpenShell reports Deleting after the final start (#9531)", () => { + it("restores the prior container when OpenShell reports Deleting after final start (#9531)", () => { const result = exactDeferredCreateResult(); + const rollbackDeps = rollbackPlanDeps(result); const events: string[] = []; const sleep = vi.fn(); const outcome = finalizeDockerGpuPatchBackup( @@ -216,12 +270,13 @@ describe("finalizeDockerGpuPatchBackup", () => { finalHandoffTimeoutSecs: 60, }, { + dockerCapture: rollbackDeps.dockerCapture, dockerStop: vi.fn(() => { events.push("stop replacement"); return { status: 0 }; }), - dockerRm: vi.fn(() => { - events.push("remove backup"); + dockerRm: vi.fn((target: string) => { + events.push(target === result.oldContainerId ? "remove backup" : "remove replacement"); return { status: 0 }; }), dockerStart: vi.fn(() => { @@ -237,14 +292,20 @@ describe("finalizeDockerGpuPatchBackup", () => { ? { status: 0, stdout: "beta 2026-08-23 10:00:00 Ready\n" } : { status: 1 }, ), - dockerRun: vi.fn(() => ({ status: 0, stdout: `${result.newContainerId}\n` })), + dockerRun: rollbackDeps.dockerRun, + dockerRunDetached: rollbackDeps.dockerRunDetached, sleep, }, ); expect(outcome).toEqual({ backupRemoved: true, - rolledBack: false, + rolledBack: true, + rollbackImageId: ROLLBACK_IMAGE_ID, + rollbackImageRemoved: false, + replacementStopConfirmed: true, + replacementRemovalConfirmed: true, + replacementPresence: "absent", replacementStoppedForCommit: true, replacementRestarted: true, lifecycleReleaseObserved: true, @@ -256,19 +317,78 @@ describe("finalizeDockerGpuPatchBackup", () => { "remove backup", "start replacement", "observe deleting", + "stop replacement", + "remove replacement", ]); expect(sleep).not.toHaveBeenCalled(); + expect(rollbackDeps.dockerRunDetached).toHaveBeenCalledOnce(); + expect(rollbackDeps.dockerRunDetached.mock.calls[0]?.[0]).toEqual( + expect.arrayContaining([ + "--name", + "openshell-alpha", + "--label", + "openshell.ai/managed-by=openshell", + "--label", + "openshell.ai/sandbox-name=alpha", + "--volume", + "/host:/container:rw", + ROLLBACK_IMAGE_ID, + ]), + ); + expect(rollbackDeps.dockerRunDetached.mock.calls[0]?.[0]).not.toContain("--gpus"); + const diagnostics = collectRollbackDiagnostics(result.newContainerId, outcome); + expect(diagnostics.summary).toContain(`rollback_image_id=${ROLLBACK_IMAGE_ID}`); + expect(diagnostics.summary).toContain("rollback_image_removed=no"); + expect(diagnostics.summary).toContain("rolled_back=yes"); + }); + + it("reports a retained rollback image when successful-handoff cleanup fails (#9531)", () => { + const result = exactDeferredCreateResult(); + const deps = readyHandoffDeps(); + const baseDockerRun = deps.dockerRun; + deps.dockerRun = vi.fn((args: readonly string[]) => + args[0] === "image" ? { status: 1, stderr: "image is busy" } : baseDockerRun(args), + ); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + + const outcome = finalizeDockerGpuPatchBackup( + { + result, + supervisorReady: true, + sandboxName: "alpha", + finalHandoffTimeoutSecs: 60, + }, + { + ...deps, + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + dockerStart: vi.fn(() => ({ status: 0 })), + }, + ); + + expect(outcome).toMatchObject({ + backupRemoved: true, + rolledBack: false, + rollbackImageId: ROLLBACK_IMAGE_ID, + rollbackImageRemoved: false, + finalHandoffAcknowledged: true, + }); + expect(warn).toHaveBeenCalledWith( + ` Could not remove temporary Docker rollback image ${ROLLBACK_IMAGE_ID}.`, + ); }); it("accepts a retiring Error row only when the exact replacement has the OpenShell label (#9962)", () => { const result = exactDeferredCreateResult(); + const rollbackDeps = rollbackPlanDeps(result); const dockerRunResults = { inspect: { status: 0, stdout: "true\n" }, ps: { status: 0, stdout: `${result.newContainerId}\n` }, } as const; - const dockerRun = vi.fn( - (args: readonly string[]) => - dockerRunResults[String(args[0]) as keyof typeof dockerRunResults], + const dockerRun = vi.fn((args: readonly string[]) => + args[0] === "commit" || args[0] === "image" + ? rollbackDeps.dockerRun(args) + : dockerRunResults[String(args[0]) as keyof typeof dockerRunResults], ); const outcome = finalizeDockerGpuPatchBackup( @@ -279,10 +399,12 @@ describe("finalizeDockerGpuPatchBackup", () => { finalHandoffTimeoutSecs: 60, }, { + dockerCapture: rollbackDeps.dockerCapture, dockerStop: vi.fn(() => ({ status: 0 })), dockerRm: vi.fn(() => ({ status: 0 })), dockerStart: vi.fn(() => ({ status: 0 })), dockerRun, + dockerRunDetached: rollbackDeps.dockerRunDetached, runCaptureOpenshell: vi.fn(() => "restored-name 2026-08-23 01:40:35 Ready\n"), runOpenshell: vi.fn((args: readonly string[]) => args[1] === "list" @@ -298,7 +420,7 @@ describe("finalizeDockerGpuPatchBackup", () => { lifecycleReleaseObserved: true, finalHandoffAcknowledged: true, }); - expect(dockerRun.mock.calls[0]?.[0]).toEqual([ + expect(dockerRun.mock.calls.find((call) => call[0][0] === "ps")?.[0]).toEqual([ "ps", "-a", "--no-trunc", @@ -319,9 +441,13 @@ describe("finalizeDockerGpuPatchBackup", () => { "multiple labeled replacements", { status: 0, stdout: `${"b".repeat(64)}\n${"c".repeat(64)}\n` }, ], - ])("rejects retiring Error when the canonical query returns %s (#9531)", (_case, query) => { + ])("restores the prior container when lifecycle release has %s (#9531)", (_case, query) => { const result = exactDeferredCreateResult(); + const rollbackDeps = rollbackPlanDeps(result); const dockerStart = vi.fn(() => ({ status: 0 })); + const dockerRun = vi.fn((args: readonly string[]) => + args[0] === "commit" || args[0] === "image" ? rollbackDeps.dockerRun(args) : query, + ); const outcome = finalizeDockerGpuPatchBackup( { @@ -331,9 +457,11 @@ describe("finalizeDockerGpuPatchBackup", () => { finalHandoffTimeoutSecs: 1, }, { + dockerCapture: rollbackDeps.dockerCapture, dockerStop: vi.fn(() => ({ status: 0 })), dockerRm: vi.fn(() => ({ status: 0 })), - dockerRun: vi.fn(() => query), + dockerRun, + dockerRunDetached: rollbackDeps.dockerRunDetached, dockerStart, runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 01:40:35 Ready\n"), runOpenshell: vi.fn(() => ({ @@ -346,10 +474,13 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(outcome).toMatchObject({ backupRemoved: true, + rolledBack: true, + rollbackImageId: ROLLBACK_IMAGE_ID, lifecycleReleaseObserved: false, replacementRestarted: false, }); expect(dockerStart).not.toHaveBeenCalled(); + expect(rollbackDeps.dockerRunDetached).toHaveBeenCalledOnce(); }); it("rolls back to the backup container when supervisor reconnect failed", () => { @@ -455,6 +586,7 @@ describe("finalizeDockerGpuPatchBackup", () => { }); it("reports backupRemoved=false when supervisor reconnect succeeded but docker rm of the backup failed", () => { + const result = exactDeferredCreateResult(); const dockerStop = vi.fn(() => ({ status: 0 })); const dockerRm = vi.fn((_name: string) => ({ status: 1, @@ -463,7 +595,7 @@ describe("finalizeDockerGpuPatchBackup", () => { const dockerStart = vi.fn(() => ({ status: 0 })); const outcome = finalizeDockerGpuPatchBackup( { - result: deferredCreateResult(), + result, supervisorReady: true, sandboxName: "alpha", finalHandoffTimeoutSecs: 60, @@ -473,28 +605,30 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(outcome).toEqual({ backupRemoved: false, rolledBack: false, + rollbackImageRemoved: true, replacementStoppedForCommit: true, replacementRestarted: true, finalHandoffAcknowledged: false, lastSandboxPhase: null, }); expect(dockerRm).toHaveBeenCalledWith( - "old-container-id", + result.oldContainerId, expect.objectContaining({ ignoreError: true }), ); expect(dockerStart).toHaveBeenCalledWith( - "new-container-id", + result.newContainerId, expect.objectContaining({ ignoreError: true }), ); }); it("fails closed when backup removal has no exit status", () => { + const result = exactDeferredCreateResult(); const dockerStop = vi.fn(() => ({ status: 0 })); const dockerRm = vi.fn((_name: string) => ({ status: null, stderr: "timed out" })); const dockerStart = vi.fn(() => ({ status: 0 })); const outcome = finalizeDockerGpuPatchBackup( { - result: deferredCreateResult(), + result, supervisorReady: true, sandboxName: "alpha", finalHandoffTimeoutSecs: 60, @@ -504,13 +638,14 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(outcome).toEqual({ backupRemoved: false, rolledBack: false, + rollbackImageRemoved: true, replacementStoppedForCommit: true, replacementRestarted: true, finalHandoffAcknowledged: false, lastSandboxPhase: null, }); expect(dockerStart).toHaveBeenCalledWith( - "new-container-id", + result.newContainerId, expect.objectContaining({ ignoreError: true }), ); }); @@ -539,10 +674,48 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(dockerStart).not.toHaveBeenCalled(); }); - it("reports a failed replacement restart after the backup is removed", () => { + it("retains the backup when Docker cannot prepare the rollback image (#9531)", () => { + const result = exactDeferredCreateResult(); + const dockerRm = vi.fn(() => ({ status: 0 })); + const dockerStart = vi.fn(() => ({ status: 0 })); + const outcome = finalizeDockerGpuPatchBackup( { - result: deferredCreateResult(), + result, + supervisorReady: true, + sandboxName: "alpha", + finalHandoffTimeoutSecs: 60, + }, + { + ...readyHandoffDeps(), + dockerCapture: rollbackPlanDeps(result).dockerCapture, + dockerRun: vi.fn(() => ({ status: 1, stderr: "commit failed" })), + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm, + dockerStart, + }, + ); + + expect(outcome).toEqual({ + backupRemoved: false, + rolledBack: false, + replacementStoppedForCommit: true, + replacementRestarted: true, + finalHandoffAcknowledged: false, + lastSandboxPhase: null, + }); + expect(dockerRm).not.toHaveBeenCalled(); + expect(dockerStart).toHaveBeenCalledWith( + result.newContainerId, + expect.objectContaining({ ignoreError: true }), + ); + }); + + it("restores the prior container when replacement restart fails after removal (#9531)", () => { + const result = exactDeferredCreateResult(); + const outcome = finalizeDockerGpuPatchBackup( + { + result, supervisorReady: true, sandboxName: "alpha", finalHandoffTimeoutSecs: 60, @@ -557,7 +730,12 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(outcome).toEqual({ backupRemoved: true, - rolledBack: false, + rolledBack: true, + rollbackImageId: ROLLBACK_IMAGE_ID, + rollbackImageRemoved: false, + replacementStopConfirmed: true, + replacementRemovalConfirmed: true, + replacementPresence: "absent", replacementStoppedForCommit: true, replacementRestarted: false, finalHandoffAcknowledged: false, @@ -565,6 +743,40 @@ describe("finalizeDockerGpuPatchBackup", () => { }); }); + it("retains the rollback image when post-commit restoration fails (#9531)", () => { + const result = exactDeferredCreateResult(); + const deps = readyHandoffDeps(); + deps.dockerRunDetached.mockReturnValue({ status: 1, stdout: "", stderr: "restore failed" }); + + const outcome = finalizeDockerGpuPatchBackup( + { + result, + supervisorReady: true, + sandboxName: "alpha", + finalHandoffTimeoutSecs: 60, + }, + { + ...deps, + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + dockerStart: vi.fn(() => ({ status: 1 })), + }, + ); + + expect(outcome).toMatchObject({ + backupRemoved: true, + rolledBack: false, + rollbackImageId: ROLLBACK_IMAGE_ID, + rollbackImageRemoved: false, + replacementPresence: "absent", + replacementRestarted: false, + }); + expect(deps.dockerRun).not.toHaveBeenCalledWith( + ["image", "rm", "--force", ROLLBACK_IMAGE_ID], + expect.anything(), + ); + }); + it("records a remaining exact-ID replacement when removal fails (#7996)", () => { const newContainerId = "a".repeat(64); const outcome = finalizeDockerGpuPatchBackup( diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index c874626d0db..c881745b110 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -11,9 +11,9 @@ // supports that natively, NemoClaw recreates the container with GPU access // and uses this module to either restore the pre-patch backup before commit or // complete the exact stop/remove/start handoff and require OpenShell's final -// Ready acknowledgement. Removing the old container is the irreversible -// commit point: later failures require a sandbox rebuild rather than automatic -// rollback. Regression coverage: +// Ready acknowledgement. Before removing the old container, NemoClaw commits +// its writable layer to a temporary image and records the exact run arguments +// needed to restore it. Regression coverage: // * src/lib/onboard/docker-gpu-patch-finalize.test.ts — direct unit tests // for exact final handoff, terminal phase, rollback, and failure outcomes. // * src/lib/onboard/docker-gpu-patch-rollback.test.ts — composed @@ -29,11 +29,23 @@ import { hasZeroDockerExitStatus } from "./docker-command-result"; import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; import { + type ResolvedDockerGpuPatchRollbackDeps, + removeReplacementContainer, resolveDockerGpuPatchRollbackDeps, rollbackToBackupContainer, } from "./docker-gpu-patch-rollback"; -import { fullDockerContainerId } from "./docker-gpu-patch-clone"; -import type { DockerGpuPatchDeps, DockerGpuPatchResult } from "./docker-gpu-patch-types"; +import { + buildDockerGpuCloneRunArgs, + buildDockerGpuCloneRunOptions, + fullDockerContainerId, + parseDockerInspectJson, +} from "./docker-gpu-patch-clone"; +import { buildDockerGpuMode } from "./docker-gpu-patch-mode"; +import type { + DockerContainerInspect, + DockerGpuPatchDeps, + DockerGpuPatchResult, +} from "./docker-gpu-patch-types"; import { waitForOpenShellFinalHandoff, waitForOpenShellSandboxLifecycleRelease, @@ -58,9 +70,11 @@ export type DockerGpuPatchFinalizeOptions = }; export type DockerGpuPatchFinalizeOutcome = { - /** True once the old container has crossed the irreversible removal boundary. */ + /** True once Docker has removed the old container. */ backupRemoved: boolean; rolledBack: boolean; + rollbackImageId?: string; + rollbackImageRemoved?: boolean; replacementStoppedForCommit?: boolean; replacementRestarted?: boolean; lifecycleReleaseObserved?: boolean; @@ -71,6 +85,109 @@ export type DockerGpuPatchFinalizeOutcome = { replacementPresence?: "absent" | "present" | "unknown"; }; +type PostCommitRollbackPlan = { + imageId: string; + oldContainerId: string; + runArgs: string[]; +}; + +const DOCKER_IMAGE_ID = /^sha256:[0-9a-f]{64}$/iu; + +function removeRollbackImage( + imageId: string, + deps: ResolvedDockerGpuPatchRollbackDeps, + options: Record, +): boolean { + return hasZeroDockerExitStatus(deps.dockerRun(["image", "rm", "--force", imageId], options)); +} + +function preparePostCommitRollback( + result: DockerGpuPatchResult, + resolved: ResolvedDockerGpuPatchRollbackDeps, + options: Record, +): PostCommitRollbackPlan | null { + const oldContainerId = fullDockerContainerId(result.oldContainerId); + if (!oldContainerId) return null; + let inspect: DockerContainerInspect; + try { + inspect = parseDockerInspectJson( + resolved.dockerCapture(["inspect", "--type", "container", oldContainerId], { + ignoreError: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }), + ); + } catch { + return null; + } + + const committed = resolved.dockerRun(["commit", oldContainerId], options); + if (!hasZeroDockerExitStatus(committed)) return null; + const imageId = String(committed.stdout ?? "").trim(); + if (!DOCKER_IMAGE_ID.test(imageId)) return null; + + try { + const cloneOptions = buildDockerGpuCloneRunOptions(inspect); + cloneOptions.containerName = result.originalName; + cloneOptions.image = imageId; + return { + imageId, + oldContainerId, + runArgs: buildDockerGpuCloneRunArgs( + inspect, + buildDockerGpuMode("startup-command"), + cloneOptions, + ), + }; + } catch { + if (!removeRollbackImage(imageId, resolved, options)) { + console.warn(` Could not remove temporary Docker rollback image ${imageId}.`); + } + return null; + } +} + +function restorePostCommitRollback( + plan: PostCommitRollbackPlan, + result: DockerGpuPatchResult, + resolved: ResolvedDockerGpuPatchRollbackDeps, + options: Record, +): Pick< + DockerGpuPatchFinalizeOutcome, + | "rolledBack" + | "rollbackImageId" + | "rollbackImageRemoved" + | "replacementStopConfirmed" + | "replacementRemovalConfirmed" + | "replacementPresence" +> { + const replacementContainerId = fullDockerContainerId(result.newContainerId); + if (!replacementContainerId) { + return { + rolledBack: false, + rollbackImageId: plan.imageId, + rollbackImageRemoved: false, + replacementStopConfirmed: false, + replacementRemovalConfirmed: false, + replacementPresence: "unknown", + }; + } + const removal = removeReplacementContainer(replacementContainerId, resolved); + const restored = + removal.replacementPresence === "absent" + ? resolved.dockerRunDetached(plan.runArgs, options) + : null; + const restoredContainerId = + restored !== null && hasZeroDockerExitStatus(restored) + ? fullDockerContainerId(String(restored.stdout ?? "")) + : null; + return { + rolledBack: restoredContainerId !== null, + rollbackImageId: plan.imageId, + rollbackImageRemoved: false, + ...removal, + }; +} + function isExactRunningReplacement( sandboxName: string, replacementContainerId: string, @@ -128,10 +245,8 @@ export function finalizeDockerGpuPatchBackup( if (options.supervisorReady) { // Stop the exact replacement before retiring the exact backup, then start // the replacement afterward. The final start is the authoritative Docker - // lifecycle event. Backup removal is the irreversible commit point; - // failures after it require a sandbox rebuild. Success is withheld until - // OpenShell reports Ready and Docker still proves the exact replacement is - // the sole running labeled container (#9531). + // lifecycle event. Keep a committed rollback image until OpenShell accepts + // the exact replacement as Ready (#9531). if (!deps.runOpenshell || !deps.runCaptureOpenshell) { return { backupRemoved: false, @@ -149,15 +264,37 @@ export function finalizeDockerGpuPatchBackup( replacementStoppedForCommit: false, }; } - const rmResult = resolved.dockerRm(options.result.oldContainerId, containerOpts); + const rollbackPlan = preparePostCommitRollback(options.result, resolved, containerOpts); + if (!rollbackPlan) { + const replacementRestarted = hasZeroDockerExitStatus( + resolved.dockerStart(options.result.newContainerId, containerOpts), + ); + return { + backupRemoved: false, + rolledBack: false, + replacementStoppedForCommit: true, + replacementRestarted, + finalHandoffAcknowledged: false, + lastSandboxPhase: null, + }; + } + const rmResult = resolved.dockerRm(rollbackPlan.oldContainerId, containerOpts); const backupRemoved = hasZeroDockerExitStatus(rmResult); if (!backupRemoved) { + const rollbackImageRemoved = removeRollbackImage( + rollbackPlan.imageId, + resolved, + containerOpts, + ); const replacementRestarted = hasZeroDockerExitStatus( resolved.dockerStart(options.result.newContainerId, containerOpts), ); return { backupRemoved: false, rolledBack: false, + ...(rollbackImageRemoved + ? { rollbackImageRemoved: true } + : { rollbackImageId: rollbackPlan.imageId, rollbackImageRemoved: false }), replacementStoppedForCommit: true, replacementRestarted, finalHandoffAcknowledged: false, @@ -190,9 +327,15 @@ export function finalizeDockerGpuPatchBackup( }, ); if (!lifecycleReleaseObserved) { + const rollback = restorePostCommitRollback( + rollbackPlan, + options.result, + resolved, + containerOpts, + ); return { backupRemoved: true, - rolledBack: false, + ...rollback, replacementStoppedForCommit: true, replacementRestarted: false, lifecycleReleaseObserved: false, @@ -203,9 +346,15 @@ export function finalizeDockerGpuPatchBackup( const startResult = resolved.dockerStart(options.result.newContainerId, containerOpts); const replacementRestarted = hasZeroDockerExitStatus(startResult); if (!replacementRestarted) { + const rollback = restorePostCommitRollback( + rollbackPlan, + options.result, + resolved, + containerOpts, + ); return { backupRemoved: true, - rolledBack: false, + ...rollback, replacementStoppedForCommit: true, replacementRestarted, finalHandoffAcknowledged: false, @@ -231,13 +380,37 @@ export function finalizeDockerGpuPatchBackup( ), }, ); + if (!acknowledgement.acknowledged) { + const rollback = restorePostCommitRollback( + rollbackPlan, + options.result, + resolved, + containerOpts, + ); + return { + backupRemoved: true, + ...rollback, + replacementStoppedForCommit: true, + replacementRestarted: true, + lifecycleReleaseObserved: true, + finalHandoffAcknowledged: false, + lastSandboxPhase: acknowledgement.lastSandboxPhase, + }; + } + const rollbackImageRemoved = removeRollbackImage(rollbackPlan.imageId, resolved, containerOpts); + if (!rollbackImageRemoved) { + console.warn(` Could not remove temporary Docker rollback image ${rollbackPlan.imageId}.`); + } return { backupRemoved: true, rolledBack: false, + ...(rollbackImageRemoved + ? { rollbackImageRemoved: true } + : { rollbackImageId: rollbackPlan.imageId, rollbackImageRemoved: false }), replacementStoppedForCommit: true, replacementRestarted: true, lifecycleReleaseObserved: true, - finalHandoffAcknowledged: acknowledgement.acknowledged, + finalHandoffAcknowledged: true, lastSandboxPhase: acknowledgement.lastSandboxPhase, }; } diff --git a/src/lib/onboard/docker-gpu-patch-recreate.test.ts b/src/lib/onboard/docker-gpu-patch-recreate.test.ts index 98867a8a3d4..279980d27bd 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.test.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.test.ts @@ -10,6 +10,8 @@ import { const OLD_CONTAINER_ID = "a".repeat(64); const NEW_CONTAINER_ID = "b".repeat(64); +const ROLLBACK_IMAGE_ID = `sha256:${"d".repeat(64)}`; +const RESTORED_CONTAINER_ID = "e".repeat(64); function dockerCaptureFixture() { const responses: Record = { @@ -27,13 +29,21 @@ describe("Docker GPU recreate orchestration", () => { ps: { status: 0, stdout: `${NEW_CONTAINER_ID}\n` }, inspect: { status: 0, stdout: "true\n" }, }; - const dockerRun = vi.fn( - (args: readonly string[]) => - dockerRunResults[String(args[0]) as keyof typeof dockerRunResults] ?? { - status: 0, - stdout: "probe-id\n", - }, - ); + const dockerRun = vi.fn((args: readonly string[]) => { + switch (args[0]) { + case "commit": + return { status: 0, stdout: `${ROLLBACK_IMAGE_ID}\n` }; + case "image": + return { status: 0 }; + default: + return ( + dockerRunResults[String(args[0]) as keyof typeof dockerRunResults] ?? { + status: 0, + stdout: "probe-id\n", + } + ); + } + }); const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: `${NEW_CONTAINER_ID}\n` })); const dockerRename = vi.fn(() => ({ status: 0 })); const dockerStop = vi.fn(() => ({ status: 0 })); @@ -112,15 +122,30 @@ describe("Docker GPU recreate orchestration", () => { ); }); - it("does not report success when the final OpenShell phase is Deleting (#9531)", () => { + it("restores the pre-patch container when final OpenShell phase is Deleting (#9531)", () => { + const dockerRunDetached = vi + .fn() + .mockReturnValueOnce({ status: 0, stdout: `${NEW_CONTAINER_ID}\n` }) + .mockReturnValueOnce({ status: 0, stdout: `${RESTORED_CONTAINER_ID}\n` }); let failure: unknown; try { recreateOpenShellDockerSandboxWithGpu( { sandboxName: "alpha", timeoutSecs: 1 }, { dockerCapture: dockerCaptureFixture(), - dockerRun: vi.fn(() => ({ status: 0, stdout: `${NEW_CONTAINER_ID}\n` })), - dockerRunDetached: vi.fn(() => ({ status: 0, stdout: `${NEW_CONTAINER_ID}\n` })), + dockerRun: vi.fn((args: readonly string[]) => { + switch (args[0]) { + case "commit": + return { status: 0, stdout: `${ROLLBACK_IMAGE_ID}\n` }; + case "image": + return { status: 0 }; + case "inspect": + return { status: 0, stdout: "true\n" }; + default: + return { status: 0, stdout: `${NEW_CONTAINER_ID}\n` }; + } + }), + dockerRunDetached, dockerRename: vi.fn(() => ({ status: 0 })), dockerStop: vi.fn(() => ({ status: 0 })), dockerRm: vi.fn(() => ({ status: 0 })), @@ -142,15 +167,20 @@ describe("Docker GPU recreate orchestration", () => { expect(failure).toBeInstanceOf(Error); expect((failure as Error).message).toContain("final replacement handoff"); - expect((failure as Error).message).toContain("automatic rollback is unavailable"); - expect((failure as Error).message).toContain("Rebuild the sandbox before retrying"); + expect((failure as Error).message).toContain("pre-patch sandbox container was restored"); expect(getDockerGpuPatchFailureContext(failure)).toMatchObject({ backupRemoved: true, oldContainerId: OLD_CONTAINER_ID, newContainerId: NEW_CONTAINER_ID, + rollbackImageId: ROLLBACK_IMAGE_ID, lastSandboxPhase: "Deleting", - rolledBack: false, + rolledBack: true, }); + expect(dockerRunDetached).toHaveBeenCalledTimes(2); + expect(dockerRunDetached.mock.calls[1]?.[0]).toEqual( + expect.arrayContaining(["--name", "openshell-alpha", ROLLBACK_IMAGE_ID]), + ); + expect(dockerRunDetached.mock.calls[1]?.[0]).not.toContain("--gpus"); }); it("can recreate during sandbox create before supervisor exec is allowed", () => { diff --git a/src/lib/onboard/docker-gpu-patch-recreate.ts b/src/lib/onboard/docker-gpu-patch-recreate.ts index 7d2310ff366..fa5a5fd2827 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.ts @@ -407,6 +407,8 @@ export function recreateOpenShellDockerSandboxContainer( : finalizeDockerGpuPatchBackup({ result: patchResult, supervisorReady: false }, deps); context.rolledBack = finalization.rolledBack; context.backupRemoved = finalization.backupRemoved; + context.rollbackImageId = finalization.rollbackImageId; + context.rollbackImageRemoved = finalization.rollbackImageRemoved; context.replacementStopConfirmed = finalization.replacementStopConfirmed; context.replacementRemovalConfirmed = finalization.replacementRemovalConfirmed; context.replacementPresence = finalization.replacementPresence; @@ -419,13 +421,19 @@ export function recreateOpenShellDockerSandboxContainer( ); } if (finalization.finalHandoffAcknowledged) return result(true); - const rebuildRequired = finalization.backupRemoved - ? " The previous container was removed at the final handoff commit point; automatic rollback is unavailable. Rebuild the sandbox before retrying." - : ""; + const rollbackDetail = finalization.rolledBack + ? " The pre-patch sandbox container was restored." + : finalization.backupRemoved + ? ` Automatic rollback failed.${ + finalization.rollbackImageId + ? ` Docker retained rollback image ${finalization.rollbackImageId}.` + : "" + }` + : ""; throw new Error( finalization.lastSandboxPhase - ? `OpenShell did not acknowledge the final replacement handoff; last sandbox phase was ${finalization.lastSandboxPhase}.${rebuildRequired}` - : `OpenShell did not acknowledge the final replacement handoff.${rebuildRequired}`, + ? `OpenShell did not acknowledge the final replacement handoff; last sandbox phase was ${finalization.lastSandboxPhase}.${rollbackDetail}` + : `OpenShell did not acknowledge the final replacement handoff.${rollbackDetail}`, ); } catch (error) { throw decoratePatchError(error instanceof Error ? error : new Error(String(error)), context); diff --git a/src/lib/onboard/docker-gpu-patch-rollback.test.ts b/src/lib/onboard/docker-gpu-patch-rollback.test.ts index 6c6af7c26ac..71f7e4581f9 100644 --- a/src/lib/onboard/docker-gpu-patch-rollback.test.ts +++ b/src/lib/onboard/docker-gpu-patch-rollback.test.ts @@ -175,6 +175,8 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { (args: readonly string[]) => captureByCommand[String(args[0])]?.(args) ?? "", ); const runByCommand: Record { status: number; stdout: string }> = { + commit: () => ({ status: 0, stdout: `sha256:${"d".repeat(64)}\n` }), + image: () => ({ status: 0, stdout: "" }), ps: () => ({ status: 0, stdout: `${alphaContainerIds().join("\n")}\n`, diff --git a/src/lib/onboard/docker-gpu-patch-rollback.ts b/src/lib/onboard/docker-gpu-patch-rollback.ts index 9422f21899a..f7ccfb2d541 100644 --- a/src/lib/onboard/docker-gpu-patch-rollback.ts +++ b/src/lib/onboard/docker-gpu-patch-rollback.ts @@ -2,9 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 import { + dockerCapture as defaultDockerCapture, dockerRename as defaultDockerRename, dockerRm as defaultDockerRm, dockerRun as defaultDockerRun, + dockerRunDetached as defaultDockerRunDetached, dockerStart as defaultDockerStart, dockerStop as defaultDockerStop, } from "../adapters/docker"; @@ -21,6 +23,7 @@ type DockerRunResult = { }; type DockerRunOptions = Record; +type DockerCaptureFn = (args: readonly string[], opts?: DockerRunOptions) => string; type DockerContainerFn = (containerName: string, opts?: DockerRunOptions) => DockerRunResult; type DockerRenameFn = ( oldContainerName: string, @@ -45,8 +48,12 @@ export type DockerGpuPatchRollbackOutcome = { replacementPresence: "absent" | "present" | "unknown"; }; +export type DockerGpuReplacementRemovalOutcome = Omit; + export type ResolvedDockerGpuPatchRollbackDeps = { + dockerCapture: DockerCaptureFn; dockerRun: DockerRunFn; + dockerRunDetached: DockerRunFn; dockerStop: DockerContainerFn; dockerRm: DockerContainerFn; dockerRename: DockerRenameFn; @@ -58,7 +65,9 @@ export function resolveDockerGpuPatchRollbackDeps( deps: DockerGpuPatchDeps, ): ResolvedDockerGpuPatchRollbackDeps { return { + dockerCapture: deps.dockerCapture ?? defaultDockerCapture, dockerRun: deps.dockerRun ?? defaultDockerRun, + dockerRunDetached: deps.dockerRunDetached ?? defaultDockerRunDetached, dockerStop: deps.dockerStop ?? defaultDockerStop, dockerRm: deps.dockerRm ?? defaultDockerRm, dockerRename: deps.dockerRename ?? defaultDockerRename, @@ -106,26 +115,38 @@ export function rollbackToBackupContainer( refs: { newContainerId: string; backupContainerName: string; originalName: string }, deps: ResolvedDockerGpuPatchRollbackDeps, ): DockerGpuPatchRollbackOutcome { + const removal = removeReplacementContainer(refs.newContainerId, deps); + const containerOpts = { + ignoreError: true, + suppressOutput: true, + timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, + }; + const restored = deps.dockerRename(refs.backupContainerName, refs.originalName, containerOpts); + const rolledBack = hasZeroDockerExitStatus(restored) + ? hasZeroDockerExitStatus(deps.dockerStart(refs.originalName, containerOpts)) + : false; + return { rolledBack, ...removal }; +} + +export function removeReplacementContainer( + containerId: string, + deps: ResolvedDockerGpuPatchRollbackDeps, +): DockerGpuReplacementRemovalOutcome { const containerOpts = { ignoreError: true, suppressOutput: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, }; const replacementStopConfirmed = hasZeroDockerExitStatus( - deps.dockerStop(refs.newContainerId, containerOpts), + deps.dockerStop(containerId, containerOpts), ); const replacementRemovalConfirmed = hasZeroDockerExitStatus( - deps.dockerRm(refs.newContainerId, containerOpts), + deps.dockerRm(containerId, containerOpts), ); const replacementPresence = replacementRemovalConfirmed ? "absent" - : observeReplacementPresence(refs.newContainerId, deps, containerOpts); - const restored = deps.dockerRename(refs.backupContainerName, refs.originalName, containerOpts); - const rolledBack = hasZeroDockerExitStatus(restored) - ? hasZeroDockerExitStatus(deps.dockerStart(refs.originalName, containerOpts)) - : false; + : observeReplacementPresence(containerId, deps, containerOpts); return { - rolledBack, replacementStopConfirmed, replacementRemovalConfirmed, replacementPresence, diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index 972cd858f54..57ea2c9dbdc 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -80,6 +80,8 @@ export type DockerGpuPatchFailureContext = { newContainerId?: string | null; backupContainerName?: string | null; backupRemoved?: boolean; + rollbackImageId?: string | null; + rollbackImageRemoved?: boolean; selectedMode?: DockerGpuPatchMode | null; modeAttempts?: DockerGpuPatchModeAttempt[]; rolledBack?: boolean; diff --git a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index fde42865d83..854331a8414 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -147,7 +147,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { expect(onPatchFailureExit).not.toHaveBeenCalled(); }); - it("rejects an explicit replacement restart failure after backup removal", async () => { + it("reports prior-container restoration after replacement restart failure (#9531)", async () => { const deps = makeDeps(); const result = deferredCreateResult(); const onPatchFailureExit = vi.fn(); @@ -162,7 +162,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { waitForSupervisor: vi.fn(() => true), finalizeBackup: vi.fn(() => ({ backupRemoved: true, - rolledBack: false, + rolledBack: true, replacementRestarted: false, })), onPatchFailureExit, @@ -171,14 +171,17 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); patch.waitForSupervisorReconnectIfNeeded(); - await expect(patch.commitAfterReady()).rejects.toThrow("automatic rollback is unavailable"); + await expect(patch.commitAfterReady()).rejects.toThrow( + "pre-patch sandbox container was restored", + ); expect(onPatchFailureExit).toHaveBeenCalledOnce(); expect(onPatchFailureExit.mock.calls[0]?.[2]?.context).toMatchObject({ backupRemoved: true, + rolledBack: true, }); }); - it("rejects final handoff when OpenShell reports Deleting after restart (#9531)", async () => { + it("reports prior-container restoration when final handoff stays Deleting (#9531)", async () => { const deps = makeDeps(); const result = deferredCreateResult(); const waitForSupervisor = vi.fn(() => true); @@ -194,7 +197,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { waitForSupervisor, finalizeBackup: vi.fn(() => ({ backupRemoved: true, - rolledBack: false, + rolledBack: true, replacementRestarted: true, finalHandoffAcknowledged: false, lastSandboxPhase: "Deleting", @@ -205,7 +208,9 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); patch.waitForSupervisorReconnectIfNeeded(); - await expect(patch.commitAfterReady()).rejects.toThrow("automatic rollback is unavailable"); + await expect(patch.commitAfterReady()).rejects.toThrow( + "pre-patch sandbox container was restored", + ); expect(waitForSupervisor).toHaveBeenCalledOnce(); expect(onPatchFailureExit).toHaveBeenCalledOnce(); diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index e4ed566917b..af39bd14c07 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -522,9 +522,15 @@ export function createDockerGpuSandboxCreatePatch( ? ` Last OpenShell phase: ${finalizeOutcome.lastSandboxPhase}.` : "" }${ - finalizeOutcome.backupRemoved - ? " The previous container was removed at the final handoff commit point; automatic rollback is unavailable. Rebuild the sandbox before retrying." - : "" + finalizeOutcome.rolledBack + ? " The pre-patch sandbox container was restored." + : finalizeOutcome.backupRemoved + ? ` Automatic rollback failed.${ + finalizeOutcome.rollbackImageId + ? ` Docker retained rollback image ${finalizeOutcome.rollbackImageId}.` + : "" + }` + : "" }`, ); cutoverFinalizationFailure = failure; @@ -535,7 +541,13 @@ export function createDockerGpuSandboxCreatePatch( context: { ...failureContext(), backupRemoved: finalizeOutcome.backupRemoved, + rollbackImageId: finalizeOutcome.rollbackImageId, + rollbackImageRemoved: finalizeOutcome.rollbackImageRemoved, lastSandboxPhase: finalizeOutcome.lastSandboxPhase, + replacementStopConfirmed: finalizeOutcome.replacementStopConfirmed, + replacementRemovalConfirmed: finalizeOutcome.replacementRemovalConfirmed, + replacementPresence: finalizeOutcome.replacementPresence, + rolledBack: finalizeOutcome.rolledBack, }, }); throw failure; diff --git a/test/process-recovery/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery/process-recovery-supervisor-relaunch.test.ts index 963bb65cc72..88b7ebc785e 100644 --- a/test/process-recovery/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery/process-recovery-supervisor-relaunch.test.ts @@ -7,6 +7,7 @@ import { checkAndRecoverSandboxProcesses } from "../../src/lib/actions/sandbox/p import { relaunchManagedSupervisorSession } from "../../src/lib/actions/sandbox/supervisor-relaunch.ts"; import * as openshellRuntime from "../../src/lib/adapters/openshell/runtime.ts"; import * as agentRuntime from "../../src/lib/agent/runtime.ts"; +import { createDockerGpuInspectFixture as inspectFixture } from "../../src/lib/onboard/__test-helpers__/docker-gpu-patch-fixtures.ts"; import { finalizeDockerGpuPatchBackup } from "../../src/lib/onboard/docker-gpu-patch-finalize.ts"; import * as registry from "../../src/lib/state/registry.ts"; @@ -479,11 +480,19 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { const dockerStart = vi.fn(() => ({ status: 0 })); const oldContainerId = "a".repeat(64); const replacementContainerId = "b".repeat(64); - const dockerRun = vi.fn((args: readonly string[]) => - args[0] === "ps" - ? { status: 0, stdout: `${replacementContainerId}\n` } - : { status: 0, stdout: "true\n" }, - ); + const rollbackImageId = `sha256:${"d".repeat(64)}`; + const dockerRun = vi.fn((args: readonly string[]) => { + switch (args[0]) { + case "commit": + return { status: 0, stdout: `${rollbackImageId}\n` }; + case "image": + return { status: 0 }; + case "ps": + return { status: 0, stdout: `${replacementContainerId}\n` }; + default: + return { status: 0, stdout: "true\n" }; + } + }); const finalizeTransaction = vi.fn( ( options: Parameters[0], @@ -491,6 +500,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { ) => finalizeDockerGpuPatchBackup(options, { ...deps, + dockerCapture: vi.fn(() => JSON.stringify([{ ...inspectFixture(), Id: oldContainerId }])), dockerRm, dockerRun, dockerStart, From 2392026aa98653847a40d9a0617032286959ff87 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 25 Aug 2026 09:05:24 -0700 Subject: [PATCH 07/12] fix(onboard): persist failed handoff recovery Signed-off-by: Aaron Erickson --- src/lib/onboard/docker-gpu-patch-constants.ts | 2 + .../onboard/docker-gpu-patch-diagnostics.ts | 7 + .../onboard/docker-gpu-patch-finalize.test.ts | 91 +++++++----- src/lib/onboard/docker-gpu-patch-finalize.ts | 135 ++++++++++++++++-- src/lib/onboard/docker-gpu-patch-mode.ts | 3 +- .../onboard/docker-gpu-patch-recreate.test.ts | 18 ++- src/lib/onboard/docker-gpu-patch-recreate.ts | 11 +- src/lib/onboard/docker-gpu-patch-rollback.ts | 10 +- src/lib/onboard/docker-gpu-patch-types.ts | 2 + src/lib/onboard/docker-gpu-sandbox-create.ts | 6 + .../docker-gpu-supervisor-reconnect.ts | 12 +- ...ocess-recovery-supervisor-relaunch.test.ts | 17 ++- 12 files changed, 242 insertions(+), 72 deletions(-) diff --git a/src/lib/onboard/docker-gpu-patch-constants.ts b/src/lib/onboard/docker-gpu-patch-constants.ts index b51378a6529..34326d94b68 100644 --- a/src/lib/onboard/docker-gpu-patch-constants.ts +++ b/src/lib/onboard/docker-gpu-patch-constants.ts @@ -1,6 +1,8 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +export { sleepSeconds } from "../core/wait"; + /** Default timeout for one Docker CLI operation in the compatibility GPU patch path. */ export const DOCKER_GPU_PATCH_TIMEOUT_MS = 30_000; diff --git a/src/lib/onboard/docker-gpu-patch-diagnostics.ts b/src/lib/onboard/docker-gpu-patch-diagnostics.ts index d832591540e..de8a34faa25 100644 --- a/src/lib/onboard/docker-gpu-patch-diagnostics.ts +++ b/src/lib/onboard/docker-gpu-patch-diagnostics.ts @@ -246,6 +246,13 @@ export function collectDockerGpuPatchDiagnostics( `backup_removed=${confirmationValue(context?.backupRemoved)}`, `rollback_image_id=${redactor.redactText(context?.rollbackImageId ?? "none")}`, `rollback_image_removed=${confirmationValue(context?.rollbackImageRemoved)}`, + `rollback_record_path=${redactor.redactText(context?.rollbackRecordPath ?? "none")}`, + `rollback_record_removed=${confirmationValue(context?.rollbackRecordRemoved)}`, + ...(context?.rollbackRecordPath + ? [ + "rollback_recovery_action=After confirming the replacement container is absent, run the Docker recoveryAction from rollback_record_path.", + ] + : []), `last_openshell_phase=${redactor.redactText(context?.lastSandboxPhase ?? "unknown")}`, `rolled_back=${cleanupPendingRollback ? "pending" : context?.rolledBack === true ? "yes" : context?.rolledBack === false ? "failed" : "no"}`, ...(context?.replacementStopConfirmed !== undefined diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index bc0f2701908..efb0524eedd 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -5,7 +5,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { describe, expect, it, vi } from "vitest"; +import { afterAll, describe, expect, it, vi } from "vitest"; import { createDockerGpuInspectFixture as inspectFixture } from "./__test-helpers__/docker-gpu-patch-fixtures"; import { collectDockerGpuPatchDiagnostics, type DockerGpuPatchResult } from "./docker-gpu-patch"; @@ -16,6 +16,13 @@ import { const ROLLBACK_IMAGE_ID = `sha256:${"d".repeat(64)}`; const RESTORED_CONTAINER_ID = "e".repeat(64); +const ROLLBACK_TEST_ROOT = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-docker-gpu-rollback-record-"), +); + +afterAll(() => { + fs.rmSync(ROLLBACK_TEST_ROOT, { recursive: true, force: true }); +}); function deferredCreateResult(): DockerGpuPatchResult { return { @@ -43,10 +50,14 @@ function exactDeferredCreateResult(): DockerGpuPatchResult { } function rollbackPlanDeps(result: DockerGpuPatchResult = exactDeferredCreateResult()) { + const homeDir = fs.mkdtempSync(path.join(ROLLBACK_TEST_ROOT, "case-")); + const inspect = inspectFixture(); + inspect.Config = { + ...inspect.Config, + Env: [...(inspect.Config?.Env ?? []), "ROLLBACK_TEST_TOKEN=super-secret-value"], + }; return { - dockerCapture: vi.fn(() => - JSON.stringify([{ ...inspectFixture(), Id: result.oldContainerId }]), - ), + dockerCapture: vi.fn(() => JSON.stringify([{ ...inspect, Id: result.oldContainerId }])), dockerRun: vi.fn((args: readonly string[]) => { switch (args[0]) { case "commit": @@ -66,9 +77,17 @@ function rollbackPlanDeps(result: DockerGpuPatchResult = exactDeferredCreateResu stdout: `${RESTORED_CONTAINER_ID}\n`, stderr: "", })), + homedir: () => homeDir, }; } +function rollbackRecordFiles(homeDir: string): string[] { + const directory = path.join(homeDir, ".nemoclaw", "recovery", "docker-gpu"); + return fs.existsSync(directory) + ? fs.readdirSync(directory).map((entry) => path.join(directory, entry)) + : []; +} + function readyHandoffDeps() { return { ...rollbackPlanDeps(), @@ -220,12 +239,14 @@ describe("finalizeDockerGpuPatchBackup", () => { backupRemoved: true, rolledBack: false, rollbackImageRemoved: true, + rollbackRecordRemoved: true, replacementStoppedForCommit: true, replacementRestarted: true, lifecycleReleaseObserved: true, finalHandoffAcknowledged: true, lastSandboxPhase: "Ready", }); + expect(rollbackRecordFiles(rollbackDeps.homedir())).toEqual([]); expect(events).toEqual([ "stop replacement", "commit rollback image", @@ -303,6 +324,7 @@ describe("finalizeDockerGpuPatchBackup", () => { rolledBack: true, rollbackImageId: ROLLBACK_IMAGE_ID, rollbackImageRemoved: false, + rollbackRecordRemoved: true, replacementStopConfirmed: true, replacementRemovalConfirmed: true, replacementPresence: "absent", @@ -336,9 +358,12 @@ describe("finalizeDockerGpuPatchBackup", () => { ]), ); expect(rollbackDeps.dockerRunDetached.mock.calls[0]?.[0]).not.toContain("--gpus"); + expect(rollbackRecordFiles(rollbackDeps.homedir())).toEqual([]); const diagnostics = collectRollbackDiagnostics(result.newContainerId, outcome); expect(diagnostics.summary).toContain(`rollback_image_id=${ROLLBACK_IMAGE_ID}`); expect(diagnostics.summary).toContain("rollback_image_removed=no"); + expect(diagnostics.summary).toContain("rollback_record_path=none"); + expect(diagnostics.summary).toContain("rollback_record_removed=yes"); expect(diagnostics.summary).toContain("rolled_back=yes"); }); @@ -371,6 +396,7 @@ describe("finalizeDockerGpuPatchBackup", () => { rolledBack: false, rollbackImageId: ROLLBACK_IMAGE_ID, rollbackImageRemoved: false, + rollbackRecordRemoved: true, finalHandoffAcknowledged: true, }); expect(warn).toHaveBeenCalledWith( @@ -585,13 +611,13 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(dockerRm).not.toHaveBeenCalled(); }); - it("reports backupRemoved=false when supervisor reconnect succeeded but docker rm of the backup failed", () => { + it.each([ + ["an explicit failure", { status: 1, stderr: "container is in use" }], + ["no exit status", { status: null, stderr: "timed out" }], + ])("retains the backup when Docker removal returns %s (#9531)", (_case, removal) => { const result = exactDeferredCreateResult(); const dockerStop = vi.fn(() => ({ status: 0 })); - const dockerRm = vi.fn((_name: string) => ({ - status: 1, - stderr: "Error response from daemon: container is in use", - })); + const dockerRm = vi.fn(() => removal); const dockerStart = vi.fn(() => ({ status: 0 })); const outcome = finalizeDockerGpuPatchBackup( { @@ -606,6 +632,7 @@ describe("finalizeDockerGpuPatchBackup", () => { backupRemoved: false, rolledBack: false, rollbackImageRemoved: true, + rollbackRecordRemoved: true, replacementStoppedForCommit: true, replacementRestarted: true, finalHandoffAcknowledged: false, @@ -621,35 +648,6 @@ describe("finalizeDockerGpuPatchBackup", () => { ); }); - it("fails closed when backup removal has no exit status", () => { - const result = exactDeferredCreateResult(); - const dockerStop = vi.fn(() => ({ status: 0 })); - const dockerRm = vi.fn((_name: string) => ({ status: null, stderr: "timed out" })); - const dockerStart = vi.fn(() => ({ status: 0 })); - const outcome = finalizeDockerGpuPatchBackup( - { - result, - supervisorReady: true, - sandboxName: "alpha", - finalHandoffTimeoutSecs: 60, - }, - { dockerStop, dockerRm, dockerStart, ...readyHandoffDeps() }, - ); - expect(outcome).toEqual({ - backupRemoved: false, - rolledBack: false, - rollbackImageRemoved: true, - replacementStoppedForCommit: true, - replacementRestarted: true, - finalHandoffAcknowledged: false, - lastSandboxPhase: null, - }); - expect(dockerStart).toHaveBeenCalledWith( - result.newContainerId, - expect.objectContaining({ ignoreError: true }), - ); - }); - it("retains the backup when the replacement cannot be stopped for the final handoff", () => { const dockerStop = vi.fn(() => ({ status: 1 })); const dockerRm = vi.fn(() => ({ status: 0 })); @@ -733,6 +731,7 @@ describe("finalizeDockerGpuPatchBackup", () => { rolledBack: true, rollbackImageId: ROLLBACK_IMAGE_ID, rollbackImageRemoved: false, + rollbackRecordRemoved: true, replacementStopConfirmed: true, replacementRemovalConfirmed: true, replacementPresence: "absent", @@ -768,6 +767,7 @@ describe("finalizeDockerGpuPatchBackup", () => { rolledBack: false, rollbackImageId: ROLLBACK_IMAGE_ID, rollbackImageRemoved: false, + rollbackRecordRemoved: false, replacementPresence: "absent", replacementRestarted: false, }); @@ -775,6 +775,21 @@ describe("finalizeDockerGpuPatchBackup", () => { ["image", "rm", "--force", ROLLBACK_IMAGE_ID], expect.anything(), ); + const recordPath = outcome.rollbackRecordPath; + expect(recordPath).toBeTypeOf("string"); + expect(fs.statSync(recordPath as string).mode & 0o777).toBe(0o600); + const record = fs.readFileSync(recordPath as string, "utf8"); + expect(record).toContain(ROLLBACK_IMAGE_ID); + expect(record).toContain('"command": "docker"'); + expect(record).toContain('"run"'); + expect(record).toContain('"--detach"'); + expect(record).not.toContain("super-secret-value"); + expect(record).not.toContain("ROLLBACK_TEST_TOKEN"); + expect(record).not.toContain('"--env"'); + expect(record).not.toContain('"--label"'); + const diagnostics = collectRollbackDiagnostics(result.newContainerId, outcome); + expect(diagnostics.summary).toContain("rollback_record_removed=no"); + expect(diagnostics.summary).toContain("rollback_recovery_action="); }); it("records a remaining exact-ID replacement when removal fails (#7996)", () => { diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index c881745b110..3c0fda11ac2 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -26,6 +26,10 @@ // and delete this module along with its callers in docker-gpu-patch.ts and // docker-gpu-sandbox-create.ts. +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + import { hasZeroDockerExitStatus } from "./docker-command-result"; import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; import { @@ -52,11 +56,6 @@ import { } from "./docker-gpu-supervisor-reconnect"; import { queryOpenShellDockerSandboxContainers } from "./openshell-docker-sandbox-containers"; -export { - restoreDockerGpuPatchBackupAfterRecreateFailure as rollbackDockerGpuPatchOnRecreateFailure, - rollbackToBackupContainer, -} from "./docker-gpu-patch-rollback"; - export type DockerGpuPatchFinalizeOptions = | { result: DockerGpuPatchResult; @@ -75,6 +74,8 @@ export type DockerGpuPatchFinalizeOutcome = { rolledBack: boolean; rollbackImageId?: string; rollbackImageRemoved?: boolean; + rollbackRecordPath?: string; + rollbackRecordRemoved?: boolean; replacementStoppedForCommit?: boolean; replacementRestarted?: boolean; lifecycleReleaseObserved?: boolean; @@ -88,10 +89,13 @@ export type DockerGpuPatchFinalizeOutcome = { type PostCommitRollbackPlan = { imageId: string; oldContainerId: string; + recordPath: string; runArgs: string[]; }; const DOCKER_IMAGE_ID = /^sha256:[0-9a-f]{64}$/iu; +const ROLLBACK_RECORD_VERSION = 1; +const REDACTED_RUN_VALUE_FLAGS = new Set(["--env", "--label"]); function removeRollbackImage( imageId: string, @@ -101,8 +105,82 @@ function removeRollbackImage( return hasZeroDockerExitStatus(deps.dockerRun(["image", "rm", "--force", imageId], options)); } +function recoveryRunArgs(runArgs: readonly string[], imageId: string): string[] { + const imageIndex = runArgs.indexOf(imageId); + if (imageIndex < 0) + throw new Error("Docker rollback arguments do not include the rollback image."); + const persisted: string[] = []; + for (let index = 0; index < imageIndex; index += 1) { + const value = runArgs[index]; + if (REDACTED_RUN_VALUE_FLAGS.has(value)) { + if (index + 1 >= imageIndex) { + throw new Error(`Docker rollback argument '${value}' has no value.`); + } + index += 1; + continue; + } + persisted.push(value); + } + return [...persisted, imageId]; +} + +function writeRollbackRecord( + sandboxName: string, + oldContainerId: string, + imageId: string, + runArgs: readonly string[], + deps: DockerGpuPatchDeps, +): string | null { + const directory = path.join( + (deps.homedir ?? os.homedir)(), + ".nemoclaw", + "recovery", + "docker-gpu", + ); + const recordPath = path.join( + directory, + `${encodeURIComponent(sandboxName)}-${oldContainerId.slice(0, 12)}.json`, + ); + try { + fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + fs.chmodSync(directory, 0o700); + fs.writeFileSync( + recordPath, + `${JSON.stringify( + { + version: ROLLBACK_RECORD_VERSION, + sandboxName, + rollbackImageId: imageId, + recoveryAction: { + command: "docker", + args: ["run", "--detach", ...recoveryRunArgs(runArgs, imageId)], + }, + }, + null, + 2, + )}\n`, + { encoding: "utf8", flag: "wx", mode: 0o600 }, + ); + fs.chmodSync(recordPath, 0o600); + return recordPath; + } catch { + return null; + } +} + +function removeRollbackRecord(recordPath: string): boolean { + try { + fs.rmSync(recordPath, { force: true }); + return !fs.existsSync(recordPath); + } catch { + return false; + } +} + function preparePostCommitRollback( result: DockerGpuPatchResult, + sandboxName: string, + deps: DockerGpuPatchDeps, resolved: ResolvedDockerGpuPatchRollbackDeps, options: Record, ): PostCommitRollbackPlan | null { @@ -129,14 +207,23 @@ function preparePostCommitRollback( const cloneOptions = buildDockerGpuCloneRunOptions(inspect); cloneOptions.containerName = result.originalName; cloneOptions.image = imageId; + const runArgs = buildDockerGpuCloneRunArgs( + inspect, + buildDockerGpuMode("startup-command"), + cloneOptions, + ); + const recordPath = writeRollbackRecord(sandboxName, oldContainerId, imageId, runArgs, deps); + if (!recordPath) { + if (!removeRollbackImage(imageId, resolved, options)) { + console.warn(` Could not remove temporary Docker rollback image ${imageId}.`); + } + return null; + } return { imageId, oldContainerId, - runArgs: buildDockerGpuCloneRunArgs( - inspect, - buildDockerGpuMode("startup-command"), - cloneOptions, - ), + recordPath, + runArgs, }; } catch { if (!removeRollbackImage(imageId, resolved, options)) { @@ -156,6 +243,8 @@ function restorePostCommitRollback( | "rolledBack" | "rollbackImageId" | "rollbackImageRemoved" + | "rollbackRecordPath" + | "rollbackRecordRemoved" | "replacementStopConfirmed" | "replacementRemovalConfirmed" | "replacementPresence" @@ -166,6 +255,8 @@ function restorePostCommitRollback( rolledBack: false, rollbackImageId: plan.imageId, rollbackImageRemoved: false, + rollbackRecordPath: plan.recordPath, + rollbackRecordRemoved: false, replacementStopConfirmed: false, replacementRemovalConfirmed: false, replacementPresence: "unknown", @@ -180,10 +271,15 @@ function restorePostCommitRollback( restored !== null && hasZeroDockerExitStatus(restored) ? fullDockerContainerId(String(restored.stdout ?? "")) : null; + const rollbackRecordRemoved = + restoredContainerId !== null && removeRollbackRecord(plan.recordPath); return { rolledBack: restoredContainerId !== null, rollbackImageId: plan.imageId, rollbackImageRemoved: false, + ...(rollbackRecordRemoved + ? { rollbackRecordRemoved: true } + : { rollbackRecordPath: plan.recordPath, rollbackRecordRemoved: false }), ...removal, }; } @@ -264,7 +360,13 @@ export function finalizeDockerGpuPatchBackup( replacementStoppedForCommit: false, }; } - const rollbackPlan = preparePostCommitRollback(options.result, resolved, containerOpts); + const rollbackPlan = preparePostCommitRollback( + options.result, + options.sandboxName, + deps, + resolved, + containerOpts, + ); if (!rollbackPlan) { const replacementRestarted = hasZeroDockerExitStatus( resolved.dockerStart(options.result.newContainerId, containerOpts), @@ -286,6 +388,7 @@ export function finalizeDockerGpuPatchBackup( resolved, containerOpts, ); + const rollbackRecordRemoved = removeRollbackRecord(rollbackPlan.recordPath); const replacementRestarted = hasZeroDockerExitStatus( resolved.dockerStart(options.result.newContainerId, containerOpts), ); @@ -295,6 +398,9 @@ export function finalizeDockerGpuPatchBackup( ...(rollbackImageRemoved ? { rollbackImageRemoved: true } : { rollbackImageId: rollbackPlan.imageId, rollbackImageRemoved: false }), + ...(rollbackRecordRemoved + ? { rollbackRecordRemoved: true } + : { rollbackRecordPath: rollbackPlan.recordPath, rollbackRecordRemoved: false }), replacementStoppedForCommit: true, replacementRestarted, finalHandoffAcknowledged: false, @@ -398,15 +504,22 @@ export function finalizeDockerGpuPatchBackup( }; } const rollbackImageRemoved = removeRollbackImage(rollbackPlan.imageId, resolved, containerOpts); + const rollbackRecordRemoved = removeRollbackRecord(rollbackPlan.recordPath); if (!rollbackImageRemoved) { console.warn(` Could not remove temporary Docker rollback image ${rollbackPlan.imageId}.`); } + if (!rollbackRecordRemoved) { + console.warn(` Could not remove Docker rollback record ${rollbackPlan.recordPath}.`); + } return { backupRemoved: true, rolledBack: false, ...(rollbackImageRemoved ? { rollbackImageRemoved: true } : { rollbackImageId: rollbackPlan.imageId, rollbackImageRemoved: false }), + ...(rollbackRecordRemoved + ? { rollbackRecordRemoved: true } + : { rollbackRecordPath: rollbackPlan.recordPath, rollbackRecordRemoved: false }), replacementStoppedForCommit: true, replacementRestarted: true, lifecycleReleaseObserved: true, diff --git a/src/lib/onboard/docker-gpu-patch-mode.ts b/src/lib/onboard/docker-gpu-patch-mode.ts index 3ab3acd970a..c86155d4192 100644 --- a/src/lib/onboard/docker-gpu-patch-mode.ts +++ b/src/lib/onboard/docker-gpu-patch-mode.ts @@ -5,9 +5,8 @@ import fs from "node:fs"; import path from "node:path"; import { dockerCapture, dockerRm, dockerRun } from "../adapters/docker"; -import { sleepSeconds } from "../core/wait"; import { hasZeroDockerExitStatus } from "./docker-command-result"; -import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; +import { DOCKER_GPU_PATCH_TIMEOUT_MS, sleepSeconds } from "./docker-gpu-patch-constants"; import { normalizeSandboxGpuDeviceForCdi } from "./sandbox-gpu-create"; import type { DockerGpuPatchBackend, diff --git a/src/lib/onboard/docker-gpu-patch-recreate.test.ts b/src/lib/onboard/docker-gpu-patch-recreate.test.ts index 279980d27bd..74be50cd87d 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.test.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.test.ts @@ -1,7 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it, vi } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterAll, describe, expect, it, vi } from "vitest"; import { createDockerGpuInspectFixture as inspectFixture } from "./__test-helpers__/docker-gpu-patch-fixtures"; import { getDockerGpuPatchFailureContext, @@ -12,6 +16,15 @@ const OLD_CONTAINER_ID = "a".repeat(64); const NEW_CONTAINER_ID = "b".repeat(64); const ROLLBACK_IMAGE_ID = `sha256:${"d".repeat(64)}`; const RESTORED_CONTAINER_ID = "e".repeat(64); +const RECREATE_TEST_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gpu-recreate-")); + +afterAll(() => { + fs.rmSync(RECREATE_TEST_ROOT, { recursive: true, force: true }); +}); + +function testHome(): string { + return fs.mkdtempSync(path.join(RECREATE_TEST_ROOT, "case-")); +} function dockerCaptureFixture() { const responses: Record = { @@ -66,6 +79,7 @@ describe("Docker GPU recreate orchestration", () => { dockerStart, runCaptureOpenshell, runOpenshell, + homedir: testHome, sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), detectSandboxFallbackDns: vi.fn(() => null), @@ -154,6 +168,7 @@ describe("Docker GPU recreate orchestration", () => { runOpenshell: vi.fn((args: readonly string[]) => args[1] === "list" ? { status: 0, stdout: "No sandboxes found.\n" } : { status: 0 }, ), + homedir: testHome, sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), detectSandboxFallbackDns: vi.fn(() => null), @@ -173,6 +188,7 @@ describe("Docker GPU recreate orchestration", () => { oldContainerId: OLD_CONTAINER_ID, newContainerId: NEW_CONTAINER_ID, rollbackImageId: ROLLBACK_IMAGE_ID, + rollbackRecordRemoved: true, lastSandboxPhase: "Deleting", rolledBack: true, }); diff --git a/src/lib/onboard/docker-gpu-patch-recreate.ts b/src/lib/onboard/docker-gpu-patch-recreate.ts index fa5a5fd2827..fbe8fa7ddc8 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.ts @@ -25,6 +25,7 @@ import { import { DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, DOCKER_GPU_PATCH_TIMEOUT_MS, + sleepSeconds, } from "./docker-gpu-patch-constants"; import { finalizeDockerGpuPatchBackup } from "./docker-gpu-patch-finalize"; import { selectDockerGpuPatchMode } from "./docker-gpu-patch-mode"; @@ -72,9 +73,7 @@ function recreateDeps(deps: DockerGpuPatchDeps): RecreateDeps { dockerRm, dockerStart, dockerStop, - sleep: (seconds: number) => { - Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Math.max(0, seconds) * 1000); - }, + sleep: sleepSeconds, now: () => new Date(), detectSandboxFallbackDns: () => detectSandboxFallbackDns(), probeContainerDns: (options) => probeContainerDns(options), @@ -409,6 +408,8 @@ export function recreateOpenShellDockerSandboxContainer( context.backupRemoved = finalization.backupRemoved; context.rollbackImageId = finalization.rollbackImageId; context.rollbackImageRemoved = finalization.rollbackImageRemoved; + context.rollbackRecordPath = finalization.rollbackRecordPath; + context.rollbackRecordRemoved = finalization.rollbackRecordRemoved; context.replacementStopConfirmed = finalization.replacementStopConfirmed; context.replacementRemovalConfirmed = finalization.replacementRemovalConfirmed; context.replacementPresence = finalization.replacementPresence; @@ -428,6 +429,10 @@ export function recreateOpenShellDockerSandboxContainer( finalization.rollbackImageId ? ` Docker retained rollback image ${finalization.rollbackImageId}.` : "" + }${ + finalization.rollbackRecordPath + ? ` Recovery action record: ${finalization.rollbackRecordPath}.` + : "" }` : ""; throw new Error( diff --git a/src/lib/onboard/docker-gpu-patch-rollback.ts b/src/lib/onboard/docker-gpu-patch-rollback.ts index f7ccfb2d541..731d473c363 100644 --- a/src/lib/onboard/docker-gpu-patch-rollback.ts +++ b/src/lib/onboard/docker-gpu-patch-rollback.ts @@ -13,7 +13,7 @@ import { import { retryUntil } from "../core/retry"; import { hasZeroDockerExitStatus } from "./docker-command-result"; import { fullDockerContainerId } from "./docker-gpu-patch-clone"; -import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; +import { DOCKER_GPU_PATCH_TIMEOUT_MS, sleepSeconds } from "./docker-gpu-patch-constants"; import type { DockerGpuPatchDeps } from "./docker-gpu-patch-types"; type DockerRunResult = { @@ -35,12 +35,6 @@ type DockerRunFn = (args: readonly string[], opts?: DockerRunOptions) => DockerR const REPLACEMENT_PRESENCE_ATTEMPTS = 3; const REPLACEMENT_PRESENCE_RETRY_MS = 500; -function sleepBeforeReplacementPresenceRetry(seconds: number): void { - if (seconds <= 0 || !Number.isFinite(seconds)) return; - const buffer = new Int32Array(new SharedArrayBuffer(4)); - Atomics.wait(buffer, 0, 0, seconds * 1000); -} - export type DockerGpuPatchRollbackOutcome = { rolledBack: boolean; replacementStopConfirmed: boolean; @@ -72,7 +66,7 @@ export function resolveDockerGpuPatchRollbackDeps( dockerRm: deps.dockerRm ?? defaultDockerRm, dockerRename: deps.dockerRename ?? defaultDockerRename, dockerStart: deps.dockerStart ?? defaultDockerStart, - sleep: deps.sleep ?? sleepBeforeReplacementPresenceRetry, + sleep: deps.sleep ?? sleepSeconds, }; } diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index 57ea2c9dbdc..a8f6d49b2a8 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -82,6 +82,8 @@ export type DockerGpuPatchFailureContext = { backupRemoved?: boolean; rollbackImageId?: string | null; rollbackImageRemoved?: boolean; + rollbackRecordPath?: string | null; + rollbackRecordRemoved?: boolean; selectedMode?: DockerGpuPatchMode | null; modeAttempts?: DockerGpuPatchModeAttempt[]; rolledBack?: boolean; diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index af39bd14c07..70f90a22314 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -529,6 +529,10 @@ export function createDockerGpuSandboxCreatePatch( finalizeOutcome.rollbackImageId ? ` Docker retained rollback image ${finalizeOutcome.rollbackImageId}.` : "" + }${ + finalizeOutcome.rollbackRecordPath + ? ` Recovery action record: ${finalizeOutcome.rollbackRecordPath}.` + : "" }` : "" }`, @@ -543,6 +547,8 @@ export function createDockerGpuSandboxCreatePatch( backupRemoved: finalizeOutcome.backupRemoved, rollbackImageId: finalizeOutcome.rollbackImageId, rollbackImageRemoved: finalizeOutcome.rollbackImageRemoved, + rollbackRecordPath: finalizeOutcome.rollbackRecordPath, + rollbackRecordRemoved: finalizeOutcome.rollbackRecordRemoved, lastSandboxPhase: finalizeOutcome.lastSandboxPhase, replacementStopConfirmed: finalizeOutcome.replacementStopConfirmed, replacementRemovalConfirmed: finalizeOutcome.replacementRemovalConfirmed, diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts index 76215a08973..0f505191187 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts @@ -25,7 +25,7 @@ import { parseLiveSandboxEntries } from "../runtime-recovery"; import { hasZeroDockerExitStatus } from "./docker-command-result"; -import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; +import { DOCKER_GPU_PATCH_TIMEOUT_MS, sleepSeconds } from "./docker-gpu-patch-constants"; import { envInt } from "./env"; const DOCKER_GPU_SUPERVISOR_RECONNECT_MIN_SECS = 900; @@ -111,7 +111,7 @@ export function waitForOpenShellSandboxLifecycleRelease( deps: DockerLifecycleReleaseDeps, ): boolean { if (!deps.runOpenshell) return false; - const sleep = deps.sleep ?? defaultSleep; + const sleep = deps.sleep ?? sleepSeconds; const deadline = Date.now() + Math.max(1, Math.round(timeoutSecs)) * 1000; const maxAttempts = Math.max(1, Math.ceil(Math.max(1, Math.round(timeoutSecs)) / 2) + 1); @@ -178,7 +178,7 @@ export function waitForOpenShellFinalHandoff( timeoutSecs: number, deps: DockerFinalHandoffDeps, ): DockerFinalHandoffAcknowledgement { - const sleep = deps.sleep ?? defaultSleep; + const sleep = deps.sleep ?? sleepSeconds; const boundedTimeoutSecs = Math.max(1, Math.round(timeoutSecs)); const deadline = Date.now() + boundedTimeoutSecs * 1000; const maxAttempts = Math.max(1, Math.ceil(boundedTimeoutSecs / 2) + 1); @@ -244,10 +244,6 @@ export function waitForOpenShellFinalHandoff( return { acknowledged: false, lastSandboxPhase }; } -function defaultSleep(seconds: number): void { - Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Math.max(0, seconds) * 1000); -} - const ANSI_RE = /\x1b\[[0-9;]*m/g; function parseSandboxListFailurePhase(output: string, sandboxName: string): string | null { @@ -283,7 +279,7 @@ export function waitForOpenShellSupervisorReconnect( deps: DockerGpuSupervisorReconnectDeps, ): boolean { if (!deps.runOpenshell) return false; - const sleep = deps.sleep ?? defaultSleep; + const sleep = deps.sleep ?? sleepSeconds; const deadline = Date.now() + Math.max(1, timeoutSecs) * 1000; const errorPhaseDebouncePolls = deps.errorPhaseDebouncePolls == null || !Number.isFinite(deps.errorPhaseDebouncePolls) diff --git a/test/process-recovery/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery/process-recovery-supervisor-relaunch.test.ts index 88b7ebc785e..3260da28ca2 100644 --- a/test/process-recovery/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery/process-recovery-supervisor-relaunch.test.ts @@ -1,7 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { afterEach, describe, expect, it, vi } from "vitest"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; import * as forwardHealth from "../../src/lib/actions/sandbox/forward-health.ts"; import { checkAndRecoverSandboxProcesses } from "../../src/lib/actions/sandbox/process-recovery.ts"; import { relaunchManagedSupervisorSession } from "../../src/lib/actions/sandbox/supervisor-relaunch.ts"; @@ -24,6 +28,12 @@ const MISSING_MANAGED_SUPERVISOR = { stdout: "", stderr: "SUPERVISOR_NOT_RUNNING", } as const; +const TEST_TEMP_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-recovery-relaunch-")); + +function testHome(): string { + return fs.mkdtempSync(path.join(TEST_TEMP_ROOT, "case-")); +} + function pinnedIdentityRefusal(sandboxName: string) { return { status: 1, @@ -37,6 +47,10 @@ afterEach(() => { vi.unstubAllEnvs(); }); +afterAll(() => { + fs.rmSync(TEST_TEMP_ROOT, { recursive: true, force: true }); +}); + function mockOpenClawSandbox(sandboxName: string, healthTimeoutSeconds = 30) { vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "openclaw", @@ -505,6 +519,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { dockerRun, dockerStart, dockerStop, + homedir: testHome, }), ); const { relaunchManagedSupervisorSessionImpl } = composedRelaunchTransaction( From 9588a38dd3bbae1b0782cc107d091ca387de5f86 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 25 Aug 2026 10:41:33 -0700 Subject: [PATCH 08/12] test(onboard): model rollback image commands Signed-off-by: Aaron Erickson --- src/lib/onboard/docker-gpu-patch-finalize.test.ts | 10 ++++++++-- test/helpers/onboard-script-mocks.cjs | 11 +++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index efb0524eedd..b78821ff0f2 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -777,8 +777,14 @@ describe("finalizeDockerGpuPatchBackup", () => { ); const recordPath = outcome.rollbackRecordPath; expect(recordPath).toBeTypeOf("string"); - expect(fs.statSync(recordPath as string).mode & 0o777).toBe(0o600); - const record = fs.readFileSync(recordPath as string, "utf8"); + const recordFd = fs.openSync(recordPath as string, "r"); + let record: string; + try { + expect(fs.fstatSync(recordFd).mode & 0o777).toBe(0o600); + record = fs.readFileSync(recordFd, "utf8"); + } finally { + fs.closeSync(recordFd); + } expect(record).toContain(ROLLBACK_IMAGE_ID); expect(record).toContain('"command": "docker"'); expect(record).toContain('"run"'); diff --git a/test/helpers/onboard-script-mocks.cjs b/test/helpers/onboard-script-mocks.cjs index 6c5fc129fb8..fe3a80c9a5e 100644 --- a/test/helpers/onboard-script-mocks.cjs +++ b/test/helpers/onboard-script-mocks.cjs @@ -193,6 +193,7 @@ const OPENCLAW_SECURITY_INVENTORY_PROBE = [ const ONBOARD_SANDBOX_OLD_CONTAINER_ID = "a".repeat(64); const ONBOARD_SANDBOX_NEW_CONTAINER_ID = "b".repeat(64); +const ONBOARD_SANDBOX_ROLLBACK_IMAGE_ID = `sha256:${"d".repeat(64)}`; const ONBOARD_SANDBOX_INSPECT = { Id: ONBOARD_SANDBOX_OLD_CONTAINER_ID, Image: `sha256:${"c".repeat(64)}`, @@ -341,6 +342,16 @@ function mockDockerSandboxLifecycleReleaseFromRunner() { let lifecycleReleased = false; const wrappedRun = (command, options) => { const normalized = normalizeCommand(command); + switch (normalized) { + case `docker commit ${ONBOARD_SANDBOX_OLD_CONTAINER_ID}`: + return { + status: 0, + stdout: Buffer.from(`${ONBOARD_SANDBOX_ROLLBACK_IMAGE_ID}\n`), + stderr: Buffer.alloc(0), + }; + case `docker image rm --force ${ONBOARD_SANDBOX_ROLLBACK_IMAGE_ID}`: + return { status: 0, stdout: Buffer.alloc(0), stderr: Buffer.alloc(0) }; + } if ( finalCommitReleased && normalized.startsWith("docker ps -a --no-trunc ") && From 54ba903bf8cc34cd1a5f556f1f11160024324d21 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 25 Aug 2026 11:20:20 -0700 Subject: [PATCH 09/12] fix(onboard): secure rollback recovery records Signed-off-by: Aaron Erickson --- .../onboard/docker-gpu-patch-finalize.test.ts | 89 ++++++++++---- src/lib/onboard/docker-gpu-patch-finalize.ts | 114 +++++++++++++++++- 2 files changed, 178 insertions(+), 25 deletions(-) diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index b78821ff0f2..ae6f51f0cb7 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -745,6 +745,22 @@ describe("finalizeDockerGpuPatchBackup", () => { it("retains the rollback image when post-commit restoration fails (#9531)", () => { const result = exactDeferredCreateResult(); const deps = readyHandoffDeps(); + const inspect = inspectFixture(); + inspect.Config = { + ...inspect.Config, + Env: [ + ...(inspect.Config?.Env ?? []).filter( + (entry) => !entry.startsWith("OPENSHELL_SANDBOX_COMMAND="), + ), + "ROLLBACK_TEST_TOKEN=super-secret-value", + "OPENSHELL_SANDBOX_COMMAND=env NEMOCLAW_EXTRA_PLACEHOLDER_KEYS=CUSTOM_PROVIDER_VALUE CUSTOM_PROVIDER_VALUE=startup-secret CHAT_UI_URL=http://127.0.0.1:8642 nemoclaw-start", + ], + Labels: { + ...inspect.Config?.Labels, + "untrusted.example/token": "label-secret", + }, + }; + deps.dockerCapture.mockReturnValue(JSON.stringify([{ ...inspect, Id: result.oldContainerId }])); deps.dockerRunDetached.mockReturnValue({ status: 1, stdout: "", stderr: "restore failed" }); const outcome = finalizeDockerGpuPatchBackup( @@ -789,15 +805,65 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(record).toContain('"command": "docker"'); expect(record).toContain('"run"'); expect(record).toContain('"--detach"'); + expect(record).toContain("OPENSHELL_ENDPOINT=http://host.openshell.internal:8080/"); + expect(record).toContain( + "OPENSHELL_SANDBOX_COMMAND=env NEMOCLAW_EXTRA_PLACEHOLDER_KEYS=CUSTOM_PROVIDER_VALUE CHAT_UI_URL=http://127.0.0.1:8642 nemoclaw-start", + ); + expect(record).toContain("openshell.ai/managed-by=openshell"); + expect(record).toContain("openshell.ai/sandbox-name=alpha"); + expect(record).toContain("openshell.ai/sandbox-id=sandbox-id"); expect(record).not.toContain("super-secret-value"); expect(record).not.toContain("ROLLBACK_TEST_TOKEN"); - expect(record).not.toContain('"--env"'); - expect(record).not.toContain('"--label"'); + expect(record).not.toContain("startup-secret"); + expect(record).not.toContain("CUSTOM_PROVIDER_VALUE=startup-secret"); + expect(record).not.toContain("OPENSHELL_TEST=1"); + expect(record).not.toContain("untrusted.example/token"); + expect(record).not.toContain("label-secret"); const diagnostics = collectRollbackDiagnostics(result.newContainerId, outcome); expect(diagnostics.summary).toContain("rollback_record_removed=no"); expect(diagnostics.summary).toContain("rollback_recovery_action="); }); + it("refuses a symlinked rollback-record path before removing the backup (#9531)", () => { + const result = exactDeferredCreateResult(); + const deps = readyHandoffDeps(); + const homeDir = deps.homedir(); + const redirectedDirectory = fs.mkdtempSync(path.join(ROLLBACK_TEST_ROOT, "redirected-")); + fs.mkdirSync(path.join(homeDir, ".nemoclaw"), { recursive: true }); + fs.symlinkSync(redirectedDirectory, path.join(homeDir, ".nemoclaw", "recovery"), "dir"); + const dockerRm = vi.fn(() => ({ status: 0 })); + const dockerStart = vi.fn(() => ({ status: 0 })); + vi.stubEnv("HOME", homeDir); + let outcome: DockerGpuPatchFinalizeOutcome; + try { + outcome = finalizeDockerGpuPatchBackup( + { + result, + supervisorReady: true, + sandboxName: "alpha", + finalHandoffTimeoutSecs: 60, + }, + { ...deps, dockerRm, dockerStart, dockerStop: vi.fn(() => ({ status: 0 })) }, + ); + } finally { + vi.unstubAllEnvs(); + } + + expect(outcome).toMatchObject({ + backupRemoved: false, + rolledBack: false, + replacementStoppedForCommit: true, + replacementRestarted: true, + finalHandoffAcknowledged: false, + }); + expect(dockerRm).not.toHaveBeenCalled(); + expect(fs.readdirSync(redirectedDirectory)).toEqual([]); + expect(deps.dockerRun).toHaveBeenCalledWith( + ["image", "rm", "--force", ROLLBACK_IMAGE_ID], + expect.anything(), + ); + }); + it("records a remaining exact-ID replacement when removal fails (#7996)", () => { const newContainerId = "a".repeat(64); const outcome = finalizeDockerGpuPatchBackup( @@ -928,23 +994,4 @@ describe("finalizeDockerGpuPatchBackup", () => { }); expect(dockerStart).not.toHaveBeenCalled(); }); - - it("fails closed when rollback start has no exit status", () => { - const outcome = finalizeDockerGpuPatchBackup( - { result: deferredCreateResult(), supervisorReady: false }, - { - dockerStop: vi.fn(() => ({ status: 0 })), - dockerRm: vi.fn(() => ({ status: 0 })), - dockerRename: vi.fn(() => ({ status: 0 })), - dockerStart: vi.fn(() => ({ status: null })), - }, - ); - expect(outcome).toEqual({ - backupRemoved: false, - rolledBack: false, - replacementStopConfirmed: true, - replacementRemovalConfirmed: true, - replacementPresence: "absent", - }); - }); }); diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index 3c0fda11ac2..af65b9753f8 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -18,7 +18,7 @@ // for exact final handoff, terminal phase, rollback, and failure outcomes. // * src/lib/onboard/docker-gpu-patch-rollback.test.ts — composed // recreate-with-rollback scenarios. -// * src/lib/onboard/docker-gpu-sandbox-create.test.ts — composed create +// * src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts — composed create // flow driving maybeApplyDuringCreate → waitForSupervisorReconnect → // finalizeBackup. // Removal condition: when OpenShell supports native Docker-driver GPU @@ -30,6 +30,8 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { shouldStripCredentialEnv } from "../security/credential-env"; +import { rejectSymlinksOnPath } from "../state/config-io"; import { hasZeroDockerExitStatus } from "./docker-command-result"; import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; import { @@ -54,7 +56,12 @@ import { waitForOpenShellFinalHandoff, waitForOpenShellSandboxLifecycleRelease, } from "./docker-gpu-supervisor-reconnect"; -import { queryOpenShellDockerSandboxContainers } from "./openshell-docker-sandbox-containers"; +import { + OPENSHELL_MANAGED_BY_LABEL, + OPENSHELL_SANDBOX_ID_LABEL, + OPENSHELL_SANDBOX_NAME_LABEL, + queryOpenShellDockerSandboxContainers, +} from "./openshell-docker-sandbox-containers"; export type DockerGpuPatchFinalizeOptions = | { @@ -95,7 +102,22 @@ type PostCommitRollbackPlan = { const DOCKER_IMAGE_ID = /^sha256:[0-9a-f]{64}$/iu; const ROLLBACK_RECORD_VERSION = 1; -const REDACTED_RUN_VALUE_FLAGS = new Set(["--env", "--label"]); +const RECOVERY_ENV_KEYS = new Set([ + "OPENSHELL_ENDPOINT", + "OPENSHELL_SANDBOX_COMMAND", + "OPENSHELL_OCI_IMAGE_USER", + "OPENSHELL_SANDBOX_UID", + "OPENSHELL_SANDBOX_GID", +]); +const RECOVERY_LABEL_KEYS = new Set([ + OPENSHELL_MANAGED_BY_LABEL, + OPENSHELL_SANDBOX_NAME_LABEL, + OPENSHELL_SANDBOX_ID_LABEL, +]); +const ENV_KEY = /^[A-Za-z_][A-Za-z0-9_]*$/u; +const OCI_IDENTITY = /^[A-Za-z0-9_.-]+(?::[A-Za-z0-9_.-]+)?$/u; +const NUMERIC_IDENTITY = /^(?:0|[1-9][0-9]*)$/u; +const NEMOCLAW_STARTUP_EXECUTABLES = new Set(["nemoclaw-start", "/usr/local/bin/nemoclaw-start"]); function removeRollbackImage( imageId: string, @@ -105,6 +127,86 @@ function removeRollbackImage( return hasZeroDockerExitStatus(deps.dockerRun(["image", "rm", "--force", imageId], options)); } +function envAssignment(value: string): { key: string; value: string } | null { + const separator = value.indexOf("="); + if (separator <= 0) return null; + const key = value.slice(0, separator); + return ENV_KEY.test(key) ? { key, value: value.slice(separator + 1) } : null; +} + +function isCredentialFreeHttpUrl(value: string): boolean { + try { + const parsed = new URL(value); + return ( + ["http:", "https:"].includes(parsed.protocol) && + parsed.username === "" && + parsed.password === "" && + parsed.search === "" && + parsed.hash === "" + ); + } catch { + return false; + } +} + +function recoverySandboxCommand(value: string): string | null { + const tokens = value.trim().split(/\s+/u).filter(Boolean); + if (tokens.length === 2 && tokens[0] === "sleep" && tokens[1] === "infinity") return value; + if (tokens.length === 1 && NEMOCLAW_STARTUP_EXECUTABLES.has(tokens[0])) return value; + if (tokens[0] !== "env") return null; + + const assignments: Array<{ key: string; value: string }> = []; + let executableIndex = 1; + while (executableIndex < tokens.length) { + const assignment = envAssignment(tokens[executableIndex]); + if (!assignment) break; + assignments.push(assignment); + executableIndex += 1; + } + if ( + executableIndex !== tokens.length - 1 || + !NEMOCLAW_STARTUP_EXECUTABLES.has(tokens[executableIndex]) + ) { + return null; + } + const extraPlaceholderKeys = new Set( + (assignments.find(({ key }) => key === "NEMOCLAW_EXTRA_PLACEHOLDER_KEYS")?.value ?? "") + .split(/[\s,]+/u) + .filter((key) => ENV_KEY.test(key)), + ); + const retained = assignments + .filter( + ({ key, value: assignmentValue }) => + !shouldStripCredentialEnv(key) && + !extraPlaceholderKeys.has(key) && + (!assignmentValue.includes("://") || isCredentialFreeHttpUrl(assignmentValue)), + ) + .map(({ key, value: assignmentValue }) => `${key}=${assignmentValue}`); + return ["env", ...retained, tokens[executableIndex]].join(" "); +} + +function recoveryRunValue(flag: string, value: string): string | null { + if (flag === "--label") { + const separator = value.indexOf("="); + const key = separator > 0 ? value.slice(0, separator) : ""; + return RECOVERY_LABEL_KEYS.has(key) ? value : null; + } + const assignment = envAssignment(value); + if (!assignment) return null; + if (flag !== "--env" || !RECOVERY_ENV_KEYS.has(assignment.key)) return null; + if (assignment.key === "OPENSHELL_SANDBOX_COMMAND") { + const command = recoverySandboxCommand(assignment.value); + return command === null ? null : `${assignment.key}=${command}`; + } + if (assignment.key === "OPENSHELL_ENDPOINT") + return isCredentialFreeHttpUrl(assignment.value) ? value : null; + if (assignment.key === "OPENSHELL_OCI_IMAGE_USER") + return OCI_IDENTITY.test(assignment.value) ? value : null; + if (assignment.key === "OPENSHELL_SANDBOX_UID" || assignment.key === "OPENSHELL_SANDBOX_GID") + return assignment.value === "" || NUMERIC_IDENTITY.test(assignment.value) ? value : null; + return value; +} + function recoveryRunArgs(runArgs: readonly string[], imageId: string): string[] { const imageIndex = runArgs.indexOf(imageId); if (imageIndex < 0) @@ -112,11 +214,13 @@ function recoveryRunArgs(runArgs: readonly string[], imageId: string): string[] const persisted: string[] = []; for (let index = 0; index < imageIndex; index += 1) { const value = runArgs[index]; - if (REDACTED_RUN_VALUE_FLAGS.has(value)) { + if (value === "--env" || value === "--label") { if (index + 1 >= imageIndex) { throw new Error(`Docker rollback argument '${value}' has no value.`); } + const persistedValue = recoveryRunValue(value, runArgs[index + 1]); index += 1; + if (persistedValue !== null) persisted.push(value, persistedValue); continue; } persisted.push(value); @@ -142,7 +246,9 @@ function writeRollbackRecord( `${encodeURIComponent(sandboxName)}-${oldContainerId.slice(0, 12)}.json`, ); try { + rejectSymlinksOnPath(directory); fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); + rejectSymlinksOnPath(directory); fs.chmodSync(directory, 0o700); fs.writeFileSync( recordPath, From cd3669bf858648bbb7bf4c5c4266ee48f9e190ba Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 25 Aug 2026 11:38:11 -0700 Subject: [PATCH 10/12] fix(recovery): report retained rollback artifacts Signed-off-by: Aaron Erickson --- src/lib/actions/sandbox/process-recovery.ts | 27 +++++++++++++++-- ...ocess-recovery-supervisor-relaunch.test.ts | 29 +++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 80e9cdf51ca..d62caad4ba7 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -499,19 +499,40 @@ function waitForFinalRelaunchManagedSupervisor( return { failure: classifyGatewayRestartFailure(probeResult), ready: false }; } +function retainedRollbackArtifactsDetail( + completion: ReturnType, +): string { + if (completion.backupRemoved !== true || completion.rolledBack !== false) return ""; + const artifacts = [ + ...(completion.rollbackImageId + ? [`rollback image ID ${JSON.stringify(completion.rollbackImageId)}`] + : []), + ...(completion.rollbackRecordPath + ? [`rollback recovery record ${JSON.stringify(completion.rollbackRecordPath)}`] + : []), + ]; + if (artifacts.length === 0) return ""; + return ` Retained ${artifacts.join(" and ")}.${ + completion.rollbackRecordPath + ? " Confirm the replacement container is absent before using the recovery record's recoveryAction command and arguments." + : "" + }`; +} + function finalRelaunchContainerFailureDetail( completion: ReturnType, ): string | null { + const rollbackArtifacts = retainedRollbackArtifactsDetail(completion); if (completion.replacementStoppedForCommit === false) { - return "Docker could not stop the replacement container for the final recovery handoff. NemoClaw did not start the primary dashboard/API host forward"; + return `Docker could not stop the replacement container for the final recovery handoff. NemoClaw did not start the primary dashboard/API host forward${rollbackArtifacts}`; } if (completion.replacementRestarted === false) { - return "Docker could not start the replacement container to complete the final recovery handoff. NemoClaw did not start the primary dashboard/API host forward"; + return `Docker could not start the replacement container to complete the final recovery handoff. NemoClaw did not start the primary dashboard/API host forward${rollbackArtifacts}`; } if (completion.finalHandoffAcknowledged === false) { return `OpenShell did not acknowledge the final replacement container handoff${ completion.lastSandboxPhase ? `; last sandbox phase was ${completion.lastSandboxPhase}` : "" - }. NemoClaw did not start the primary dashboard/API host forward`; + }. NemoClaw did not start the primary dashboard/API host forward${rollbackArtifacts}`; } return null; } diff --git a/test/process-recovery/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery/process-recovery-supervisor-relaunch.test.ts index 3260da28ca2..1cd6a48e6b8 100644 --- a/test/process-recovery/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery/process-recovery-supervisor-relaunch.test.ts @@ -653,6 +653,29 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { finalPinnedAction: () => ACCEPTED_MANAGED_PROBE, finalReadinessReady: true, }, + { + condition: "automatic rollback after the final container handoff fails", + finalizeOutcome: () => ({ + backupRemoved: true, + finalHandoffAcknowledged: false, + lastSandboxPhase: "Deleting", + replacementRestarted: true, + rollbackImageId: `sha256:${"d".repeat(64)}`, + rollbackRecordPath: "/home/test/.nemoclaw/recovery/docker-gpu/recovery.json", + rolledBack: false, + stateRestored: true, + }), + expectedDetail: "OpenShell did not acknowledge the final replacement container handoff", + expectedDetailFragments: [ + `sha256:${"d".repeat(64)}`, + "/home/test/.nemoclaw/recovery/docker-gpu/recovery.json", + "Confirm the replacement container is absent", + "recoveryAction command and arguments", + ], + expectedReadinessCalls: 1, + finalPinnedAction: () => ACCEPTED_MANAGED_PROBE, + finalReadinessReady: true, + }, { condition: "the final OpenShell readiness check fails", finalizeOutcome: () => ({ @@ -715,6 +738,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { "does not start the primary dashboard/API host forward when $condition (#9364)", ({ expectedDetail, + expectedDetailFragments, expectedReadinessCalls, finalPinnedAction, finalReadinessReady, @@ -773,6 +797,11 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { expect("recoveryFailureDetail" in result ? result.recoveryFailureDetail : "").not.toContain( "opaque-", ); + const detail = "recoveryFailureDetail" in result ? result.recoveryFailureDetail : ""; + expect(detail).toContain(expectedDetailFragments?.[0] ?? ""); + expect(detail).toContain(expectedDetailFragments?.[1] ?? ""); + expect(detail).toContain(expectedDetailFragments?.[2] ?? ""); + expect(detail).toContain(expectedDetailFragments?.[3] ?? ""); expect(result).not.toHaveProperty("forwardRecoveryFailed"); expect(finalize).toHaveBeenCalledOnce(); expect(finalize).toHaveBeenCalledWith(true); From f79330c3c99e64c65cad47d8aadd6a2b405d20d5 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 25 Aug 2026 12:46:15 -0700 Subject: [PATCH 11/12] fix(onboard): restore bounded handoff contract Signed-off-by: Aaron Erickson --- src/lib/actions/sandbox/process-recovery.ts | 27 +- .../sandbox/supervisor-relaunch.test.ts | 6 +- src/lib/adapters/openshell/client.test.ts | 16 + src/lib/adapters/openshell/client.ts | 12 +- .../openshell/process-tree-timeout.ts | 44 ++ src/lib/adapters/openshell/runtime.ts | 6 + src/lib/onboard/docker-gpu-patch-constants.ts | 2 - .../onboard/docker-gpu-patch-diagnostics.ts | 9 - .../onboard/docker-gpu-patch-finalize.test.ts | 404 +++------------- src/lib/onboard/docker-gpu-patch-finalize.ts | 434 +----------------- src/lib/onboard/docker-gpu-patch-mode.ts | 3 +- .../onboard/docker-gpu-patch-recreate.test.ts | 74 +-- src/lib/onboard/docker-gpu-patch-recreate.ts | 29 +- .../onboard/docker-gpu-patch-rollback.test.ts | 2 - src/lib/onboard/docker-gpu-patch-rollback.ts | 47 +- src/lib/onboard/docker-gpu-patch-types.ts | 4 - ...ocker-gpu-sandbox-create-lifecycle.test.ts | 17 +- src/lib/onboard/docker-gpu-sandbox-create.ts | 24 +- .../docker-gpu-supervisor-reconnect.test.ts | 33 +- .../docker-gpu-supervisor-reconnect.ts | 21 +- src/lib/onboard/openshell-cli.ts | 26 +- src/lib/onboard/sandbox-create-launch.test.ts | 33 ++ test/helpers/onboard-script-mocks.cjs | 11 - ...ocess-recovery-supervisor-relaunch.test.ts | 66 +-- 24 files changed, 322 insertions(+), 1028 deletions(-) create mode 100644 src/lib/adapters/openshell/process-tree-timeout.ts diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 069f0e599b3..8b879c928df 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -499,40 +499,19 @@ function waitForFinalRelaunchManagedSupervisor( return { failure: classifyGatewayRestartFailure(probeResult), ready: false }; } -function retainedRollbackArtifactsDetail( - completion: ReturnType, -): string { - if (completion.backupRemoved !== true || completion.rolledBack !== false) return ""; - const artifacts = [ - ...(completion.rollbackImageId - ? [`rollback image ID ${JSON.stringify(completion.rollbackImageId)}`] - : []), - ...(completion.rollbackRecordPath - ? [`rollback recovery record ${JSON.stringify(completion.rollbackRecordPath)}`] - : []), - ]; - if (artifacts.length === 0) return ""; - return ` Retained ${artifacts.join(" and ")}.${ - completion.rollbackRecordPath - ? " Confirm the replacement container is absent before using the recovery record's recoveryAction command and arguments." - : "" - }`; -} - function finalRelaunchContainerFailureDetail( completion: ReturnType, ): string | null { - const rollbackArtifacts = retainedRollbackArtifactsDetail(completion); if (completion.replacementStoppedForCommit === false) { - return `Docker could not stop the replacement container for the final recovery handoff. NemoClaw did not start the primary dashboard/API host forward${rollbackArtifacts}`; + return "Docker could not stop the replacement container for the final recovery handoff. NemoClaw did not start the primary dashboard/API host forward"; } if (completion.replacementRestarted === false) { - return `Docker could not start the replacement container to complete the final recovery handoff. NemoClaw did not start the primary dashboard/API host forward${rollbackArtifacts}`; + return "Docker could not start the replacement container to complete the final recovery handoff. NemoClaw did not start the primary dashboard/API host forward"; } if (completion.finalHandoffAcknowledged === false) { return `OpenShell did not acknowledge the final replacement container handoff${ completion.lastSandboxPhase ? `; last sandbox phase was ${completion.lastSandboxPhase}` : "" - }. NemoClaw did not start the primary dashboard/API host forward${rollbackArtifacts}`; + }. NemoClaw did not start the primary dashboard/API host forward`; } return null; } diff --git a/src/lib/actions/sandbox/supervisor-relaunch.test.ts b/src/lib/actions/sandbox/supervisor-relaunch.test.ts index 438b72af361..8a3d48bcedb 100644 --- a/src/lib/actions/sandbox/supervisor-relaunch.test.ts +++ b/src/lib/actions/sandbox/supervisor-relaunch.test.ts @@ -508,13 +508,13 @@ describe("relaunchManagedSupervisorSession", () => { }); }); - it("restores the prior container and retains state backup when final handoff fails (#9531)", () => { + it("retains the restored state backup when final handoff is not acknowledged (#9531)", () => { const deps = baseDeps({ finalize: vi.fn(() => ({ backupRemoved: true, finalHandoffAcknowledged: false, lastSandboxPhase: "Deleting", - rolledBack: true, + rolledBack: false, })), }); const relaunch = relaunchManagedSupervisorSession("alpha", { quiet: true, deps }); @@ -523,7 +523,7 @@ describe("relaunchManagedSupervisorSession", () => { backupRemoved: true, finalHandoffAcknowledged: false, lastSandboxPhase: "Deleting", - rolledBack: true, + rolledBack: false, stateRestored: true, }); expect(deps.removeBackup).not.toHaveBeenCalled(); diff --git a/src/lib/adapters/openshell/client.test.ts b/src/lib/adapters/openshell/client.test.ts index 038be1d759d..6f2a83da194 100644 --- a/src/lib/adapters/openshell/client.test.ts +++ b/src/lib/adapters/openshell/client.test.ts @@ -17,6 +17,7 @@ import { stripAnsi, versionGte, } from "./client"; +import { processTreeBoundedOpenshellInvocation } from "./process-tree-timeout"; interface SpawnResultSpec { status: number | null; @@ -51,6 +52,21 @@ function exitWithCode(code: number): never { } describe("openshell helpers", () => { + it("wraps a synchronous Linux probe in a process-group timeout (#10238)", () => { + expect( + processTreeBoundedOpenshellInvocation( + "/opt/openshell", + ["sandbox", "list"], + { killProcessTreeOnTimeout: true, timeout: 1000 }, + { platform: "linux", timeoutExecutableExists: () => true }, + ), + ).toEqual({ + binary: "/usr/bin/timeout", + args: ["--signal=KILL", "0.75s", "/opt/openshell", "sandbox", "list"], + killSignal: "SIGKILL", + }); + }); + afterEach(() => { vi.unstubAllEnvs(); }); diff --git a/src/lib/adapters/openshell/client.ts b/src/lib/adapters/openshell/client.ts index 012546d0422..14ca2134699 100644 --- a/src/lib/adapters/openshell/client.ts +++ b/src/lib/adapters/openshell/client.ts @@ -12,6 +12,7 @@ import { import { redirectInheritedChildStdoutToStderr } from "../../cli/stdout-guard"; import { buildSubprocessEnv } from "../../subprocess-env"; +import { processTreeBoundedOpenshellInvocation } from "./process-tree-timeout"; export { openshellSandboxSshHost, resolveOpenshellSandboxSshHost } from "./sandbox-ssh-host"; @@ -28,6 +29,7 @@ interface OpenshellSpawnOptions { env?: NodeJS.ProcessEnv; replaceEnv?: boolean; timeout?: number; + killProcessTreeOnTimeout?: boolean; ignoreError?: boolean; spawnSyncImpl?: OpenshellSpawnSync; errorLine?: (message: string) => void; @@ -53,6 +55,7 @@ export interface RunOpenshellOptions extends OpenshellSpawnOptions { export interface CaptureOpenshellOptions extends OpenshellSpawnOptions { includeStderr?: boolean; includeStreams?: boolean; + killSignal?: SpawnSyncOptions["killSignal"]; maxBuffer?: number; } @@ -207,14 +210,15 @@ export function runOpenshellCommand( opts: RunOpenshellOptions = {}, ): SpawnSyncReturns { const spawnSyncImpl = opts.spawnSyncImpl ?? spawnSync; - const result = spawnSyncImpl(binary, args, { + const bounded = processTreeBoundedOpenshellInvocation(binary, args, opts); + const result = spawnSyncImpl(bounded.binary, bounded.args, { cwd: opts.cwd, env: openshellSpawnEnv(opts), encoding: "utf-8", stdio: redirectInheritedChildStdoutToStderr(opts.stdio ?? "inherit"), input: opts.input, timeout: opts.timeout, - killSignal: opts.killSignal, + killSignal: bounded.killSignal, maxBuffer: opts.maxBuffer, }); if (result.error) { @@ -236,12 +240,14 @@ export function captureOpenshellCommand( opts: CaptureOpenshellOptions = {}, ): CaptureOpenshellResult { const spawnSyncImpl = opts.spawnSyncImpl ?? spawnSync; - const result = spawnSyncImpl(binary, args, { + const bounded = processTreeBoundedOpenshellInvocation(binary, args, opts); + const result = spawnSyncImpl(bounded.binary, bounded.args, { cwd: opts.cwd, env: openshellSpawnEnv(opts), encoding: "utf-8", stdio: ["ignore", "pipe", "pipe"], timeout: opts.timeout, + killSignal: bounded.killSignal, maxBuffer: opts.maxBuffer, }); if (result.error) { diff --git a/src/lib/adapters/openshell/process-tree-timeout.ts b/src/lib/adapters/openshell/process-tree-timeout.ts new file mode 100644 index 00000000000..2e945613c40 --- /dev/null +++ b/src/lib/adapters/openshell/process-tree-timeout.ts @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { SpawnSyncOptions } from "node:child_process"; +import fs from "node:fs"; + +type ProcessTreeTimeoutDeps = { + platform?: NodeJS.Platform; + timeoutExecutableExists?: (pathname: string) => boolean; +}; + +export type ProcessTreeTimeoutOptions = { + killProcessTreeOnTimeout?: boolean; + killSignal?: SpawnSyncOptions["killSignal"]; + timeout?: number; +}; + +export function processTreeBoundedOpenshellInvocation( + binary: string, + args: readonly string[], + opts: ProcessTreeTimeoutOptions, + deps: ProcessTreeTimeoutDeps = {}, +): { binary: string; args: string[]; killSignal?: SpawnSyncOptions["killSignal"] } { + if (opts.killProcessTreeOnTimeout !== true) { + return { binary, args: [...args], killSignal: opts.killSignal }; + } + const timeoutMs = Number(opts.timeout); + const platform = deps.platform ?? process.platform; + const timeoutExecutableExists = deps.timeoutExecutableExists ?? fs.existsSync; + if ( + platform !== "linux" || + !timeoutExecutableExists("/usr/bin/timeout") || + !Number.isFinite(timeoutMs) || + timeoutMs <= 0 + ) { + return { binary, args: [...args], killSignal: "SIGKILL" }; + } + const groupTimeoutSeconds = Math.max(1, Math.floor(timeoutMs) - 250) / 1000; + return { + binary: "/usr/bin/timeout", + args: ["--signal=KILL", `${String(groupTimeoutSeconds)}s`, binary, ...args], + killSignal: "SIGKILL", + }; +} diff --git a/src/lib/adapters/openshell/runtime.ts b/src/lib/adapters/openshell/runtime.ts index 87a21b47359..c6dc0435afb 100644 --- a/src/lib/adapters/openshell/runtime.ts +++ b/src/lib/adapters/openshell/runtime.ts @@ -31,6 +31,7 @@ type RunnerOptions = { includeStreams?: boolean; timeout?: number; killSignal?: NodeJS.Signals; + killProcessTreeOnTimeout?: boolean; maxBuffer?: number; }; @@ -59,6 +60,7 @@ export function runOpenshell(args: CommandArgs, opts: RunnerOptions = {}) { ignoreError: opts.ignoreError, timeout: opts.timeout, killSignal: opts.killSignal, + killProcessTreeOnTimeout: opts.killProcessTreeOnTimeout, maxBuffer: opts.maxBuffer, errorLine: console.error, exit: (code: number) => process.exit(code), @@ -79,6 +81,8 @@ export function captureOpenshell(args: CommandArgs, opts: RunnerOptions = {}) { includeStderr: opts.includeStderr, includeStreams: opts.includeStreams, timeout: opts.timeout, + killSignal: opts.killSignal, + killProcessTreeOnTimeout: opts.killProcessTreeOnTimeout, maxBuffer: opts.maxBuffer, errorLine: console.error, exit: (code: number) => process.exit(code), @@ -98,6 +102,8 @@ export function captureResolvedOpenshell(args: CommandArgs, opts: RunnerOptions includeStderr: opts.includeStderr, includeStreams: opts.includeStreams, timeout: opts.timeout, + killSignal: opts.killSignal, + killProcessTreeOnTimeout: opts.killProcessTreeOnTimeout, maxBuffer: opts.maxBuffer, errorLine: console.error, exit: (code: number) => process.exit(code), diff --git a/src/lib/onboard/docker-gpu-patch-constants.ts b/src/lib/onboard/docker-gpu-patch-constants.ts index 34326d94b68..b51378a6529 100644 --- a/src/lib/onboard/docker-gpu-patch-constants.ts +++ b/src/lib/onboard/docker-gpu-patch-constants.ts @@ -1,8 +1,6 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -export { sleepSeconds } from "../core/wait"; - /** Default timeout for one Docker CLI operation in the compatibility GPU patch path. */ export const DOCKER_GPU_PATCH_TIMEOUT_MS = 30_000; diff --git a/src/lib/onboard/docker-gpu-patch-diagnostics.ts b/src/lib/onboard/docker-gpu-patch-diagnostics.ts index de8a34faa25..cfb61b78ef2 100644 --- a/src/lib/onboard/docker-gpu-patch-diagnostics.ts +++ b/src/lib/onboard/docker-gpu-patch-diagnostics.ts @@ -244,15 +244,6 @@ export function collectDockerGpuPatchDiagnostics( `new_container_id=${redactor.redactText(context?.newContainerId ?? "unknown")}`, `backup_container_name=${redactor.redactText(context?.backupContainerName ?? "none")}`, `backup_removed=${confirmationValue(context?.backupRemoved)}`, - `rollback_image_id=${redactor.redactText(context?.rollbackImageId ?? "none")}`, - `rollback_image_removed=${confirmationValue(context?.rollbackImageRemoved)}`, - `rollback_record_path=${redactor.redactText(context?.rollbackRecordPath ?? "none")}`, - `rollback_record_removed=${confirmationValue(context?.rollbackRecordRemoved)}`, - ...(context?.rollbackRecordPath - ? [ - "rollback_recovery_action=After confirming the replacement container is absent, run the Docker recoveryAction from rollback_record_path.", - ] - : []), `last_openshell_phase=${redactor.redactText(context?.lastSandboxPhase ?? "unknown")}`, `rolled_back=${cleanupPendingRollback ? "pending" : context?.rolledBack === true ? "yes" : context?.rolledBack === false ? "failed" : "no"}`, ...(context?.replacementStopConfirmed !== undefined diff --git a/src/lib/onboard/docker-gpu-patch-finalize.test.ts b/src/lib/onboard/docker-gpu-patch-finalize.test.ts index ae6f51f0cb7..5c917ebe1cb 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.test.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.test.ts @@ -5,25 +5,14 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { afterAll, describe, expect, it, vi } from "vitest"; +import { describe, expect, it, vi } from "vitest"; -import { createDockerGpuInspectFixture as inspectFixture } from "./__test-helpers__/docker-gpu-patch-fixtures"; import { collectDockerGpuPatchDiagnostics, type DockerGpuPatchResult } from "./docker-gpu-patch"; import { type DockerGpuPatchFinalizeOutcome, finalizeDockerGpuPatchBackup, } from "./docker-gpu-patch-finalize"; -const ROLLBACK_IMAGE_ID = `sha256:${"d".repeat(64)}`; -const RESTORED_CONTAINER_ID = "e".repeat(64); -const ROLLBACK_TEST_ROOT = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-docker-gpu-rollback-record-"), -); - -afterAll(() => { - fs.rmSync(ROLLBACK_TEST_ROOT, { recursive: true, force: true }); -}); - function deferredCreateResult(): DockerGpuPatchResult { return { applied: true, @@ -49,48 +38,8 @@ function exactDeferredCreateResult(): DockerGpuPatchResult { }; } -function rollbackPlanDeps(result: DockerGpuPatchResult = exactDeferredCreateResult()) { - const homeDir = fs.mkdtempSync(path.join(ROLLBACK_TEST_ROOT, "case-")); - const inspect = inspectFixture(); - inspect.Config = { - ...inspect.Config, - Env: [...(inspect.Config?.Env ?? []), "ROLLBACK_TEST_TOKEN=super-secret-value"], - }; - return { - dockerCapture: vi.fn(() => JSON.stringify([{ ...inspect, Id: result.oldContainerId }])), - dockerRun: vi.fn((args: readonly string[]) => { - switch (args[0]) { - case "commit": - return { status: 0, stdout: `${ROLLBACK_IMAGE_ID}\n` }; - case "image": - return { status: 0 }; - case "ps": - return { status: 0, stdout: `${result.newContainerId}\n` }; - case "inspect": - return { status: 0, stdout: "true\n" }; - default: - return { status: 1, stderr: "unexpected Docker command" }; - } - }), - dockerRunDetached: vi.fn((_args: readonly string[]) => ({ - status: 0, - stdout: `${RESTORED_CONTAINER_ID}\n`, - stderr: "", - })), - homedir: () => homeDir, - }; -} - -function rollbackRecordFiles(homeDir: string): string[] { - const directory = path.join(homeDir, ".nemoclaw", "recovery", "docker-gpu"); - return fs.existsSync(directory) - ? fs.readdirSync(directory).map((entry) => path.join(directory, entry)) - : []; -} - function readyHandoffDeps() { return { - ...rollbackPlanDeps(), runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 10:00:00 Ready\n"), runOpenshell: vi.fn((args: readonly string[]) => args[1] === "list" @@ -162,7 +111,6 @@ describe("finalizeDockerGpuPatchBackup", () => { it("uses one exact stop, remove, start, and Ready acknowledgement handoff (#9531)", () => { const result = exactDeferredCreateResult(); - const rollbackDeps = rollbackPlanDeps(result); const events: string[] = []; const dockerStop = vi.fn(() => { events.push("stop replacement"); @@ -200,19 +148,9 @@ describe("finalizeDockerGpuPatchBackup", () => { inspect: { event: "confirm running replacement", result: { status: 0, stdout: "true\n" } }, } as const; const dockerRun = vi.fn((args: readonly string[]) => { - switch (args[0]) { - case "commit": - events.push("commit rollback image"); - return rollbackDeps.dockerRun(args); - case "image": - events.push("remove rollback image"); - return rollbackDeps.dockerRun(args); - default: { - const response = dockerResults[String(args[0]) as keyof typeof dockerResults]; - events.push(response.event); - return response.result; - } - } + const response = dockerResults[String(args[0]) as keyof typeof dockerResults]; + events.push(response.event); + return response.result; }); const outcome = finalizeDockerGpuPatchBackup( @@ -223,11 +161,9 @@ describe("finalizeDockerGpuPatchBackup", () => { finalHandoffTimeoutSecs: 60, }, { - dockerCapture: rollbackDeps.dockerCapture, dockerStop, dockerRm, dockerRun, - dockerRunDetached: rollbackDeps.dockerRunDetached, dockerStart, runCaptureOpenshell, runOpenshell, @@ -238,18 +174,14 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(outcome).toEqual({ backupRemoved: true, rolledBack: false, - rollbackImageRemoved: true, - rollbackRecordRemoved: true, replacementStoppedForCommit: true, replacementRestarted: true, lifecycleReleaseObserved: true, finalHandoffAcknowledged: true, lastSandboxPhase: "Ready", }); - expect(rollbackRecordFiles(rollbackDeps.homedir())).toEqual([]); expect(events).toEqual([ "stop replacement", - "commit rollback image", "remove backup", "observe lifecycle release", "start replacement", @@ -257,7 +189,6 @@ describe("finalizeDockerGpuPatchBackup", () => { "exec ready", "confirm sole replacement", "confirm running replacement", - "remove rollback image", ]); expect(dockerStop).toHaveBeenCalledWith( result.newContainerId, @@ -267,20 +198,14 @@ describe("finalizeDockerGpuPatchBackup", () => { result.oldContainerId, expect.objectContaining({ ignoreError: true }), ); - const commitCall = dockerRun.mock.calls.findIndex((call) => call[0][0] === "commit"); - expect(commitCall).toBeGreaterThanOrEqual(0); - expect(dockerRun.mock.invocationCallOrder[commitCall]).toBeLessThan( - dockerRm.mock.invocationCallOrder[0], - ); expect(dockerStart).toHaveBeenCalledWith( result.newContainerId, expect.objectContaining({ ignoreError: true }), ); }); - it("restores the prior container when OpenShell reports Deleting after final start (#9531)", () => { + it("fails immediately when OpenShell reports Deleting after the final start (#9531)", () => { const result = exactDeferredCreateResult(); - const rollbackDeps = rollbackPlanDeps(result); const events: string[] = []; const sleep = vi.fn(); const outcome = finalizeDockerGpuPatchBackup( @@ -291,13 +216,12 @@ describe("finalizeDockerGpuPatchBackup", () => { finalHandoffTimeoutSecs: 60, }, { - dockerCapture: rollbackDeps.dockerCapture, dockerStop: vi.fn(() => { events.push("stop replacement"); return { status: 0 }; }), - dockerRm: vi.fn((target: string) => { - events.push(target === result.oldContainerId ? "remove backup" : "remove replacement"); + dockerRm: vi.fn(() => { + events.push("remove backup"); return { status: 0 }; }), dockerStart: vi.fn(() => { @@ -313,21 +237,14 @@ describe("finalizeDockerGpuPatchBackup", () => { ? { status: 0, stdout: "beta 2026-08-23 10:00:00 Ready\n" } : { status: 1 }, ), - dockerRun: rollbackDeps.dockerRun, - dockerRunDetached: rollbackDeps.dockerRunDetached, + dockerRun: vi.fn(() => ({ status: 0, stdout: `${result.newContainerId}\n` })), sleep, }, ); expect(outcome).toEqual({ backupRemoved: true, - rolledBack: true, - rollbackImageId: ROLLBACK_IMAGE_ID, - rollbackImageRemoved: false, - rollbackRecordRemoved: true, - replacementStopConfirmed: true, - replacementRemovalConfirmed: true, - replacementPresence: "absent", + rolledBack: false, replacementStoppedForCommit: true, replacementRestarted: true, lifecycleReleaseObserved: true, @@ -339,82 +256,19 @@ describe("finalizeDockerGpuPatchBackup", () => { "remove backup", "start replacement", "observe deleting", - "stop replacement", - "remove replacement", ]); expect(sleep).not.toHaveBeenCalled(); - expect(rollbackDeps.dockerRunDetached).toHaveBeenCalledOnce(); - expect(rollbackDeps.dockerRunDetached.mock.calls[0]?.[0]).toEqual( - expect.arrayContaining([ - "--name", - "openshell-alpha", - "--label", - "openshell.ai/managed-by=openshell", - "--label", - "openshell.ai/sandbox-name=alpha", - "--volume", - "/host:/container:rw", - ROLLBACK_IMAGE_ID, - ]), - ); - expect(rollbackDeps.dockerRunDetached.mock.calls[0]?.[0]).not.toContain("--gpus"); - expect(rollbackRecordFiles(rollbackDeps.homedir())).toEqual([]); - const diagnostics = collectRollbackDiagnostics(result.newContainerId, outcome); - expect(diagnostics.summary).toContain(`rollback_image_id=${ROLLBACK_IMAGE_ID}`); - expect(diagnostics.summary).toContain("rollback_image_removed=no"); - expect(diagnostics.summary).toContain("rollback_record_path=none"); - expect(diagnostics.summary).toContain("rollback_record_removed=yes"); - expect(diagnostics.summary).toContain("rolled_back=yes"); - }); - - it("reports a retained rollback image when successful-handoff cleanup fails (#9531)", () => { - const result = exactDeferredCreateResult(); - const deps = readyHandoffDeps(); - const baseDockerRun = deps.dockerRun; - deps.dockerRun = vi.fn((args: readonly string[]) => - args[0] === "image" ? { status: 1, stderr: "image is busy" } : baseDockerRun(args), - ); - const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); - - const outcome = finalizeDockerGpuPatchBackup( - { - result, - supervisorReady: true, - sandboxName: "alpha", - finalHandoffTimeoutSecs: 60, - }, - { - ...deps, - dockerStop: vi.fn(() => ({ status: 0 })), - dockerRm: vi.fn(() => ({ status: 0 })), - dockerStart: vi.fn(() => ({ status: 0 })), - }, - ); - - expect(outcome).toMatchObject({ - backupRemoved: true, - rolledBack: false, - rollbackImageId: ROLLBACK_IMAGE_ID, - rollbackImageRemoved: false, - rollbackRecordRemoved: true, - finalHandoffAcknowledged: true, - }); - expect(warn).toHaveBeenCalledWith( - ` Could not remove temporary Docker rollback image ${ROLLBACK_IMAGE_ID}.`, - ); }); it("accepts a retiring Error row only when the exact replacement has the OpenShell label (#9962)", () => { const result = exactDeferredCreateResult(); - const rollbackDeps = rollbackPlanDeps(result); const dockerRunResults = { inspect: { status: 0, stdout: "true\n" }, ps: { status: 0, stdout: `${result.newContainerId}\n` }, } as const; - const dockerRun = vi.fn((args: readonly string[]) => - args[0] === "commit" || args[0] === "image" - ? rollbackDeps.dockerRun(args) - : dockerRunResults[String(args[0]) as keyof typeof dockerRunResults], + const dockerRun = vi.fn( + (args: readonly string[]) => + dockerRunResults[String(args[0]) as keyof typeof dockerRunResults], ); const outcome = finalizeDockerGpuPatchBackup( @@ -425,12 +279,10 @@ describe("finalizeDockerGpuPatchBackup", () => { finalHandoffTimeoutSecs: 60, }, { - dockerCapture: rollbackDeps.dockerCapture, dockerStop: vi.fn(() => ({ status: 0 })), dockerRm: vi.fn(() => ({ status: 0 })), dockerStart: vi.fn(() => ({ status: 0 })), dockerRun, - dockerRunDetached: rollbackDeps.dockerRunDetached, runCaptureOpenshell: vi.fn(() => "restored-name 2026-08-23 01:40:35 Ready\n"), runOpenshell: vi.fn((args: readonly string[]) => args[1] === "list" @@ -446,7 +298,7 @@ describe("finalizeDockerGpuPatchBackup", () => { lifecycleReleaseObserved: true, finalHandoffAcknowledged: true, }); - expect(dockerRun.mock.calls.find((call) => call[0][0] === "ps")?.[0]).toEqual([ + expect(dockerRun.mock.calls[0]?.[0]).toEqual([ "ps", "-a", "--no-trunc", @@ -467,13 +319,9 @@ describe("finalizeDockerGpuPatchBackup", () => { "multiple labeled replacements", { status: 0, stdout: `${"b".repeat(64)}\n${"c".repeat(64)}\n` }, ], - ])("restores the prior container when lifecycle release has %s (#9531)", (_case, query) => { + ])("rejects retiring Error when the canonical query returns %s (#9531)", (_case, query) => { const result = exactDeferredCreateResult(); - const rollbackDeps = rollbackPlanDeps(result); const dockerStart = vi.fn(() => ({ status: 0 })); - const dockerRun = vi.fn((args: readonly string[]) => - args[0] === "commit" || args[0] === "image" ? rollbackDeps.dockerRun(args) : query, - ); const outcome = finalizeDockerGpuPatchBackup( { @@ -483,11 +331,9 @@ describe("finalizeDockerGpuPatchBackup", () => { finalHandoffTimeoutSecs: 1, }, { - dockerCapture: rollbackDeps.dockerCapture, dockerStop: vi.fn(() => ({ status: 0 })), dockerRm: vi.fn(() => ({ status: 0 })), - dockerRun, - dockerRunDetached: rollbackDeps.dockerRunDetached, + dockerRun: vi.fn(() => query), dockerStart, runCaptureOpenshell: vi.fn(() => "alpha 2026-08-23 01:40:35 Ready\n"), runOpenshell: vi.fn(() => ({ @@ -500,13 +346,10 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(outcome).toMatchObject({ backupRemoved: true, - rolledBack: true, - rollbackImageId: ROLLBACK_IMAGE_ID, lifecycleReleaseObserved: false, replacementRestarted: false, }); expect(dockerStart).not.toHaveBeenCalled(); - expect(rollbackDeps.dockerRunDetached).toHaveBeenCalledOnce(); }); it("rolls back to the backup container when supervisor reconnect failed", () => { @@ -611,17 +454,16 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(dockerRm).not.toHaveBeenCalled(); }); - it.each([ - ["an explicit failure", { status: 1, stderr: "container is in use" }], - ["no exit status", { status: null, stderr: "timed out" }], - ])("retains the backup when Docker removal returns %s (#9531)", (_case, removal) => { - const result = exactDeferredCreateResult(); + it("reports backupRemoved=false when supervisor reconnect succeeded but docker rm of the backup failed", () => { const dockerStop = vi.fn(() => ({ status: 0 })); - const dockerRm = vi.fn(() => removal); + const dockerRm = vi.fn((_name: string) => ({ + status: 1, + stderr: "Error response from daemon: container is in use", + })); const dockerStart = vi.fn(() => ({ status: 0 })); const outcome = finalizeDockerGpuPatchBackup( { - result, + result: deferredCreateResult(), supervisorReady: true, sandboxName: "alpha", finalHandoffTimeoutSecs: 60, @@ -631,28 +473,25 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(outcome).toEqual({ backupRemoved: false, rolledBack: false, - rollbackImageRemoved: true, - rollbackRecordRemoved: true, replacementStoppedForCommit: true, replacementRestarted: true, finalHandoffAcknowledged: false, lastSandboxPhase: null, }); expect(dockerRm).toHaveBeenCalledWith( - result.oldContainerId, + "old-container-id", expect.objectContaining({ ignoreError: true }), ); expect(dockerStart).toHaveBeenCalledWith( - result.newContainerId, + "new-container-id", expect.objectContaining({ ignoreError: true }), ); }); - it("retains the backup when the replacement cannot be stopped for the final handoff", () => { - const dockerStop = vi.fn(() => ({ status: 1 })); - const dockerRm = vi.fn(() => ({ status: 0 })); + it("fails closed when backup removal has no exit status", () => { + const dockerStop = vi.fn(() => ({ status: 0 })); + const dockerRm = vi.fn((_name: string) => ({ status: null, stderr: "timed out" })); const dockerStart = vi.fn(() => ({ status: 0 })); - const outcome = finalizeDockerGpuPatchBackup( { result: deferredCreateResult(), @@ -662,58 +501,48 @@ describe("finalizeDockerGpuPatchBackup", () => { }, { dockerStop, dockerRm, dockerStart, ...readyHandoffDeps() }, ); - expect(outcome).toEqual({ backupRemoved: false, rolledBack: false, - replacementStoppedForCommit: false, + replacementStoppedForCommit: true, + replacementRestarted: true, + finalHandoffAcknowledged: false, + lastSandboxPhase: null, }); - expect(dockerRm).not.toHaveBeenCalled(); - expect(dockerStart).not.toHaveBeenCalled(); + expect(dockerStart).toHaveBeenCalledWith( + "new-container-id", + expect.objectContaining({ ignoreError: true }), + ); }); - it("retains the backup when Docker cannot prepare the rollback image (#9531)", () => { - const result = exactDeferredCreateResult(); + it("retains the backup when the replacement cannot be stopped for the final handoff", () => { + const dockerStop = vi.fn(() => ({ status: 1 })); const dockerRm = vi.fn(() => ({ status: 0 })); const dockerStart = vi.fn(() => ({ status: 0 })); const outcome = finalizeDockerGpuPatchBackup( { - result, + result: deferredCreateResult(), supervisorReady: true, sandboxName: "alpha", finalHandoffTimeoutSecs: 60, }, - { - ...readyHandoffDeps(), - dockerCapture: rollbackPlanDeps(result).dockerCapture, - dockerRun: vi.fn(() => ({ status: 1, stderr: "commit failed" })), - dockerStop: vi.fn(() => ({ status: 0 })), - dockerRm, - dockerStart, - }, + { dockerStop, dockerRm, dockerStart, ...readyHandoffDeps() }, ); expect(outcome).toEqual({ backupRemoved: false, rolledBack: false, - replacementStoppedForCommit: true, - replacementRestarted: true, - finalHandoffAcknowledged: false, - lastSandboxPhase: null, + replacementStoppedForCommit: false, }); expect(dockerRm).not.toHaveBeenCalled(); - expect(dockerStart).toHaveBeenCalledWith( - result.newContainerId, - expect.objectContaining({ ignoreError: true }), - ); + expect(dockerStart).not.toHaveBeenCalled(); }); - it("restores the prior container when replacement restart fails after removal (#9531)", () => { - const result = exactDeferredCreateResult(); + it("reports a failed replacement restart after the backup is removed", () => { const outcome = finalizeDockerGpuPatchBackup( { - result, + result: deferredCreateResult(), supervisorReady: true, sandboxName: "alpha", finalHandoffTimeoutSecs: 60, @@ -728,13 +557,7 @@ describe("finalizeDockerGpuPatchBackup", () => { expect(outcome).toEqual({ backupRemoved: true, - rolledBack: true, - rollbackImageId: ROLLBACK_IMAGE_ID, - rollbackImageRemoved: false, - rollbackRecordRemoved: true, - replacementStopConfirmed: true, - replacementRemovalConfirmed: true, - replacementPresence: "absent", + rolledBack: false, replacementStoppedForCommit: true, replacementRestarted: false, finalHandoffAcknowledged: false, @@ -742,128 +565,6 @@ describe("finalizeDockerGpuPatchBackup", () => { }); }); - it("retains the rollback image when post-commit restoration fails (#9531)", () => { - const result = exactDeferredCreateResult(); - const deps = readyHandoffDeps(); - const inspect = inspectFixture(); - inspect.Config = { - ...inspect.Config, - Env: [ - ...(inspect.Config?.Env ?? []).filter( - (entry) => !entry.startsWith("OPENSHELL_SANDBOX_COMMAND="), - ), - "ROLLBACK_TEST_TOKEN=super-secret-value", - "OPENSHELL_SANDBOX_COMMAND=env NEMOCLAW_EXTRA_PLACEHOLDER_KEYS=CUSTOM_PROVIDER_VALUE CUSTOM_PROVIDER_VALUE=startup-secret CHAT_UI_URL=http://127.0.0.1:8642 nemoclaw-start", - ], - Labels: { - ...inspect.Config?.Labels, - "untrusted.example/token": "label-secret", - }, - }; - deps.dockerCapture.mockReturnValue(JSON.stringify([{ ...inspect, Id: result.oldContainerId }])); - deps.dockerRunDetached.mockReturnValue({ status: 1, stdout: "", stderr: "restore failed" }); - - const outcome = finalizeDockerGpuPatchBackup( - { - result, - supervisorReady: true, - sandboxName: "alpha", - finalHandoffTimeoutSecs: 60, - }, - { - ...deps, - dockerStop: vi.fn(() => ({ status: 0 })), - dockerRm: vi.fn(() => ({ status: 0 })), - dockerStart: vi.fn(() => ({ status: 1 })), - }, - ); - - expect(outcome).toMatchObject({ - backupRemoved: true, - rolledBack: false, - rollbackImageId: ROLLBACK_IMAGE_ID, - rollbackImageRemoved: false, - rollbackRecordRemoved: false, - replacementPresence: "absent", - replacementRestarted: false, - }); - expect(deps.dockerRun).not.toHaveBeenCalledWith( - ["image", "rm", "--force", ROLLBACK_IMAGE_ID], - expect.anything(), - ); - const recordPath = outcome.rollbackRecordPath; - expect(recordPath).toBeTypeOf("string"); - const recordFd = fs.openSync(recordPath as string, "r"); - let record: string; - try { - expect(fs.fstatSync(recordFd).mode & 0o777).toBe(0o600); - record = fs.readFileSync(recordFd, "utf8"); - } finally { - fs.closeSync(recordFd); - } - expect(record).toContain(ROLLBACK_IMAGE_ID); - expect(record).toContain('"command": "docker"'); - expect(record).toContain('"run"'); - expect(record).toContain('"--detach"'); - expect(record).toContain("OPENSHELL_ENDPOINT=http://host.openshell.internal:8080/"); - expect(record).toContain( - "OPENSHELL_SANDBOX_COMMAND=env NEMOCLAW_EXTRA_PLACEHOLDER_KEYS=CUSTOM_PROVIDER_VALUE CHAT_UI_URL=http://127.0.0.1:8642 nemoclaw-start", - ); - expect(record).toContain("openshell.ai/managed-by=openshell"); - expect(record).toContain("openshell.ai/sandbox-name=alpha"); - expect(record).toContain("openshell.ai/sandbox-id=sandbox-id"); - expect(record).not.toContain("super-secret-value"); - expect(record).not.toContain("ROLLBACK_TEST_TOKEN"); - expect(record).not.toContain("startup-secret"); - expect(record).not.toContain("CUSTOM_PROVIDER_VALUE=startup-secret"); - expect(record).not.toContain("OPENSHELL_TEST=1"); - expect(record).not.toContain("untrusted.example/token"); - expect(record).not.toContain("label-secret"); - const diagnostics = collectRollbackDiagnostics(result.newContainerId, outcome); - expect(diagnostics.summary).toContain("rollback_record_removed=no"); - expect(diagnostics.summary).toContain("rollback_recovery_action="); - }); - - it("refuses a symlinked rollback-record path before removing the backup (#9531)", () => { - const result = exactDeferredCreateResult(); - const deps = readyHandoffDeps(); - const homeDir = deps.homedir(); - const redirectedDirectory = fs.mkdtempSync(path.join(ROLLBACK_TEST_ROOT, "redirected-")); - fs.mkdirSync(path.join(homeDir, ".nemoclaw"), { recursive: true }); - fs.symlinkSync(redirectedDirectory, path.join(homeDir, ".nemoclaw", "recovery"), "dir"); - const dockerRm = vi.fn(() => ({ status: 0 })); - const dockerStart = vi.fn(() => ({ status: 0 })); - vi.stubEnv("HOME", homeDir); - let outcome: DockerGpuPatchFinalizeOutcome; - try { - outcome = finalizeDockerGpuPatchBackup( - { - result, - supervisorReady: true, - sandboxName: "alpha", - finalHandoffTimeoutSecs: 60, - }, - { ...deps, dockerRm, dockerStart, dockerStop: vi.fn(() => ({ status: 0 })) }, - ); - } finally { - vi.unstubAllEnvs(); - } - - expect(outcome).toMatchObject({ - backupRemoved: false, - rolledBack: false, - replacementStoppedForCommit: true, - replacementRestarted: true, - finalHandoffAcknowledged: false, - }); - expect(dockerRm).not.toHaveBeenCalled(); - expect(fs.readdirSync(redirectedDirectory)).toEqual([]); - expect(deps.dockerRun).toHaveBeenCalledWith( - ["image", "rm", "--force", ROLLBACK_IMAGE_ID], - expect.anything(), - ); - }); - it("records a remaining exact-ID replacement when removal fails (#7996)", () => { const newContainerId = "a".repeat(64); const outcome = finalizeDockerGpuPatchBackup( @@ -994,4 +695,23 @@ describe("finalizeDockerGpuPatchBackup", () => { }); expect(dockerStart).not.toHaveBeenCalled(); }); + + it("fails closed when rollback start has no exit status", () => { + const outcome = finalizeDockerGpuPatchBackup( + { result: deferredCreateResult(), supervisorReady: false }, + { + dockerStop: vi.fn(() => ({ status: 0 })), + dockerRm: vi.fn(() => ({ status: 0 })), + dockerRename: vi.fn(() => ({ status: 0 })), + dockerStart: vi.fn(() => ({ status: null })), + }, + ); + expect(outcome).toEqual({ + backupRemoved: false, + rolledBack: false, + replacementStopConfirmed: true, + replacementRemovalConfirmed: true, + replacementPresence: "absent", + }); + }); }); diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index af65b9753f8..c874626d0db 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -11,14 +11,14 @@ // supports that natively, NemoClaw recreates the container with GPU access // and uses this module to either restore the pre-patch backup before commit or // complete the exact stop/remove/start handoff and require OpenShell's final -// Ready acknowledgement. Before removing the old container, NemoClaw commits -// its writable layer to a temporary image and records the exact run arguments -// needed to restore it. Regression coverage: +// Ready acknowledgement. Removing the old container is the irreversible +// commit point: later failures require a sandbox rebuild rather than automatic +// rollback. Regression coverage: // * src/lib/onboard/docker-gpu-patch-finalize.test.ts — direct unit tests // for exact final handoff, terminal phase, rollback, and failure outcomes. // * src/lib/onboard/docker-gpu-patch-rollback.test.ts — composed // recreate-with-rollback scenarios. -// * src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts — composed create +// * src/lib/onboard/docker-gpu-sandbox-create.test.ts — composed create // flow driving maybeApplyDuringCreate → waitForSupervisorReconnect → // finalizeBackup. // Removal condition: when OpenShell supports native Docker-driver GPU @@ -26,42 +26,24 @@ // and delete this module along with its callers in docker-gpu-patch.ts and // docker-gpu-sandbox-create.ts. -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { shouldStripCredentialEnv } from "../security/credential-env"; -import { rejectSymlinksOnPath } from "../state/config-io"; import { hasZeroDockerExitStatus } from "./docker-command-result"; import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; import { - type ResolvedDockerGpuPatchRollbackDeps, - removeReplacementContainer, resolveDockerGpuPatchRollbackDeps, rollbackToBackupContainer, } from "./docker-gpu-patch-rollback"; -import { - buildDockerGpuCloneRunArgs, - buildDockerGpuCloneRunOptions, - fullDockerContainerId, - parseDockerInspectJson, -} from "./docker-gpu-patch-clone"; -import { buildDockerGpuMode } from "./docker-gpu-patch-mode"; -import type { - DockerContainerInspect, - DockerGpuPatchDeps, - DockerGpuPatchResult, -} from "./docker-gpu-patch-types"; +import { fullDockerContainerId } from "./docker-gpu-patch-clone"; +import type { DockerGpuPatchDeps, DockerGpuPatchResult } from "./docker-gpu-patch-types"; import { waitForOpenShellFinalHandoff, waitForOpenShellSandboxLifecycleRelease, } from "./docker-gpu-supervisor-reconnect"; -import { - OPENSHELL_MANAGED_BY_LABEL, - OPENSHELL_SANDBOX_ID_LABEL, - OPENSHELL_SANDBOX_NAME_LABEL, - queryOpenShellDockerSandboxContainers, -} from "./openshell-docker-sandbox-containers"; +import { queryOpenShellDockerSandboxContainers } from "./openshell-docker-sandbox-containers"; + +export { + restoreDockerGpuPatchBackupAfterRecreateFailure as rollbackDockerGpuPatchOnRecreateFailure, + rollbackToBackupContainer, +} from "./docker-gpu-patch-rollback"; export type DockerGpuPatchFinalizeOptions = | { @@ -76,13 +58,9 @@ export type DockerGpuPatchFinalizeOptions = }; export type DockerGpuPatchFinalizeOutcome = { - /** True once Docker has removed the old container. */ + /** True once the old container has crossed the irreversible removal boundary. */ backupRemoved: boolean; rolledBack: boolean; - rollbackImageId?: string; - rollbackImageRemoved?: boolean; - rollbackRecordPath?: string; - rollbackRecordRemoved?: boolean; replacementStoppedForCommit?: boolean; replacementRestarted?: boolean; lifecycleReleaseObserved?: boolean; @@ -93,303 +71,6 @@ export type DockerGpuPatchFinalizeOutcome = { replacementPresence?: "absent" | "present" | "unknown"; }; -type PostCommitRollbackPlan = { - imageId: string; - oldContainerId: string; - recordPath: string; - runArgs: string[]; -}; - -const DOCKER_IMAGE_ID = /^sha256:[0-9a-f]{64}$/iu; -const ROLLBACK_RECORD_VERSION = 1; -const RECOVERY_ENV_KEYS = new Set([ - "OPENSHELL_ENDPOINT", - "OPENSHELL_SANDBOX_COMMAND", - "OPENSHELL_OCI_IMAGE_USER", - "OPENSHELL_SANDBOX_UID", - "OPENSHELL_SANDBOX_GID", -]); -const RECOVERY_LABEL_KEYS = new Set([ - OPENSHELL_MANAGED_BY_LABEL, - OPENSHELL_SANDBOX_NAME_LABEL, - OPENSHELL_SANDBOX_ID_LABEL, -]); -const ENV_KEY = /^[A-Za-z_][A-Za-z0-9_]*$/u; -const OCI_IDENTITY = /^[A-Za-z0-9_.-]+(?::[A-Za-z0-9_.-]+)?$/u; -const NUMERIC_IDENTITY = /^(?:0|[1-9][0-9]*)$/u; -const NEMOCLAW_STARTUP_EXECUTABLES = new Set(["nemoclaw-start", "/usr/local/bin/nemoclaw-start"]); - -function removeRollbackImage( - imageId: string, - deps: ResolvedDockerGpuPatchRollbackDeps, - options: Record, -): boolean { - return hasZeroDockerExitStatus(deps.dockerRun(["image", "rm", "--force", imageId], options)); -} - -function envAssignment(value: string): { key: string; value: string } | null { - const separator = value.indexOf("="); - if (separator <= 0) return null; - const key = value.slice(0, separator); - return ENV_KEY.test(key) ? { key, value: value.slice(separator + 1) } : null; -} - -function isCredentialFreeHttpUrl(value: string): boolean { - try { - const parsed = new URL(value); - return ( - ["http:", "https:"].includes(parsed.protocol) && - parsed.username === "" && - parsed.password === "" && - parsed.search === "" && - parsed.hash === "" - ); - } catch { - return false; - } -} - -function recoverySandboxCommand(value: string): string | null { - const tokens = value.trim().split(/\s+/u).filter(Boolean); - if (tokens.length === 2 && tokens[0] === "sleep" && tokens[1] === "infinity") return value; - if (tokens.length === 1 && NEMOCLAW_STARTUP_EXECUTABLES.has(tokens[0])) return value; - if (tokens[0] !== "env") return null; - - const assignments: Array<{ key: string; value: string }> = []; - let executableIndex = 1; - while (executableIndex < tokens.length) { - const assignment = envAssignment(tokens[executableIndex]); - if (!assignment) break; - assignments.push(assignment); - executableIndex += 1; - } - if ( - executableIndex !== tokens.length - 1 || - !NEMOCLAW_STARTUP_EXECUTABLES.has(tokens[executableIndex]) - ) { - return null; - } - const extraPlaceholderKeys = new Set( - (assignments.find(({ key }) => key === "NEMOCLAW_EXTRA_PLACEHOLDER_KEYS")?.value ?? "") - .split(/[\s,]+/u) - .filter((key) => ENV_KEY.test(key)), - ); - const retained = assignments - .filter( - ({ key, value: assignmentValue }) => - !shouldStripCredentialEnv(key) && - !extraPlaceholderKeys.has(key) && - (!assignmentValue.includes("://") || isCredentialFreeHttpUrl(assignmentValue)), - ) - .map(({ key, value: assignmentValue }) => `${key}=${assignmentValue}`); - return ["env", ...retained, tokens[executableIndex]].join(" "); -} - -function recoveryRunValue(flag: string, value: string): string | null { - if (flag === "--label") { - const separator = value.indexOf("="); - const key = separator > 0 ? value.slice(0, separator) : ""; - return RECOVERY_LABEL_KEYS.has(key) ? value : null; - } - const assignment = envAssignment(value); - if (!assignment) return null; - if (flag !== "--env" || !RECOVERY_ENV_KEYS.has(assignment.key)) return null; - if (assignment.key === "OPENSHELL_SANDBOX_COMMAND") { - const command = recoverySandboxCommand(assignment.value); - return command === null ? null : `${assignment.key}=${command}`; - } - if (assignment.key === "OPENSHELL_ENDPOINT") - return isCredentialFreeHttpUrl(assignment.value) ? value : null; - if (assignment.key === "OPENSHELL_OCI_IMAGE_USER") - return OCI_IDENTITY.test(assignment.value) ? value : null; - if (assignment.key === "OPENSHELL_SANDBOX_UID" || assignment.key === "OPENSHELL_SANDBOX_GID") - return assignment.value === "" || NUMERIC_IDENTITY.test(assignment.value) ? value : null; - return value; -} - -function recoveryRunArgs(runArgs: readonly string[], imageId: string): string[] { - const imageIndex = runArgs.indexOf(imageId); - if (imageIndex < 0) - throw new Error("Docker rollback arguments do not include the rollback image."); - const persisted: string[] = []; - for (let index = 0; index < imageIndex; index += 1) { - const value = runArgs[index]; - if (value === "--env" || value === "--label") { - if (index + 1 >= imageIndex) { - throw new Error(`Docker rollback argument '${value}' has no value.`); - } - const persistedValue = recoveryRunValue(value, runArgs[index + 1]); - index += 1; - if (persistedValue !== null) persisted.push(value, persistedValue); - continue; - } - persisted.push(value); - } - return [...persisted, imageId]; -} - -function writeRollbackRecord( - sandboxName: string, - oldContainerId: string, - imageId: string, - runArgs: readonly string[], - deps: DockerGpuPatchDeps, -): string | null { - const directory = path.join( - (deps.homedir ?? os.homedir)(), - ".nemoclaw", - "recovery", - "docker-gpu", - ); - const recordPath = path.join( - directory, - `${encodeURIComponent(sandboxName)}-${oldContainerId.slice(0, 12)}.json`, - ); - try { - rejectSymlinksOnPath(directory); - fs.mkdirSync(directory, { recursive: true, mode: 0o700 }); - rejectSymlinksOnPath(directory); - fs.chmodSync(directory, 0o700); - fs.writeFileSync( - recordPath, - `${JSON.stringify( - { - version: ROLLBACK_RECORD_VERSION, - sandboxName, - rollbackImageId: imageId, - recoveryAction: { - command: "docker", - args: ["run", "--detach", ...recoveryRunArgs(runArgs, imageId)], - }, - }, - null, - 2, - )}\n`, - { encoding: "utf8", flag: "wx", mode: 0o600 }, - ); - fs.chmodSync(recordPath, 0o600); - return recordPath; - } catch { - return null; - } -} - -function removeRollbackRecord(recordPath: string): boolean { - try { - fs.rmSync(recordPath, { force: true }); - return !fs.existsSync(recordPath); - } catch { - return false; - } -} - -function preparePostCommitRollback( - result: DockerGpuPatchResult, - sandboxName: string, - deps: DockerGpuPatchDeps, - resolved: ResolvedDockerGpuPatchRollbackDeps, - options: Record, -): PostCommitRollbackPlan | null { - const oldContainerId = fullDockerContainerId(result.oldContainerId); - if (!oldContainerId) return null; - let inspect: DockerContainerInspect; - try { - inspect = parseDockerInspectJson( - resolved.dockerCapture(["inspect", "--type", "container", oldContainerId], { - ignoreError: true, - timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, - }), - ); - } catch { - return null; - } - - const committed = resolved.dockerRun(["commit", oldContainerId], options); - if (!hasZeroDockerExitStatus(committed)) return null; - const imageId = String(committed.stdout ?? "").trim(); - if (!DOCKER_IMAGE_ID.test(imageId)) return null; - - try { - const cloneOptions = buildDockerGpuCloneRunOptions(inspect); - cloneOptions.containerName = result.originalName; - cloneOptions.image = imageId; - const runArgs = buildDockerGpuCloneRunArgs( - inspect, - buildDockerGpuMode("startup-command"), - cloneOptions, - ); - const recordPath = writeRollbackRecord(sandboxName, oldContainerId, imageId, runArgs, deps); - if (!recordPath) { - if (!removeRollbackImage(imageId, resolved, options)) { - console.warn(` Could not remove temporary Docker rollback image ${imageId}.`); - } - return null; - } - return { - imageId, - oldContainerId, - recordPath, - runArgs, - }; - } catch { - if (!removeRollbackImage(imageId, resolved, options)) { - console.warn(` Could not remove temporary Docker rollback image ${imageId}.`); - } - return null; - } -} - -function restorePostCommitRollback( - plan: PostCommitRollbackPlan, - result: DockerGpuPatchResult, - resolved: ResolvedDockerGpuPatchRollbackDeps, - options: Record, -): Pick< - DockerGpuPatchFinalizeOutcome, - | "rolledBack" - | "rollbackImageId" - | "rollbackImageRemoved" - | "rollbackRecordPath" - | "rollbackRecordRemoved" - | "replacementStopConfirmed" - | "replacementRemovalConfirmed" - | "replacementPresence" -> { - const replacementContainerId = fullDockerContainerId(result.newContainerId); - if (!replacementContainerId) { - return { - rolledBack: false, - rollbackImageId: plan.imageId, - rollbackImageRemoved: false, - rollbackRecordPath: plan.recordPath, - rollbackRecordRemoved: false, - replacementStopConfirmed: false, - replacementRemovalConfirmed: false, - replacementPresence: "unknown", - }; - } - const removal = removeReplacementContainer(replacementContainerId, resolved); - const restored = - removal.replacementPresence === "absent" - ? resolved.dockerRunDetached(plan.runArgs, options) - : null; - const restoredContainerId = - restored !== null && hasZeroDockerExitStatus(restored) - ? fullDockerContainerId(String(restored.stdout ?? "")) - : null; - const rollbackRecordRemoved = - restoredContainerId !== null && removeRollbackRecord(plan.recordPath); - return { - rolledBack: restoredContainerId !== null, - rollbackImageId: plan.imageId, - rollbackImageRemoved: false, - ...(rollbackRecordRemoved - ? { rollbackRecordRemoved: true } - : { rollbackRecordPath: plan.recordPath, rollbackRecordRemoved: false }), - ...removal, - }; -} - function isExactRunningReplacement( sandboxName: string, replacementContainerId: string, @@ -447,8 +128,10 @@ export function finalizeDockerGpuPatchBackup( if (options.supervisorReady) { // Stop the exact replacement before retiring the exact backup, then start // the replacement afterward. The final start is the authoritative Docker - // lifecycle event. Keep a committed rollback image until OpenShell accepts - // the exact replacement as Ready (#9531). + // lifecycle event. Backup removal is the irreversible commit point; + // failures after it require a sandbox rebuild. Success is withheld until + // OpenShell reports Ready and Docker still proves the exact replacement is + // the sole running labeled container (#9531). if (!deps.runOpenshell || !deps.runCaptureOpenshell) { return { backupRemoved: false, @@ -466,47 +149,15 @@ export function finalizeDockerGpuPatchBackup( replacementStoppedForCommit: false, }; } - const rollbackPlan = preparePostCommitRollback( - options.result, - options.sandboxName, - deps, - resolved, - containerOpts, - ); - if (!rollbackPlan) { - const replacementRestarted = hasZeroDockerExitStatus( - resolved.dockerStart(options.result.newContainerId, containerOpts), - ); - return { - backupRemoved: false, - rolledBack: false, - replacementStoppedForCommit: true, - replacementRestarted, - finalHandoffAcknowledged: false, - lastSandboxPhase: null, - }; - } - const rmResult = resolved.dockerRm(rollbackPlan.oldContainerId, containerOpts); + const rmResult = resolved.dockerRm(options.result.oldContainerId, containerOpts); const backupRemoved = hasZeroDockerExitStatus(rmResult); if (!backupRemoved) { - const rollbackImageRemoved = removeRollbackImage( - rollbackPlan.imageId, - resolved, - containerOpts, - ); - const rollbackRecordRemoved = removeRollbackRecord(rollbackPlan.recordPath); const replacementRestarted = hasZeroDockerExitStatus( resolved.dockerStart(options.result.newContainerId, containerOpts), ); return { backupRemoved: false, rolledBack: false, - ...(rollbackImageRemoved - ? { rollbackImageRemoved: true } - : { rollbackImageId: rollbackPlan.imageId, rollbackImageRemoved: false }), - ...(rollbackRecordRemoved - ? { rollbackRecordRemoved: true } - : { rollbackRecordPath: rollbackPlan.recordPath, rollbackRecordRemoved: false }), replacementStoppedForCommit: true, replacementRestarted, finalHandoffAcknowledged: false, @@ -539,15 +190,9 @@ export function finalizeDockerGpuPatchBackup( }, ); if (!lifecycleReleaseObserved) { - const rollback = restorePostCommitRollback( - rollbackPlan, - options.result, - resolved, - containerOpts, - ); return { backupRemoved: true, - ...rollback, + rolledBack: false, replacementStoppedForCommit: true, replacementRestarted: false, lifecycleReleaseObserved: false, @@ -558,15 +203,9 @@ export function finalizeDockerGpuPatchBackup( const startResult = resolved.dockerStart(options.result.newContainerId, containerOpts); const replacementRestarted = hasZeroDockerExitStatus(startResult); if (!replacementRestarted) { - const rollback = restorePostCommitRollback( - rollbackPlan, - options.result, - resolved, - containerOpts, - ); return { backupRemoved: true, - ...rollback, + rolledBack: false, replacementStoppedForCommit: true, replacementRestarted, finalHandoffAcknowledged: false, @@ -592,44 +231,13 @@ export function finalizeDockerGpuPatchBackup( ), }, ); - if (!acknowledgement.acknowledged) { - const rollback = restorePostCommitRollback( - rollbackPlan, - options.result, - resolved, - containerOpts, - ); - return { - backupRemoved: true, - ...rollback, - replacementStoppedForCommit: true, - replacementRestarted: true, - lifecycleReleaseObserved: true, - finalHandoffAcknowledged: false, - lastSandboxPhase: acknowledgement.lastSandboxPhase, - }; - } - const rollbackImageRemoved = removeRollbackImage(rollbackPlan.imageId, resolved, containerOpts); - const rollbackRecordRemoved = removeRollbackRecord(rollbackPlan.recordPath); - if (!rollbackImageRemoved) { - console.warn(` Could not remove temporary Docker rollback image ${rollbackPlan.imageId}.`); - } - if (!rollbackRecordRemoved) { - console.warn(` Could not remove Docker rollback record ${rollbackPlan.recordPath}.`); - } return { backupRemoved: true, rolledBack: false, - ...(rollbackImageRemoved - ? { rollbackImageRemoved: true } - : { rollbackImageId: rollbackPlan.imageId, rollbackImageRemoved: false }), - ...(rollbackRecordRemoved - ? { rollbackRecordRemoved: true } - : { rollbackRecordPath: rollbackPlan.recordPath, rollbackRecordRemoved: false }), replacementStoppedForCommit: true, replacementRestarted: true, lifecycleReleaseObserved: true, - finalHandoffAcknowledged: true, + finalHandoffAcknowledged: acknowledgement.acknowledged, lastSandboxPhase: acknowledgement.lastSandboxPhase, }; } diff --git a/src/lib/onboard/docker-gpu-patch-mode.ts b/src/lib/onboard/docker-gpu-patch-mode.ts index c86155d4192..3ab3acd970a 100644 --- a/src/lib/onboard/docker-gpu-patch-mode.ts +++ b/src/lib/onboard/docker-gpu-patch-mode.ts @@ -5,8 +5,9 @@ import fs from "node:fs"; import path from "node:path"; import { dockerCapture, dockerRm, dockerRun } from "../adapters/docker"; +import { sleepSeconds } from "../core/wait"; import { hasZeroDockerExitStatus } from "./docker-command-result"; -import { DOCKER_GPU_PATCH_TIMEOUT_MS, sleepSeconds } from "./docker-gpu-patch-constants"; +import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; import { normalizeSandboxGpuDeviceForCdi } from "./sandbox-gpu-create"; import type { DockerGpuPatchBackend, diff --git a/src/lib/onboard/docker-gpu-patch-recreate.test.ts b/src/lib/onboard/docker-gpu-patch-recreate.test.ts index 74be50cd87d..98867a8a3d4 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.test.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.test.ts @@ -1,11 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { afterAll, describe, expect, it, vi } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { createDockerGpuInspectFixture as inspectFixture } from "./__test-helpers__/docker-gpu-patch-fixtures"; import { getDockerGpuPatchFailureContext, @@ -14,17 +10,6 @@ import { const OLD_CONTAINER_ID = "a".repeat(64); const NEW_CONTAINER_ID = "b".repeat(64); -const ROLLBACK_IMAGE_ID = `sha256:${"d".repeat(64)}`; -const RESTORED_CONTAINER_ID = "e".repeat(64); -const RECREATE_TEST_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gpu-recreate-")); - -afterAll(() => { - fs.rmSync(RECREATE_TEST_ROOT, { recursive: true, force: true }); -}); - -function testHome(): string { - return fs.mkdtempSync(path.join(RECREATE_TEST_ROOT, "case-")); -} function dockerCaptureFixture() { const responses: Record = { @@ -42,21 +27,13 @@ describe("Docker GPU recreate orchestration", () => { ps: { status: 0, stdout: `${NEW_CONTAINER_ID}\n` }, inspect: { status: 0, stdout: "true\n" }, }; - const dockerRun = vi.fn((args: readonly string[]) => { - switch (args[0]) { - case "commit": - return { status: 0, stdout: `${ROLLBACK_IMAGE_ID}\n` }; - case "image": - return { status: 0 }; - default: - return ( - dockerRunResults[String(args[0]) as keyof typeof dockerRunResults] ?? { - status: 0, - stdout: "probe-id\n", - } - ); - } - }); + const dockerRun = vi.fn( + (args: readonly string[]) => + dockerRunResults[String(args[0]) as keyof typeof dockerRunResults] ?? { + status: 0, + stdout: "probe-id\n", + }, + ); const dockerRunDetached = vi.fn(() => ({ status: 0, stdout: `${NEW_CONTAINER_ID}\n` })); const dockerRename = vi.fn(() => ({ status: 0 })); const dockerStop = vi.fn(() => ({ status: 0 })); @@ -79,7 +56,6 @@ describe("Docker GPU recreate orchestration", () => { dockerStart, runCaptureOpenshell, runOpenshell, - homedir: testHome, sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), detectSandboxFallbackDns: vi.fn(() => null), @@ -136,30 +112,15 @@ describe("Docker GPU recreate orchestration", () => { ); }); - it("restores the pre-patch container when final OpenShell phase is Deleting (#9531)", () => { - const dockerRunDetached = vi - .fn() - .mockReturnValueOnce({ status: 0, stdout: `${NEW_CONTAINER_ID}\n` }) - .mockReturnValueOnce({ status: 0, stdout: `${RESTORED_CONTAINER_ID}\n` }); + it("does not report success when the final OpenShell phase is Deleting (#9531)", () => { let failure: unknown; try { recreateOpenShellDockerSandboxWithGpu( { sandboxName: "alpha", timeoutSecs: 1 }, { dockerCapture: dockerCaptureFixture(), - dockerRun: vi.fn((args: readonly string[]) => { - switch (args[0]) { - case "commit": - return { status: 0, stdout: `${ROLLBACK_IMAGE_ID}\n` }; - case "image": - return { status: 0 }; - case "inspect": - return { status: 0, stdout: "true\n" }; - default: - return { status: 0, stdout: `${NEW_CONTAINER_ID}\n` }; - } - }), - dockerRunDetached, + dockerRun: vi.fn(() => ({ status: 0, stdout: `${NEW_CONTAINER_ID}\n` })), + dockerRunDetached: vi.fn(() => ({ status: 0, stdout: `${NEW_CONTAINER_ID}\n` })), dockerRename: vi.fn(() => ({ status: 0 })), dockerStop: vi.fn(() => ({ status: 0 })), dockerRm: vi.fn(() => ({ status: 0 })), @@ -168,7 +129,6 @@ describe("Docker GPU recreate orchestration", () => { runOpenshell: vi.fn((args: readonly string[]) => args[1] === "list" ? { status: 0, stdout: "No sandboxes found.\n" } : { status: 0 }, ), - homedir: testHome, sleep: vi.fn(), now: () => new Date("2026-05-12T00:00:00Z"), detectSandboxFallbackDns: vi.fn(() => null), @@ -182,21 +142,15 @@ describe("Docker GPU recreate orchestration", () => { expect(failure).toBeInstanceOf(Error); expect((failure as Error).message).toContain("final replacement handoff"); - expect((failure as Error).message).toContain("pre-patch sandbox container was restored"); + expect((failure as Error).message).toContain("automatic rollback is unavailable"); + expect((failure as Error).message).toContain("Rebuild the sandbox before retrying"); expect(getDockerGpuPatchFailureContext(failure)).toMatchObject({ backupRemoved: true, oldContainerId: OLD_CONTAINER_ID, newContainerId: NEW_CONTAINER_ID, - rollbackImageId: ROLLBACK_IMAGE_ID, - rollbackRecordRemoved: true, lastSandboxPhase: "Deleting", - rolledBack: true, + rolledBack: false, }); - expect(dockerRunDetached).toHaveBeenCalledTimes(2); - expect(dockerRunDetached.mock.calls[1]?.[0]).toEqual( - expect.arrayContaining(["--name", "openshell-alpha", ROLLBACK_IMAGE_ID]), - ); - expect(dockerRunDetached.mock.calls[1]?.[0]).not.toContain("--gpus"); }); it("can recreate during sandbox create before supervisor exec is allowed", () => { diff --git a/src/lib/onboard/docker-gpu-patch-recreate.ts b/src/lib/onboard/docker-gpu-patch-recreate.ts index fbe8fa7ddc8..7d2310ff366 100644 --- a/src/lib/onboard/docker-gpu-patch-recreate.ts +++ b/src/lib/onboard/docker-gpu-patch-recreate.ts @@ -25,7 +25,6 @@ import { import { DOCKER_GPU_PATCH_STOP_TIMEOUT_MS, DOCKER_GPU_PATCH_TIMEOUT_MS, - sleepSeconds, } from "./docker-gpu-patch-constants"; import { finalizeDockerGpuPatchBackup } from "./docker-gpu-patch-finalize"; import { selectDockerGpuPatchMode } from "./docker-gpu-patch-mode"; @@ -73,7 +72,9 @@ function recreateDeps(deps: DockerGpuPatchDeps): RecreateDeps { dockerRm, dockerStart, dockerStop, - sleep: sleepSeconds, + sleep: (seconds: number) => { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Math.max(0, seconds) * 1000); + }, now: () => new Date(), detectSandboxFallbackDns: () => detectSandboxFallbackDns(), probeContainerDns: (options) => probeContainerDns(options), @@ -406,10 +407,6 @@ export function recreateOpenShellDockerSandboxContainer( : finalizeDockerGpuPatchBackup({ result: patchResult, supervisorReady: false }, deps); context.rolledBack = finalization.rolledBack; context.backupRemoved = finalization.backupRemoved; - context.rollbackImageId = finalization.rollbackImageId; - context.rollbackImageRemoved = finalization.rollbackImageRemoved; - context.rollbackRecordPath = finalization.rollbackRecordPath; - context.rollbackRecordRemoved = finalization.rollbackRecordRemoved; context.replacementStopConfirmed = finalization.replacementStopConfirmed; context.replacementRemovalConfirmed = finalization.replacementRemovalConfirmed; context.replacementPresence = finalization.replacementPresence; @@ -422,23 +419,13 @@ export function recreateOpenShellDockerSandboxContainer( ); } if (finalization.finalHandoffAcknowledged) return result(true); - const rollbackDetail = finalization.rolledBack - ? " The pre-patch sandbox container was restored." - : finalization.backupRemoved - ? ` Automatic rollback failed.${ - finalization.rollbackImageId - ? ` Docker retained rollback image ${finalization.rollbackImageId}.` - : "" - }${ - finalization.rollbackRecordPath - ? ` Recovery action record: ${finalization.rollbackRecordPath}.` - : "" - }` - : ""; + const rebuildRequired = finalization.backupRemoved + ? " The previous container was removed at the final handoff commit point; automatic rollback is unavailable. Rebuild the sandbox before retrying." + : ""; throw new Error( finalization.lastSandboxPhase - ? `OpenShell did not acknowledge the final replacement handoff; last sandbox phase was ${finalization.lastSandboxPhase}.${rollbackDetail}` - : `OpenShell did not acknowledge the final replacement handoff.${rollbackDetail}`, + ? `OpenShell did not acknowledge the final replacement handoff; last sandbox phase was ${finalization.lastSandboxPhase}.${rebuildRequired}` + : `OpenShell did not acknowledge the final replacement handoff.${rebuildRequired}`, ); } catch (error) { throw decoratePatchError(error instanceof Error ? error : new Error(String(error)), context); diff --git a/src/lib/onboard/docker-gpu-patch-rollback.test.ts b/src/lib/onboard/docker-gpu-patch-rollback.test.ts index 71f7e4581f9..6c6af7c26ac 100644 --- a/src/lib/onboard/docker-gpu-patch-rollback.test.ts +++ b/src/lib/onboard/docker-gpu-patch-rollback.test.ts @@ -175,8 +175,6 @@ describe("recreateOpenShellDockerSandboxWithGpu rollback path", () => { (args: readonly string[]) => captureByCommand[String(args[0])]?.(args) ?? "", ); const runByCommand: Record { status: number; stdout: string }> = { - commit: () => ({ status: 0, stdout: `sha256:${"d".repeat(64)}\n` }), - image: () => ({ status: 0, stdout: "" }), ps: () => ({ status: 0, stdout: `${alphaContainerIds().join("\n")}\n`, diff --git a/src/lib/onboard/docker-gpu-patch-rollback.ts b/src/lib/onboard/docker-gpu-patch-rollback.ts index 731d473c363..9422f21899a 100644 --- a/src/lib/onboard/docker-gpu-patch-rollback.ts +++ b/src/lib/onboard/docker-gpu-patch-rollback.ts @@ -2,18 +2,16 @@ // SPDX-License-Identifier: Apache-2.0 import { - dockerCapture as defaultDockerCapture, dockerRename as defaultDockerRename, dockerRm as defaultDockerRm, dockerRun as defaultDockerRun, - dockerRunDetached as defaultDockerRunDetached, dockerStart as defaultDockerStart, dockerStop as defaultDockerStop, } from "../adapters/docker"; import { retryUntil } from "../core/retry"; import { hasZeroDockerExitStatus } from "./docker-command-result"; import { fullDockerContainerId } from "./docker-gpu-patch-clone"; -import { DOCKER_GPU_PATCH_TIMEOUT_MS, sleepSeconds } from "./docker-gpu-patch-constants"; +import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; import type { DockerGpuPatchDeps } from "./docker-gpu-patch-types"; type DockerRunResult = { @@ -23,7 +21,6 @@ type DockerRunResult = { }; type DockerRunOptions = Record; -type DockerCaptureFn = (args: readonly string[], opts?: DockerRunOptions) => string; type DockerContainerFn = (containerName: string, opts?: DockerRunOptions) => DockerRunResult; type DockerRenameFn = ( oldContainerName: string, @@ -35,6 +32,12 @@ type DockerRunFn = (args: readonly string[], opts?: DockerRunOptions) => DockerR const REPLACEMENT_PRESENCE_ATTEMPTS = 3; const REPLACEMENT_PRESENCE_RETRY_MS = 500; +function sleepBeforeReplacementPresenceRetry(seconds: number): void { + if (seconds <= 0 || !Number.isFinite(seconds)) return; + const buffer = new Int32Array(new SharedArrayBuffer(4)); + Atomics.wait(buffer, 0, 0, seconds * 1000); +} + export type DockerGpuPatchRollbackOutcome = { rolledBack: boolean; replacementStopConfirmed: boolean; @@ -42,12 +45,8 @@ export type DockerGpuPatchRollbackOutcome = { replacementPresence: "absent" | "present" | "unknown"; }; -export type DockerGpuReplacementRemovalOutcome = Omit; - export type ResolvedDockerGpuPatchRollbackDeps = { - dockerCapture: DockerCaptureFn; dockerRun: DockerRunFn; - dockerRunDetached: DockerRunFn; dockerStop: DockerContainerFn; dockerRm: DockerContainerFn; dockerRename: DockerRenameFn; @@ -59,14 +58,12 @@ export function resolveDockerGpuPatchRollbackDeps( deps: DockerGpuPatchDeps, ): ResolvedDockerGpuPatchRollbackDeps { return { - dockerCapture: deps.dockerCapture ?? defaultDockerCapture, dockerRun: deps.dockerRun ?? defaultDockerRun, - dockerRunDetached: deps.dockerRunDetached ?? defaultDockerRunDetached, dockerStop: deps.dockerStop ?? defaultDockerStop, dockerRm: deps.dockerRm ?? defaultDockerRm, dockerRename: deps.dockerRename ?? defaultDockerRename, dockerStart: deps.dockerStart ?? defaultDockerStart, - sleep: deps.sleep ?? sleepSeconds, + sleep: deps.sleep ?? sleepBeforeReplacementPresenceRetry, }; } @@ -109,38 +106,26 @@ export function rollbackToBackupContainer( refs: { newContainerId: string; backupContainerName: string; originalName: string }, deps: ResolvedDockerGpuPatchRollbackDeps, ): DockerGpuPatchRollbackOutcome { - const removal = removeReplacementContainer(refs.newContainerId, deps); - const containerOpts = { - ignoreError: true, - suppressOutput: true, - timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, - }; - const restored = deps.dockerRename(refs.backupContainerName, refs.originalName, containerOpts); - const rolledBack = hasZeroDockerExitStatus(restored) - ? hasZeroDockerExitStatus(deps.dockerStart(refs.originalName, containerOpts)) - : false; - return { rolledBack, ...removal }; -} - -export function removeReplacementContainer( - containerId: string, - deps: ResolvedDockerGpuPatchRollbackDeps, -): DockerGpuReplacementRemovalOutcome { const containerOpts = { ignoreError: true, suppressOutput: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, }; const replacementStopConfirmed = hasZeroDockerExitStatus( - deps.dockerStop(containerId, containerOpts), + deps.dockerStop(refs.newContainerId, containerOpts), ); const replacementRemovalConfirmed = hasZeroDockerExitStatus( - deps.dockerRm(containerId, containerOpts), + deps.dockerRm(refs.newContainerId, containerOpts), ); const replacementPresence = replacementRemovalConfirmed ? "absent" - : observeReplacementPresence(containerId, deps, containerOpts); + : observeReplacementPresence(refs.newContainerId, deps, containerOpts); + const restored = deps.dockerRename(refs.backupContainerName, refs.originalName, containerOpts); + const rolledBack = hasZeroDockerExitStatus(restored) + ? hasZeroDockerExitStatus(deps.dockerStart(refs.originalName, containerOpts)) + : false; return { + rolledBack, replacementStopConfirmed, replacementRemovalConfirmed, replacementPresence, diff --git a/src/lib/onboard/docker-gpu-patch-types.ts b/src/lib/onboard/docker-gpu-patch-types.ts index a8f6d49b2a8..972cd858f54 100644 --- a/src/lib/onboard/docker-gpu-patch-types.ts +++ b/src/lib/onboard/docker-gpu-patch-types.ts @@ -80,10 +80,6 @@ export type DockerGpuPatchFailureContext = { newContainerId?: string | null; backupContainerName?: string | null; backupRemoved?: boolean; - rollbackImageId?: string | null; - rollbackImageRemoved?: boolean; - rollbackRecordPath?: string | null; - rollbackRecordRemoved?: boolean; selectedMode?: DockerGpuPatchMode | null; modeAttempts?: DockerGpuPatchModeAttempt[]; rolledBack?: boolean; diff --git a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts index 854331a8414..fde42865d83 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts @@ -147,7 +147,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { expect(onPatchFailureExit).not.toHaveBeenCalled(); }); - it("reports prior-container restoration after replacement restart failure (#9531)", async () => { + it("rejects an explicit replacement restart failure after backup removal", async () => { const deps = makeDeps(); const result = deferredCreateResult(); const onPatchFailureExit = vi.fn(); @@ -162,7 +162,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { waitForSupervisor: vi.fn(() => true), finalizeBackup: vi.fn(() => ({ backupRemoved: true, - rolledBack: true, + rolledBack: false, replacementRestarted: false, })), onPatchFailureExit, @@ -171,17 +171,14 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); patch.waitForSupervisorReconnectIfNeeded(); - await expect(patch.commitAfterReady()).rejects.toThrow( - "pre-patch sandbox container was restored", - ); + await expect(patch.commitAfterReady()).rejects.toThrow("automatic rollback is unavailable"); expect(onPatchFailureExit).toHaveBeenCalledOnce(); expect(onPatchFailureExit.mock.calls[0]?.[2]?.context).toMatchObject({ backupRemoved: true, - rolledBack: true, }); }); - it("reports prior-container restoration when final handoff stays Deleting (#9531)", async () => { + it("rejects final handoff when OpenShell reports Deleting after restart (#9531)", async () => { const deps = makeDeps(); const result = deferredCreateResult(); const waitForSupervisor = vi.fn(() => true); @@ -197,7 +194,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { waitForSupervisor, finalizeBackup: vi.fn(() => ({ backupRemoved: true, - rolledBack: true, + rolledBack: false, replacementRestarted: true, finalHandoffAcknowledged: false, lastSandboxPhase: "Deleting", @@ -208,9 +205,7 @@ describe("createDockerGpuSandboxCreatePatch composed flow", () => { patch.maybeApplyDuringCreate(); patch.waitForSupervisorReconnectIfNeeded(); - await expect(patch.commitAfterReady()).rejects.toThrow( - "pre-patch sandbox container was restored", - ); + await expect(patch.commitAfterReady()).rejects.toThrow("automatic rollback is unavailable"); expect(waitForSupervisor).toHaveBeenCalledOnce(); expect(onPatchFailureExit).toHaveBeenCalledOnce(); diff --git a/src/lib/onboard/docker-gpu-sandbox-create.ts b/src/lib/onboard/docker-gpu-sandbox-create.ts index 70f90a22314..e4ed566917b 100644 --- a/src/lib/onboard/docker-gpu-sandbox-create.ts +++ b/src/lib/onboard/docker-gpu-sandbox-create.ts @@ -522,19 +522,9 @@ export function createDockerGpuSandboxCreatePatch( ? ` Last OpenShell phase: ${finalizeOutcome.lastSandboxPhase}.` : "" }${ - finalizeOutcome.rolledBack - ? " The pre-patch sandbox container was restored." - : finalizeOutcome.backupRemoved - ? ` Automatic rollback failed.${ - finalizeOutcome.rollbackImageId - ? ` Docker retained rollback image ${finalizeOutcome.rollbackImageId}.` - : "" - }${ - finalizeOutcome.rollbackRecordPath - ? ` Recovery action record: ${finalizeOutcome.rollbackRecordPath}.` - : "" - }` - : "" + finalizeOutcome.backupRemoved + ? " The previous container was removed at the final handoff commit point; automatic rollback is unavailable. Rebuild the sandbox before retrying." + : "" }`, ); cutoverFinalizationFailure = failure; @@ -545,15 +535,7 @@ export function createDockerGpuSandboxCreatePatch( context: { ...failureContext(), backupRemoved: finalizeOutcome.backupRemoved, - rollbackImageId: finalizeOutcome.rollbackImageId, - rollbackImageRemoved: finalizeOutcome.rollbackImageRemoved, - rollbackRecordPath: finalizeOutcome.rollbackRecordPath, - rollbackRecordRemoved: finalizeOutcome.rollbackRecordRemoved, lastSandboxPhase: finalizeOutcome.lastSandboxPhase, - replacementStopConfirmed: finalizeOutcome.replacementStopConfirmed, - replacementRemovalConfirmed: finalizeOutcome.replacementRemovalConfirmed, - replacementPresence: finalizeOutcome.replacementPresence, - rolledBack: finalizeOutcome.rolledBack, }, }); throw failure; diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts index a197f82343a..86a84d9cee2 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.test.ts @@ -14,18 +14,27 @@ import { describe("Docker GPU final lifecycle release", () => { it("requires corroboration for a retiring lifecycle row (#9962)", () => { const corroborate = vi.fn(() => true); + const runOpenshell = vi.fn(() => ({ + status: 0, + stdout: "alpha 2026-08-23 01:40:35 Deleting\n", + })); expect( waitForOpenShellSandboxLifecycleRelease("alpha", 1, { - runOpenshell: vi.fn(() => ({ - status: 0, - stdout: "alpha 2026-08-23 01:40:35 Deleting\n", - })), + runOpenshell, sleep: vi.fn(), soleLabeledReplacementCorroboratesRetiringPhase: corroborate, }), ).toBe(true); expect(corroborate).toHaveBeenCalledOnce(); + expect(runOpenshell).toHaveBeenCalledWith( + ["sandbox", "list"], + expect.objectContaining({ + killProcessTreeOnTimeout: true, + killSignal: "SIGKILL", + timeout: expect.any(Number), + }), + ); }); it("does not accept an uncorroborated retiring lifecycle row (#9962)", () => { @@ -84,6 +93,22 @@ describe("Docker GPU final handoff acknowledgement", () => { "exec ready", "confirm exact replacement", ]); + expect(runCaptureOpenshell).toHaveBeenCalledWith( + ["sandbox", "list"], + expect.objectContaining({ + killProcessTreeOnTimeout: true, + killSignal: "SIGKILL", + timeout: expect.any(Number), + }), + ); + expect(runOpenshell).toHaveBeenCalledWith( + ["sandbox", "exec", "-n", "alpha", "--", "true"], + expect.objectContaining({ + killProcessTreeOnTimeout: true, + killSignal: "SIGKILL", + timeout: expect.any(Number), + }), + ); }); it("treats Deleting after the replacement start as terminal (#9531)", () => { diff --git a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts index 0f505191187..9f41580f7ee 100644 --- a/src/lib/onboard/docker-gpu-supervisor-reconnect.ts +++ b/src/lib/onboard/docker-gpu-supervisor-reconnect.ts @@ -25,7 +25,7 @@ import { parseLiveSandboxEntries } from "../runtime-recovery"; import { hasZeroDockerExitStatus } from "./docker-command-result"; -import { DOCKER_GPU_PATCH_TIMEOUT_MS, sleepSeconds } from "./docker-gpu-patch-constants"; +import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; import { envInt } from "./env"; const DOCKER_GPU_SUPERVISOR_RECONNECT_MIN_SECS = 900; @@ -99,6 +99,10 @@ export type DockerFinalHandoffAcknowledgement = { }; const FINAL_HANDOFF_TERMINAL_PHASES = new Set(["Deleting", "Failed", "CrashLoopBackOff"]); +const PROCESS_TREE_BOUNDED_OPENSHELL_OPTIONS = { + killProcessTreeOnTimeout: true, + killSignal: "SIGKILL", +} as const; /** * Wait for OpenShell to retire the pre-replacement lifecycle record before @@ -111,7 +115,7 @@ export function waitForOpenShellSandboxLifecycleRelease( deps: DockerLifecycleReleaseDeps, ): boolean { if (!deps.runOpenshell) return false; - const sleep = deps.sleep ?? sleepSeconds; + const sleep = deps.sleep ?? defaultSleep; const deadline = Date.now() + Math.max(1, Math.round(timeoutSecs)) * 1000; const maxAttempts = Math.max(1, Math.ceil(Math.max(1, Math.round(timeoutSecs)) / 2) + 1); @@ -120,6 +124,7 @@ export function waitForOpenShellSandboxLifecycleRelease( if (remainingMs <= 0) break; const result = deps.runOpenshell(["sandbox", "list"], { ignoreError: true, + ...PROCESS_TREE_BOUNDED_OPENSHELL_OPTIONS, suppressOutput: true, timeout: Math.min(DOCKER_GPU_PATCH_TIMEOUT_MS, remainingMs), }); @@ -178,7 +183,7 @@ export function waitForOpenShellFinalHandoff( timeoutSecs: number, deps: DockerFinalHandoffDeps, ): DockerFinalHandoffAcknowledgement { - const sleep = deps.sleep ?? sleepSeconds; + const sleep = deps.sleep ?? defaultSleep; const boundedTimeoutSecs = Math.max(1, Math.round(timeoutSecs)); const deadline = Date.now() + boundedTimeoutSecs * 1000; const maxAttempts = Math.max(1, Math.ceil(boundedTimeoutSecs / 2) + 1); @@ -191,6 +196,7 @@ export function waitForOpenShellFinalHandoff( try { listOutput = deps.runCaptureOpenshell(["sandbox", "list"], { ignoreError: true, + ...PROCESS_TREE_BOUNDED_OPENSHELL_OPTIONS, suppressOutput: true, timeout: Math.min(DOCKER_GPU_PATCH_TIMEOUT_MS, remainingBeforeListMs), }); @@ -217,6 +223,7 @@ export function waitForOpenShellFinalHandoff( if (remainingBeforeExecMs <= 0) break; const execResult = deps.runOpenshell(["sandbox", "exec", "-n", sandboxName, "--", "true"], { ignoreError: true, + ...PROCESS_TREE_BOUNDED_OPENSHELL_OPTIONS, suppressOutput: true, timeout: Math.min(DOCKER_GPU_PATCH_TIMEOUT_MS, remainingBeforeExecMs), }); @@ -244,6 +251,10 @@ export function waitForOpenShellFinalHandoff( return { acknowledged: false, lastSandboxPhase }; } +function defaultSleep(seconds: number): void { + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, Math.max(0, seconds) * 1000); +} + const ANSI_RE = /\x1b\[[0-9;]*m/g; function parseSandboxListFailurePhase(output: string, sandboxName: string): string | null { @@ -264,6 +275,7 @@ function sandboxListShowsErrorPhase( try { const list = runCaptureOpenshell(["sandbox", "list"], { ignoreError: true, + ...PROCESS_TREE_BOUNDED_OPENSHELL_OPTIONS, suppressOutput: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, }); @@ -279,7 +291,7 @@ export function waitForOpenShellSupervisorReconnect( deps: DockerGpuSupervisorReconnectDeps, ): boolean { if (!deps.runOpenshell) return false; - const sleep = deps.sleep ?? sleepSeconds; + const sleep = deps.sleep ?? defaultSleep; const deadline = Date.now() + Math.max(1, timeoutSecs) * 1000; const errorPhaseDebouncePolls = deps.errorPhaseDebouncePolls == null || !Number.isFinite(deps.errorPhaseDebouncePolls) @@ -291,6 +303,7 @@ export function waitForOpenShellSupervisorReconnect( while (Date.now() <= deadline) { const result = deps.runOpenshell(["sandbox", "exec", "-n", sandboxName, "--", "true"], { ignoreError: true, + ...PROCESS_TREE_BOUNDED_OPENSHELL_OPTIONS, suppressOutput: true, timeout: DOCKER_GPU_PATCH_TIMEOUT_MS, }); diff --git a/src/lib/onboard/openshell-cli.ts b/src/lib/onboard/openshell-cli.ts index 69bf9ef0fca..9abf61ae6e4 100644 --- a/src/lib/onboard/openshell-cli.ts +++ b/src/lib/onboard/openshell-cli.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { processTreeBoundedOpenshellInvocation } from "../adapters/openshell/process-tree-timeout"; import { resolveOpenshell } from "../adapters/openshell/resolve"; import { ROOT, run, runCapture, shellQuote } from "../runner"; import { @@ -19,6 +20,8 @@ export interface OpenshellCliDeps { setCachedBinary(binary: string): void; getGatewayPort(): number; getDockerDriverGatewayEndpoint(): string; + runCommand?: typeof run; + runCaptureCommand?: typeof runCapture; } export interface OpenshellCliHelpers { @@ -34,6 +37,9 @@ export interface OpenshellCliHelpers { } export function createOpenshellCliHelpers(deps: OpenshellCliDeps): OpenshellCliHelpers { + const runCommand = deps.runCommand ?? run; + const runCaptureCommand = deps.runCaptureCommand ?? runCapture; + function getOpenshellBinary(): string { const cached = deps.getCachedBinary(); if (cached) return cached; @@ -60,12 +66,28 @@ export function createOpenshellCliHelpers(deps: OpenshellCliDeps): OpenshellCliH return [openshellBinary, ...args]; } + function processTreeBoundedInvocation(args: string[], opts: any) { + const [binary, ...commandArgs] = openshellArgv(args, opts); + const { killProcessTreeOnTimeout, ...runnerOptions } = opts; + const bounded = processTreeBoundedOpenshellInvocation(binary, commandArgs, { + killProcessTreeOnTimeout, + killSignal: runnerOptions.killSignal, + timeout: runnerOptions.timeout, + }); + return { + argv: [bounded.binary, ...bounded.args], + runnerOptions: { ...runnerOptions, killSignal: bounded.killSignal }, + }; + } + function runOpenshell(args: string[], opts: any = {}) { - return run(openshellArgv(args, opts), opts); + const invocation = processTreeBoundedInvocation(args, opts); + return runCommand(invocation.argv, invocation.runnerOptions); } function runCaptureOpenshell(args: string[], opts: any = {}) { - return runCapture(openshellArgv(args, opts), opts); + const invocation = processTreeBoundedInvocation(args, opts); + return runCaptureCommand(invocation.argv, invocation.runnerOptions); } function captureOpenshell(args: string[], opts: CaptureOpenshellOptions = {}) { diff --git a/src/lib/onboard/sandbox-create-launch.test.ts b/src/lib/onboard/sandbox-create-launch.test.ts index 7bddef4eee7..a873639748c 100644 --- a/src/lib/onboard/sandbox-create-launch.test.ts +++ b/src/lib/onboard/sandbox-create-launch.test.ts @@ -565,6 +565,39 @@ describe("prepareSandboxCreateLaunch", () => { } }); + it("routes bounded OpenShell probes through the process-tree timeout adapter (#10238)", () => { + const runCommand = vi.fn((_command: readonly string[], _options?: unknown) => { + return { status: 0 } as never; + }); + const helpers = createOpenshellCliHelpers({ + getCachedBinary: () => "/opt/openshell", + setCachedBinary: vi.fn(), + getGatewayPort: () => 31818, + getDockerDriverGatewayEndpoint: () => "http://127.0.0.1:31818", + runCommand, + }); + + helpers.runOpenshell(["sandbox", "list"], { + ignoreError: true, + killProcessTreeOnTimeout: true, + timeout: 1000, + }); + + const expectedArgv = + process.platform === "linux" && fs.existsSync("/usr/bin/timeout") + ? ["/usr/bin/timeout", "--signal=KILL", "0.75s", "/opt/openshell", "sandbox", "list"] + : ["/opt/openshell", "sandbox", "list"]; + expect(runCommand).toHaveBeenCalledWith( + expectedArgv, + expect.objectContaining({ + ignoreError: true, + killSignal: "SIGKILL", + timeout: 1000, + }), + ); + expect(runCommand.mock.calls[0]?.[1]).not.toHaveProperty("killProcessTreeOnTimeout"); + }); + it("forwards the validated sandbox name into the Deep Agents Code sandbox create env", () => { const result = prepareSandboxCreateLaunch({ agent: { name: "langchain-deepagents-code" } as any, diff --git a/test/helpers/onboard-script-mocks.cjs b/test/helpers/onboard-script-mocks.cjs index b9fe6bf02b1..a7978657307 100644 --- a/test/helpers/onboard-script-mocks.cjs +++ b/test/helpers/onboard-script-mocks.cjs @@ -249,7 +249,6 @@ const OPENCLAW_SECURITY_INVENTORY_PROBE = [ const ONBOARD_SANDBOX_OLD_CONTAINER_ID = "a".repeat(64); const ONBOARD_SANDBOX_NEW_CONTAINER_ID = "b".repeat(64); -const ONBOARD_SANDBOX_ROLLBACK_IMAGE_ID = `sha256:${"d".repeat(64)}`; const ONBOARD_SANDBOX_INSPECT = { Id: ONBOARD_SANDBOX_OLD_CONTAINER_ID, Image: `sha256:${"c".repeat(64)}`, @@ -398,16 +397,6 @@ function mockDockerSandboxLifecycleReleaseFromRunner() { let lifecycleReleased = false; const wrappedRun = (command, options) => { const normalized = normalizeCommand(command); - switch (normalized) { - case `docker commit ${ONBOARD_SANDBOX_OLD_CONTAINER_ID}`: - return { - status: 0, - stdout: Buffer.from(`${ONBOARD_SANDBOX_ROLLBACK_IMAGE_ID}\n`), - stderr: Buffer.alloc(0), - }; - case `docker image rm --force ${ONBOARD_SANDBOX_ROLLBACK_IMAGE_ID}`: - return { status: 0, stdout: Buffer.alloc(0), stderr: Buffer.alloc(0) }; - } if ( finalCommitReleased && normalized.startsWith("docker ps -a --no-trunc ") && diff --git a/test/process-recovery/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery/process-recovery-supervisor-relaunch.test.ts index 23911cf54c7..5debdc72994 100644 --- a/test/process-recovery/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery/process-recovery-supervisor-relaunch.test.ts @@ -1,17 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; - -import { afterAll, afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import * as forwardHealth from "../../src/lib/actions/sandbox/forward-health.ts"; import { checkAndRecoverSandboxProcesses } from "../../src/lib/actions/sandbox/process-recovery.ts"; import { relaunchManagedSupervisorSession } from "../../src/lib/actions/sandbox/supervisor-relaunch.ts"; import * as openshellRuntime from "../../src/lib/adapters/openshell/runtime.ts"; import * as agentRuntime from "../../src/lib/agent/runtime.ts"; -import { createDockerGpuInspectFixture as inspectFixture } from "../../src/lib/onboard/__test-helpers__/docker-gpu-patch-fixtures.ts"; import { finalizeDockerGpuPatchBackup } from "../../src/lib/onboard/docker-gpu-patch-finalize.ts"; import * as registry from "../../src/lib/state/registry.ts"; @@ -28,12 +23,6 @@ const MISSING_MANAGED_SUPERVISOR = { stdout: "", stderr: "SUPERVISOR_NOT_RUNNING", } as const; -const TEST_TEMP_ROOT = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-recovery-relaunch-")); - -function testHome(): string { - return fs.mkdtempSync(path.join(TEST_TEMP_ROOT, "case-")); -} - function pinnedIdentityRefusal(sandboxName: string) { return { status: 1, @@ -47,10 +36,6 @@ afterEach(() => { vi.unstubAllEnvs(); }); -afterAll(() => { - fs.rmSync(TEST_TEMP_ROOT, { recursive: true, force: true }); -}); - function mockOpenClawSandbox(sandboxName: string, healthTimeoutSeconds = 30) { vi.spyOn(agentRuntime, "getSessionAgent").mockReturnValue({ name: "openclaw", @@ -494,19 +479,11 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { const dockerStart = vi.fn(() => ({ status: 0 })); const oldContainerId = "a".repeat(64); const replacementContainerId = "b".repeat(64); - const rollbackImageId = `sha256:${"d".repeat(64)}`; - const dockerRun = vi.fn((args: readonly string[]) => { - switch (args[0]) { - case "commit": - return { status: 0, stdout: `${rollbackImageId}\n` }; - case "image": - return { status: 0 }; - case "ps": - return { status: 0, stdout: `${replacementContainerId}\n` }; - default: - return { status: 0, stdout: "true\n" }; - } - }); + const dockerRun = vi.fn((args: readonly string[]) => + args[0] === "ps" + ? { status: 0, stdout: `${replacementContainerId}\n` } + : { status: 0, stdout: "true\n" }, + ); const finalizeTransaction = vi.fn( ( options: Parameters[0], @@ -514,12 +491,10 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { ) => finalizeDockerGpuPatchBackup(options, { ...deps, - dockerCapture: vi.fn(() => JSON.stringify([{ ...inspectFixture(), Id: oldContainerId }])), dockerRm, dockerRun, dockerStart, dockerStop, - homedir: testHome, }), ); const { relaunchManagedSupervisorSessionImpl } = composedRelaunchTransaction( @@ -653,29 +628,6 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { finalPinnedAction: () => ACCEPTED_MANAGED_PROBE, finalReadinessReady: true, }, - { - condition: "automatic rollback after the final container handoff fails", - finalizeOutcome: () => ({ - backupRemoved: true, - finalHandoffAcknowledged: false, - lastSandboxPhase: "Deleting", - replacementRestarted: true, - rollbackImageId: `sha256:${"d".repeat(64)}`, - rollbackRecordPath: "/home/test/.nemoclaw/recovery/docker-gpu/recovery.json", - rolledBack: false, - stateRestored: true, - }), - expectedDetail: "OpenShell did not acknowledge the final replacement container handoff", - expectedDetailFragments: [ - `sha256:${"d".repeat(64)}`, - "/home/test/.nemoclaw/recovery/docker-gpu/recovery.json", - "Confirm the replacement container is absent", - "recoveryAction command and arguments", - ], - expectedReadinessCalls: 1, - finalPinnedAction: () => ACCEPTED_MANAGED_PROBE, - finalReadinessReady: true, - }, { condition: "the final OpenShell readiness check fails", finalizeOutcome: () => ({ @@ -738,7 +690,6 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { "does not start the primary dashboard/API host forward when $condition (#9364)", ({ expectedDetail, - expectedDetailFragments, expectedReadinessCalls, finalPinnedAction, finalReadinessReady, @@ -797,11 +748,6 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { expect("recoveryFailureDetail" in result ? result.recoveryFailureDetail : "").not.toContain( "opaque-", ); - const detail = "recoveryFailureDetail" in result ? result.recoveryFailureDetail : ""; - expect(detail).toContain(expectedDetailFragments?.[0] ?? ""); - expect(detail).toContain(expectedDetailFragments?.[1] ?? ""); - expect(detail).toContain(expectedDetailFragments?.[2] ?? ""); - expect(detail).toContain(expectedDetailFragments?.[3] ?? ""); expect(result).not.toHaveProperty("forwardRecoveryFailed"); expect(finalize).toHaveBeenCalledOnce(); expect(finalize).toHaveBeenCalledWith(true); From 87368cfc6193f24747566dc357dd02eef156a54d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 25 Aug 2026 13:12:39 -0700 Subject: [PATCH 12/12] fix(recovery): preserve bounded handoff diagnostics Signed-off-by: Aaron Erickson --- src/lib/actions/sandbox/process-recovery.ts | 5 ++++ .../openshell/process-tree-timeout.ts | 7 ++++++ src/lib/onboard/docker-gpu-patch-finalize.ts | 7 +++--- ...ocess-recovery-supervisor-relaunch.test.ts | 25 ++++++++++++++++++- 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 8b879c928df..1240c665d95 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -505,6 +505,9 @@ function finalRelaunchContainerFailureDetail( if (completion.replacementStoppedForCommit === false) { return "Docker could not stop the replacement container for the final recovery handoff. NemoClaw did not start the primary dashboard/API host forward"; } + if (completion.lifecycleReleaseObserved === false) { + return "OpenShell did not retire the previous lifecycle record before the final replacement start. NemoClaw did not start the primary dashboard/API host forward"; + } if (completion.replacementRestarted === false) { return "Docker could not start the replacement container to complete the final recovery handoff. NemoClaw did not start the primary dashboard/API host forward"; } @@ -765,6 +768,8 @@ function recoverSandboxProcesses( captureOpenshell(args, { ignoreError: true, includeStderr: true, + killProcessTreeOnTimeout: options?.killProcessTreeOnTimeout === true, + killSignal: options?.killSignal === "SIGKILL" ? "SIGKILL" : undefined, timeout: typeof options?.timeout === "number" ? options.timeout : OPENSHELL_PROBE_TIMEOUT_MS, }).output, diff --git a/src/lib/adapters/openshell/process-tree-timeout.ts b/src/lib/adapters/openshell/process-tree-timeout.ts index 2e945613c40..cf494aa81ba 100644 --- a/src/lib/adapters/openshell/process-tree-timeout.ts +++ b/src/lib/adapters/openshell/process-tree-timeout.ts @@ -15,6 +15,13 @@ export type ProcessTreeTimeoutOptions = { timeout?: number; }; +/** + * #10238 ownership decision: NemoClaw owns the wall-clock bound at its + * synchronous OpenShell adapter boundary. This remains opt-in for lifecycle + * probes; ordinary commands keep their existing execution behavior. Linux uses + * a process-group timeout so descendants cannot retain captured pipes after the + * direct CLI is killed. Real DGX Spark confirmation remains tracked by #10238. + */ export function processTreeBoundedOpenshellInvocation( binary: string, args: readonly string[], diff --git a/src/lib/onboard/docker-gpu-patch-finalize.ts b/src/lib/onboard/docker-gpu-patch-finalize.ts index c874626d0db..7a1c09fe3c1 100644 --- a/src/lib/onboard/docker-gpu-patch-finalize.ts +++ b/src/lib/onboard/docker-gpu-patch-finalize.ts @@ -18,13 +18,14 @@ // for exact final handoff, terminal phase, rollback, and failure outcomes. // * src/lib/onboard/docker-gpu-patch-rollback.test.ts — composed // recreate-with-rollback scenarios. -// * src/lib/onboard/docker-gpu-sandbox-create.test.ts — composed create +// * src/lib/onboard/docker-gpu-sandbox-create-lifecycle.test.ts — composed create // flow driving maybeApplyDuringCreate → waitForSupervisorReconnect → // finalizeBackup. // Removal condition: when OpenShell supports native Docker-driver GPU // creation/reconnect, drop the NemoClaw post-create container recreation -// and delete this module along with its callers in docker-gpu-patch.ts and -// docker-gpu-sandbox-create.ts. +// and delete this module along with its direct callers in +// docker-gpu-patch-recreate.ts, docker-gpu-sandbox-create.ts, and +// src/lib/actions/sandbox/supervisor-relaunch.ts. import { hasZeroDockerExitStatus } from "./docker-command-result"; import { DOCKER_GPU_PATCH_TIMEOUT_MS } from "./docker-gpu-patch-constants"; diff --git a/test/process-recovery/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery/process-recovery-supervisor-relaunch.test.ts index 5debdc72994..e110f53b2a6 100644 --- a/test/process-recovery/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery/process-recovery-supervisor-relaunch.test.ts @@ -523,7 +523,8 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { options?.beforeProbe?.(1000) === true, ); vi.spyOn(forwardHealth, "isLocalForwardReachable").mockImplementation(() => forwardStarted); - vi.spyOn(openshellRuntime, "captureOpenshell").mockImplementation((args) => { + const captureOpenshell = vi.spyOn(openshellRuntime, "captureOpenshell"); + captureOpenshell.mockImplementation((args) => { const responses = { "forward list": () => ({ status: 0, @@ -588,6 +589,13 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { ["forward", "start", "--background", "18789", "legacy-handoff-box"], expect.objectContaining({ ignoreError: true }), ); + expect(captureOpenshell).toHaveBeenCalledWith( + ["sandbox", "list"], + expect.objectContaining({ + killProcessTreeOnTimeout: true, + killSignal: "SIGKILL", + }), + ); }); it.each([ @@ -604,6 +612,21 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { finalPinnedAction: () => ACCEPTED_MANAGED_PROBE, finalReadinessReady: true, }, + { + condition: "OpenShell does not retire the previous lifecycle record", + finalizeOutcome: () => ({ + backupRemoved: true, + lifecycleReleaseObserved: false, + replacementRestarted: false, + replacementStoppedForCommit: true, + rolledBack: false, + stateRestored: true, + }), + expectedDetail: "OpenShell did not retire the previous lifecycle record", + expectedReadinessCalls: 1, + finalPinnedAction: () => ACCEPTED_MANAGED_PROBE, + finalReadinessReady: true, + }, { condition: "Docker cannot start the replacement container", finalizeOutcome: () => ({