diff --git a/src/lib/onboard/docker-driver-sandbox-recovery.test.ts b/src/lib/onboard/docker-driver-sandbox-recovery.test.ts index 493f3a580e2..6662d8acd1c 100644 --- a/src/lib/onboard/docker-driver-sandbox-recovery.test.ts +++ b/src/lib/onboard/docker-driver-sandbox-recovery.test.ts @@ -72,6 +72,42 @@ describe("recoverDockerDriverSandbox — running original (no-op)", () => { }); }); +describe("recoverDockerDriverSandbox — paused original (unpause)", () => { + it("unpauses a paused container and reports via=unpaused-original (not started-running-original)", () => { + const start = vi.fn(fakeStart(0)); + const unpause = vi.fn(fakeStart(0)); + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture("openshell-e2e-x\tUp 3 hours (Paused)\n"), + dockerStart: start, + dockerUnpause: unpause, + }); + expect(result).toEqual({ + recovered: true, + via: "unpaused-original", + containerName: "openshell-e2e-x", + }); + expect(unpause).toHaveBeenCalledTimes(1); + expect(unpause).toHaveBeenCalledWith( + "openshell-e2e-x", + expect.objectContaining({ ignoreError: true }), + ); + expect(start).not.toHaveBeenCalled(); + }); + + it("reports recovered=false with a detail when docker unpause fails", () => { + const unpause = vi.fn(fakeStart(1)); + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture("openshell-e2e-x\tUp 3 hours (Paused)\n"), + dockerStart: vi.fn(fakeStart(0)), + dockerUnpause: unpause, + }); + expect(result.recovered).toBe(false); + expect(result.via).toBeNull(); + expect(result.detail).toContain("docker unpause"); + expect(unpause).toHaveBeenCalledTimes(1); + }); +}); + describe("recoverDockerDriverSandbox — stopped original (start)", () => { it("starts the labeled container and reports started-stopped-original", () => { const start = vi.fn(fakeStart(0)); diff --git a/src/lib/onboard/docker-driver-sandbox-recovery.ts b/src/lib/onboard/docker-driver-sandbox-recovery.ts index fa87e03432f..429660d6eb4 100644 --- a/src/lib/onboard/docker-driver-sandbox-recovery.ts +++ b/src/lib/onboard/docker-driver-sandbox-recovery.ts @@ -32,6 +32,9 @@ function loadDockerStart(): DockerStartFn { function loadDockerRename(): DockerRenameFn { return (require("../adapters/docker") as { dockerRename: DockerRenameFn }).dockerRename; } +function loadDockerUnpause(): DockerStartFn { + return (require("../adapters/docker") as { dockerUnpause: DockerStartFn }).dockerUnpause; +} /** * Active Docker-driver sandbox recovery (#4423 part 2). @@ -75,6 +78,7 @@ const MAX_DOCKER_CONTAINER_NAME_LENGTH = 253; */ export type DockerDriverRecoveryVia = | "started-running-original" // labeled container was already running; nothing to do. + | "unpaused-original" // labeled container was paused; `docker unpause` resumed it. | "started-stopped-original" // labeled container existed but was stopped; `docker start`. | "renamed-and-started-backup"; // only a `*-nemoclaw-gpu-backup-*` sibling existed; rename back + start. @@ -94,6 +98,7 @@ export interface DockerDriverRecoveryDeps { /** `docker ps -a --filter ... --format ...` runner. */ dockerCapture?: (args: readonly string[], opts?: Record) => string; dockerStart?: (name: string, opts?: Record) => { status?: number | null }; + dockerUnpause?: (name: string, opts?: Record) => { status?: number | null }; dockerRename?: ( oldName: string, newName: string, @@ -114,12 +119,21 @@ function depsWithDefaults(deps: DockerDriverRecoveryDeps) { return { dockerCapture: deps.dockerCapture ?? ((args, opts) => loadDockerCapture()(args, opts)), dockerStart: deps.dockerStart ?? ((name, opts) => loadDockerStart()(name, opts)), + dockerUnpause: deps.dockerUnpause ?? ((name, opts) => loadDockerUnpause()(name, opts)), dockerRename: deps.dockerRename ?? ((oldName, newName, opts) => loadDockerRename()(oldName, newName, opts)), now: deps.now ?? (() => Date.now()), }; } +// Paused containers report `Up N ... (Paused)` from `docker ps` and are +// classified `running` because the status starts with `Up`, but a paused +// container is frozen: `docker unpause` (not `start`) is the verb that +// resumes it. Mirrors the pre-check in `src/lib/actions/sandbox/start.ts`. +function isPausedStatus(status: string): boolean { + return status.startsWith("Up") && status.endsWith("(Paused)"); +} + function isBackupSiblingName(name: string): boolean { return /-nemoclaw-gpu-backup-\d+$/.test(name); } @@ -231,6 +245,25 @@ export function recoverDockerDriverSandbox( const { runningOriginal, stoppedOriginal, backup } = classifyCandidate(containers); if (runningOriginal) { + if (isPausedStatus(runningOriginal.status)) { + const unpause = d.dockerUnpause(runningOriginal.name, { + ignoreError: true, + timeout: DOCKER_OPERATION_TIMEOUT_MS, + }); + if ((unpause.status ?? 1) !== 0) { + return { + recovered: false, + via: null, + containerName: runningOriginal.name, + detail: `docker unpause ${runningOriginal.name} failed (exit ${unpause.status ?? "unknown"}).`, + }; + } + return { + recovered: true, + via: "unpaused-original", + containerName: runningOriginal.name, + }; + } return { recovered: true, via: "started-running-original",