Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 36 additions & 0 deletions src/lib/onboard/docker-driver-sandbox-recovery.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,42 @@ describe("recoverDockerDriverSandbox — running original (no-op)", () => {
});
});

describe("recoverDockerDriverSandbox — paused original (unpause)", () => {
it("unpauses a paused container and reports via=unpaused-original (not started-running-original)", () => {
const start = vi.fn(fakeStart(0));
const unpause = vi.fn(fakeStart(0));
const result = recoverDockerDriverSandbox("e2e-x", {
dockerCapture: fakeCapture("openshell-e2e-x\tUp 3 hours (Paused)\n"),
dockerStart: start,
dockerUnpause: unpause,
});
expect(result).toEqual({
recovered: true,
via: "unpaused-original",
containerName: "openshell-e2e-x",
});
expect(unpause).toHaveBeenCalledTimes(1);
expect(unpause).toHaveBeenCalledWith(
"openshell-e2e-x",
expect.objectContaining({ ignoreError: true }),
);
expect(start).not.toHaveBeenCalled();
});

it("reports recovered=false with a detail when docker unpause fails", () => {
const unpause = vi.fn(fakeStart(1));
const result = recoverDockerDriverSandbox("e2e-x", {
dockerCapture: fakeCapture("openshell-e2e-x\tUp 3 hours (Paused)\n"),
dockerStart: vi.fn(fakeStart(0)),
dockerUnpause: unpause,
});
expect(result.recovered).toBe(false);
expect(result.via).toBeNull();
expect(result.detail).toContain("docker unpause");
expect(unpause).toHaveBeenCalledTimes(1);
});
});

describe("recoverDockerDriverSandbox — stopped original (start)", () => {
it("starts the labeled container and reports started-stopped-original", () => {
const start = vi.fn(fakeStart(0));
Expand Down
33 changes: 33 additions & 0 deletions src/lib/onboard/docker-driver-sandbox-recovery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ function loadDockerStart(): DockerStartFn {
function loadDockerRename(): DockerRenameFn {
return (require("../adapters/docker") as { dockerRename: DockerRenameFn }).dockerRename;
}
function loadDockerUnpause(): DockerStartFn {
return (require("../adapters/docker") as { dockerUnpause: DockerStartFn }).dockerUnpause;
}

/**
* Active Docker-driver sandbox recovery (#4423 part 2).
Expand Down Expand Up @@ -75,6 +78,7 @@ const MAX_DOCKER_CONTAINER_NAME_LENGTH = 253;
*/
export type DockerDriverRecoveryVia =
| "started-running-original" // labeled container was already running; nothing to do.
| "unpaused-original" // labeled container was paused; `docker unpause` resumed it.
| "started-stopped-original" // labeled container existed but was stopped; `docker start`.
| "renamed-and-started-backup"; // only a `*-nemoclaw-gpu-backup-*` sibling existed; rename back + start.

Expand All @@ -94,6 +98,7 @@ export interface DockerDriverRecoveryDeps {
/** `docker ps -a --filter ... --format ...` runner. */
dockerCapture?: (args: readonly string[], opts?: Record<string, unknown>) => string;
dockerStart?: (name: string, opts?: Record<string, unknown>) => { status?: number | null };
dockerUnpause?: (name: string, opts?: Record<string, unknown>) => { status?: number | null };
dockerRename?: (
oldName: string,
newName: string,
Expand All @@ -114,12 +119,21 @@ function depsWithDefaults(deps: DockerDriverRecoveryDeps) {
return {
dockerCapture: deps.dockerCapture ?? ((args, opts) => loadDockerCapture()(args, opts)),
dockerStart: deps.dockerStart ?? ((name, opts) => loadDockerStart()(name, opts)),
dockerUnpause: deps.dockerUnpause ?? ((name, opts) => loadDockerUnpause()(name, opts)),
dockerRename:
deps.dockerRename ?? ((oldName, newName, opts) => loadDockerRename()(oldName, newName, opts)),
now: deps.now ?? (() => Date.now()),
};
}

// Paused containers report `Up N ... (Paused)` from `docker ps` and are
// classified `running` because the status starts with `Up`, but a paused
// container is frozen: `docker unpause` (not `start`) is the verb that
// resumes it. Mirrors the pre-check in `src/lib/actions/sandbox/start.ts`.
function isPausedStatus(status: string): boolean {
return status.startsWith("Up") && status.endsWith("(Paused)");
}

function isBackupSiblingName(name: string): boolean {
return /-nemoclaw-gpu-backup-\d+$/.test(name);
}
Expand Down Expand Up @@ -231,6 +245,25 @@ export function recoverDockerDriverSandbox(
const { runningOriginal, stoppedOriginal, backup } = classifyCandidate(containers);

if (runningOriginal) {
if (isPausedStatus(runningOriginal.status)) {
const unpause = d.dockerUnpause(runningOriginal.name, {
ignoreError: true,
timeout: DOCKER_OPERATION_TIMEOUT_MS,
});
if ((unpause.status ?? 1) !== 0) {
return {
recovered: false,
via: null,
containerName: runningOriginal.name,
detail: `docker unpause ${runningOriginal.name} failed (exit ${unpause.status ?? "unknown"}).`,
};
}
return {
recovered: true,
via: "unpaused-original",
containerName: runningOriginal.name,
};
}
return {
recovered: true,
via: "started-running-original",
Expand Down
Loading