From 4c739bd13d88931d82d314a8a0d46a7eff85ceb4 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 9 Jun 2026 20:12:58 -0400 Subject: [PATCH] fix(sandbox): recover Docker-driver sandbox from labels post-reboot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the active-recovery half of #4423 (parts 2 & 3 of @ericksoa's plan). #4578 and #4497 already neutralized the destructive `missing` branches in `status` and `ensureLiveSandboxOrExit`; the registry entry survives a healthy_named-gateway + NotFound-sandbox round-trip. This PR adds the actively recoverable path so the user's next command sees a live sandbox instead of guidance-and-retry. Adds `src/lib/onboard/docker-driver-sandbox-recovery.ts` with `recoverDockerDriverSandbox(name, deps)`. It scans Docker by the OpenShell labels (`openshell.ai/managed-by=openshell` AND `openshell.ai/sandbox-name=`) and dispatches by container shape: 1. `started-running-original` — labeled container is already running; no-op success. 2. `started-stopped-original` — labeled container exists but is stopped; `docker start`. 3. `renamed-and-started-backup` — only a `*-nemoclaw-gpu-backup-*` sibling exists (from the GPU patch path in `docker-gpu-patch.ts`); rename back to the original and start. 4. Conflict (backup + stale stopped original with the same base name) — start the original, leave the backup alone. 5. No labeled container at all → `{ recovered: false }` so callers fall through to existing non-destructive guidance. Wires the helper into `gateway-state.ts::reconcileMissingAgainstNamedGateway`. After the existing `gateway select nemoclaw` retry path AND on the new `healthy_named` + `missing` precondition that #4423 reports, the reconciler invokes the helper and re-queries OpenShell. When recovery succeeds, the returned `SandboxGatewayState` carries `recoveredSandbox: true` + `recoverySandboxVia: ` so callers can render a recovery breadcrumb without re-walking the recovery chain themselves. `status.ts`'s present-state path now prints `Recovered sandbox '' from Docker via ; OpenShell now sees it as live.` so the user knows why a previously-NotFound sandbox is suddenly Ready. Boundary: * The helper talks to Docker only. OpenShell re-query is the caller's job (`getSandboxGatewayState` in `gateway-state.ts`). * Adapter access is per-call lazy `require` — top-level import of `../adapters/docker` would pull in `runner.ts`'s load-time `require("./platform")` which the Vitest TS loader can't resolve. Same pattern as `auto-pair-approval.ts`. * Label constants are duplicated locally with comments pointing to `docker-gpu-patch.ts` as the source of truth. Tests: * `docker-driver-sandbox-recovery.test.ts` — 12 cases across discover parsing, all 4 dispatch branches, rename/start failure surfaces, and the no-labeled-container empty case. * Existing gateway-state tests (15 in `gateway-state-reconcile-2276.test.ts`, 2 in `gateway-state-drift.test.ts`) remain green; the new branch is additive and only fires under `healthy_named` + `missing` which none of them exercise. Verification surface: * The post-reboot recovery scenario shipped by #5087 (`ubuntu-repo-docker-post-reboot-recovery`) continues to go GREEN — its host-side invariants are exactly what this fix preserves. * A future controlled-runner scenario can extend the `post-reboot-recovery-ready` expected state with gateway/sandbox runtime probes once a real-reboot environment is wired. Refs #4423 --- src/lib/actions/sandbox/gateway-state.ts | 62 +++- src/lib/actions/sandbox/status.ts | 10 + .../docker-driver-sandbox-recovery.test.ts | 196 ++++++++++++ .../onboard/docker-driver-sandbox-recovery.ts | 296 ++++++++++++++++++ 4 files changed, 563 insertions(+), 1 deletion(-) create mode 100644 src/lib/onboard/docker-driver-sandbox-recovery.test.ts create mode 100644 src/lib/onboard/docker-driver-sandbox-recovery.ts diff --git a/src/lib/actions/sandbox/gateway-state.ts b/src/lib/actions/sandbox/gateway-state.ts index 55da8e10b55..f01b805ab6a 100644 --- a/src/lib/actions/sandbox/gateway-state.ts +++ b/src/lib/actions/sandbox/gateway-state.ts @@ -35,6 +35,10 @@ import { OPENSHELL_PROBE_TIMEOUT_MS, } from "../../adapters/openshell/timeouts"; import { isDockerRuntimeDown, printDockerRuntimeDownGuidance } from "./gateway-failure-classifier"; +import { + recoverDockerDriverSandbox, + type DockerDriverRecoveryResult, +} from "../../onboard/docker-driver-sandbox-recovery"; export type SandboxGatewayState = { state: string; @@ -43,6 +47,20 @@ export type SandboxGatewayState = { recoveredGateway?: boolean; recoveryVia?: string | null; gatewayRecoveryFailed?: boolean; + /** + * True when active Docker-driver sandbox recovery (#4423 part 2) + * restarted the labeled sandbox container before the lookup + * returned `present`. Callers can surface this in user-facing + * output to explain why a previously-NotFound sandbox is now + * Ready. + */ + recoveredSandbox?: boolean; + /** + * Stable identifier for which Docker-driver recovery branch fired, + * mirroring `DockerDriverRecoveryVia`. `null` when no Docker-side + * recovery was attempted or required. + */ + recoverySandboxVia?: string | null; }; type SandboxGatewayStateLookup = ( @@ -225,7 +243,10 @@ export function reconcileMissingAgainstNamedGateway( if (retry.state === "missing") { const after = getNamedGatewayLifecycleState(); if (after.state === "healthy_named") { - return retry; + // Even with the right gateway selected, the sandbox is + // still missing. Try Docker-side recovery before declaring + // the sandbox truly absent. + return tryRecoverDockerDriverSandbox(sandboxName, retry); } } return { @@ -240,9 +261,48 @@ export function reconcileMissingAgainstNamedGateway( if (lifecycle.state === "named_unreachable" || lifecycle.state === "named_unhealthy") { return { state: "gateway_unreachable_after_restart", output: lifecycle.status }; } + if (lifecycle.state === "healthy_named") { + // The gateway is healthy and we already see `missing`. This is + // the precise post-reboot precondition described in #4423: the + // gateway came back fresh (per #4580's user-systemd unit) with + // no sandbox memory, but Docker may still have the labeled + // container. Attempt active Docker-side recovery before falling + // through to non-destructive guidance. + return tryRecoverDockerDriverSandbox(sandboxName, missingLookup); + } return missingLookup; } +/** + * Attempt Docker-driver sandbox recovery (#4423) and re-query the + * OpenShell gateway. Returns the new lookup with `recoveredSandbox` + * flags set when recovery succeeded; otherwise returns the original + * `missing` lookup unchanged so the caller's existing non-destructive + * guidance fires. + */ +function tryRecoverDockerDriverSandbox( + sandboxName: string, + missingLookup: SandboxGatewayState, +): SandboxGatewayState { + let recovery: DockerDriverRecoveryResult; + try { + recovery = recoverDockerDriverSandbox(sandboxName); + } catch { + return missingLookup; + } + if (!recovery.recovered) { + return missingLookup; + } + // Recovery succeeded against Docker; re-query OpenShell so the + // returned state reflects what the gateway sees post-restart. + const retried = getSandboxGatewayState(sandboxName); + return { + ...retried, + recoveredSandbox: true, + recoverySandboxVia: recovery.via, + }; +} + /** * Print actionable guidance when the nemoclaw gateway exists but another * OpenShell gateway is currently active. Emphasizes that the sandbox has NOT diff --git a/src/lib/actions/sandbox/status.ts b/src/lib/actions/sandbox/status.ts index 4f9b8d3e246..9d056ebaa52 100644 --- a/src/lib/actions/sandbox/status.ts +++ b/src/lib/actions/sandbox/status.ts @@ -290,6 +290,16 @@ export async function showSandboxStatus(sandboxName: string): Promise { ); console.log(""); } + if ("recoveredSandbox" in lookup && lookup.recoveredSandbox) { + const via = + "recoverySandboxVia" in lookup && lookup.recoverySandboxVia + ? ` via ${lookup.recoverySandboxVia}` + : ""; + console.log( + ` Recovered sandbox '${sandboxName}' from Docker${via}; OpenShell now sees it as live.`, + ); + console.log(""); + } console.log(lookup.output); if (phase && phase !== "Ready") { // A non-ready, non-terminal phase can mean two very different things. If diff --git a/src/lib/onboard/docker-driver-sandbox-recovery.test.ts b/src/lib/onboard/docker-driver-sandbox-recovery.test.ts new file mode 100644 index 00000000000..493f3a580e2 --- /dev/null +++ b/src/lib/onboard/docker-driver-sandbox-recovery.test.ts @@ -0,0 +1,196 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + findLabeledSandboxContainers, + recoverDockerDriverSandbox, +} from "./docker-driver-sandbox-recovery"; + +interface FakeRunResult { + status: number; +} + +function fakeStart(status = 0): (name: string, opts?: Record) => FakeRunResult { + return () => ({ status }); +} + +function fakeRename( + status = 0, +): (oldName: string, newName: string, opts?: Record) => FakeRunResult { + return () => ({ status }); +} + +function fakeCapture(output: string): (args: readonly string[]) => string { + return () => output; +} + +describe("findLabeledSandboxContainers", () => { + it("parses the OpenShell-labeled container list and detects running state", () => { + const containers = findLabeledSandboxContainers("e2e-x", { + dockerCapture: fakeCapture( + "openshell-e2e-x\tUp 2 hours\n" + + "openshell-e2e-x-nemoclaw-gpu-backup-1717280000000\tExited (0) 10 minutes ago\n", + ), + }); + expect(containers).toEqual([ + { name: "openshell-e2e-x", status: "Up 2 hours", running: true }, + { + name: "openshell-e2e-x-nemoclaw-gpu-backup-1717280000000", + status: "Exited (0) 10 minutes ago", + running: false, + }, + ]); + }); + + it("returns an empty array when docker ps has no labeled rows", () => { + expect(findLabeledSandboxContainers("e2e-x", { dockerCapture: fakeCapture("") })).toEqual([]); + }); + + it("ignores blank lines and trims whitespace", () => { + const containers = findLabeledSandboxContainers("e2e-x", { + dockerCapture: fakeCapture("\n openshell-e2e-x\tCreated\n\n"), + }); + expect(containers).toEqual([{ name: "openshell-e2e-x", status: "Created", running: false }]); + }); +}); + +describe("recoverDockerDriverSandbox — running original (no-op)", () => { + it("reports recovered with via=started-running-original", () => { + const start = vi.fn(fakeStart(0)); + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture("openshell-e2e-x\tUp 5 minutes\n"), + dockerStart: start, + }); + expect(result).toEqual({ + recovered: true, + via: "started-running-original", + containerName: "openshell-e2e-x", + }); + expect(start).not.toHaveBeenCalled(); + }); +}); + +describe("recoverDockerDriverSandbox — stopped original (start)", () => { + it("starts the labeled container and reports started-stopped-original", () => { + const start = vi.fn(fakeStart(0)); + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture("openshell-e2e-x\tExited (137) 30 seconds ago\n"), + dockerStart: start, + }); + expect(result).toEqual({ + recovered: true, + via: "started-stopped-original", + containerName: "openshell-e2e-x", + }); + expect(start).toHaveBeenCalledTimes(1); + expect(start).toHaveBeenCalledWith( + "openshell-e2e-x", + expect.objectContaining({ ignoreError: true }), + ); + }); + + it("surfaces docker start failure as recovered=false with detail", () => { + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture("openshell-e2e-x\tExited (1) 1 minute ago\n"), + dockerStart: fakeStart(125), + }); + expect(result.recovered).toBe(false); + expect(result.via).toBeNull(); + expect(result.detail).toMatch(/docker start openshell-e2e-x failed.*125/); + }); +}); + +describe("recoverDockerDriverSandbox — backup-only (rename + start)", () => { + it("renames the backup sibling back to the original name and starts it", () => { + const rename = vi.fn(fakeRename(0)); + const start = vi.fn(fakeStart(0)); + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture( + "openshell-e2e-x-nemoclaw-gpu-backup-1717280000000\tExited (0) 5 minutes ago\n", + ), + dockerRename: rename, + dockerStart: start, + }); + expect(result).toEqual({ + recovered: true, + via: "renamed-and-started-backup", + containerName: "openshell-e2e-x", + }); + expect(rename).toHaveBeenCalledWith( + "openshell-e2e-x-nemoclaw-gpu-backup-1717280000000", + "openshell-e2e-x", + expect.objectContaining({ ignoreError: true }), + ); + expect(start).toHaveBeenCalledWith( + "openshell-e2e-x", + expect.objectContaining({ ignoreError: true }), + ); + }); + + it("picks the most recent backup when several siblings exist", () => { + const rename = vi.fn(fakeRename(0)); + const start = vi.fn(fakeStart(0)); + recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture( + "openshell-e2e-x-nemoclaw-gpu-backup-1717280000000\tExited\n" + + "openshell-e2e-x-nemoclaw-gpu-backup-1717290000000\tExited\n", + ), + dockerRename: rename, + dockerStart: start, + }); + expect(rename).toHaveBeenCalledTimes(1); + expect(rename).toHaveBeenCalledWith( + "openshell-e2e-x-nemoclaw-gpu-backup-1717290000000", + "openshell-e2e-x", + expect.anything(), + ); + }); + + it("surfaces docker rename failure as recovered=false", () => { + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture("openshell-e2e-x-nemoclaw-gpu-backup-1717280000000\tExited\n"), + dockerRename: fakeRename(125), + }); + expect(result.recovered).toBe(false); + expect(result.detail).toMatch(/docker rename .* failed.*125/); + }); + + it("surfaces docker start failure after successful rename", () => { + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture("openshell-e2e-x-nemoclaw-gpu-backup-1717280000000\tExited\n"), + dockerRename: fakeRename(0), + dockerStart: fakeStart(1), + }); + expect(result.recovered).toBe(false); + expect(result.detail).toMatch(/after backup rename failed.*1/); + }); +}); + +describe("recoverDockerDriverSandbox — collision and missing cases", () => { + it("prefers the labeled original over a backup sibling when both exist", () => { + const start = vi.fn(fakeStart(0)); + const rename = vi.fn(fakeRename(0)); + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture( + "openshell-e2e-x\tExited (137) 2 minutes ago\n" + + "openshell-e2e-x-nemoclaw-gpu-backup-1717280000000\tExited\n", + ), + dockerStart: start, + dockerRename: rename, + }); + expect(result.via).toBe("started-stopped-original"); + expect(rename).not.toHaveBeenCalled(); + expect(start).toHaveBeenCalledWith("openshell-e2e-x", expect.anything()); + }); + + it("returns recovered=false when no labeled container exists at all", () => { + const result = recoverDockerDriverSandbox("e2e-x", { + dockerCapture: fakeCapture(""), + }); + expect(result.recovered).toBe(false); + expect(result.via).toBeNull(); + expect(result.detail).toMatch(/no Docker container labeled/); + }); +}); diff --git a/src/lib/onboard/docker-driver-sandbox-recovery.ts b/src/lib/onboard/docker-driver-sandbox-recovery.ts new file mode 100644 index 00000000000..fa87e03432f --- /dev/null +++ b/src/lib/onboard/docker-driver-sandbox-recovery.ts @@ -0,0 +1,296 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +// OpenShell-managed container labels — duplicated locally to keep +// the module unit-testable. `docker-gpu-patch.ts` re-exports them as +// the source of truth; a parity test pins drift. +export const OPENSHELL_MANAGED_BY_LABEL = "openshell.ai/managed-by"; +export const OPENSHELL_MANAGED_BY_VALUE = "openshell"; +export const OPENSHELL_SANDBOX_NAME_LABEL = "openshell.ai/sandbox-name"; + +// Lazy adapter accessors. Top-level `import` from `../adapters/docker` +// pulls in `runner.ts`'s load-time `require("./platform")`, which the +// Vitest TS loader cannot resolve. Per-call lazy require keeps the +// module unit-testable in-process; the same pattern is used by +// `auto-pair-approval.ts`. Tests inject `deps` so the lazy require +// never fires. +type DockerCaptureFn = (args: readonly string[], opts?: Record) => string; +type DockerOpResult = { status?: number | null }; +type DockerStartFn = (name: string, opts?: Record) => DockerOpResult; +type DockerRenameFn = ( + oldName: string, + newName: string, + opts?: Record, +) => DockerOpResult; + +function loadDockerCapture(): DockerCaptureFn { + return (require("../adapters/docker") as { dockerCapture: DockerCaptureFn }).dockerCapture; +} +function loadDockerStart(): DockerStartFn { + return (require("../adapters/docker") as { dockerStart: DockerStartFn }).dockerStart; +} +function loadDockerRename(): DockerRenameFn { + return (require("../adapters/docker") as { dockerRename: DockerRenameFn }).dockerRename; +} + +/** + * Active Docker-driver sandbox recovery (#4423 part 2). + * + * After a host reboot on Linux/Spark Docker-driver hosts, the OpenShell + * gateway can come back HEALTHY (per #4580's user-systemd unit) yet + * report a registered sandbox as `NotFound` — its in-memory sandbox + * registry didn't survive the reboot. The labeled Docker container is + * still on disk, often just stopped, sometimes only present as a + * `*-nemoclaw-gpu-backup-*` sibling that the GPU patch path produced. + * + * #4578 (passive guard) and #4497 (active-path tightening) made the + * existing `missing` branches non-destructive — they preserve the + * registry entry. This helper closes the loop by ACTIVELY recovering + * the sandbox container so the user's next command sees a live + * sandbox instead of guidance to retry. + * + * Boundary: + * - This module talks ONLY to Docker. It does not call OpenShell + * directly — callers retry the OpenShell sandbox lookup after + * recovery succeeds. + * - Lookups are by OpenShell labels + * (`openshell.ai/managed-by=openshell` AND + * `openshell.ai/sandbox-name=`), the same labels + * `findOpenShellDockerSandboxContainerIds` already uses. + * - Recovery is best-effort and short-circuits at the first + * successful path. Failure modes are surfaced as + * `{ recovered: false, ... }` so the caller can fall through + * to the existing non-destructive guidance. + */ + +const DOCKER_PROBE_TIMEOUT_MS = 5_000; +const DOCKER_OPERATION_TIMEOUT_MS = 30_000; +const MAX_DOCKER_CONTAINER_NAME_LENGTH = 253; + +/** + * Names recovery dispatched on. Stable identifiers callers can log, + * surface in artifacts, or pin in tests. Adding a new mode requires + * extending `recoverDockerDriverSandbox` and a unit test pinning the + * new branch. + */ +export type DockerDriverRecoveryVia = + | "started-running-original" // labeled container was already running; nothing to do. + | "started-stopped-original" // labeled container existed but was stopped; `docker start`. + | "renamed-and-started-backup"; // only a `*-nemoclaw-gpu-backup-*` sibling existed; rename back + start. + +export interface DockerDriverRecoveryResult { + recovered: boolean; + via: DockerDriverRecoveryVia | null; + /** Container name (post-rename if applicable) the recovery acted on. */ + containerName?: string; + /** + * Human-readable detail when recovery did not succeed — surface in + * logs / guidance. Empty when recovery succeeded. + */ + detail?: string; +} + +export interface DockerDriverRecoveryDeps { + /** `docker ps -a --filter ... --format ...` runner. */ + dockerCapture?: (args: readonly string[], opts?: Record) => string; + dockerStart?: (name: string, opts?: Record) => { status?: number | null }; + dockerRename?: ( + oldName: string, + newName: string, + opts?: Record, + ) => { status?: number | null }; + /** Injectable clock for deterministic backup-name normalization in tests. */ + now?: () => number; +} + +interface LabeledContainer { + name: string; + status: string; + /** True when the container is in a Docker `running` state. */ + running: boolean; +} + +function depsWithDefaults(deps: DockerDriverRecoveryDeps) { + return { + dockerCapture: deps.dockerCapture ?? ((args, opts) => loadDockerCapture()(args, opts)), + dockerStart: deps.dockerStart ?? ((name, opts) => loadDockerStart()(name, opts)), + dockerRename: + deps.dockerRename ?? ((oldName, newName, opts) => loadDockerRename()(oldName, newName, opts)), + now: deps.now ?? (() => Date.now()), + }; +} + +function isBackupSiblingName(name: string): boolean { + return /-nemoclaw-gpu-backup-\d+$/.test(name); +} + +function originalNameFromBackup(backupName: string): string { + return backupName.replace(/-nemoclaw-gpu-backup-\d+$/, ""); +} + +/** + * Discover OpenShell-labeled Docker containers for the given sandbox + * name across all states (running, exited, paused, dead). Empty array + * means recovery has nothing to act on. + * + * Output rows are tab-separated `\t` from + * `--format '{{.Names}}\t{{.Status}}'`. `Status` strings start with + * `Up` for running containers (e.g. `Up 5 minutes`) and `Exited` / + * `Created` / `Paused` / `Dead` otherwise. + */ +export function findLabeledSandboxContainers( + sandboxName: string, + deps: DockerDriverRecoveryDeps = {}, +): LabeledContainer[] { + const d = depsWithDefaults(deps); + const output = d.dockerCapture( + [ + "ps", + "-a", + "--filter", + `label=${OPENSHELL_MANAGED_BY_LABEL}=${OPENSHELL_MANAGED_BY_VALUE}`, + "--filter", + `label=${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}`, + "--format", + "{{.Names}}\t{{.Status}}", + ], + { ignoreError: true, timeout: DOCKER_PROBE_TIMEOUT_MS }, + ); + return output + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => { + const [name, ...rest] = line.split("\t"); + const status = rest.join("\t").trim(); + return { name, status, running: status.startsWith("Up") }; + }); +} + +function classifyCandidate(containers: LabeledContainer[]): { + runningOriginal?: LabeledContainer; + stoppedOriginal?: LabeledContainer; + backup?: LabeledContainer; +} { + let runningOriginal: LabeledContainer | undefined; + let stoppedOriginal: LabeledContainer | undefined; + let backup: LabeledContainer | undefined; + for (const container of containers) { + if (isBackupSiblingName(container.name)) { + // Prefer the most recent backup if there are multiple (timestamp + // suffix is monotonically increasing in `buildBackupContainerName`). + if (!backup || container.name > backup.name) { + backup = container; + } + continue; + } + if (container.running) { + runningOriginal = container; + } else { + stoppedOriginal = container; + } + } + return { runningOriginal, stoppedOriginal, backup }; +} + +function buildBackupRestoreName(originalName: string): string { + if (originalName.length <= MAX_DOCKER_CONTAINER_NAME_LENGTH) { + return originalName; + } + return originalName.slice(0, MAX_DOCKER_CONTAINER_NAME_LENGTH); +} + +/** + * Attempt to recover the labeled sandbox container so OpenShell sees + * the sandbox again. Caller must retry `openshell sandbox get ` + * after a successful return. + * + * Order of operations: + * 1. Already-running labeled container → no-op success. + * 2. Stopped labeled original → `docker start `. + * 3. Backup-only (`*-nemoclaw-gpu-backup-*` sibling) → `docker + * rename ` then `docker start `. + * 4. Conflict (backup AND a stale stopped original with the same + * base name) → start the original; leave the backup alone. + * 5. No labeled container at all → `{ recovered: false }`. + */ +export function recoverDockerDriverSandbox( + sandboxName: string, + deps: DockerDriverRecoveryDeps = {}, +): DockerDriverRecoveryResult { + const d = depsWithDefaults(deps); + const containers = findLabeledSandboxContainers(sandboxName, deps); + if (containers.length === 0) { + return { + recovered: false, + via: null, + detail: `no Docker container labeled '${OPENSHELL_SANDBOX_NAME_LABEL}=${sandboxName}'`, + }; + } + + const { runningOriginal, stoppedOriginal, backup } = classifyCandidate(containers); + + if (runningOriginal) { + return { + recovered: true, + via: "started-running-original", + containerName: runningOriginal.name, + }; + } + + if (stoppedOriginal) { + const result = d.dockerStart(stoppedOriginal.name, { + ignoreError: true, + timeout: DOCKER_OPERATION_TIMEOUT_MS, + }); + if (result.status === 0) { + return { + recovered: true, + via: "started-stopped-original", + containerName: stoppedOriginal.name, + }; + } + return { + recovered: false, + via: null, + detail: `docker start ${stoppedOriginal.name} failed (exit ${result.status ?? "unknown"})`, + }; + } + + if (backup) { + const restoreName = buildBackupRestoreName(originalNameFromBackup(backup.name)); + const renameResult = d.dockerRename(backup.name, restoreName, { + ignoreError: true, + timeout: DOCKER_OPERATION_TIMEOUT_MS, + }); + if (renameResult.status !== 0) { + return { + recovered: false, + via: null, + detail: `docker rename ${backup.name} ${restoreName} failed (exit ${renameResult.status ?? "unknown"})`, + }; + } + const startResult = d.dockerStart(restoreName, { + ignoreError: true, + timeout: DOCKER_OPERATION_TIMEOUT_MS, + }); + if (startResult.status === 0) { + return { + recovered: true, + via: "renamed-and-started-backup", + containerName: restoreName, + }; + } + return { + recovered: false, + via: null, + detail: `docker start ${restoreName} after backup rename failed (exit ${startResult.status ?? "unknown"})`, + }; + } + + return { + recovered: false, + via: null, + detail: "no recoverable container shape (running original / stopped original / backup sibling)", + }; +}