Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 61 additions & 1 deletion src/lib/actions/sandbox/gateway-state.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,10 @@ import {
OPENSHELL_PROBE_TIMEOUT_MS,
} from "../../adapters/openshell/timeouts";
import { isDockerRuntimeDown, printDockerRuntimeDownGuidance } from "./gateway-failure-classifier";
import {
recoverDockerDriverSandbox,
type DockerDriverRecoveryResult,
} from "../../onboard/docker-driver-sandbox-recovery";

export type SandboxGatewayState = {
state: string;
Expand All @@ -43,6 +47,20 @@ export type SandboxGatewayState = {
recoveredGateway?: boolean;
recoveryVia?: string | null;
gatewayRecoveryFailed?: boolean;
/**
* True when active Docker-driver sandbox recovery (#4423 part 2)
* restarted the labeled sandbox container before the lookup
* returned `present`. Callers can surface this in user-facing
* output to explain why a previously-NotFound sandbox is now
* Ready.
*/
recoveredSandbox?: boolean;
/**
* Stable identifier for which Docker-driver recovery branch fired,
* mirroring `DockerDriverRecoveryVia`. `null` when no Docker-side
* recovery was attempted or required.
*/
recoverySandboxVia?: string | null;
};

type SandboxGatewayStateLookup = (
Expand Down Expand Up @@ -225,7 +243,10 @@ export function reconcileMissingAgainstNamedGateway(
if (retry.state === "missing") {
const after = getNamedGatewayLifecycleState();
if (after.state === "healthy_named") {
return retry;
// Even with the right gateway selected, the sandbox is
// still missing. Try Docker-side recovery before declaring
// the sandbox truly absent.
return tryRecoverDockerDriverSandbox(sandboxName, retry);
}
}
return {
Expand All @@ -240,9 +261,48 @@ export function reconcileMissingAgainstNamedGateway(
if (lifecycle.state === "named_unreachable" || lifecycle.state === "named_unhealthy") {
return { state: "gateway_unreachable_after_restart", output: lifecycle.status };
}
if (lifecycle.state === "healthy_named") {
// The gateway is healthy and we already see `missing`. This is
// the precise post-reboot precondition described in #4423: the
// gateway came back fresh (per #4580's user-systemd unit) with
// no sandbox memory, but Docker may still have the labeled
// container. Attempt active Docker-side recovery before falling
// through to non-destructive guidance.
return tryRecoverDockerDriverSandbox(sandboxName, missingLookup);
}
return missingLookup;
}

/**
* Attempt Docker-driver sandbox recovery (#4423) and re-query the
* OpenShell gateway. Returns the new lookup with `recoveredSandbox`
* flags set when recovery succeeded; otherwise returns the original
* `missing` lookup unchanged so the caller's existing non-destructive
* guidance fires.
*/
function tryRecoverDockerDriverSandbox(
sandboxName: string,
missingLookup: SandboxGatewayState,
): SandboxGatewayState {
let recovery: DockerDriverRecoveryResult;
try {
recovery = recoverDockerDriverSandbox(sandboxName);
} catch {
return missingLookup;
}
if (!recovery.recovered) {
return missingLookup;
}
// Recovery succeeded against Docker; re-query OpenShell so the
// returned state reflects what the gateway sees post-restart.
const retried = getSandboxGatewayState(sandboxName);
return {
...retried,
recoveredSandbox: true,
recoverySandboxVia: recovery.via,
};
}

/**
* Print actionable guidance when the nemoclaw gateway exists but another
* OpenShell gateway is currently active. Emphasizes that the sandbox has NOT
Expand Down
10 changes: 10 additions & 0 deletions src/lib/actions/sandbox/status.ts
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,16 @@ export async function showSandboxStatus(sandboxName: string): Promise<void> {
);
console.log("");
}
if ("recoveredSandbox" in lookup && lookup.recoveredSandbox) {
const via =
"recoverySandboxVia" in lookup && lookup.recoverySandboxVia
? ` via ${lookup.recoverySandboxVia}`
: "";
console.log(
` Recovered sandbox '${sandboxName}' from Docker${via}; OpenShell now sees it as live.`,
);
console.log("");
}
console.log(lookup.output);
if (phase && phase !== "Ready") {
// A non-ready, non-terminal phase can mean two very different things. If
Expand Down
196 changes: 196 additions & 0 deletions src/lib/onboard/docker-driver-sandbox-recovery.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,196 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { describe, expect, it, vi } from "vitest";

import {
findLabeledSandboxContainers,
recoverDockerDriverSandbox,
} from "./docker-driver-sandbox-recovery";

interface FakeRunResult {
status: number;
}

function fakeStart(status = 0): (name: string, opts?: Record<string, unknown>) => FakeRunResult {
return () => ({ status });
}

function fakeRename(
status = 0,
): (oldName: string, newName: string, opts?: Record<string, unknown>) => FakeRunResult {
return () => ({ status });
}

function fakeCapture(output: string): (args: readonly string[]) => string {
return () => output;
}
Comment on lines +15 to +27

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor | ⚡ Quick win

Prefix unused parameters with underscore.

The returned arrow functions in fakeStart, fakeRename, and fakeCapture receive parameters that are never used. According to the coding guidelines, unused variables must be prefixed with underscore.

🔧 Proposed fix
-function fakeStart(status = 0): (name: string, opts?: Record<string, unknown>) => FakeRunResult {
-  return () => ({ status });
+function fakeStart(status = 0): (_name: string, _opts?: Record<string, unknown>) => FakeRunResult {
+  return (_name, _opts) => ({ status });
 }
 
 function fakeRename(
   status = 0,
-): (oldName: string, newName: string, opts?: Record<string, unknown>) => FakeRunResult {
-  return () => ({ status });
+): (_oldName: string, _newName: string, _opts?: Record<string, unknown>) => FakeRunResult {
+  return (_oldName, _newName, _opts) => ({ status });
 }
 
-function fakeCapture(output: string): (args: readonly string[]) => string {
-  return () => output;
+function fakeCapture(output: string): (_args: readonly string[]) => string {
+  return (_args) => output;
 }
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
function fakeStart(status = 0): (name: string, opts?: Record<string, unknown>) => FakeRunResult {
return () => ({ status });
}
function fakeRename(
status = 0,
): (oldName: string, newName: string, opts?: Record<string, unknown>) => FakeRunResult {
return () => ({ status });
}
function fakeCapture(output: string): (args: readonly string[]) => string {
return () => output;
}
function fakeStart(status = 0): (_name: string, _opts?: Record<string, unknown>) => FakeRunResult {
return (_name, _opts) => ({ status });
}
function fakeRename(
status = 0,
): (_oldName: string, _newName: string, _opts?: Record<string, unknown>) => FakeRunResult {
return (_oldName, _newName, _opts) => ({ status });
}
function fakeCapture(output: string): (_args: readonly string[]) => string {
return (_args) => output;
}
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/lib/onboard/docker-driver-sandbox-recovery.test.ts` around lines 15 - 27,
The returned arrow functions in fakeStart, fakeRename, and fakeCapture declare
parameters that are unused; update their parameter names to be prefixed with an
underscore to satisfy the lint rule (e.g., in fakeStart change (name, opts?) to
(_name, _opts?), in fakeRename change (oldName, newName, opts?) to (_oldName,
_newName, _opts?), and in fakeCapture change (args) to (_args)); keep the same
types and return values, only rename the parameters.

Source: Coding guidelines


describe("findLabeledSandboxContainers", () => {
it("parses the OpenShell-labeled container list and detects running state", () => {
const containers = findLabeledSandboxContainers("e2e-x", {
dockerCapture: fakeCapture(
"openshell-e2e-x\tUp 2 hours\n" +
"openshell-e2e-x-nemoclaw-gpu-backup-1717280000000\tExited (0) 10 minutes ago\n",
),
});
expect(containers).toEqual([
{ name: "openshell-e2e-x", status: "Up 2 hours", running: true },
{
name: "openshell-e2e-x-nemoclaw-gpu-backup-1717280000000",
status: "Exited (0) 10 minutes ago",
running: false,
},
]);
});

it("returns an empty array when docker ps has no labeled rows", () => {
expect(findLabeledSandboxContainers("e2e-x", { dockerCapture: fakeCapture("") })).toEqual([]);
});

it("ignores blank lines and trims whitespace", () => {
const containers = findLabeledSandboxContainers("e2e-x", {
dockerCapture: fakeCapture("\n openshell-e2e-x\tCreated\n\n"),
});
expect(containers).toEqual([{ name: "openshell-e2e-x", status: "Created", running: false }]);
});
});

describe("recoverDockerDriverSandbox — running original (no-op)", () => {
it("reports recovered with via=started-running-original", () => {
const start = vi.fn(fakeStart(0));
const result = recoverDockerDriverSandbox("e2e-x", {
dockerCapture: fakeCapture("openshell-e2e-x\tUp 5 minutes\n"),
dockerStart: start,
});
expect(result).toEqual({
recovered: true,
via: "started-running-original",
containerName: "openshell-e2e-x",
});
expect(start).not.toHaveBeenCalled();
});
});

describe("recoverDockerDriverSandbox — stopped original (start)", () => {
it("starts the labeled container and reports started-stopped-original", () => {
const start = vi.fn(fakeStart(0));
const result = recoverDockerDriverSandbox("e2e-x", {
dockerCapture: fakeCapture("openshell-e2e-x\tExited (137) 30 seconds ago\n"),
dockerStart: start,
});
expect(result).toEqual({
recovered: true,
via: "started-stopped-original",
containerName: "openshell-e2e-x",
});
expect(start).toHaveBeenCalledTimes(1);
expect(start).toHaveBeenCalledWith(
"openshell-e2e-x",
expect.objectContaining({ ignoreError: true }),
);
});

it("surfaces docker start failure as recovered=false with detail", () => {
const result = recoverDockerDriverSandbox("e2e-x", {
dockerCapture: fakeCapture("openshell-e2e-x\tExited (1) 1 minute ago\n"),
dockerStart: fakeStart(125),
});
expect(result.recovered).toBe(false);
expect(result.via).toBeNull();
expect(result.detail).toMatch(/docker start openshell-e2e-x failed.*125/);
});
});

describe("recoverDockerDriverSandbox — backup-only (rename + start)", () => {
it("renames the backup sibling back to the original name and starts it", () => {
const rename = vi.fn(fakeRename(0));
const start = vi.fn(fakeStart(0));
const result = recoverDockerDriverSandbox("e2e-x", {
dockerCapture: fakeCapture(
"openshell-e2e-x-nemoclaw-gpu-backup-1717280000000\tExited (0) 5 minutes ago\n",
),
dockerRename: rename,
dockerStart: start,
});
expect(result).toEqual({
recovered: true,
via: "renamed-and-started-backup",
containerName: "openshell-e2e-x",
});
expect(rename).toHaveBeenCalledWith(
"openshell-e2e-x-nemoclaw-gpu-backup-1717280000000",
"openshell-e2e-x",
expect.objectContaining({ ignoreError: true }),
);
expect(start).toHaveBeenCalledWith(
"openshell-e2e-x",
expect.objectContaining({ ignoreError: true }),
);
});

it("picks the most recent backup when several siblings exist", () => {
const rename = vi.fn(fakeRename(0));
const start = vi.fn(fakeStart(0));
recoverDockerDriverSandbox("e2e-x", {
dockerCapture: fakeCapture(
"openshell-e2e-x-nemoclaw-gpu-backup-1717280000000\tExited\n" +
"openshell-e2e-x-nemoclaw-gpu-backup-1717290000000\tExited\n",
),
dockerRename: rename,
dockerStart: start,
});
expect(rename).toHaveBeenCalledTimes(1);
expect(rename).toHaveBeenCalledWith(
"openshell-e2e-x-nemoclaw-gpu-backup-1717290000000",
"openshell-e2e-x",
expect.anything(),
);
});

it("surfaces docker rename failure as recovered=false", () => {
const result = recoverDockerDriverSandbox("e2e-x", {
dockerCapture: fakeCapture("openshell-e2e-x-nemoclaw-gpu-backup-1717280000000\tExited\n"),
dockerRename: fakeRename(125),
});
expect(result.recovered).toBe(false);
expect(result.detail).toMatch(/docker rename .* failed.*125/);
});

it("surfaces docker start failure after successful rename", () => {
const result = recoverDockerDriverSandbox("e2e-x", {
dockerCapture: fakeCapture("openshell-e2e-x-nemoclaw-gpu-backup-1717280000000\tExited\n"),
dockerRename: fakeRename(0),
dockerStart: fakeStart(1),
});
expect(result.recovered).toBe(false);
expect(result.detail).toMatch(/after backup rename failed.*1/);
});
});

describe("recoverDockerDriverSandbox — collision and missing cases", () => {
it("prefers the labeled original over a backup sibling when both exist", () => {
const start = vi.fn(fakeStart(0));
const rename = vi.fn(fakeRename(0));
const result = recoverDockerDriverSandbox("e2e-x", {
dockerCapture: fakeCapture(
"openshell-e2e-x\tExited (137) 2 minutes ago\n" +
"openshell-e2e-x-nemoclaw-gpu-backup-1717280000000\tExited\n",
),
dockerStart: start,
dockerRename: rename,
});
expect(result.via).toBe("started-stopped-original");
expect(rename).not.toHaveBeenCalled();
expect(start).toHaveBeenCalledWith("openshell-e2e-x", expect.anything());
});

it("returns recovered=false when no labeled container exists at all", () => {
const result = recoverDockerDriverSandbox("e2e-x", {
dockerCapture: fakeCapture(""),
});
expect(result.recovered).toBe(false);
expect(result.via).toBeNull();
expect(result.detail).toMatch(/no Docker container labeled/);
});
});
Loading