Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/manage-sandboxes/recover-rebuild-sandboxes.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@ This path preserves the sandbox workspace and repairs the agent runtime and host
If the container is paused, follow the printed `docker unpause` guidance instead.
If the container is missing or OpenShell reports another terminal phase such as `Failed`, follow the printed `rebuild --yes` guidance so NemoClaw can recreate the sandbox from its recorded metadata.

<AgentOnly variant="openclaw,hermes">
The `start` command returns success only after it authenticates the recovered agent runtime, OpenShell reports the sandbox ready, and required host forwards pass their checks.
If any check fails, the command keeps the existing container, exits nonzero, identifies the failure, and tells you to run `recover` before retrying `start`.
</AgentOnly>

<AgentOnly variant="openclaw">
If the sandbox has shields up and the OpenClaw gateway does not start after the container restarts, lower shields before you rebuild:

Expand Down
29 changes: 26 additions & 3 deletions src/lib/actions/sandbox/connect.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ import {
import { isWsl } from "../../platform";
import { ROOT } from "../../runner";
import * as sandboxVersion from "../../sandbox/version";
import { redact } from "../../security/redact";
import { redact, redactFull } from "../../security/redact";
import {
isSandboxReady,
isTerminalSandboxPhase,
Expand Down Expand Up @@ -95,6 +95,20 @@ export type SandboxConnectOptions = {
probeOnly?: boolean;
};

export type SandboxStartupRecoveryResult = {
processCheck: ReturnType<typeof checkAndRecoverSandboxProcesses>;
recoveryFailureDetail: string | null;
recoveryFailureLayer: GatewayRestartFailureLayer | null;
};

export function sanitizeSandboxStartupRecoveryDetail(raw: string): string {
return redactFull(raw)
.replace(/[\u0000-\u001f\u007f-\u009f]/gu, " ")
.replace(/\s+/gu, " ")
.trim()
.slice(0, 240);
}

type SpawnLikeResult = {
status: number | null;
signal?: NodeJS.Signals | null;
Expand Down Expand Up @@ -925,8 +939,17 @@ function maybeEnsureHermesToolGatewayBroker(sb: SandboxEntry | null): void {
}
}

export function restoreSandboxStartupState(sandboxName: string): void {
checkAndRecoverSandboxProcesses(sandboxName, { quiet: true });
export function restoreSandboxStartupState(sandboxName: string): SandboxStartupRecoveryResult {
let recoveryFailureDetail: string | null = null;
let recoveryFailureLayer: GatewayRestartFailureLayer | null = null;
const processCheck = checkAndRecoverSandboxProcesses(sandboxName, {
quiet: true,
onRecoveryFailureLayer: (layer, detail) => {
recoveryFailureLayer = layer;
recoveryFailureDetail = detail ?? null;
},
});
return { processCheck, recoveryFailureDetail, recoveryFailureLayer };
}

function restoreInteractiveTerminal(): void {
Expand Down
33 changes: 20 additions & 13 deletions src/lib/actions/sandbox/process-recovery.ts
Original file line number Diff line number Diff line change
Expand Up @@ -445,7 +445,7 @@ function recoverSandboxProcesses(
requestGatewaySupervisorAction?: typeof executeGatewaySupervisorAction;
requestPinnedGatewaySupervisorAction?: RequestPinnedGatewaySupervisorAction;
relaunchManagedSupervisorSessionImpl?: typeof relaunchManagedSupervisorSession;
onFailureLayer?: (layer: GatewayRestartFailureLayer) => void;
onFailureLayer?: (layer: GatewayRestartFailureLayer, detail: string) => void;
} = {},
): SandboxProcessRecovery | null {
const agent = agentRuntime.getSessionAgent(sandboxName);
Expand Down Expand Up @@ -483,7 +483,7 @@ function recoverSandboxProcesses(
sleepSeconds(retryIntervalSeconds);
}
const failure = classifyGatewayRestartFailure(execResult);
onFailureLayer?.(failure.layer);
onFailureLayer?.(failure.layer, failure.detail);
if (
failure.layer === "supervisor not running" &&
isExactlyMissingManagedSupervisor(execResult)
Expand Down Expand Up @@ -1078,7 +1078,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
isSandboxGatewayRunningImpl?: typeof isSandboxGatewayRunning;
waitForRecreatedSandboxOpenShellReadyImpl?: typeof waitForRecreatedSandboxOpenShellReady;
isWsl?: boolean;
onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null) => void;
onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null, detail?: string) => void;
} = {},
) {
const recoveryAgent = agentRuntime.getSessionAgent(sandboxName);
Expand Down Expand Up @@ -1252,13 +1252,15 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
}

let managedRecoveryFailureLayer: GatewayRestartFailureLayer | null = null;
let managedRecoveryFailureDetail: string | null = null;
const recovery = recoverSandboxProcesses(sandboxName, {
quiet,
requestGatewaySupervisorAction,
requestPinnedGatewaySupervisorAction,
relaunchManagedSupervisorSessionImpl,
onFailureLayer: (layer) => {
onFailureLayer: (layer, detail) => {
managedRecoveryFailureLayer = layer;
managedRecoveryFailureDetail = detail;
},
});
if (recovery !== null) {
Expand Down Expand Up @@ -1353,7 +1355,10 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
managedRecoveryFailureLayer,
);
}
onRecoveryFailureLayer?.(managedRecoveryFailureLayer);
onRecoveryFailureLayer?.(
managedRecoveryFailureLayer,
managedRecoveryFailureDetail ?? undefined,
);
if (relaunchedManagedHealthFailureDetail) {
return {
checked: true,
Expand All @@ -1366,14 +1371,16 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
}
return { checked: true, wasRunning: false, recovered: false, forwardRecovered: false };
}
// State restore crosses the OpenShell SSH boundary. Prove the replacement
// is both identity-pinned and registered before asking finalize(true) to
// mutate it; otherwise a slow control-plane handoff can make a healthy
// replacement look like a failed restore and trigger rollback.
const readinessFailureDetail = relaunch
// Host-forward recovery requires an OpenShell-ready sandbox. Managed
// recovery has already passed its authenticated control and health gates;
// a replacement also rechecks its pinned identity before readiness.
const recoveryRequiresReadiness = recovery.kind === "managed" || relaunch;
const readinessFailureDetail = recoveryRequiresReadiness
? (() => {
const readinessOptions: RecreatedSandboxOpenShellReadyOptions = {
beforeProbe: (timeoutMs) => confirmRelaunchedManagedHealth?.(timeoutMs) ?? null,
beforeProbe: relaunch
? (timeoutMs) => confirmRelaunchedManagedHealth?.(timeoutMs) ?? null
: undefined,
};
const readiness =
waitForRecreatedSandboxOpenShellReadyImpl === waitForRecreatedSandboxOpenShellReady
Expand Down Expand Up @@ -1528,7 +1535,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
printHostManagedGatewayRecoveryHints(sandboxName, recoveryAgent, managedRecoveryFailureLayer);
}

onRecoveryFailureLayer?.(managedRecoveryFailureLayer);
onRecoveryFailureLayer?.(managedRecoveryFailureLayer, managedRecoveryFailureDetail ?? undefined);
return { checked: true, wasRunning: false, recovered: false, forwardRecovered: false };
}

Expand All @@ -1542,7 +1549,7 @@ export function checkAndRecoverSandboxProcesses(
isSandboxGatewayRunningImpl?: typeof isSandboxGatewayRunning;
waitForRecreatedSandboxOpenShellReadyImpl?: typeof waitForRecreatedSandboxOpenShellReady;
isWsl?: boolean;
onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null) => void;
onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null, detail?: string) => void;
} = {},
) {
return withTimerBoundShieldsMutationLock(sandboxName, "gateway process recovery", () =>
Expand Down
68 changes: 63 additions & 5 deletions src/lib/actions/sandbox/start.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,26 @@ import {
} from "../../onboard/runtime-provider/docker";
import { createRuntimeProviderBundleRegistry } from "../../onboard/runtime-provider/registry";
import type { SandboxEntry } from "../../state/registry";
import type { SandboxStartupRecoveryResult } from "./connect";
import { restoreStoppedSandboxStartupState, type SandboxStartDeps, startSandbox } from "./start";

function sandbox(values: Partial<SandboxEntry> = {}): SandboxEntry {
return { name: "my-sandbox", ...values };
}

function startupRecovery(
processCheck: SandboxStartupRecoveryResult["processCheck"] = {
checked: true,
wasRunning: true,
recovered: false,
forwardRecovered: false,
},
recoveryFailureLayer: SandboxStartupRecoveryResult["recoveryFailureLayer"] = null,
recoveryFailureDetail: string | null = null,
): SandboxStartupRecoveryResult {
return { processCheck, recoveryFailureDetail, recoveryFailureLayer };
}

function harness(overrides: Partial<SandboxStartDeps> = {}) {
const getSandbox = vi.fn<NonNullable<SandboxStartDeps["getSandbox"]>>(() => sandbox());
const isDockerRuntimeDown = vi.fn<DockerRuntimeProviderDependencies["isRuntimeDown"]>(
Expand Down Expand Up @@ -45,7 +59,9 @@ function harness(overrides: Partial<SandboxStartDeps> = {}) {
const verifyGateway = vi.fn<NonNullable<SandboxStartDeps["verifyGateway"]>>(() =>
Promise.resolve(),
);
const restoreStartupState = vi.fn<NonNullable<SandboxStartDeps["restoreStartupState"]>>();
const restoreStartupState = vi.fn<NonNullable<SandboxStartDeps["restoreStartupState"]>>(() =>
startupRecovery(),
);
const log = vi.fn<(message: string) => void>();
const runtimeProviders = createRuntimeProviderBundleRegistry([
[
Expand Down Expand Up @@ -85,9 +101,10 @@ function harness(overrides: Partial<SandboxStartDeps> = {}) {
describe("startSandbox", () => {
it("restores sealed access before recovering sandbox processes (#8112)", () => {
const restoreAccess = vi.fn();
const restoreProcesses = vi.fn();
const recovery = startupRecovery();
const restoreProcesses = vi.fn(() => recovery);

restoreStoppedSandboxStartupState("my-sandbox", {
const result = restoreStoppedSandboxStartupState("my-sandbox", {
agent: "openclaw",
restoreLockedStartupAccess: restoreAccess,
restoreProcessState: restoreProcesses,
Expand All @@ -98,20 +115,23 @@ describe("startSandbox", () => {
expect(restoreAccess.mock.invocationCallOrder[0]).toBeLessThan(
restoreProcesses.mock.invocationCallOrder[0],
);
expect(result).toBe(recovery);
});

it("keeps Hermes sealed state untouched while recovering sandbox processes (#8112)", () => {
const restoreAccess = vi.fn();
const restoreProcesses = vi.fn();
const recovery = startupRecovery();
const restoreProcesses = vi.fn(() => recovery);

restoreStoppedSandboxStartupState("my-sandbox", {
const result = restoreStoppedSandboxStartupState("my-sandbox", {
agent: "hermes",
restoreLockedStartupAccess: restoreAccess,
restoreProcessState: restoreProcesses,
});

expect(restoreAccess).not.toHaveBeenCalled();
expect(restoreProcesses).toHaveBeenCalledWith("my-sandbox");
expect(result).toBe(recovery);
});

it("restores startup state before probing readiness after a stopped container starts (#8112)", async () => {
Expand Down Expand Up @@ -152,6 +172,44 @@ describe("startSandbox", () => {
);
});

it.each([
[
"openclaw",
"managed gateway recovery",
startupRecovery(
{ checked: true, wasRunning: false, recovered: false, forwardRecovered: false },
"supervisor unavailable",
"SUPERVISOR_UNAVAILABLE Authorization: Bearer opaque-token-8662",
),
/supervisor unavailable/iu,
"opaque-token-8662",
],
[
"hermes",
"OpenShell readiness",
startupRecovery({
checked: true,
wasRunning: false,
recovered: false,
forwardRecovered: false,
forwardRecoveryFailed: true,
forwardRecoveryFailureDetail: "the sandbox did not become ready in OpenShell",
}),
/did not become ready in OpenShell/iu,
"opaque-token-8662",
],
] as const)("propagates an actionable %s %s failure (#8662)", async (agent, _layer, recovery, expected, secret) => {
const h = harness();
h.getSandbox.mockReturnValue(sandbox({ agent }));
h.restoreStartupState.mockReturnValue(recovery);

const failure = await startSandbox("my-sandbox", h.deps).catch((error) => String(error));
expect(failure).toMatch(expected);
expect(failure).toMatch(/nemoclaw my-sandbox recover/iu);
expect(failure).not.toContain(secret);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
expect(h.verifyGateway).not.toHaveBeenCalled();
});

it("reports the started container by name (#6026)", async () => {
const h = harness();

Expand Down
58 changes: 51 additions & 7 deletions src/lib/actions/sandbox/start.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

import { cliName } from "../../onboard/branding";
import {
CURRENT_RUNTIME_PROVIDER_BUNDLES,
type RuntimeProviderBundleRegistry,
Expand All @@ -19,9 +20,11 @@ function verifyGateway(sandboxName: string): Promise<void> {
return connectSandbox(sandboxName, { probeOnly: true });
}

function restoreProcessState(sandboxName: string): void {
type SandboxStartupRecoveryResult = import("./connect").SandboxStartupRecoveryResult;

function restoreProcessState(sandboxName: string): SandboxStartupRecoveryResult {
const { restoreSandboxStartupState } = require("./connect") as typeof import("./connect");
restoreSandboxStartupState(sandboxName);
return restoreSandboxStartupState(sandboxName);
}

function restoreLockedStartupAccess(sandboxName: string): void {
Expand All @@ -33,28 +36,62 @@ function restoreLockedStartupAccess(sandboxName: string): void {
export interface SandboxStartupStateDeps {
agent?: SandboxEntry["agent"];
restoreLockedStartupAccess?: (sandboxName: string) => void;
restoreProcessState?: (sandboxName: string) => void;
restoreProcessState?: (sandboxName: string) => SandboxStartupRecoveryResult;
}

export function restoreStoppedSandboxStartupState(
sandboxName: string,
deps: SandboxStartupStateDeps = {},
): void {
): SandboxStartupRecoveryResult {
if ((deps.agent ?? "openclaw") === "openclaw") {
(deps.restoreLockedStartupAccess ?? restoreLockedStartupAccess)(sandboxName);
}
(deps.restoreProcessState ?? restoreProcessState)(sandboxName);
return (deps.restoreProcessState ?? restoreProcessState)(sandboxName);
}

export interface SandboxStartDeps {
environment?: NodeJS.ProcessEnv;
getSandbox?: typeof registry.getSandbox;
runtimeProviders?: RuntimeProviderBundleRegistry;
restoreStartupState?: (sandboxName: string) => void;
restoreStartupState?: (sandboxName: string) => SandboxStartupRecoveryResult;
verifyGateway?: (sandboxName: string) => Promise<void>;
log?: (message: string) => void;
}

function displayRecoveryDetail(value: unknown): string {
const detail = value instanceof Error && value.message ? value.message : String(value);
const { sanitizeSandboxStartupRecoveryDetail } =
require("./connect") as typeof import("./connect");
return sanitizeSandboxStartupRecoveryDetail(detail);
}

function startupRecoveryFailure(result: SandboxStartupRecoveryResult): string | null {
const check = result.processCheck;
if (!check.checked) return "managed agent gateway inspection did not complete";
if ("runtime" in check && check.runtime === "terminal") return null;
if ("secretBoundaryRefused" in check && check.secretBoundaryRefused) {
return `secret-boundary refusal: ${displayRecoveryDetail(check.secretBoundaryReason)}`;
}
if ("mcpReconciliationRefused" in check && check.mcpReconciliationRefused) {
return `MCP reconciliation refusal: ${displayRecoveryDetail(check.mcpReconciliationReason)}`;
}
if ("forwardRecoveryFailed" in check && check.forwardRecoveryFailed) {
return displayRecoveryDetail(check.forwardRecoveryFailureDetail);
}
if (check.wasRunning || check.recovered) return null;
const layer = result.recoveryFailureLayer ?? "managed agent gateway recovery";
return result.recoveryFailureDetail
? `${layer}: ${displayRecoveryDetail(result.recoveryFailureDetail)}`
: `${layer}: the agent gateway did not recover`;
}

function preservedSandboxRecoveryError(sandboxName: string, detail: unknown): Error {
return new Error(
`Sandbox '${sandboxName}' started, but startup recovery failed: ${displayRecoveryDetail(detail)}. ` +
`The existing sandbox was preserved. Run \`${cliName()} ${sandboxName} recover\`, then retry \`${cliName()} ${sandboxName} start\`.`,
);
}

/**
* Restart a stopped sandbox through the lifecycle facet bound to its durable
* provider identity, then restore startup state before verifying readiness and
Expand Down Expand Up @@ -93,7 +130,14 @@ export async function startSandbox(
restoreStoppedSandboxStartupState(sandboxNameToRestore, {
agent: resolved.sandbox.agent,
}));
restoreStartupState(name);
let recovery: SandboxStartupRecoveryResult;
try {
recovery = restoreStartupState(name);
} catch (error) {
throw preservedSandboxRecoveryError(name, error);
}
Comment on lines +132 to +134

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📐 Maintainability & Code Quality | 🟠 Major | ⚡ Quick win

Import the auto-restore sentinel instead of redeclaring the literal.

Line 16 declares SHIELDS_STARTUP_AUTO_RESTORE_REQUIRED with the same string value that src/lib/shields/index.ts line 1109 declares independently. The two constants are not linked. If one value changes, this branch stops matching, and the operator receives the generic recover guidance instead of the shields up guidance that can actually clear the state. Nothing fails at build time.

Export the constant from the Shields module and import it here so one definition governs both sides.

♻️ Proposed change

In src/lib/shields/index.ts, export the existing constant:

-const SHIELDS_STARTUP_AUTO_RESTORE_REQUIRED = "NEMOCLAW_SHIELDS_AUTO_RESTORE_REQUIRED";
+export const SHIELDS_STARTUP_AUTO_RESTORE_REQUIRED = "NEMOCLAW_SHIELDS_AUTO_RESTORE_REQUIRED";

In src/lib/actions/sandbox/start.ts, drop the local copy and read it through the existing lazy loader:

-const SHIELDS_STARTUP_AUTO_RESTORE_REQUIRED = "NEMOCLAW_SHIELDS_AUTO_RESTORE_REQUIRED";
       if (
         error &&
         typeof error === "object" &&
         "code" in error &&
-        error.code === SHIELDS_STARTUP_AUTO_RESTORE_REQUIRED
+        error.code ===
+          sandboxStartDependencies.loadShields().SHIELDS_STARTUP_AUTO_RESTORE_REQUIRED
       ) {
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
} catch (error) {
if (
error &&
typeof error === "object" &&
"code" in error &&
error.code === SHIELDS_STARTUP_AUTO_RESTORE_REQUIRED
) {
throw new Error(
`Sandbox '${name}' started, but its expired Shields auto-restore must finish before startup recovery. ` +
`The existing sandbox was preserved. Run \`${cliName()} ${name} shields up\`, then retry \`${cliName()} ${name} start\`.`,
);
}
throw new Error(
`Sandbox '${name}' started, but startup state restoration failed: ${sanitizedRecoveryDetail(error)}. ` +
`The existing sandbox was preserved. Run \`${cliName()} ${name} recover\` and retry \`${cliName()} ${name} start\`.`,
);
}
} catch (error) {
if (
error &&
typeof error === "object" &&
"code" in error &&
error.code ===
sandboxStartDependencies.loadShields().SHIELDS_STARTUP_AUTO_RESTORE_REQUIRED
) {
throw new Error(
`Sandbox '${name}' started, but its expired Shields auto-restore must finish before startup recovery. ` +
`The existing sandbox was preserved. Run \`${cliName()} ${name} shields up\`, then retry \`${cliName()} ${name} start\`.`,
);
}
throw new Error(
`Sandbox '${name}' started, but startup state restoration failed: ${sanitizedRecoveryDetail(error)}. ` +
`The existing sandbox was preserved. Run \`${cliName()} ${name} recover\` and retry \`${cliName()} ${name} start\`.`,
);
}
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/lib/actions/sandbox/start.ts` around lines 307 - 323, Export the existing
SHIELDS_STARTUP_AUTO_RESTORE_REQUIRED constant from the Shields module, remove
the duplicate local declaration in the sandbox start action, and retrieve the
shared sentinel through the existing lazy loader used by start.ts so the catch
branch matches the canonical value and preserves the specialized shields up
guidance.

const failure = startupRecoveryFailure(recovery);
if (failure) throw preservedSandboxRecoveryError(name, failure);
log(" Checking gateway health and host forwards…");
await (deps.verifyGateway ?? verifyGateway)(name);
});
Expand Down
Loading
Loading