diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx
index c0d95dc2fe9..26c22171b57 100644
--- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx
+++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx
@@ -25,6 +25,11 @@ This path preserves the sandbox workspace and repairs the agent runtime and host
If the container is paused, follow the printed `docker unpause` guidance instead.
If the container is missing or OpenShell reports another terminal phase such as `Failed`, follow the printed `rebuild --yes` guidance so NemoClaw can recreate the sandbox from its recorded metadata.
+
+The `start` command returns success only after it authenticates the recovered agent runtime, OpenShell reports the sandbox ready, and host-side port forwards pass their checks.
+If any check fails, the command keeps the existing container, exits nonzero, identifies the failure, and tells you to run `recover` before retrying `start`.
+
+
If the sandbox has shields up and the OpenClaw gateway does not start after the container restarts, lower shields before you rebuild:
diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts
index 81a92fdc23f..689d7d22278 100644
--- a/src/lib/actions/sandbox/connect.ts
+++ b/src/lib/actions/sandbox/connect.ts
@@ -38,7 +38,7 @@ import {
import { isWsl } from "../../platform";
import { ROOT } from "../../runner";
import * as sandboxVersion from "../../sandbox/version";
-import { redact } from "../../security/redact";
+import { redact, redactFull } from "../../security/redact";
import {
isSandboxReady,
isTerminalSandboxPhase,
@@ -95,6 +95,19 @@ export type SandboxConnectOptions = {
probeOnly?: boolean;
};
+export type SandboxStartupRecoveryResult = ReturnType & {
+ recoveryFailureDetail?: string | null;
+ recoveryFailureLayer?: GatewayRestartFailureLayer | null;
+};
+
+export function sanitizeSandboxStartupRecoveryDetail(raw: string): string {
+ return redactFull(raw)
+ .replace(/[\u0000-\u001f\u007f-\u009f]/gu, " ")
+ .replace(/\s+/gu, " ")
+ .trim()
+ .slice(0, 240);
+}
+
type SpawnLikeResult = {
status: number | null;
signal?: NodeJS.Signals | null;
@@ -925,8 +938,17 @@ function maybeEnsureHermesToolGatewayBroker(sb: SandboxEntry | null): void {
}
}
-export function restoreSandboxStartupState(sandboxName: string): void {
- checkAndRecoverSandboxProcesses(sandboxName, { quiet: true });
+export function restoreSandboxStartupState(sandboxName: string): SandboxStartupRecoveryResult {
+ let recoveryFailureDetail: string | null = null;
+ let recoveryFailureLayer: GatewayRestartFailureLayer | null = null;
+ const processCheck = checkAndRecoverSandboxProcesses(sandboxName, {
+ quiet: true,
+ onRecoveryFailureLayer: (layer, detail) => {
+ recoveryFailureLayer = layer;
+ recoveryFailureDetail = detail ?? null;
+ },
+ });
+ return { ...processCheck, recoveryFailureDetail, recoveryFailureLayer };
}
function restoreInteractiveTerminal(): void {
diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts
index 3adc1954473..1e375750a54 100644
--- a/src/lib/actions/sandbox/process-recovery.ts
+++ b/src/lib/actions/sandbox/process-recovery.ts
@@ -445,7 +445,7 @@ function recoverSandboxProcesses(
requestGatewaySupervisorAction?: typeof executeGatewaySupervisorAction;
requestPinnedGatewaySupervisorAction?: RequestPinnedGatewaySupervisorAction;
relaunchManagedSupervisorSessionImpl?: typeof relaunchManagedSupervisorSession;
- onFailureLayer?: (layer: GatewayRestartFailureLayer) => void;
+ onFailureLayer?: (layer: GatewayRestartFailureLayer, detail: string) => void;
} = {},
): SandboxProcessRecovery | null {
const agent = agentRuntime.getSessionAgent(sandboxName);
@@ -483,7 +483,7 @@ function recoverSandboxProcesses(
sleepSeconds(retryIntervalSeconds);
}
const failure = classifyGatewayRestartFailure(execResult);
- onFailureLayer?.(failure.layer);
+ onFailureLayer?.(failure.layer, failure.detail);
if (
failure.layer === "supervisor not running" &&
isExactlyMissingManagedSupervisor(execResult)
@@ -1078,7 +1078,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
isSandboxGatewayRunningImpl?: typeof isSandboxGatewayRunning;
waitForRecreatedSandboxOpenShellReadyImpl?: typeof waitForRecreatedSandboxOpenShellReady;
isWsl?: boolean;
- onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null) => void;
+ onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null, detail?: string) => void;
} = {},
) {
const recoveryAgent = agentRuntime.getSessionAgent(sandboxName);
@@ -1252,13 +1252,15 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
}
let managedRecoveryFailureLayer: GatewayRestartFailureLayer | null = null;
+ let managedRecoveryFailureDetail: string | null = null;
const recovery = recoverSandboxProcesses(sandboxName, {
quiet,
requestGatewaySupervisorAction,
requestPinnedGatewaySupervisorAction,
relaunchManagedSupervisorSessionImpl,
- onFailureLayer: (layer) => {
+ onFailureLayer: (layer, detail) => {
managedRecoveryFailureLayer = layer;
+ managedRecoveryFailureDetail = detail;
},
});
if (recovery !== null) {
@@ -1353,7 +1355,10 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
managedRecoveryFailureLayer,
);
}
- onRecoveryFailureLayer?.(managedRecoveryFailureLayer);
+ onRecoveryFailureLayer?.(
+ managedRecoveryFailureLayer,
+ managedRecoveryFailureDetail ?? undefined,
+ );
if (relaunchedManagedHealthFailureDetail) {
return {
checked: true,
@@ -1366,14 +1371,16 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
}
return { checked: true, wasRunning: false, recovered: false, forwardRecovered: false };
}
- // State restore crosses the OpenShell SSH boundary. Prove the replacement
- // is both identity-pinned and registered before asking finalize(true) to
- // mutate it; otherwise a slow control-plane handoff can make a healthy
- // replacement look like a failed restore and trigger rollback.
- const readinessFailureDetail = relaunch
+ // Host-forward recovery requires an OpenShell-ready sandbox. Managed
+ // recovery has already passed its authenticated control and health gates;
+ // a replacement also rechecks its pinned identity before readiness.
+ const recoveryRequiresReadiness = recovery.kind === "managed" || relaunch;
+ const readinessFailureDetail = recoveryRequiresReadiness
? (() => {
const readinessOptions: RecreatedSandboxOpenShellReadyOptions = {
- beforeProbe: (timeoutMs) => confirmRelaunchedManagedHealth?.(timeoutMs) ?? null,
+ beforeProbe: relaunch
+ ? (timeoutMs) => confirmRelaunchedManagedHealth?.(timeoutMs) ?? null
+ : undefined,
};
const readiness =
waitForRecreatedSandboxOpenShellReadyImpl === waitForRecreatedSandboxOpenShellReady
@@ -1528,7 +1535,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
printHostManagedGatewayRecoveryHints(sandboxName, recoveryAgent, managedRecoveryFailureLayer);
}
- onRecoveryFailureLayer?.(managedRecoveryFailureLayer);
+ onRecoveryFailureLayer?.(managedRecoveryFailureLayer, managedRecoveryFailureDetail ?? undefined);
return { checked: true, wasRunning: false, recovered: false, forwardRecovered: false };
}
@@ -1542,7 +1549,7 @@ export function checkAndRecoverSandboxProcesses(
isSandboxGatewayRunningImpl?: typeof isSandboxGatewayRunning;
waitForRecreatedSandboxOpenShellReadyImpl?: typeof waitForRecreatedSandboxOpenShellReady;
isWsl?: boolean;
- onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null) => void;
+ onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null, detail?: string) => void;
} = {},
) {
return withTimerBoundShieldsMutationLock(sandboxName, "gateway process recovery", () =>
diff --git a/src/lib/actions/sandbox/start.test.ts b/src/lib/actions/sandbox/start.test.ts
index e6ebea0d12b..2037a957c3e 100644
--- a/src/lib/actions/sandbox/start.test.ts
+++ b/src/lib/actions/sandbox/start.test.ts
@@ -10,12 +10,22 @@ import {
} from "../../onboard/runtime-provider/docker";
import { createRuntimeProviderBundleRegistry } from "../../onboard/runtime-provider/registry";
import type { SandboxEntry } from "../../state/registry";
+import type { SandboxStartupRecoveryResult } from "./connect";
import { restoreStoppedSandboxStartupState, type SandboxStartDeps, startSandbox } from "./start";
function sandbox(values: Partial = {}): SandboxEntry {
return { name: "my-sandbox", ...values };
}
+const SUCCESSFUL_RECOVERY = {
+ checked: true,
+ wasRunning: true,
+ recovered: false,
+ forwardRecovered: false,
+} as const satisfies SandboxStartupRecoveryResult;
+const FAILED_RECOVERY = { ...SUCCESSFUL_RECOVERY, wasRunning: false } as const;
+const REDACTED_TOKEN = "opaque-token-8662";
+
function harness(overrides: Partial = {}) {
const getSandbox = vi.fn>(() => sandbox());
const isDockerRuntimeDown = vi.fn(
@@ -45,7 +55,9 @@ function harness(overrides: Partial = {}) {
const verifyGateway = vi.fn>(() =>
Promise.resolve(),
);
- const restoreStartupState = vi.fn>();
+ const restoreStartupState = vi.fn>(
+ () => SUCCESSFUL_RECOVERY,
+ );
const log = vi.fn<(message: string) => void>();
const runtimeProviders = createRuntimeProviderBundleRegistry([
[
@@ -85,9 +97,10 @@ function harness(overrides: Partial = {}) {
describe("startSandbox", () => {
it("restores sealed access before recovering sandbox processes (#8112)", () => {
const restoreAccess = vi.fn();
- const restoreProcesses = vi.fn();
+ const recovery = SUCCESSFUL_RECOVERY;
+ const restoreProcesses = vi.fn(() => recovery);
- restoreStoppedSandboxStartupState("my-sandbox", {
+ const result = restoreStoppedSandboxStartupState("my-sandbox", {
agent: "openclaw",
restoreLockedStartupAccess: restoreAccess,
restoreProcessState: restoreProcesses,
@@ -98,11 +111,12 @@ describe("startSandbox", () => {
expect(restoreAccess.mock.invocationCallOrder[0]).toBeLessThan(
restoreProcesses.mock.invocationCallOrder[0],
);
+ expect(result).toBe(recovery);
});
it("keeps Hermes sealed state untouched while recovering sandbox processes (#8112)", () => {
const restoreAccess = vi.fn();
- const restoreProcesses = vi.fn();
+ const restoreProcesses = vi.fn(() => SUCCESSFUL_RECOVERY);
restoreStoppedSandboxStartupState("my-sandbox", {
agent: "hermes",
@@ -133,13 +147,11 @@ describe("startSandbox", () => {
);
});
- it("attempts startup restoration again when start is rerun after a failure (#8112)", async () => {
+ it("retries startup after a structured recovery failure (#8662)", async () => {
const h = harness();
- h.restoreStartupState.mockImplementationOnce(() => {
- throw new Error("restore failed");
- });
+ h.restoreStartupState.mockReturnValueOnce(FAILED_RECOVERY);
- await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow("restore failed");
+ await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow("gateway did not recover");
expect(h.verifyGateway).not.toHaveBeenCalled();
const result = await startSandbox("my-sandbox", h.deps);
@@ -152,6 +164,39 @@ describe("startSandbox", () => {
);
});
+ it.each([
+ [
+ "openclaw",
+ "managed gateway recovery",
+ {
+ ...FAILED_RECOVERY,
+ recoveryFailureLayer: "supervisor unavailable",
+ recoveryFailureDetail: `SUPERVISOR_UNAVAILABLE Authorization: Bearer ${REDACTED_TOKEN}`,
+ },
+ /supervisor unavailable/iu,
+ ],
+ [
+ "hermes",
+ "OpenShell readiness",
+ {
+ ...FAILED_RECOVERY,
+ forwardRecoveryFailed: true,
+ forwardRecoveryFailureDetail: `the sandbox did not become ready in OpenShell: token=${REDACTED_TOKEN}`,
+ },
+ /did not become ready in OpenShell/iu,
+ ],
+ ] as const)("propagates an actionable %s %s failure (#8662)", async (agent, _layer, recovery, expected) => {
+ const h = harness();
+ h.getSandbox.mockReturnValue(sandbox({ agent }));
+ h.restoreStartupState.mockReturnValue(recovery);
+
+ const failure = await startSandbox("my-sandbox", h.deps).catch((error) => String(error));
+ expect(failure).toMatch(expected);
+ expect(failure).toMatch(/nemoclaw my-sandbox recover/iu);
+ expect(failure).not.toContain(REDACTED_TOKEN);
+ expect(h.verifyGateway).not.toHaveBeenCalled();
+ });
+
it("reports the started container by name (#6026)", async () => {
const h = harness();
diff --git a/src/lib/actions/sandbox/start.ts b/src/lib/actions/sandbox/start.ts
index 7e904befd0a..aeb74de90b6 100644
--- a/src/lib/actions/sandbox/start.ts
+++ b/src/lib/actions/sandbox/start.ts
@@ -1,6 +1,7 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
+import { cliName } from "../../onboard/branding";
import {
CURRENT_RUNTIME_PROVIDER_BUNDLES,
type RuntimeProviderBundleRegistry,
@@ -19,9 +20,11 @@ function verifyGateway(sandboxName: string): Promise {
return connectSandbox(sandboxName, { probeOnly: true });
}
-function restoreProcessState(sandboxName: string): void {
+type SandboxStartupRecoveryResult = import("./connect").SandboxStartupRecoveryResult;
+
+function restoreProcessState(sandboxName: string): SandboxStartupRecoveryResult {
const { restoreSandboxStartupState } = require("./connect") as typeof import("./connect");
- restoreSandboxStartupState(sandboxName);
+ return restoreSandboxStartupState(sandboxName);
}
function restoreLockedStartupAccess(sandboxName: string): void {
@@ -33,28 +36,58 @@ function restoreLockedStartupAccess(sandboxName: string): void {
export interface SandboxStartupStateDeps {
agent?: SandboxEntry["agent"];
restoreLockedStartupAccess?: (sandboxName: string) => void;
- restoreProcessState?: (sandboxName: string) => void;
+ restoreProcessState?: (sandboxName: string) => SandboxStartupRecoveryResult;
}
export function restoreStoppedSandboxStartupState(
sandboxName: string,
deps: SandboxStartupStateDeps = {},
-): void {
+): SandboxStartupRecoveryResult {
if ((deps.agent ?? "openclaw") === "openclaw") {
(deps.restoreLockedStartupAccess ?? restoreLockedStartupAccess)(sandboxName);
}
- (deps.restoreProcessState ?? restoreProcessState)(sandboxName);
+ return (deps.restoreProcessState ?? restoreProcessState)(sandboxName);
}
export interface SandboxStartDeps {
environment?: NodeJS.ProcessEnv;
getSandbox?: typeof registry.getSandbox;
runtimeProviders?: RuntimeProviderBundleRegistry;
- restoreStartupState?: (sandboxName: string) => void;
+ restoreStartupState?: (sandboxName: string) => SandboxStartupRecoveryResult;
verifyGateway?: (sandboxName: string) => Promise;
log?: (message: string) => void;
}
+function startupRecoveryFailure(check: SandboxStartupRecoveryResult): string | null {
+ if (!check.checked) return "managed agent gateway inspection did not complete";
+ if ("runtime" in check && check.runtime === "terminal") return null;
+ if ("secretBoundaryRefused" in check && check.secretBoundaryRefused) {
+ return `secret-boundary refusal: ${String(check.secretBoundaryReason)}`;
+ }
+ if ("mcpReconciliationRefused" in check && check.mcpReconciliationRefused) {
+ return `MCP reconciliation refusal: ${String(check.mcpReconciliationReason)}`;
+ }
+ if ("forwardRecoveryFailed" in check && check.forwardRecoveryFailed) {
+ return String(check.forwardRecoveryFailureDetail);
+ }
+ if (check.wasRunning || check.recovered) return null;
+ const layer = check.recoveryFailureLayer ?? "managed agent gateway recovery";
+ return check.recoveryFailureDetail
+ ? `${layer}: ${check.recoveryFailureDetail}`
+ : `${layer}: the agent gateway did not recover`;
+}
+
+function preservedSandboxRecoveryError(sandboxName: string, detail: unknown): Error {
+ const { sanitizeSandboxStartupRecoveryDetail } =
+ require("./connect") as typeof import("./connect");
+ const rawDetail = detail instanceof Error && detail.message ? detail.message : String(detail);
+ const safeDetail = sanitizeSandboxStartupRecoveryDetail(rawDetail);
+ return new Error(
+ `Sandbox '${sandboxName}' started, but startup recovery failed: ${safeDetail}. ` +
+ `The existing sandbox was preserved. Run \`${cliName()} ${sandboxName} recover\`, then retry \`${cliName()} ${sandboxName} start\`.`,
+ );
+}
+
/**
* Restart a stopped sandbox through the lifecycle facet bound to its durable
* provider identity, then restore startup state before verifying readiness and
@@ -93,7 +126,14 @@ export async function startSandbox(
restoreStoppedSandboxStartupState(sandboxNameToRestore, {
agent: resolved.sandbox.agent,
}));
- restoreStartupState(name);
+ let recovery: SandboxStartupRecoveryResult;
+ try {
+ recovery = restoreStartupState(name);
+ } catch (error) {
+ throw preservedSandboxRecoveryError(name, error);
+ }
+ const failure = startupRecoveryFailure(recovery);
+ if (failure) throw preservedSandboxRecoveryError(name, failure);
log(" Checking gateway health and host forwards…");
await (deps.verifyGateway ?? verifyGateway)(name);
});
diff --git a/src/lib/onboard/runtime-provider/podman.test.ts b/src/lib/onboard/runtime-provider/podman.test.ts
index 0fa5c627211..a21b23829ed 100644
--- a/src/lib/onboard/runtime-provider/podman.test.ts
+++ b/src/lib/onboard/runtime-provider/podman.test.ts
@@ -26,6 +26,12 @@ import {
const AGENTS = ["openclaw", "hermes", "langchain-deepagents-code"] as const;
const CONTAINER_ID = "a".repeat(64);
const AUTHORITY_ID = "test:podman-socket";
+const SUCCESSFUL_RECOVERY = {
+ checked: true,
+ wasRunning: true,
+ recovered: false,
+ forwardRecovered: false,
+} as const;
function hostDoctorEngine(authorityId = AUTHORITY_ID): ContainerEngine {
return {
@@ -146,7 +152,7 @@ describe("dormant Podman runtime provider", () => {
)("runs basic CPU start and stop for %s through an injected bundle", async (agent) => {
const runtime = providerHarness(agent);
const verifyGateway = vi.fn(async () => undefined);
- const restoreStartupState = vi.fn();
+ const restoreStartupState = vi.fn(() => SUCCESSFUL_RECOVERY);
const stopSandboxChannels = vi.fn();
await expect(
@@ -188,7 +194,7 @@ describe("dormant Podman runtime provider", () => {
startSandbox(runtime.sandboxName, {
getSandbox: () => runtime.entry,
runtimeProviders: runtime.providers,
- restoreStartupState: vi.fn(),
+ restoreStartupState: vi.fn(() => SUCCESSFUL_RECOVERY),
verifyGateway,
log: vi.fn(),
}),
diff --git a/test/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery-supervisor-relaunch.test.ts
index 7bad729a80a..3ac2a0fb066 100644
--- a/test/process-recovery-supervisor-relaunch.test.ts
+++ b/test/process-recovery-supervisor-relaunch.test.ts
@@ -281,6 +281,52 @@ describe("waitForManagedGatewaySupervisor", () => {
});
describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => {
+ it("checks managed recovery and OpenShell readiness before starting host forwards (#8662)", () => {
+ mockOpenClawSandbox("stopped-box");
+ setImmediateRecoveryPolling();
+ const order: string[] = [];
+ const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => {
+ order.push(action);
+ return {
+ status: 0,
+ stdout: `v1 ${"a".repeat(64)} complete ok 0 4242\nGATEWAY_PID=4242`,
+ stderr: "",
+ };
+ });
+ const waitForRecreatedSandboxOpenShellReadyImpl = vi.fn(() => {
+ order.push("OpenShell readiness");
+ return true;
+ });
+ const relaunchManagedSupervisorSessionImpl = vi.fn(() => null);
+ vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true);
+ vi.spyOn(openshellRuntime, "captureOpenshell")
+ .mockReturnValueOnce({ status: 0, output: "SANDBOX BIND PORT PID STATUS" })
+ .mockReturnValue({
+ status: 0,
+ output: "SANDBOX BIND PORT PID STATUS\nstopped-box 127.0.0.1 18789 12345 running",
+ });
+ vi.spyOn(openshellRuntime, "runOpenshell")
+ .mockReturnValueOnce({ status: 0 } as never)
+ .mockImplementationOnce(() => {
+ order.push("host forward");
+ return { status: 0 } as never;
+ });
+
+ const result = checkAndRecoverSandboxProcesses("stopped-box", {
+ quiet: true,
+ isSandboxGatewayRunningImpl: () => false,
+ requestGatewaySupervisorAction,
+ relaunchManagedSupervisorSessionImpl,
+ waitForRecreatedSandboxOpenShellReadyImpl,
+ });
+
+ expect(result).toMatchObject({ checked: true, recovered: true, forwardRecovered: true });
+ expect(result).toHaveProperty("managedControlCompletion.disposition", "ok");
+ expect(order).toContain("OpenShell readiness");
+ expect(order.indexOf("OpenShell readiness")).toBeLessThan(order.indexOf("host forward"));
+ expect(relaunchManagedSupervisorSessionImpl).not.toHaveBeenCalled();
+ });
+
it("does not turn ambiguous supervisor unavailability into a container mutation", () => {
mockOpenClawSandbox("ambiguous-box");
setImmediateRecoveryPolling();