diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index c0d95dc2fe9..26c22171b57 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -25,6 +25,11 @@ This path preserves the sandbox workspace and repairs the agent runtime and host If the container is paused, follow the printed `docker unpause` guidance instead. If the container is missing or OpenShell reports another terminal phase such as `Failed`, follow the printed `rebuild --yes` guidance so NemoClaw can recreate the sandbox from its recorded metadata. + +The `start` command returns success only after it authenticates the recovered agent runtime, OpenShell reports the sandbox ready, and host-side port forwards pass their checks. +If any check fails, the command keeps the existing container, exits nonzero, identifies the failure, and tells you to run `recover` before retrying `start`. + + If the sandbox has shields up and the OpenClaw gateway does not start after the container restarts, lower shields before you rebuild: diff --git a/src/lib/actions/sandbox/connect.ts b/src/lib/actions/sandbox/connect.ts index 81a92fdc23f..689d7d22278 100644 --- a/src/lib/actions/sandbox/connect.ts +++ b/src/lib/actions/sandbox/connect.ts @@ -38,7 +38,7 @@ import { import { isWsl } from "../../platform"; import { ROOT } from "../../runner"; import * as sandboxVersion from "../../sandbox/version"; -import { redact } from "../../security/redact"; +import { redact, redactFull } from "../../security/redact"; import { isSandboxReady, isTerminalSandboxPhase, @@ -95,6 +95,19 @@ export type SandboxConnectOptions = { probeOnly?: boolean; }; +export type SandboxStartupRecoveryResult = ReturnType & { + recoveryFailureDetail?: string | null; + recoveryFailureLayer?: GatewayRestartFailureLayer | null; +}; + +export function sanitizeSandboxStartupRecoveryDetail(raw: string): string { + return redactFull(raw) + .replace(/[\u0000-\u001f\u007f-\u009f]/gu, " ") + .replace(/\s+/gu, " ") + .trim() + .slice(0, 240); +} + type SpawnLikeResult = { status: number | null; signal?: NodeJS.Signals | null; @@ -925,8 +938,17 @@ function maybeEnsureHermesToolGatewayBroker(sb: SandboxEntry | null): void { } } -export function restoreSandboxStartupState(sandboxName: string): void { - checkAndRecoverSandboxProcesses(sandboxName, { quiet: true }); +export function restoreSandboxStartupState(sandboxName: string): SandboxStartupRecoveryResult { + let recoveryFailureDetail: string | null = null; + let recoveryFailureLayer: GatewayRestartFailureLayer | null = null; + const processCheck = checkAndRecoverSandboxProcesses(sandboxName, { + quiet: true, + onRecoveryFailureLayer: (layer, detail) => { + recoveryFailureLayer = layer; + recoveryFailureDetail = detail ?? null; + }, + }); + return { ...processCheck, recoveryFailureDetail, recoveryFailureLayer }; } function restoreInteractiveTerminal(): void { diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index 3adc1954473..1e375750a54 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -445,7 +445,7 @@ function recoverSandboxProcesses( requestGatewaySupervisorAction?: typeof executeGatewaySupervisorAction; requestPinnedGatewaySupervisorAction?: RequestPinnedGatewaySupervisorAction; relaunchManagedSupervisorSessionImpl?: typeof relaunchManagedSupervisorSession; - onFailureLayer?: (layer: GatewayRestartFailureLayer) => void; + onFailureLayer?: (layer: GatewayRestartFailureLayer, detail: string) => void; } = {}, ): SandboxProcessRecovery | null { const agent = agentRuntime.getSessionAgent(sandboxName); @@ -483,7 +483,7 @@ function recoverSandboxProcesses( sleepSeconds(retryIntervalSeconds); } const failure = classifyGatewayRestartFailure(execResult); - onFailureLayer?.(failure.layer); + onFailureLayer?.(failure.layer, failure.detail); if ( failure.layer === "supervisor not running" && isExactlyMissingManagedSupervisor(execResult) @@ -1078,7 +1078,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( isSandboxGatewayRunningImpl?: typeof isSandboxGatewayRunning; waitForRecreatedSandboxOpenShellReadyImpl?: typeof waitForRecreatedSandboxOpenShellReady; isWsl?: boolean; - onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null) => void; + onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null, detail?: string) => void; } = {}, ) { const recoveryAgent = agentRuntime.getSessionAgent(sandboxName); @@ -1252,13 +1252,15 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( } let managedRecoveryFailureLayer: GatewayRestartFailureLayer | null = null; + let managedRecoveryFailureDetail: string | null = null; const recovery = recoverSandboxProcesses(sandboxName, { quiet, requestGatewaySupervisorAction, requestPinnedGatewaySupervisorAction, relaunchManagedSupervisorSessionImpl, - onFailureLayer: (layer) => { + onFailureLayer: (layer, detail) => { managedRecoveryFailureLayer = layer; + managedRecoveryFailureDetail = detail; }, }); if (recovery !== null) { @@ -1353,7 +1355,10 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( managedRecoveryFailureLayer, ); } - onRecoveryFailureLayer?.(managedRecoveryFailureLayer); + onRecoveryFailureLayer?.( + managedRecoveryFailureLayer, + managedRecoveryFailureDetail ?? undefined, + ); if (relaunchedManagedHealthFailureDetail) { return { checked: true, @@ -1366,14 +1371,16 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( } return { checked: true, wasRunning: false, recovered: false, forwardRecovered: false }; } - // State restore crosses the OpenShell SSH boundary. Prove the replacement - // is both identity-pinned and registered before asking finalize(true) to - // mutate it; otherwise a slow control-plane handoff can make a healthy - // replacement look like a failed restore and trigger rollback. - const readinessFailureDetail = relaunch + // Host-forward recovery requires an OpenShell-ready sandbox. Managed + // recovery has already passed its authenticated control and health gates; + // a replacement also rechecks its pinned identity before readiness. + const recoveryRequiresReadiness = recovery.kind === "managed" || relaunch; + const readinessFailureDetail = recoveryRequiresReadiness ? (() => { const readinessOptions: RecreatedSandboxOpenShellReadyOptions = { - beforeProbe: (timeoutMs) => confirmRelaunchedManagedHealth?.(timeoutMs) ?? null, + beforeProbe: relaunch + ? (timeoutMs) => confirmRelaunchedManagedHealth?.(timeoutMs) ?? null + : undefined, }; const readiness = waitForRecreatedSandboxOpenShellReadyImpl === waitForRecreatedSandboxOpenShellReady @@ -1528,7 +1535,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( printHostManagedGatewayRecoveryHints(sandboxName, recoveryAgent, managedRecoveryFailureLayer); } - onRecoveryFailureLayer?.(managedRecoveryFailureLayer); + onRecoveryFailureLayer?.(managedRecoveryFailureLayer, managedRecoveryFailureDetail ?? undefined); return { checked: true, wasRunning: false, recovered: false, forwardRecovered: false }; } @@ -1542,7 +1549,7 @@ export function checkAndRecoverSandboxProcesses( isSandboxGatewayRunningImpl?: typeof isSandboxGatewayRunning; waitForRecreatedSandboxOpenShellReadyImpl?: typeof waitForRecreatedSandboxOpenShellReady; isWsl?: boolean; - onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null) => void; + onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null, detail?: string) => void; } = {}, ) { return withTimerBoundShieldsMutationLock(sandboxName, "gateway process recovery", () => diff --git a/src/lib/actions/sandbox/start.test.ts b/src/lib/actions/sandbox/start.test.ts index e6ebea0d12b..2037a957c3e 100644 --- a/src/lib/actions/sandbox/start.test.ts +++ b/src/lib/actions/sandbox/start.test.ts @@ -10,12 +10,22 @@ import { } from "../../onboard/runtime-provider/docker"; import { createRuntimeProviderBundleRegistry } from "../../onboard/runtime-provider/registry"; import type { SandboxEntry } from "../../state/registry"; +import type { SandboxStartupRecoveryResult } from "./connect"; import { restoreStoppedSandboxStartupState, type SandboxStartDeps, startSandbox } from "./start"; function sandbox(values: Partial = {}): SandboxEntry { return { name: "my-sandbox", ...values }; } +const SUCCESSFUL_RECOVERY = { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, +} as const satisfies SandboxStartupRecoveryResult; +const FAILED_RECOVERY = { ...SUCCESSFUL_RECOVERY, wasRunning: false } as const; +const REDACTED_TOKEN = "opaque-token-8662"; + function harness(overrides: Partial = {}) { const getSandbox = vi.fn>(() => sandbox()); const isDockerRuntimeDown = vi.fn( @@ -45,7 +55,9 @@ function harness(overrides: Partial = {}) { const verifyGateway = vi.fn>(() => Promise.resolve(), ); - const restoreStartupState = vi.fn>(); + const restoreStartupState = vi.fn>( + () => SUCCESSFUL_RECOVERY, + ); const log = vi.fn<(message: string) => void>(); const runtimeProviders = createRuntimeProviderBundleRegistry([ [ @@ -85,9 +97,10 @@ function harness(overrides: Partial = {}) { describe("startSandbox", () => { it("restores sealed access before recovering sandbox processes (#8112)", () => { const restoreAccess = vi.fn(); - const restoreProcesses = vi.fn(); + const recovery = SUCCESSFUL_RECOVERY; + const restoreProcesses = vi.fn(() => recovery); - restoreStoppedSandboxStartupState("my-sandbox", { + const result = restoreStoppedSandboxStartupState("my-sandbox", { agent: "openclaw", restoreLockedStartupAccess: restoreAccess, restoreProcessState: restoreProcesses, @@ -98,11 +111,12 @@ describe("startSandbox", () => { expect(restoreAccess.mock.invocationCallOrder[0]).toBeLessThan( restoreProcesses.mock.invocationCallOrder[0], ); + expect(result).toBe(recovery); }); it("keeps Hermes sealed state untouched while recovering sandbox processes (#8112)", () => { const restoreAccess = vi.fn(); - const restoreProcesses = vi.fn(); + const restoreProcesses = vi.fn(() => SUCCESSFUL_RECOVERY); restoreStoppedSandboxStartupState("my-sandbox", { agent: "hermes", @@ -133,13 +147,11 @@ describe("startSandbox", () => { ); }); - it("attempts startup restoration again when start is rerun after a failure (#8112)", async () => { + it("retries startup after a structured recovery failure (#8662)", async () => { const h = harness(); - h.restoreStartupState.mockImplementationOnce(() => { - throw new Error("restore failed"); - }); + h.restoreStartupState.mockReturnValueOnce(FAILED_RECOVERY); - await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow("restore failed"); + await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow("gateway did not recover"); expect(h.verifyGateway).not.toHaveBeenCalled(); const result = await startSandbox("my-sandbox", h.deps); @@ -152,6 +164,39 @@ describe("startSandbox", () => { ); }); + it.each([ + [ + "openclaw", + "managed gateway recovery", + { + ...FAILED_RECOVERY, + recoveryFailureLayer: "supervisor unavailable", + recoveryFailureDetail: `SUPERVISOR_UNAVAILABLE Authorization: Bearer ${REDACTED_TOKEN}`, + }, + /supervisor unavailable/iu, + ], + [ + "hermes", + "OpenShell readiness", + { + ...FAILED_RECOVERY, + forwardRecoveryFailed: true, + forwardRecoveryFailureDetail: `the sandbox did not become ready in OpenShell: token=${REDACTED_TOKEN}`, + }, + /did not become ready in OpenShell/iu, + ], + ] as const)("propagates an actionable %s %s failure (#8662)", async (agent, _layer, recovery, expected) => { + const h = harness(); + h.getSandbox.mockReturnValue(sandbox({ agent })); + h.restoreStartupState.mockReturnValue(recovery); + + const failure = await startSandbox("my-sandbox", h.deps).catch((error) => String(error)); + expect(failure).toMatch(expected); + expect(failure).toMatch(/nemoclaw my-sandbox recover/iu); + expect(failure).not.toContain(REDACTED_TOKEN); + expect(h.verifyGateway).not.toHaveBeenCalled(); + }); + it("reports the started container by name (#6026)", async () => { const h = harness(); diff --git a/src/lib/actions/sandbox/start.ts b/src/lib/actions/sandbox/start.ts index 7e904befd0a..aeb74de90b6 100644 --- a/src/lib/actions/sandbox/start.ts +++ b/src/lib/actions/sandbox/start.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { cliName } from "../../onboard/branding"; import { CURRENT_RUNTIME_PROVIDER_BUNDLES, type RuntimeProviderBundleRegistry, @@ -19,9 +20,11 @@ function verifyGateway(sandboxName: string): Promise { return connectSandbox(sandboxName, { probeOnly: true }); } -function restoreProcessState(sandboxName: string): void { +type SandboxStartupRecoveryResult = import("./connect").SandboxStartupRecoveryResult; + +function restoreProcessState(sandboxName: string): SandboxStartupRecoveryResult { const { restoreSandboxStartupState } = require("./connect") as typeof import("./connect"); - restoreSandboxStartupState(sandboxName); + return restoreSandboxStartupState(sandboxName); } function restoreLockedStartupAccess(sandboxName: string): void { @@ -33,28 +36,58 @@ function restoreLockedStartupAccess(sandboxName: string): void { export interface SandboxStartupStateDeps { agent?: SandboxEntry["agent"]; restoreLockedStartupAccess?: (sandboxName: string) => void; - restoreProcessState?: (sandboxName: string) => void; + restoreProcessState?: (sandboxName: string) => SandboxStartupRecoveryResult; } export function restoreStoppedSandboxStartupState( sandboxName: string, deps: SandboxStartupStateDeps = {}, -): void { +): SandboxStartupRecoveryResult { if ((deps.agent ?? "openclaw") === "openclaw") { (deps.restoreLockedStartupAccess ?? restoreLockedStartupAccess)(sandboxName); } - (deps.restoreProcessState ?? restoreProcessState)(sandboxName); + return (deps.restoreProcessState ?? restoreProcessState)(sandboxName); } export interface SandboxStartDeps { environment?: NodeJS.ProcessEnv; getSandbox?: typeof registry.getSandbox; runtimeProviders?: RuntimeProviderBundleRegistry; - restoreStartupState?: (sandboxName: string) => void; + restoreStartupState?: (sandboxName: string) => SandboxStartupRecoveryResult; verifyGateway?: (sandboxName: string) => Promise; log?: (message: string) => void; } +function startupRecoveryFailure(check: SandboxStartupRecoveryResult): string | null { + if (!check.checked) return "managed agent gateway inspection did not complete"; + if ("runtime" in check && check.runtime === "terminal") return null; + if ("secretBoundaryRefused" in check && check.secretBoundaryRefused) { + return `secret-boundary refusal: ${String(check.secretBoundaryReason)}`; + } + if ("mcpReconciliationRefused" in check && check.mcpReconciliationRefused) { + return `MCP reconciliation refusal: ${String(check.mcpReconciliationReason)}`; + } + if ("forwardRecoveryFailed" in check && check.forwardRecoveryFailed) { + return String(check.forwardRecoveryFailureDetail); + } + if (check.wasRunning || check.recovered) return null; + const layer = check.recoveryFailureLayer ?? "managed agent gateway recovery"; + return check.recoveryFailureDetail + ? `${layer}: ${check.recoveryFailureDetail}` + : `${layer}: the agent gateway did not recover`; +} + +function preservedSandboxRecoveryError(sandboxName: string, detail: unknown): Error { + const { sanitizeSandboxStartupRecoveryDetail } = + require("./connect") as typeof import("./connect"); + const rawDetail = detail instanceof Error && detail.message ? detail.message : String(detail); + const safeDetail = sanitizeSandboxStartupRecoveryDetail(rawDetail); + return new Error( + `Sandbox '${sandboxName}' started, but startup recovery failed: ${safeDetail}. ` + + `The existing sandbox was preserved. Run \`${cliName()} ${sandboxName} recover\`, then retry \`${cliName()} ${sandboxName} start\`.`, + ); +} + /** * Restart a stopped sandbox through the lifecycle facet bound to its durable * provider identity, then restore startup state before verifying readiness and @@ -93,7 +126,14 @@ export async function startSandbox( restoreStoppedSandboxStartupState(sandboxNameToRestore, { agent: resolved.sandbox.agent, })); - restoreStartupState(name); + let recovery: SandboxStartupRecoveryResult; + try { + recovery = restoreStartupState(name); + } catch (error) { + throw preservedSandboxRecoveryError(name, error); + } + const failure = startupRecoveryFailure(recovery); + if (failure) throw preservedSandboxRecoveryError(name, failure); log(" Checking gateway health and host forwards…"); await (deps.verifyGateway ?? verifyGateway)(name); }); diff --git a/src/lib/onboard/runtime-provider/podman.test.ts b/src/lib/onboard/runtime-provider/podman.test.ts index 0fa5c627211..a21b23829ed 100644 --- a/src/lib/onboard/runtime-provider/podman.test.ts +++ b/src/lib/onboard/runtime-provider/podman.test.ts @@ -26,6 +26,12 @@ import { const AGENTS = ["openclaw", "hermes", "langchain-deepagents-code"] as const; const CONTAINER_ID = "a".repeat(64); const AUTHORITY_ID = "test:podman-socket"; +const SUCCESSFUL_RECOVERY = { + checked: true, + wasRunning: true, + recovered: false, + forwardRecovered: false, +} as const; function hostDoctorEngine(authorityId = AUTHORITY_ID): ContainerEngine { return { @@ -146,7 +152,7 @@ describe("dormant Podman runtime provider", () => { )("runs basic CPU start and stop for %s through an injected bundle", async (agent) => { const runtime = providerHarness(agent); const verifyGateway = vi.fn(async () => undefined); - const restoreStartupState = vi.fn(); + const restoreStartupState = vi.fn(() => SUCCESSFUL_RECOVERY); const stopSandboxChannels = vi.fn(); await expect( @@ -188,7 +194,7 @@ describe("dormant Podman runtime provider", () => { startSandbox(runtime.sandboxName, { getSandbox: () => runtime.entry, runtimeProviders: runtime.providers, - restoreStartupState: vi.fn(), + restoreStartupState: vi.fn(() => SUCCESSFUL_RECOVERY), verifyGateway, log: vi.fn(), }), diff --git a/test/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery-supervisor-relaunch.test.ts index 7bad729a80a..3ac2a0fb066 100644 --- a/test/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery-supervisor-relaunch.test.ts @@ -281,6 +281,52 @@ describe("waitForManagedGatewaySupervisor", () => { }); describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { + it("checks managed recovery and OpenShell readiness before starting host forwards (#8662)", () => { + mockOpenClawSandbox("stopped-box"); + setImmediateRecoveryPolling(); + const order: string[] = []; + const requestGatewaySupervisorAction = vi.fn((_name: string, action: string) => { + order.push(action); + return { + status: 0, + stdout: `v1 ${"a".repeat(64)} complete ok 0 4242\nGATEWAY_PID=4242`, + stderr: "", + }; + }); + const waitForRecreatedSandboxOpenShellReadyImpl = vi.fn(() => { + order.push("OpenShell readiness"); + return true; + }); + const relaunchManagedSupervisorSessionImpl = vi.fn(() => null); + vi.spyOn(forwardHealth, "isLocalForwardReachable").mockReturnValue(true); + vi.spyOn(openshellRuntime, "captureOpenshell") + .mockReturnValueOnce({ status: 0, output: "SANDBOX BIND PORT PID STATUS" }) + .mockReturnValue({ + status: 0, + output: "SANDBOX BIND PORT PID STATUS\nstopped-box 127.0.0.1 18789 12345 running", + }); + vi.spyOn(openshellRuntime, "runOpenshell") + .mockReturnValueOnce({ status: 0 } as never) + .mockImplementationOnce(() => { + order.push("host forward"); + return { status: 0 } as never; + }); + + const result = checkAndRecoverSandboxProcesses("stopped-box", { + quiet: true, + isSandboxGatewayRunningImpl: () => false, + requestGatewaySupervisorAction, + relaunchManagedSupervisorSessionImpl, + waitForRecreatedSandboxOpenShellReadyImpl, + }); + + expect(result).toMatchObject({ checked: true, recovered: true, forwardRecovered: true }); + expect(result).toHaveProperty("managedControlCompletion.disposition", "ok"); + expect(order).toContain("OpenShell readiness"); + expect(order.indexOf("OpenShell readiness")).toBeLessThan(order.indexOf("host forward")); + expect(relaunchManagedSupervisorSessionImpl).not.toHaveBeenCalled(); + }); + it("does not turn ambiguous supervisor unavailability into a container mutation", () => { mockOpenClawSandbox("ambiguous-box"); setImmediateRecoveryPolling();