fix(sandbox): restore protected stopped sandboxes

docs/manage-sandboxes/recover-rebuild-sandboxes.mdx

@@ -25,6 +25,11 @@ This path preserves the sandbox workspace and repairs the agent runtime and host
 If the container is paused, follow the printed `docker unpause` guidance instead.
 If the container is missing or OpenShell reports another terminal phase such as `Failed`, follow the printed `rebuild --yes` guidance so NemoClaw can recreate the sandbox from its recorded metadata.
 
+<AgentOnly variant="openclaw,hermes">
+The `start` command returns success only after it authenticates the recovered agent runtime, OpenShell reports the sandbox ready, and host-side port forwards pass their checks.
+If any check fails, the command keeps the existing container, exits nonzero, identifies the failure, and tells you to run `recover` before retrying `start`.
+</AgentOnly>
+
 <AgentOnly variant="openclaw">
 If the sandbox has shields up and the OpenClaw gateway does not start after the container restarts, lower shields before you rebuild:
 

src/lib/actions/sandbox/connect.ts

@@ -38,7 +38,7 @@ import {
 import { isWsl } from "../../platform";
 import { ROOT } from "../../runner";
 import * as sandboxVersion from "../../sandbox/version";
-import { redact } from "../../security/redact";
+import { redact, redactFull } from "../../security/redact";
 import {
   isSandboxReady,
   isTerminalSandboxPhase,
@@ -95,6 +95,19 @@ export type SandboxConnectOptions = {
   probeOnly?: boolean;
 };
 
+export type SandboxStartupRecoveryResult = ReturnType<typeof checkAndRecoverSandboxProcesses> & {
+  recoveryFailureDetail?: string | null;
+  recoveryFailureLayer?: GatewayRestartFailureLayer | null;
+};
+
+export function sanitizeSandboxStartupRecoveryDetail(raw: string): string {
+  return redactFull(raw)
+    .replace(/[\u0000-\u001f\u007f-\u009f]/gu, " ")
+    .replace(/\s+/gu, " ")
+    .trim()
+    .slice(0, 240);
+}
+
 type SpawnLikeResult = {
   status: number | null;
   signal?: NodeJS.Signals | null;
@@ -925,8 +938,17 @@ function maybeEnsureHermesToolGatewayBroker(sb: SandboxEntry | null): void {
   }
 }
 
-export function restoreSandboxStartupState(sandboxName: string): void {
-  checkAndRecoverSandboxProcesses(sandboxName, { quiet: true });
+export function restoreSandboxStartupState(sandboxName: string): SandboxStartupRecoveryResult {
+  let recoveryFailureDetail: string | null = null;
+  let recoveryFailureLayer: GatewayRestartFailureLayer | null = null;
+  const processCheck = checkAndRecoverSandboxProcesses(sandboxName, {
+    quiet: true,
+    onRecoveryFailureLayer: (layer, detail) => {
+      recoveryFailureLayer = layer;
+      recoveryFailureDetail = detail ?? null;
+    },
+  });
+  return { ...processCheck, recoveryFailureDetail, recoveryFailureLayer };
 }
 
 function restoreInteractiveTerminal(): void {

src/lib/actions/sandbox/process-recovery.ts

@@ -445,7 +445,7 @@ function recoverSandboxProcesses(
     requestGatewaySupervisorAction?: typeof executeGatewaySupervisorAction;
     requestPinnedGatewaySupervisorAction?: RequestPinnedGatewaySupervisorAction;
     relaunchManagedSupervisorSessionImpl?: typeof relaunchManagedSupervisorSession;
-    onFailureLayer?: (layer: GatewayRestartFailureLayer) => void;
+    onFailureLayer?: (layer: GatewayRestartFailureLayer, detail: string) => void;
   } = {},
 ): SandboxProcessRecovery | null {
   const agent = agentRuntime.getSessionAgent(sandboxName);
@@ -483,7 +483,7 @@ function recoverSandboxProcesses(
       sleepSeconds(retryIntervalSeconds);
     }
     const failure = classifyGatewayRestartFailure(execResult);
-    onFailureLayer?.(failure.layer);
+    onFailureLayer?.(failure.layer, failure.detail);
     if (
       failure.layer === "supervisor not running" &&
       isExactlyMissingManagedSupervisor(execResult)
@@ -1078,7 +1078,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
     isSandboxGatewayRunningImpl?: typeof isSandboxGatewayRunning;
     waitForRecreatedSandboxOpenShellReadyImpl?: typeof waitForRecreatedSandboxOpenShellReady;
     isWsl?: boolean;
-    onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null) => void;
+    onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null, detail?: string) => void;
   } = {},
 ) {
   const recoveryAgent = agentRuntime.getSessionAgent(sandboxName);
@@ -1252,13 +1252,15 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
   }
 
   let managedRecoveryFailureLayer: GatewayRestartFailureLayer | null = null;
+  let managedRecoveryFailureDetail: string | null = null;
   const recovery = recoverSandboxProcesses(sandboxName, {
     quiet,
     requestGatewaySupervisorAction,
     requestPinnedGatewaySupervisorAction,
     relaunchManagedSupervisorSessionImpl,
-    onFailureLayer: (layer) => {
+    onFailureLayer: (layer, detail) => {
       managedRecoveryFailureLayer = layer;
+      managedRecoveryFailureDetail = detail;
     },
   });
   if (recovery !== null) {
@@ -1353,7 +1355,10 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
           managedRecoveryFailureLayer,
         );
       }
-      onRecoveryFailureLayer?.(managedRecoveryFailureLayer);
+      onRecoveryFailureLayer?.(
+        managedRecoveryFailureLayer,
+        managedRecoveryFailureDetail ?? undefined,
+      );
       if (relaunchedManagedHealthFailureDetail) {
         return {
           checked: true,
@@ -1366,14 +1371,16 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
       }
       return { checked: true, wasRunning: false, recovered: false, forwardRecovered: false };
     }
-    // State restore crosses the OpenShell SSH boundary. Prove the replacement
-    // is both identity-pinned and registered before asking finalize(true) to
-    // mutate it; otherwise a slow control-plane handoff can make a healthy
-    // replacement look like a failed restore and trigger rollback.
-    const readinessFailureDetail = relaunch
+    // Host-forward recovery requires an OpenShell-ready sandbox. Managed
+    // recovery has already passed its authenticated control and health gates;
+    // a replacement also rechecks its pinned identity before readiness.
+    const recoveryRequiresReadiness = recovery.kind === "managed" || relaunch;
+    const readinessFailureDetail = recoveryRequiresReadiness
       ? (() => {
           const readinessOptions: RecreatedSandboxOpenShellReadyOptions = {
-            beforeProbe: (timeoutMs) => confirmRelaunchedManagedHealth?.(timeoutMs) ?? null,
+            beforeProbe: relaunch
+              ? (timeoutMs) => confirmRelaunchedManagedHealth?.(timeoutMs) ?? null
+              : undefined,
           };
           const readiness =
             waitForRecreatedSandboxOpenShellReadyImpl === waitForRecreatedSandboxOpenShellReady
@@ -1528,7 +1535,7 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
     printHostManagedGatewayRecoveryHints(sandboxName, recoveryAgent, managedRecoveryFailureLayer);
   }
 
-  onRecoveryFailureLayer?.(managedRecoveryFailureLayer);
+  onRecoveryFailureLayer?.(managedRecoveryFailureLayer, managedRecoveryFailureDetail ?? undefined);
   return { checked: true, wasRunning: false, recovered: false, forwardRecovered: false };
 }
 
@@ -1542,7 +1549,7 @@ export function checkAndRecoverSandboxProcesses(
     isSandboxGatewayRunningImpl?: typeof isSandboxGatewayRunning;
     waitForRecreatedSandboxOpenShellReadyImpl?: typeof waitForRecreatedSandboxOpenShellReady;
     isWsl?: boolean;
-    onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null) => void;
+    onRecoveryFailureLayer?: (layer: GatewayRestartFailureLayer | null, detail?: string) => void;
   } = {},
 ) {
   return withTimerBoundShieldsMutationLock(sandboxName, "gateway process recovery", () =>

src/lib/actions/sandbox/start.test.ts

@@ -10,12 +10,22 @@ import {
 } from "../../onboard/runtime-provider/docker";
 import { createRuntimeProviderBundleRegistry } from "../../onboard/runtime-provider/registry";
 import type { SandboxEntry } from "../../state/registry";
+import type { SandboxStartupRecoveryResult } from "./connect";
 import { restoreStoppedSandboxStartupState, type SandboxStartDeps, startSandbox } from "./start";
 
 function sandbox(values: Partial<SandboxEntry> = {}): SandboxEntry {
   return { name: "my-sandbox", ...values };
 }
 
+const SUCCESSFUL_RECOVERY = {
+  checked: true,
+  wasRunning: true,
+  recovered: false,
+  forwardRecovered: false,
+} as const satisfies SandboxStartupRecoveryResult;
+const FAILED_RECOVERY = { ...SUCCESSFUL_RECOVERY, wasRunning: false } as const;
+const REDACTED_TOKEN = "opaque-token-8662";
+
 function harness(overrides: Partial<SandboxStartDeps> = {}) {
   const getSandbox = vi.fn<NonNullable<SandboxStartDeps["getSandbox"]>>(() => sandbox());
   const isDockerRuntimeDown = vi.fn<DockerRuntimeProviderDependencies["isRuntimeDown"]>(
@@ -45,7 +55,9 @@ function harness(overrides: Partial<SandboxStartDeps> = {}) {
   const verifyGateway = vi.fn<NonNullable<SandboxStartDeps["verifyGateway"]>>(() =>
     Promise.resolve(),
   );
-  const restoreStartupState = vi.fn<NonNullable<SandboxStartDeps["restoreStartupState"]>>();
+  const restoreStartupState = vi.fn<NonNullable<SandboxStartDeps["restoreStartupState"]>>(
+    () => SUCCESSFUL_RECOVERY,
+  );
   const log = vi.fn<(message: string) => void>();
   const runtimeProviders = createRuntimeProviderBundleRegistry([
     [
@@ -85,9 +97,10 @@ function harness(overrides: Partial<SandboxStartDeps> = {}) {
 describe("startSandbox", () => {
   it("restores sealed access before recovering sandbox processes (#8112)", () => {
     const restoreAccess = vi.fn();
-    const restoreProcesses = vi.fn();
+    const recovery = SUCCESSFUL_RECOVERY;
+    const restoreProcesses = vi.fn(() => recovery);
 
-    restoreStoppedSandboxStartupState("my-sandbox", {
+    const result = restoreStoppedSandboxStartupState("my-sandbox", {
       agent: "openclaw",
       restoreLockedStartupAccess: restoreAccess,
       restoreProcessState: restoreProcesses,
@@ -98,11 +111,12 @@ describe("startSandbox", () => {
     expect(restoreAccess.mock.invocationCallOrder[0]).toBeLessThan(
       restoreProcesses.mock.invocationCallOrder[0],
     );
+    expect(result).toBe(recovery);
   });
 
   it("keeps Hermes sealed state untouched while recovering sandbox processes (#8112)", () => {
     const restoreAccess = vi.fn();
-    const restoreProcesses = vi.fn();
+    const restoreProcesses = vi.fn(() => SUCCESSFUL_RECOVERY);
 
     restoreStoppedSandboxStartupState("my-sandbox", {
       agent: "hermes",
@@ -133,13 +147,11 @@ describe("startSandbox", () => {
     );
   });
 
-  it("attempts startup restoration again when start is rerun after a failure (#8112)", async () => {
+  it("retries startup after a structured recovery failure (#8662)", async () => {
     const h = harness();
-    h.restoreStartupState.mockImplementationOnce(() => {
-      throw new Error("restore failed");
-    });
+    h.restoreStartupState.mockReturnValueOnce(FAILED_RECOVERY);
 
-    await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow("restore failed");
+    await expect(startSandbox("my-sandbox", h.deps)).rejects.toThrow("gateway did not recover");
     expect(h.verifyGateway).not.toHaveBeenCalled();
 
     const result = await startSandbox("my-sandbox", h.deps);
@@ -152,6 +164,39 @@ describe("startSandbox", () => {
     );
   });
 
+  it.each([
+    [
+      "openclaw",
+      "managed gateway recovery",
+      {
+        ...FAILED_RECOVERY,
+        recoveryFailureLayer: "supervisor unavailable",
+        recoveryFailureDetail: `SUPERVISOR_UNAVAILABLE Authorization: Bearer ${REDACTED_TOKEN}`,
+      },
+      /supervisor unavailable/iu,
+    ],
+    [
+      "hermes",
+      "OpenShell readiness",
+      {
+        ...FAILED_RECOVERY,
+        forwardRecoveryFailed: true,
+        forwardRecoveryFailureDetail: `the sandbox did not become ready in OpenShell: token=${REDACTED_TOKEN}`,
+      },
+      /did not become ready in OpenShell/iu,
+    ],
+  ] as const)("propagates an actionable %s %s failure (#8662)", async (agent, _layer, recovery, expected) => {
+    const h = harness();
+    h.getSandbox.mockReturnValue(sandbox({ agent }));
+    h.restoreStartupState.mockReturnValue(recovery);
+
+    const failure = await startSandbox("my-sandbox", h.deps).catch((error) => String(error));
+    expect(failure).toMatch(expected);
+    expect(failure).toMatch(/nemoclaw my-sandbox recover/iu);
+    expect(failure).not.toContain(REDACTED_TOKEN);
+    expect(h.verifyGateway).not.toHaveBeenCalled();
+  });
+
   it("reports the started container by name (#6026)", async () => {
     const h = harness();
 

src/lib/actions/sandbox/start.ts

@@ -1,6 +1,7 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+import { cliName } from "../../onboard/branding";
 import {
   CURRENT_RUNTIME_PROVIDER_BUNDLES,
   type RuntimeProviderBundleRegistry,
@@ -19,9 +20,11 @@ function verifyGateway(sandboxName: string): Promise<void> {
   return connectSandbox(sandboxName, { probeOnly: true });
 }
 
-function restoreProcessState(sandboxName: string): void {
+type SandboxStartupRecoveryResult = import("./connect").SandboxStartupRecoveryResult;
+
+function restoreProcessState(sandboxName: string): SandboxStartupRecoveryResult {
   const { restoreSandboxStartupState } = require("./connect") as typeof import("./connect");
-  restoreSandboxStartupState(sandboxName);
+  return restoreSandboxStartupState(sandboxName);
 }
 
 function restoreLockedStartupAccess(sandboxName: string): void {
@@ -33,28 +36,58 @@ function restoreLockedStartupAccess(sandboxName: string): void {
 export interface SandboxStartupStateDeps {
   agent?: SandboxEntry["agent"];
   restoreLockedStartupAccess?: (sandboxName: string) => void;
-  restoreProcessState?: (sandboxName: string) => void;
+  restoreProcessState?: (sandboxName: string) => SandboxStartupRecoveryResult;
 }
 
 export function restoreStoppedSandboxStartupState(
   sandboxName: string,
   deps: SandboxStartupStateDeps = {},
-): void {
+): SandboxStartupRecoveryResult {
   if ((deps.agent ?? "openclaw") === "openclaw") {
     (deps.restoreLockedStartupAccess ?? restoreLockedStartupAccess)(sandboxName);
   }
-  (deps.restoreProcessState ?? restoreProcessState)(sandboxName);
+  return (deps.restoreProcessState ?? restoreProcessState)(sandboxName);
 }
 
 export interface SandboxStartDeps {
   environment?: NodeJS.ProcessEnv;
   getSandbox?: typeof registry.getSandbox;
   runtimeProviders?: RuntimeProviderBundleRegistry;
-  restoreStartupState?: (sandboxName: string) => void;
+  restoreStartupState?: (sandboxName: string) => SandboxStartupRecoveryResult;
   verifyGateway?: (sandboxName: string) => Promise<void>;
   log?: (message: string) => void;
 }
 
+function startupRecoveryFailure(check: SandboxStartupRecoveryResult): string | null {
+  if (!check.checked) return "managed agent gateway inspection did not complete";
+  if ("runtime" in check && check.runtime === "terminal") return null;
+  if ("secretBoundaryRefused" in check && check.secretBoundaryRefused) {
+    return `secret-boundary refusal: ${String(check.secretBoundaryReason)}`;
+  }
+  if ("mcpReconciliationRefused" in check && check.mcpReconciliationRefused) {
+    return `MCP reconciliation refusal: ${String(check.mcpReconciliationReason)}`;
+  }
+  if ("forwardRecoveryFailed" in check && check.forwardRecoveryFailed) {
+    return String(check.forwardRecoveryFailureDetail);
+  }
+  if (check.wasRunning || check.recovered) return null;
+  const layer = check.recoveryFailureLayer ?? "managed agent gateway recovery";
+  return check.recoveryFailureDetail
+    ? `${layer}: ${check.recoveryFailureDetail}`
+    : `${layer}: the agent gateway did not recover`;
+}
+
+function preservedSandboxRecoveryError(sandboxName: string, detail: unknown): Error {
+  const { sanitizeSandboxStartupRecoveryDetail } =
+    require("./connect") as typeof import("./connect");
+  const rawDetail = detail instanceof Error && detail.message ? detail.message : String(detail);
+  const safeDetail = sanitizeSandboxStartupRecoveryDetail(rawDetail);
+  return new Error(
+    `Sandbox '${sandboxName}' started, but startup recovery failed: ${safeDetail}. ` +
+      `The existing sandbox was preserved. Run \`${cliName()} ${sandboxName} recover\`, then retry \`${cliName()} ${sandboxName} start\`.`,
+  );
+}
+
 /**
  * Restart a stopped sandbox through the lifecycle facet bound to its durable
  * provider identity, then restore startup state before verifying readiness and
@@ -93,7 +126,14 @@ export async function startSandbox(
         restoreStoppedSandboxStartupState(sandboxNameToRestore, {
           agent: resolved.sandbox.agent,
         }));
-    restoreStartupState(name);
+    let recovery: SandboxStartupRecoveryResult;
+    try {
+      recovery = restoreStartupState(name);
+    } catch (error) {
+      throw preservedSandboxRecoveryError(name, error);
+    }
+    const failure = startupRecoveryFailure(recovery);
+    if (failure) throw preservedSandboxRecoveryError(name, failure);
     log("  Checking gateway health and host forwards…");
     await (deps.verifyGateway ?? verifyGateway)(name);
   });