fix(sandbox): honor recovery wait override

ci/source-architecture-budget.json

@@ -41,7 +41,7 @@
       "src/lib/actions/sandbox/destroy.ts": 29,
       "src/lib/actions/sandbox/doctor.ts": 29,
       "src/lib/actions/sandbox/policy-channel.ts": 29,
-      "src/lib/actions/sandbox/process-recovery.ts": 22,
+      "src/lib/actions/sandbox/process-recovery.ts": 21,
       "src/lib/actions/sandbox/rebuild-pipeline.ts": 28,
       "src/lib/actions/sandbox/snapshot.ts": 39,
       "src/lib/actions/uninstall/run-plan.ts": 25,

docs/inference/configure-inference-timeouts.mdx

@@ -3,13 +3,13 @@
 # SPDX-License-Identifier: Apache-2.0
 title: "Configure Inference Timeouts"
 sidebar-title: "Configure Timeouts"
-description: "Configure NemoClaw inference request, provider-validation, and sandbox-readiness timeouts."
-description-agent: "Configures inference-related time budgets. Use when local validation, agent requests, or sandbox readiness exceeds the default timeout."
-keywords: ["nemoclaw inference timeout", "local inference timeout", "sandbox ready timeout"]
+description: "Configure NemoClaw inference request, provider-validation, sandbox-readiness, and recovery timeouts."
+description-agent: "Configures inference, sandbox-readiness, and recovery time budgets. Use when provider validation, agent requests, sandbox readiness, or recovery exceeds its default timeout."
+keywords: ["nemoclaw inference timeout", "local inference timeout", "sandbox ready timeout", "gateway recovery wait"]
 content:
   type: "how_to"
 ---
-NemoClaw uses separate time budgets for agent requests, local provider validation, and sandbox readiness.
+NemoClaw uses separate time budgets for agent requests, local provider validation, sandbox readiness, and recovery.
 Change the budget that matches the phase that times out.
 
 ## Choose the Timeout
@@ -20,12 +20,16 @@ Use the error location to select the correct setting.
 |---|---|---|
 | `NEMOCLAW_AGENT_TIMEOUT` | OpenClaw per-request inference | `600` seconds |
 | `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT` | Ollama, vLLM, NIM, and compatible-endpoint validation during onboarding | `180` seconds |
-| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | Image build, gateway upload, and in-sandbox boot after creation; OpenShell command re-registration after policy application or after OpenClaw or Hermes managed recovery recreates the sandbox | `180` seconds |
+| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | Image build, gateway upload, and in-sandbox boot after creation | `180` seconds |
+| `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` | OpenShell command re-registration after policy application, plus gateway health and re-registration during managed OpenClaw or Hermes recovery | `30`, `90`, or `120` seconds, depending on the recovery phase |
 
 The readiness timeout does not govern inference requests or provider validation.
 
-Set each value to a whole number of seconds; a fractional value is rounded.
-NemoClaw falls back to the default shown above for any value it cannot use, including a negative one — a negative budget is treated as an invalid setting rather than as a request to give up immediately.
+`NEMOCLAW_AGENT_TIMEOUT` requires a positive integer; onboarding rejects any other value.
+`NEMOCLAW_LOCAL_INFERENCE_TIMEOUT` and `NEMOCLAW_SANDBOX_READY_TIMEOUT` accept finite, nonnegative seconds, round fractional values, and use their defaults for invalid or negative values.
+`NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` accepts finite, nonnegative seconds and preserves fractional values.
+A valid value overrides the internal budget for the current recovery phase.
+An unset, blank, invalid, or negative value uses 30 seconds for OpenClaw gateway health, 90 seconds for Hermes gateway health, and 120 seconds for recreated-sandbox OpenShell registration when the recovery path does not supply another budget.
 
 ## Increase the OpenClaw Request Timeout
 
@@ -50,6 +54,12 @@ Hermes does not consume the OpenClaw-only `NEMOCLAW_AGENT_TIMEOUT` setting.
 
 </AgentOnly>
 
+<AgentOnly variant="deepagents">
+
+Deep Agents does not consume the OpenClaw-only `NEMOCLAW_AGENT_TIMEOUT` setting.
+
+</AgentOnly>
+
 ## Increase the Local Validation Timeout
 
 Raise `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT` when the inference-server validation probe needs more than 180 seconds.
@@ -67,22 +77,38 @@ This variable does not extend the later sandbox-readiness wait.
 
 Raise `NEMOCLAW_SANDBOX_READY_TIMEOUT` when onboarding creates the sandbox but image build, upload, or boot exceeds 180 seconds.
 This can occur during a first run with cold caches or on a remote VM over a slow link.
-Onboarding also uses this budget to confirm that the sandbox can execute commands again after applying policy presets.
-
-<AgentOnly variant="openclaw,hermes">
-
-The same budget applies when `start` or `recover` transactionally recreates a managed sandbox and waits for OpenShell to re-register it.
-
-</AgentOnly>
 
 ```bash
 export NEMOCLAW_SANDBOX_READY_TIMEOUT=600
 $$nemoclaw onboard
 ```
 
+## Increase the Recovery Wait
+
+Set `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` when OpenShell needs more than 120 seconds to re-register the sandbox after onboarding applies policy presets.
+
+<AgentOnly variant="openclaw">
+
+Managed OpenClaw gateway health uses 30 seconds by default.
+
+</AgentOnly>
+
+<AgentOnly variant="hermes">
+
+Managed Hermes gateway health uses 90 seconds by default.
+
+</AgentOnly>
+
 <AgentOnly variant="openclaw,hermes">
 
-For an existing sandbox, export the variable before the `start` or `recover` command that performs the recreation.
+Set `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` before `start` or `recover` to extend the agent-specific gateway-health wait and, when recovery recreates the sandbox, the 120-second OpenShell re-registration wait.
+
+```bash
+export NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS=300
+$$nemoclaw <sandbox-name> recover
+```
+
+A valid finite, nonnegative recovery override takes precedence over internal per-agent and per-call-site budgets.
 
 </AgentOnly>
 

docs/manage-sandboxes/recover-rebuild-sandboxes.mdx

@@ -90,9 +90,10 @@ It commits only after the replacement identity, state restoration, gateway resta
 NemoClaw removes the temporary state backup after a successful restore or rollback.
 If state restoration and rollback both fail, it retains the backup and prints host recovery guidance.
 Mounted state remains available, but a committed swap does not retain other writable-layer changes.
-NemoClaw uses the `NEMOCLAW_SANDBOX_READY_TIMEOUT` budget (180 seconds by default) for OpenShell re-registration.
-If re-registration, state restoration, or a later gateway check fails, NemoClaw attempts to roll back the replacement.
-The primary dashboard or API host forward stays stopped.
+After a transactional recreation, NemoClaw waits 120 seconds for OpenShell to re-register the sandbox before state restoration and replacement commit.
+Set `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` before the recovery command to change this budget.
+A definitive managed-health failure still stops immediately.
+If re-registration, state restoration, or a later gateway check fails, NemoClaw attempts to roll back the replacement and leaves the primary dashboard or API host forward stopped.
 
 For the controller topology, trust boundary, and fail-closed conditions, refer to [Understand Gateway Lifecycle Control](../configure-sandboxes/understand-gateway-lifecycle-control).
 If recovery cannot repair a sandbox that needs credentials or a current controller contract, rebuild it.

docs/reference/commands.mdx

@@ -3999,24 +3999,23 @@ Defaults are sized for typical hardware; override only if you see false-positive
 
 ### Onboard and Sandbox Readiness Timeouts
 
-The following environment variables tune onboard-time wall-clock limits.
-`NEMOCLAW_SANDBOX_READY_TIMEOUT` also covers OpenShell command re-registration after onboarding applies policy presets.
-
-<AgentOnly variant="openclaw,hermes">
-`NEMOCLAW_SANDBOX_READY_TIMEOUT` also applies when managed recovery transactionally recreates an existing sandbox.
-</AgentOnly>
-Set them before running `$$nemoclaw onboard` if a slow connection or large model pull risks tripping the default.
+The following environment variables tune onboard-time and recovery wall-clock limits.
+Set the onboarding variables before running `$$nemoclaw onboard` if a slow connection or large model pull risks tripping the default.
 
 | Variable | Default | Purpose |
 |----------|---------|---------|
 | `NEMOCLAW_OLLAMA_PULL_TIMEOUT` | `1800` (30 minutes) | Wall-clock timeout for `ollama pull` during onboard, in seconds. Accepts integer or float values. Already-downloaded layers are kept; re-running the pull resumes them. |
 | `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT` | `180` | Wall-clock timeout for the inference-server validation probe during onboard, in seconds. Raise on slow networks or for very large prompts. |
-| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | `180` | Wall-clock timeout for post-create readiness and OpenShell command re-registration after policy application, in seconds. Raise when the sandbox image build, gateway upload, in-sandbox boot, or post-policy re-registration exceeds the default (typical on 70B+ models, first-time gateway uploads over slow links, or DGX Station / remote-VM first runs). When the post-create deadline expires, onboarding deletes an orphaned sandbox and prints the retry hint. |
+| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | `180` | Wall-clock timeout for post-create readiness, in seconds. Raise when the sandbox image build, gateway upload, or in-sandbox boot exceeds the default (typical on 70B+ models, first-time gateway uploads over slow links, or DGX Station / remote-VM first runs). When the post-create deadline expires, onboarding deletes an orphaned sandbox and prints the retry hint. |
 | `NEMOCLAW_SANDBOX_READY_ERROR_DEBOUNCE` | `30` | Consecutive `Error`-phase polls the post-create readiness wait tolerates before treating `Error` as terminal. Polling starts at 250ms and backs off to a 2-second cap, while `NEMOCLAW_SANDBOX_READY_TIMEOUT` remains the overall deadline. The gateway can briefly report a just-created sandbox in `Error` while it re-registers the sandbox (seen on DGX Spark); the debounce lets that transient recover to `Ready`. `Failed` and `CrashLoopBackOff` always fail immediately. Set to `1` to restore fast-fail on the first `Error` poll. |
+| `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` | `30`, `90`, or `120`, depending on the recovery phase | Wall-clock timeout for OpenShell command re-registration after policy application, plus gateway health and re-registration during managed OpenClaw or Hermes recovery. A valid finite, nonnegative value overrides the internal budget for the current recovery phase. |
+
+An unset, blank, invalid, or negative `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` value uses 30 seconds for OpenClaw gateway health and 90 seconds for Hermes gateway health.
+Recreated-sandbox OpenShell registration uses 120 seconds when the recovery path does not supply another budget.
 
 <AgentOnly variant="openclaw,hermes">
 
-For managed recovery, the same timeout covers OpenShell re-registration before state restoration and replacement commit.
+For managed recovery, `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` covers OpenShell re-registration before state restoration and replacement commit.
 When the deadline expires, NemoClaw attempts to roll back the replacement and leaves the primary dashboard or API host forward stopped.
 
 </AgentOnly>
@@ -4027,6 +4026,17 @@ export NEMOCLAW_SANDBOX_READY_TIMEOUT=600
 $$nemoclaw onboard
 ```
 
+<AgentOnly variant="openclaw,hermes">
+
+Set the recovery override before a `start` or `recover` command that can recreate the sandbox.
+
+```bash
+export NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS=300
+$$nemoclaw <sandbox-name> recover
+```
+
+</AgentOnly>
+
 If the Ollama pull or post-create readiness timeout fires, onboarding emits the elapsed budget plus a hint to raise the relevant variable.
 The Ollama pull preserves its partial download for the next attempt.
 The post-create readiness wait deletes the orphaned sandbox first so the next `$$nemoclaw onboard` starts clean.

src/lib/actions/sandbox/process-recovery.test.ts

@@ -497,7 +497,7 @@ describe("recreated sandbox OpenShell readiness", () => {
     expect(sleeps).toEqual([3, 3]);
   });
 
-  it("does not let the legacy gateway timeout shorten the sandbox readiness budget (#7273)", () => {
+  it("lets the recovery wait override replace an explicit readiness budget", () => {
     vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS", "1");
     vi.stubEnv("NEMOCLAW_SANDBOX_READY_TIMEOUT", "6");
     const captureOpenshellImpl = vi.fn(() => ({
@@ -516,8 +516,8 @@ describe("recreated sandbox OpenShell readiness", () => {
         timeoutSeconds: Number(process.env.NEMOCLAW_SANDBOX_READY_TIMEOUT),
       }),
     ).toBe(false);
-    expect(captureOpenshellImpl).toHaveBeenCalledTimes(3);
-    expect(sleeps).toEqual([3, 3]);
+    expect(captureOpenshellImpl).toHaveBeenCalledOnce();
+    expect(sleeps).toEqual([]);
   });
 });
 

src/lib/actions/sandbox/process-recovery.ts

@@ -16,7 +16,6 @@ import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts";
 import * as agentRuntime from "../../agent/runtime";
 import { G, R } from "../../cli/terminal-style";
 import { sleepSeconds, waitUntil } from "../../core/wait";
-import { SANDBOX_READY_TIMEOUT_SECS } from "../../onboard/env";
 import { ROOT, shellQuote } from "../../runner";
 import {
   isDirectSandboxFallbackUnavailableError,
@@ -819,15 +818,16 @@ function waitForRecreatedSandboxOpenShellReadyResult(
   const capture = options.captureOpenshellImpl ?? captureOpenshell;
   const now = options.nowImpl ?? Date.now;
   const sleep = options.sleepImpl ?? sleepSeconds;
-  const timeoutSeconds =
+  const requestedTimeoutSeconds =
     typeof options.timeoutSeconds === "number" &&
     Number.isFinite(options.timeoutSeconds) &&
     options.timeoutSeconds >= 0
       ? options.timeoutSeconds
-      : readNonNegativeNumberEnv(
-          "NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS",
-          GATEWAY_RECOVERY_WAIT_DEFAULT_SECONDS,
-        );
+      : GATEWAY_RECOVERY_WAIT_DEFAULT_SECONDS;
+  const timeoutSeconds = readNonNegativeNumberEnv(
+    "NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS",
+    requestedTimeoutSeconds,
+  );
   const intervalSeconds = readNonNegativeNumberEnv(
     "NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS",
     options.intervalSeconds ?? 3,
@@ -1449,7 +1449,6 @@ function checkAndRecoverSandboxProcessesWithoutHostLock(
       ? (() => {
           const readinessOptions: RecreatedSandboxOpenShellReadyOptions = {
             beforeProbe: (timeoutMs) => confirmRelaunchedManagedHealth?.(timeoutMs) ?? null,
-            timeoutSeconds: SANDBOX_READY_TIMEOUT_SECS,
           };
           const readiness =
             waitForRecreatedSandboxOpenShellReadyImpl === waitForRecreatedSandboxOpenShellReady

src/lib/onboard/finalization-deps.test.ts

@@ -1,10 +1,41 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-import { afterEach, describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it, vi } from "vitest";
 
 import type { VerifyDeploymentResult } from "../verify-deployment";
-import { finalizationHandlerDeps } from "./finalization-deps";
+import { finalizationHandlerDeps, finalizationHandlerRuntime } from "./finalization-deps";
+
+describe("finalizationHandlerDeps.waitForSandboxControlPlaneReady", () => {
+  afterEach(() => {
+    vi.clearAllMocks();
+    vi.restoreAllMocks();
+    vi.unstubAllEnvs();
+  });
+
+  it("delegates timeout selection to the recovery readiness helper", () => {
+    vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS", "75");
+    vi.stubEnv("NEMOCLAW_SANDBOX_READY_TIMEOUT", "180");
+    let effectiveTimeoutSeconds: number | undefined;
+    const waitForRecreatedSandboxOpenShellReady = vi.fn(
+      (_name: string, options: { timeoutSeconds?: number } = {}) => {
+        const requestedTimeoutSeconds = options.timeoutSeconds ?? 120;
+        effectiveTimeoutSeconds = Number(
+          process.env.NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS ?? requestedTimeoutSeconds,
+        );
+        return true;
+      },
+    );
+    vi.spyOn(finalizationHandlerRuntime, "loadProcessRecovery").mockReturnValue({
+      checkAndRecoverSandboxProcesses: vi.fn(),
+      waitForRecreatedSandboxOpenShellReady,
+    });
+
+    expect(finalizationHandlerDeps.waitForSandboxControlPlaneReady("policy-box")).toBe(true);
+    expect(waitForRecreatedSandboxOpenShellReady).toHaveBeenCalledWith("policy-box");
+    expect(effectiveTimeoutSeconds).toBe(75);
+  });
+});
 
 describe("finalizationHandlerDeps.reportDeploymentReadiness", () => {
   const originalExitCode = process.exitCode;

src/lib/onboard/finalization-deps.ts

@@ -6,18 +6,23 @@
 // entrypoint stays lean (codebase-growth-guardrails). The lazy `require` calls
 // avoid an import cycle: connect.ts and process-recovery.ts both pull in
 // onboard helpers, so they must not be statically imported here.
+type ProcessRecoveryDeps = Pick<
+  typeof import("../actions/sandbox/process-recovery"),
+  "checkAndRecoverSandboxProcesses" | "waitForRecreatedSandboxOpenShellReady"
+>;
+
+export const finalizationHandlerRuntime = {
+  loadProcessRecovery: () => require("../actions/sandbox/process-recovery") as ProcessRecoveryDeps,
+};
+
 export const finalizationHandlerDeps = {
   waitForSandboxControlPlaneReady(name: string): boolean {
-    const processRecovery: typeof import("../actions/sandbox/process-recovery") =
-      require("../actions/sandbox/process-recovery");
-    const { SANDBOX_READY_TIMEOUT_SECS }: typeof import("./env") = require("./env");
-    return processRecovery.waitForRecreatedSandboxOpenShellReady(name, {
-      timeoutSeconds: SANDBOX_READY_TIMEOUT_SECS,
-    });
+    return finalizationHandlerRuntime
+      .loadProcessRecovery()
+      .waitForRecreatedSandboxOpenShellReady(name);
   },
   checkAndRecoverSandboxProcesses(name: string, options: { quiet: boolean }): void {
-    const processRecovery: typeof import("../actions/sandbox/process-recovery") =
-      require("../actions/sandbox/process-recovery");
+    const processRecovery = finalizationHandlerRuntime.loadProcessRecovery();
     processRecovery.checkAndRecoverSandboxProcesses(name, options);
   },
   // Best-effort device-approval sweep that clears pending allowlisted