diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 1b39fa40c48..bc0f4b62fd1 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -41,7 +41,7 @@ "src/lib/actions/sandbox/destroy.ts": 29, "src/lib/actions/sandbox/doctor.ts": 29, "src/lib/actions/sandbox/policy-channel.ts": 29, - "src/lib/actions/sandbox/process-recovery.ts": 22, + "src/lib/actions/sandbox/process-recovery.ts": 21, "src/lib/actions/sandbox/rebuild-pipeline.ts": 28, "src/lib/actions/sandbox/snapshot.ts": 39, "src/lib/actions/uninstall/run-plan.ts": 25, diff --git a/docs/inference/configure-inference-timeouts.mdx b/docs/inference/configure-inference-timeouts.mdx index ef08e3bdeba..4b11d563b5d 100644 --- a/docs/inference/configure-inference-timeouts.mdx +++ b/docs/inference/configure-inference-timeouts.mdx @@ -3,13 +3,13 @@ # SPDX-License-Identifier: Apache-2.0 title: "Configure Inference Timeouts" sidebar-title: "Configure Timeouts" -description: "Configure NemoClaw inference request, provider-validation, and sandbox-readiness timeouts." -description-agent: "Configures inference-related time budgets. Use when local validation, agent requests, or sandbox readiness exceeds the default timeout." -keywords: ["nemoclaw inference timeout", "local inference timeout", "sandbox ready timeout"] +description: "Configure NemoClaw inference request, provider-validation, sandbox-readiness, and recovery timeouts." +description-agent: "Configures inference, sandbox-readiness, and recovery time budgets. Use when provider validation, agent requests, sandbox readiness, or recovery exceeds its default timeout." +keywords: ["nemoclaw inference timeout", "local inference timeout", "sandbox ready timeout", "gateway recovery wait"] content: type: "how_to" --- -NemoClaw uses separate time budgets for agent requests, local provider validation, and sandbox readiness. +NemoClaw uses separate time budgets for agent requests, local provider validation, sandbox readiness, and recovery. Change the budget that matches the phase that times out. ## Choose the Timeout @@ -20,12 +20,16 @@ Use the error location to select the correct setting. |---|---|---| | `NEMOCLAW_AGENT_TIMEOUT` | OpenClaw per-request inference | `600` seconds | | `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT` | Ollama, vLLM, NIM, and compatible-endpoint validation during onboarding | `180` seconds | -| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | Image build, gateway upload, and in-sandbox boot after creation; OpenShell command re-registration after policy application or after OpenClaw or Hermes managed recovery recreates the sandbox | `180` seconds | +| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | Image build, gateway upload, and in-sandbox boot after creation | `180` seconds | +| `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` | OpenShell command re-registration after policy application, plus gateway health and re-registration during managed OpenClaw or Hermes recovery | `30`, `90`, or `120` seconds, depending on the recovery phase | The readiness timeout does not govern inference requests or provider validation. -Set each value to a whole number of seconds; a fractional value is rounded. -NemoClaw falls back to the default shown above for any value it cannot use, including a negative one — a negative budget is treated as an invalid setting rather than as a request to give up immediately. +`NEMOCLAW_AGENT_TIMEOUT` requires a positive integer; onboarding rejects any other value. +`NEMOCLAW_LOCAL_INFERENCE_TIMEOUT` and `NEMOCLAW_SANDBOX_READY_TIMEOUT` accept finite, nonnegative seconds, round fractional values, and use their defaults for invalid or negative values. +`NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` accepts finite, nonnegative seconds and preserves fractional values. +A valid value overrides the internal budget for the current recovery phase. +An unset, blank, invalid, or negative value uses 30 seconds for OpenClaw gateway health, 90 seconds for Hermes gateway health, and 120 seconds for recreated-sandbox OpenShell registration when the recovery path does not supply another budget. ## Increase the OpenClaw Request Timeout @@ -50,6 +54,12 @@ Hermes does not consume the OpenClaw-only `NEMOCLAW_AGENT_TIMEOUT` setting. + + +Deep Agents does not consume the OpenClaw-only `NEMOCLAW_AGENT_TIMEOUT` setting. + + + ## Increase the Local Validation Timeout Raise `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT` when the inference-server validation probe needs more than 180 seconds. @@ -67,22 +77,38 @@ This variable does not extend the later sandbox-readiness wait. Raise `NEMOCLAW_SANDBOX_READY_TIMEOUT` when onboarding creates the sandbox but image build, upload, or boot exceeds 180 seconds. This can occur during a first run with cold caches or on a remote VM over a slow link. -Onboarding also uses this budget to confirm that the sandbox can execute commands again after applying policy presets. - - - -The same budget applies when `start` or `recover` transactionally recreates a managed sandbox and waits for OpenShell to re-register it. - - ```bash export NEMOCLAW_SANDBOX_READY_TIMEOUT=600 $$nemoclaw onboard ``` +## Increase the Recovery Wait + +Set `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` when OpenShell needs more than 120 seconds to re-register the sandbox after onboarding applies policy presets. + + + +Managed OpenClaw gateway health uses 30 seconds by default. + + + + + +Managed Hermes gateway health uses 90 seconds by default. + + + -For an existing sandbox, export the variable before the `start` or `recover` command that performs the recreation. +Set `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` before `start` or `recover` to extend the agent-specific gateway-health wait and, when recovery recreates the sandbox, the 120-second OpenShell re-registration wait. + +```bash +export NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS=300 +$$nemoclaw recover +``` + +A valid finite, nonnegative recovery override takes precedence over internal per-agent and per-call-site budgets. diff --git a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx index 28928333c41..4b18bf4170a 100644 --- a/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx +++ b/docs/manage-sandboxes/recover-rebuild-sandboxes.mdx @@ -90,9 +90,10 @@ It commits only after the replacement identity, state restoration, gateway resta NemoClaw removes the temporary state backup after a successful restore or rollback. If state restoration and rollback both fail, it retains the backup and prints host recovery guidance. Mounted state remains available, but a committed swap does not retain other writable-layer changes. -NemoClaw uses the `NEMOCLAW_SANDBOX_READY_TIMEOUT` budget (180 seconds by default) for OpenShell re-registration. -If re-registration, state restoration, or a later gateway check fails, NemoClaw attempts to roll back the replacement. -The primary dashboard or API host forward stays stopped. +After a transactional recreation, NemoClaw waits 120 seconds for OpenShell to re-register the sandbox before state restoration and replacement commit. +Set `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` before the recovery command to change this budget. +A definitive managed-health failure still stops immediately. +If re-registration, state restoration, or a later gateway check fails, NemoClaw attempts to roll back the replacement and leaves the primary dashboard or API host forward stopped. For the controller topology, trust boundary, and fail-closed conditions, refer to [Understand Gateway Lifecycle Control](../configure-sandboxes/understand-gateway-lifecycle-control). If recovery cannot repair a sandbox that needs credentials or a current controller contract, rebuild it. diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index 1acd7964833..4c2fb030d45 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -3999,24 +3999,23 @@ Defaults are sized for typical hardware; override only if you see false-positive ### Onboard and Sandbox Readiness Timeouts -The following environment variables tune onboard-time wall-clock limits. -`NEMOCLAW_SANDBOX_READY_TIMEOUT` also covers OpenShell command re-registration after onboarding applies policy presets. - - -`NEMOCLAW_SANDBOX_READY_TIMEOUT` also applies when managed recovery transactionally recreates an existing sandbox. - -Set them before running `$$nemoclaw onboard` if a slow connection or large model pull risks tripping the default. +The following environment variables tune onboard-time and recovery wall-clock limits. +Set the onboarding variables before running `$$nemoclaw onboard` if a slow connection or large model pull risks tripping the default. | Variable | Default | Purpose | |----------|---------|---------| | `NEMOCLAW_OLLAMA_PULL_TIMEOUT` | `1800` (30 minutes) | Wall-clock timeout for `ollama pull` during onboard, in seconds. Accepts integer or float values. Already-downloaded layers are kept; re-running the pull resumes them. | | `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT` | `180` | Wall-clock timeout for the inference-server validation probe during onboard, in seconds. Raise on slow networks or for very large prompts. | -| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | `180` | Wall-clock timeout for post-create readiness and OpenShell command re-registration after policy application, in seconds. Raise when the sandbox image build, gateway upload, in-sandbox boot, or post-policy re-registration exceeds the default (typical on 70B+ models, first-time gateway uploads over slow links, or DGX Station / remote-VM first runs). When the post-create deadline expires, onboarding deletes an orphaned sandbox and prints the retry hint. | +| `NEMOCLAW_SANDBOX_READY_TIMEOUT` | `180` | Wall-clock timeout for post-create readiness, in seconds. Raise when the sandbox image build, gateway upload, or in-sandbox boot exceeds the default (typical on 70B+ models, first-time gateway uploads over slow links, or DGX Station / remote-VM first runs). When the post-create deadline expires, onboarding deletes an orphaned sandbox and prints the retry hint. | | `NEMOCLAW_SANDBOX_READY_ERROR_DEBOUNCE` | `30` | Consecutive `Error`-phase polls the post-create readiness wait tolerates before treating `Error` as terminal. Polling starts at 250ms and backs off to a 2-second cap, while `NEMOCLAW_SANDBOX_READY_TIMEOUT` remains the overall deadline. The gateway can briefly report a just-created sandbox in `Error` while it re-registers the sandbox (seen on DGX Spark); the debounce lets that transient recover to `Ready`. `Failed` and `CrashLoopBackOff` always fail immediately. Set to `1` to restore fast-fail on the first `Error` poll. | +| `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` | `30`, `90`, or `120`, depending on the recovery phase | Wall-clock timeout for OpenShell command re-registration after policy application, plus gateway health and re-registration during managed OpenClaw or Hermes recovery. A valid finite, nonnegative value overrides the internal budget for the current recovery phase. | + +An unset, blank, invalid, or negative `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` value uses 30 seconds for OpenClaw gateway health and 90 seconds for Hermes gateway health. +Recreated-sandbox OpenShell registration uses 120 seconds when the recovery path does not supply another budget. -For managed recovery, the same timeout covers OpenShell re-registration before state restoration and replacement commit. +For managed recovery, `NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS` covers OpenShell re-registration before state restoration and replacement commit. When the deadline expires, NemoClaw attempts to roll back the replacement and leaves the primary dashboard or API host forward stopped. @@ -4027,6 +4026,17 @@ export NEMOCLAW_SANDBOX_READY_TIMEOUT=600 $$nemoclaw onboard ``` + + +Set the recovery override before a `start` or `recover` command that can recreate the sandbox. + +```bash +export NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS=300 +$$nemoclaw recover +``` + + + If the Ollama pull or post-create readiness timeout fires, onboarding emits the elapsed budget plus a hint to raise the relevant variable. The Ollama pull preserves its partial download for the next attempt. The post-create readiness wait deletes the orphaned sandbox first so the next `$$nemoclaw onboard` starts clean. diff --git a/src/lib/actions/sandbox/process-recovery.test.ts b/src/lib/actions/sandbox/process-recovery.test.ts index b17274a9bde..e8b07d81201 100644 --- a/src/lib/actions/sandbox/process-recovery.test.ts +++ b/src/lib/actions/sandbox/process-recovery.test.ts @@ -497,7 +497,7 @@ describe("recreated sandbox OpenShell readiness", () => { expect(sleeps).toEqual([3, 3]); }); - it("does not let the legacy gateway timeout shorten the sandbox readiness budget (#7273)", () => { + it("lets the recovery wait override replace an explicit readiness budget", () => { vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS", "1"); vi.stubEnv("NEMOCLAW_SANDBOX_READY_TIMEOUT", "6"); const captureOpenshellImpl = vi.fn(() => ({ @@ -516,8 +516,8 @@ describe("recreated sandbox OpenShell readiness", () => { timeoutSeconds: Number(process.env.NEMOCLAW_SANDBOX_READY_TIMEOUT), }), ).toBe(false); - expect(captureOpenshellImpl).toHaveBeenCalledTimes(3); - expect(sleeps).toEqual([3, 3]); + expect(captureOpenshellImpl).toHaveBeenCalledOnce(); + expect(sleeps).toEqual([]); }); }); diff --git a/src/lib/actions/sandbox/process-recovery.ts b/src/lib/actions/sandbox/process-recovery.ts index fb2da1de2fd..845c6361f57 100644 --- a/src/lib/actions/sandbox/process-recovery.ts +++ b/src/lib/actions/sandbox/process-recovery.ts @@ -16,7 +16,6 @@ import { OPENSHELL_PROBE_TIMEOUT_MS } from "../../adapters/openshell/timeouts"; import * as agentRuntime from "../../agent/runtime"; import { G, R } from "../../cli/terminal-style"; import { sleepSeconds, waitUntil } from "../../core/wait"; -import { SANDBOX_READY_TIMEOUT_SECS } from "../../onboard/env"; import { ROOT, shellQuote } from "../../runner"; import { isDirectSandboxFallbackUnavailableError, @@ -819,15 +818,16 @@ function waitForRecreatedSandboxOpenShellReadyResult( const capture = options.captureOpenshellImpl ?? captureOpenshell; const now = options.nowImpl ?? Date.now; const sleep = options.sleepImpl ?? sleepSeconds; - const timeoutSeconds = + const requestedTimeoutSeconds = typeof options.timeoutSeconds === "number" && Number.isFinite(options.timeoutSeconds) && options.timeoutSeconds >= 0 ? options.timeoutSeconds - : readNonNegativeNumberEnv( - "NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS", - GATEWAY_RECOVERY_WAIT_DEFAULT_SECONDS, - ); + : GATEWAY_RECOVERY_WAIT_DEFAULT_SECONDS; + const timeoutSeconds = readNonNegativeNumberEnv( + "NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS", + requestedTimeoutSeconds, + ); const intervalSeconds = readNonNegativeNumberEnv( "NEMOCLAW_GATEWAY_RECOVERY_POLL_INTERVAL_SECONDS", options.intervalSeconds ?? 3, @@ -1449,7 +1449,6 @@ function checkAndRecoverSandboxProcessesWithoutHostLock( ? (() => { const readinessOptions: RecreatedSandboxOpenShellReadyOptions = { beforeProbe: (timeoutMs) => confirmRelaunchedManagedHealth?.(timeoutMs) ?? null, - timeoutSeconds: SANDBOX_READY_TIMEOUT_SECS, }; const readiness = waitForRecreatedSandboxOpenShellReadyImpl === waitForRecreatedSandboxOpenShellReady diff --git a/src/lib/onboard/finalization-deps.test.ts b/src/lib/onboard/finalization-deps.test.ts index c9bcffa9f5a..575bf9fbf45 100644 --- a/src/lib/onboard/finalization-deps.test.ts +++ b/src/lib/onboard/finalization-deps.test.ts @@ -1,10 +1,41 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import type { VerifyDeploymentResult } from "../verify-deployment"; -import { finalizationHandlerDeps } from "./finalization-deps"; +import { finalizationHandlerDeps, finalizationHandlerRuntime } from "./finalization-deps"; + +describe("finalizationHandlerDeps.waitForSandboxControlPlaneReady", () => { + afterEach(() => { + vi.clearAllMocks(); + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + }); + + it("delegates timeout selection to the recovery readiness helper", () => { + vi.stubEnv("NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS", "75"); + vi.stubEnv("NEMOCLAW_SANDBOX_READY_TIMEOUT", "180"); + let effectiveTimeoutSeconds: number | undefined; + const waitForRecreatedSandboxOpenShellReady = vi.fn( + (_name: string, options: { timeoutSeconds?: number } = {}) => { + const requestedTimeoutSeconds = options.timeoutSeconds ?? 120; + effectiveTimeoutSeconds = Number( + process.env.NEMOCLAW_GATEWAY_RECOVERY_WAIT_SECONDS ?? requestedTimeoutSeconds, + ); + return true; + }, + ); + vi.spyOn(finalizationHandlerRuntime, "loadProcessRecovery").mockReturnValue({ + checkAndRecoverSandboxProcesses: vi.fn(), + waitForRecreatedSandboxOpenShellReady, + }); + + expect(finalizationHandlerDeps.waitForSandboxControlPlaneReady("policy-box")).toBe(true); + expect(waitForRecreatedSandboxOpenShellReady).toHaveBeenCalledWith("policy-box"); + expect(effectiveTimeoutSeconds).toBe(75); + }); +}); describe("finalizationHandlerDeps.reportDeploymentReadiness", () => { const originalExitCode = process.exitCode; diff --git a/src/lib/onboard/finalization-deps.ts b/src/lib/onboard/finalization-deps.ts index 35b01d2a46d..915e1f7c125 100644 --- a/src/lib/onboard/finalization-deps.ts +++ b/src/lib/onboard/finalization-deps.ts @@ -6,18 +6,23 @@ // entrypoint stays lean (codebase-growth-guardrails). The lazy `require` calls // avoid an import cycle: connect.ts and process-recovery.ts both pull in // onboard helpers, so they must not be statically imported here. +type ProcessRecoveryDeps = Pick< + typeof import("../actions/sandbox/process-recovery"), + "checkAndRecoverSandboxProcesses" | "waitForRecreatedSandboxOpenShellReady" +>; + +export const finalizationHandlerRuntime = { + loadProcessRecovery: () => require("../actions/sandbox/process-recovery") as ProcessRecoveryDeps, +}; + export const finalizationHandlerDeps = { waitForSandboxControlPlaneReady(name: string): boolean { - const processRecovery: typeof import("../actions/sandbox/process-recovery") = - require("../actions/sandbox/process-recovery"); - const { SANDBOX_READY_TIMEOUT_SECS }: typeof import("./env") = require("./env"); - return processRecovery.waitForRecreatedSandboxOpenShellReady(name, { - timeoutSeconds: SANDBOX_READY_TIMEOUT_SECS, - }); + return finalizationHandlerRuntime + .loadProcessRecovery() + .waitForRecreatedSandboxOpenShellReady(name); }, checkAndRecoverSandboxProcesses(name: string, options: { quiet: boolean }): void { - const processRecovery: typeof import("../actions/sandbox/process-recovery") = - require("../actions/sandbox/process-recovery"); + const processRecovery = finalizationHandlerRuntime.loadProcessRecovery(); processRecovery.checkAndRecoverSandboxProcesses(name, options); }, // Best-effort device-approval sweep that clears pending allowlisted diff --git a/test/process-recovery-supervisor-relaunch.test.ts b/test/process-recovery-supervisor-relaunch.test.ts index e7fc6128022..a42ba3c5641 100644 --- a/test/process-recovery-supervisor-relaunch.test.ts +++ b/test/process-recovery-supervisor-relaunch.test.ts @@ -575,7 +575,12 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { stdout: "GATEWAY_PID=4242\n", stderr: "", })); - const waitForRecreatedSandboxOpenShellReadyImpl = vi.fn(() => true); + const waitForRecreatedSandboxOpenShellReadyImpl = vi.fn( + ( + _name: string, + _options?: { beforeProbe?: (timeoutMs: number) => boolean | null; timeoutSeconds?: number }, + ) => true, + ); const runOpenshell = vi.spyOn(openshellRuntime, "runOpenshell"); const result = checkAndRecoverSandboxProcesses("restore-failed-box", { @@ -598,7 +603,10 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { expect(waitForRecreatedSandboxOpenShellReadyImpl).toHaveBeenCalledOnce(); expect(waitForRecreatedSandboxOpenShellReadyImpl).toHaveBeenCalledWith( "restore-failed-box", - expect.objectContaining({ beforeProbe: expect.any(Function), timeoutSeconds: 180 }), + expect.objectContaining({ beforeProbe: expect.any(Function) }), + ); + expect(waitForRecreatedSandboxOpenShellReadyImpl.mock.calls[0]?.[1]).not.toHaveProperty( + "timeoutSeconds", ); expect(runOpenshell).not.toHaveBeenCalled(); }); @@ -737,7 +745,7 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { ); }); - it("uses the sandbox readiness budget after a longer gateway health wait (#7273)", () => { + it("uses the shared recreate-readiness budget after a longer gateway health wait", () => { mockOpenClawSandbox("unready-box", 600); setImmediateRecoveryPolling(); const finalize = vi.fn(() => ({ backupRemoved: true, rolledBack: false })); @@ -755,7 +763,12 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { stdout: "GATEWAY_PID=4242\n", stderr: "", })); - const waitForRecreatedSandboxOpenShellReadyImpl = vi.fn(() => false); + const waitForRecreatedSandboxOpenShellReadyImpl = vi.fn( + ( + _name: string, + _options?: { beforeProbe?: (timeoutMs: number) => boolean | null; timeoutSeconds?: number }, + ) => false, + ); const runOpenshell = vi.spyOn(openshellRuntime, "runOpenshell"); const result = checkAndRecoverSandboxProcesses("unready-box", { @@ -779,7 +792,10 @@ describe("checkAndRecoverSandboxProcesses supervisor relaunch", () => { expect(finalize).toHaveBeenCalledWith(false); expect(waitForRecreatedSandboxOpenShellReadyImpl).toHaveBeenCalledWith( "unready-box", - expect.objectContaining({ beforeProbe: expect.any(Function), timeoutSeconds: 180 }), + expect.objectContaining({ beforeProbe: expect.any(Function) }), + ); + expect(waitForRecreatedSandboxOpenShellReadyImpl.mock.calls[0]?.[1]).not.toHaveProperty( + "timeoutSeconds", ); expect(runOpenshell).not.toHaveBeenCalled(); });