perf(onboard): replace fixed readiness polls with deadline waits

docs/reference/commands.mdx

@@ -3626,7 +3626,7 @@ Set them before running `$$nemoclaw onboard` if a slow connection or large model
 | `NEMOCLAW_OLLAMA_PULL_TIMEOUT` | `1800` (30 minutes) | Wall-clock timeout for `ollama pull` during onboard, in seconds. Accepts integer or float values. Already-downloaded layers are kept; re-running the pull resumes them. |
 | `NEMOCLAW_LOCAL_INFERENCE_TIMEOUT` | `180` | Wall-clock timeout for the inference-server validation probe during onboard, in seconds. Raise on slow networks or for very large prompts. |
 | `NEMOCLAW_SANDBOX_READY_TIMEOUT` | `180` | Wall-clock timeout for the post-create readiness wait, in seconds. Raise when the sandbox image build, gateway upload, or in-sandbox boot exceeds the default (typical on 70B+ models, first-time gateway uploads over slow links, or DGX Station / remote-VM first runs). When the deadline expires onboarding deletes the orphaned sandbox and prints the retry hint. |
-| `NEMOCLAW_SANDBOX_READY_ERROR_DEBOUNCE` | `30` | Consecutive `Error`-phase polls (2s apart, so ~60s by default) the post-create readiness wait tolerates before treating `Error` as terminal. The gateway can briefly report a just-created sandbox in `Error` while it re-registers the sandbox (seen on DGX Spark); the debounce lets that transient recover to `Ready`. `Failed` and `CrashLoopBackOff` always fail immediately. Set to `1` to restore fast-fail on the first `Error` poll. |
+| `NEMOCLAW_SANDBOX_READY_ERROR_DEBOUNCE` | `30` | Consecutive `Error`-phase polls the post-create readiness wait tolerates before treating `Error` as terminal. Polling starts at 250ms and backs off to a 2-second cap, while `NEMOCLAW_SANDBOX_READY_TIMEOUT` remains the overall deadline. The gateway can briefly report a just-created sandbox in `Error` while it re-registers the sandbox (seen on DGX Spark); the debounce lets that transient recover to `Ready`. `Failed` and `CrashLoopBackOff` always fail immediately. Set to `1` to restore fast-fail on the first `Error` poll. |
 
 ```bash
 export NEMOCLAW_OLLAMA_PULL_TIMEOUT=3600

docs/reference/troubleshooting.mdx

@@ -1496,16 +1496,22 @@ Onboarding ends with:
 On a fresh onboard the OpenShell gateway can (re)start its supervisor session and re-register the just-created sandbox.
 During that window `openshell sandbox list` briefly reports the sandbox in the transient `Error` phase before it flips to `Ready`, as seen on DGX Spark when supervisor restart races the sandbox bootstrap.
 
-NemoClaw tolerates a bounded run of consecutive `Error` polls (default 30 polls / ~60s) so this transient recovers on its own; only `Error` that persists past the debounce window is treated as terminal. `Failed` and `CrashLoopBackOff` are always terminal and fail immediately.
+NemoClaw polls immediately, starts retrying after 250ms, and backs off to a 2-second cap.
+It tolerates 30 consecutive `Error` observations by default so this transient recovers on its own.
+Only `Error` that persists through the debounce count is terminal, unless the overall `NEMOCLAW_SANDBOX_READY_TIMEOUT` deadline expires first.
+`Failed` and `CrashLoopBackOff` are always terminal and fail immediately.
 
-If your host needs a longer window (slower re-registration), raise the debounce; to fail fast on the first `Error` poll, set it to `1`:
+If your host needs more observations for slower re-registration, raise the debounce.
+Raise `NEMOCLAW_SANDBOX_READY_TIMEOUT` too if the overall deadline is too short.
+To fail fast on the first `Error` poll, set the debounce to `1`:
 
 ```bash
-export NEMOCLAW_SANDBOX_READY_ERROR_DEBOUNCE=60   # tolerate ~120s of transient Error
+export NEMOCLAW_SANDBOX_READY_ERROR_DEBOUNCE=1
 $$nemoclaw onboard
 ```
 
-If the failure persists after the debounce, the sandbox is genuinely stuck — inspect the retained diagnostics and gateway state:
+If the failure persists after the debounce, the sandbox is stuck.
+Inspect the retained diagnostics and gateway state:
 
 ```bash
 openshell sandbox list

src/lib/agent/gateway-readiness.test.ts

@@ -0,0 +1,50 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it, vi } from "vitest";
+
+import { waitForAgentGatewayReady } from "./gateway-readiness";
+
+function clock() {
+  let nowMs = 10_000;
+  const sleepSeconds = vi.fn((seconds: number) => {
+    nowMs += seconds * 1000;
+  });
+  return { now: () => nowMs, sleepSeconds };
+}
+
+describe("waitForAgentGatewayReady", () => {
+  it("returns immediately when the agent gateway is already ready", () => {
+    const time = clock();
+    const probe = vi.fn(() => true);
+
+    expect(waitForAgentGatewayReady({ timeoutSeconds: 60, probe, ...time })).toBe(true);
+    expect(probe).toHaveBeenCalledOnce();
+    expect(time.sleepSeconds).not.toHaveBeenCalled();
+  });
+
+  it("retries quickly and succeeds before the deadline", () => {
+    const time = clock();
+    const probe = vi.fn().mockReturnValueOnce(false).mockReturnValue(true);
+
+    expect(waitForAgentGatewayReady({ timeoutSeconds: 60, probe, ...time })).toBe(true);
+    expect(probe).toHaveBeenCalledTimes(2);
+    expect(time.sleepSeconds).toHaveBeenCalledWith(0.25);
+  });
+
+  it("uses the full short deadline before timing out", () => {
+    const time = clock();
+    const probe = vi.fn(() => false);
+
+    expect(waitForAgentGatewayReady({ timeoutSeconds: 0.1, probe, ...time })).toBe(false);
+    expect(time.sleepSeconds.mock.calls.reduce((total, [seconds]) => total + seconds, 0)).toBe(0.1);
+  });
+
+  it("does not probe when the configured deadline is zero", () => {
+    const time = clock();
+    const probe = vi.fn(() => true);
+
+    expect(waitForAgentGatewayReady({ timeoutSeconds: 0, probe, ...time })).toBe(false);
+    expect(probe).not.toHaveBeenCalled();
+  });
+});

src/lib/agent/gateway-readiness.ts

@@ -0,0 +1,20 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { waitUntil } from "../core/wait";
+import { createReadinessWaitOptions } from "../onboard/readiness-wait";
+
+export function waitForAgentGatewayReady(options: {
+  timeoutSeconds: number;
+  probe: () => boolean;
+  sleepSeconds: (seconds: number) => void;
+  now?: () => number;
+}): boolean {
+  const waitOptions = createReadinessWaitOptions({
+    budgetMs: Math.max(0, options.timeoutSeconds * 1000),
+    maxIntervalMs: 3_000,
+    now: options.now,
+    sleep: (ms) => options.sleepSeconds(ms / 1000),
+  });
+  return waitOptions !== null && waitUntil(options.probe, waitOptions);
+}

src/lib/agent/onboard.test.ts

@@ -7,6 +7,7 @@ import type { AgentDefinition } from "./defs";
 import {
   collectHermesStartupDiagnostics,
   handleAgentSetup,
+  type OnboardContext,
   printDashboardUi,
   verifyAgentBinaryAvailable,
 } from "./onboard";
@@ -269,7 +270,10 @@ describe("printDashboardUi with port 8642 outside the chat UI (#2078)", () => {
 });
 
 describe("agent setup session boundaries", () => {
-  function createAgentSetupContext(runCaptureOpenshell = vi.fn(() => "")) {
+  function createAgentSetupContext(
+    runCaptureOpenshell: OnboardContext["runCaptureOpenshell"] = vi.fn(() => ""),
+    timing: Pick<OnboardContext, "now" | "sleepSeconds"> = {},
+  ) {
     return {
       context: {
         step: vi.fn(),
@@ -280,10 +284,15 @@ describe("agent setup session boundaries", () => {
         recordStepComplete: vi.fn(async () => undefined),
         recordStepFailed: vi.fn(async () => undefined),
         skippedStepMessage: vi.fn(),
+        ...timing,
       },
     };
   }
 
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
   it("records resume success through the supplied completion boundary", async () => {
     const runCaptureOpenshell = vi.fn(() => "ok");
     const { context } = createAgentSetupContext(runCaptureOpenshell);
@@ -320,6 +329,86 @@ describe("agent setup session boundaries", () => {
     });
     expect(context.recordStepFailed).not.toHaveBeenCalled();
   });
+
+  it("retries a configured gateway probe through the supplied scheduler", async () => {
+    let nowMs = 0;
+    const sleepSeconds = vi.fn((seconds: number) => {
+      nowMs += seconds * 1000;
+    });
+    const runCaptureOpenshell = vi
+      .fn<OnboardContext["runCaptureOpenshell"]>(() => "ok")
+      .mockReturnValueOnce("NEMOCLAW_AGENT_BINARY_CHECK:ok")
+      .mockReturnValueOnce("");
+    const { context } = createAgentSetupContext(runCaptureOpenshell, {
+      now: () => nowMs,
+      sleepSeconds,
+    });
+
+    await handleAgentSetup(
+      "sandbox-x",
+      "model-x",
+      "provider-x",
+      makeAgent({
+        healthProbe: { url: "http://127.0.0.1:19000/", port: 19000, timeout_seconds: 1 },
+      }),
+      false,
+      null,
+      context,
+    );
+
+    expect(runCaptureOpenshell.mock.calls.filter(([args]) => args.includes("curl"))).toHaveLength(
+      2,
+    );
+    expect(sleepSeconds).toHaveBeenCalledWith(0.25);
+    expect(context.recordStepComplete).toHaveBeenCalledWith("agent_setup", {
+      sandboxName: "sandbox-x",
+      provider: "provider-x",
+      model: "model-x",
+    });
+    expect(context.recordStepFailed).not.toHaveBeenCalled();
+  });
+
+  it("records gateway failure when the configured deadline expires", async () => {
+    let nowMs = 0;
+    const sleepSeconds = vi.fn((seconds: number) => {
+      nowMs += seconds * 1000;
+    });
+    const runCaptureOpenshell = vi
+      .fn<OnboardContext["runCaptureOpenshell"]>(() => "")
+      .mockReturnValueOnce("NEMOCLAW_AGENT_BINARY_CHECK:ok");
+    const { context } = createAgentSetupContext(runCaptureOpenshell, {
+      now: () => nowMs,
+      sleepSeconds,
+    });
+    vi.spyOn(console, "error").mockImplementation(() => {});
+    vi.spyOn(process, "exit").mockImplementation(((code?: number) => {
+      throw new Error(`process.exit:${String(code)}`);
+    }) as typeof process.exit);
+
+    await expect(
+      handleAgentSetup(
+        "sandbox-x",
+        "model-x",
+        "provider-x",
+        makeAgent({
+          healthProbe: { url: "http://127.0.0.1:19000/", port: 19000, timeout_seconds: 1 },
+        }),
+        false,
+        null,
+        context,
+      ),
+    ).rejects.toThrow("process.exit:1");
+
+    expect(
+      runCaptureOpenshell.mock.calls.filter(([args]) => args.includes("curl")).length,
+    ).toBeGreaterThan(1);
+    expect(sleepSeconds).toHaveBeenCalledWith(0.25);
+    expect(context.recordStepFailed).toHaveBeenCalledWith(
+      "agent_setup",
+      "Agent gateway did not respond within 1s",
+    );
+    expect(context.recordStepComplete).not.toHaveBeenCalled();
+  });
 });
 
 describe("handleAgentSetup guards", () => {

src/lib/agent/onboard.ts

@@ -17,6 +17,7 @@ import * as baseImage from "./base-image";
 import { describeAgentBinaryFailure, verifyAgentBinaryAvailable } from "./binary-availability";
 import { printOptionalDashboardUi } from "./dashboard-ui";
 import { type AgentDefinition, isTerminalAgent, loadAgent, resolveAgentName } from "./defs";
+import { waitForAgentGatewayReady } from "./gateway-readiness";
 import { runAgentSmokeCommands } from "./terminal-smoke";
 import { enforceTerminalAgentVersion } from "./terminal-version-enforcement";
 import { printBearerTokenApiAccess } from "./web-auth-ui";
@@ -35,6 +36,8 @@ export interface OnboardContext {
   recordStepComplete: (stepName: string, updates: LooseObject) => Promise<unknown>;
   recordStepFailed: (stepName: string, message: string | null) => Promise<unknown>;
   skippedStepMessage: (stepName: string, sandboxName: string) => void;
+  now?: () => number;
+  sleepSeconds?: (seconds: number) => void;
 }
 
 // Keep these compatibility exports as ordinary writable functions. Focused
@@ -329,29 +332,27 @@ export async function handleAgentSetup(
   const probe = agent.healthProbe;
   if (probe?.url) {
     const timeoutSecs = probe.timeout_seconds || 60;
-    const pollInterval = 3;
-    const maxAttempts = Math.ceil(timeoutSecs / pollInterval);
     console.log(`  Waiting for ${agent.displayName} gateway (up to ${timeoutSecs}s)...`);
-    let healthy = false;
-    for (let i = 0; i < maxAttempts; i++) {
-      const result = runCaptureOpenshell(
-        [
-          "sandbox",
-          "exec",
-          "-n",
-          sandboxName,
-          "--",
-          "curl",
-          ...buildValidatedCurlCommandArgs(["-sf", "--max-time", "3", probe.url]),
-        ],
-        { ignoreError: true },
-      );
-      if (isHealthProbeOk(result)) {
-        healthy = true;
-        break;
-      }
-      sleep(pollInterval);
-    }
+    const healthy = waitForAgentGatewayReady({
+      timeoutSeconds: timeoutSecs,
+      now: ctx.now,
+      sleepSeconds: ctx.sleepSeconds ?? sleep,
+      probe: () => {
+        const result = runCaptureOpenshell(
+          [
+            "sandbox",
+            "exec",
+            "-n",
+            sandboxName,
+            "--",
+            "curl",
+            ...buildValidatedCurlCommandArgs(["-sf", "--max-time", "3", probe.url]),
+          ],
+          { ignoreError: true },
+        );
+        return isHealthProbeOk(result);
+      },
+    });
     if (healthy) {
       console.log(`  \u2713 ${agent.displayName} gateway is healthy`);
     } else {