From 14f86a52cc5b05aa673234149f7c7290198c46e2 Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Tue, 14 Jul 2026 12:34:11 +0000 Subject: [PATCH 01/12] fix(onboard): report an unhealthy inference route as not ready Signed-off-by: Tinson Lai --- src/lib/inference/local.test.ts | 22 ++++++ src/lib/inference/local.ts | 11 +++ src/lib/onboard.ts | 12 ++-- src/lib/onboard/dashboard.ts | 4 +- src/lib/onboard/finalization-deps.test.ts | 35 +++++++++ src/lib/onboard/finalization-deps.ts | 6 ++ .../machine/handlers/finalization.test.ts | 39 +++++++++- .../onboard/machine/handlers/finalization.ts | 11 ++- src/lib/verify-deployment.test.ts | 17 +++++ src/lib/verify-deployment.ts | 72 +++++++++++++------ test/helpers/onboard-final-flow-phases.ts | 2 + 11 files changed, 195 insertions(+), 36 deletions(-) create mode 100644 src/lib/onboard/finalization-deps.test.ts diff --git a/src/lib/inference/local.test.ts b/src/lib/inference/local.test.ts index 74d472c70fb..8343b86f55c 100644 --- a/src/lib/inference/local.test.ts +++ b/src/lib/inference/local.test.ts @@ -39,10 +39,32 @@ import { probeOllamaAuthProxyHealth, QWEN3_6_OLLAMA_MODEL, resetOllamaContainerPortCache, + rewriteHostLoopbackForSandbox, validateLocalProvider, validateOllamaModel, } from "./local"; +describe("rewriteHostLoopbackForSandbox", () => { + it("rewrites a loopback host to the sandbox-facing gateway alias, preserving port, path, query, and fragment", () => { + expect(rewriteHostLoopbackForSandbox("http://127.0.0.1:8000/v1")).toBe( + "http://host.openshell.internal:8000/v1", + ); + expect(rewriteHostLoopbackForSandbox("http://localhost/v1?x=1#frag")).toBe( + "http://host.openshell.internal/v1?x=1#frag", + ); + }); + + it("leaves a non-loopback or empty endpoint unchanged", () => { + expect(rewriteHostLoopbackForSandbox("http://host.openshell.internal:8000/v1")).toBe( + "http://host.openshell.internal:8000/v1", + ); + expect(rewriteHostLoopbackForSandbox("https://api.example.com/v1")).toBe( + "https://api.example.com/v1", + ); + expect(rewriteHostLoopbackForSandbox("")).toBe(""); + }); +}); + describe("local inference helpers", () => { const originalSandboxHostUrl = process.env[LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV]; const originalPath = process.env.PATH; diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index 3f839927005..3c4af5a3495 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -60,6 +60,17 @@ export function resetOllamaContainerPortCache(): void { } export const HOST_GATEWAY_URL = "http://host.openshell.internal"; + +export function rewriteHostLoopbackForSandbox(url: string): string { + if (!url || !/localhost|127\.0\.0\.1/.test(url)) return url; + try { + const parsed = new URL(url); + const port = parsed.port ? `:${parsed.port}` : ""; + return `${HOST_GATEWAY_URL}${port}${parsed.pathname}${parsed.search}${parsed.hash}`; + } catch { + return url; + } +} export const LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV = "NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL"; export const CONTAINER_REACHABILITY_IMAGE = "curlimages/curl:8.10.1"; // These tags are convenience aliases for callers that want to refer to a diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 2b197d9abfc..af2a6acf81e 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -3116,13 +3116,7 @@ async function handleRoutedSelection( state.provider = bp.provider_name || "nvidia-router"; state.model = bp.model; - const { HOST_GATEWAY_URL } = require("./inference/local"); - const routerEndpointUrl = bp.endpoint || ""; - state.endpointUrl = routerEndpointUrl; - if (routerEndpointUrl.match(/localhost|127\.0\.0\.1/)) { - const u = new URL(routerEndpointUrl); - state.endpointUrl = `${HOST_GATEWAY_URL}:${u.port}${u.pathname}`; - } + state.endpointUrl = localInference.rewriteHostLoopbackForSandbox(bp.endpoint || ""); state.preferredInferenceApi = "openai-completions"; state.assertRouteCompatible?.(); @@ -3331,7 +3325,9 @@ async function handleRemoteProviderSelection(args: RemoteProviderSelectionArgs, if (navigation === "exit") { exitOnboardFromPrompt(); } - state.endpointUrl = normalizeProviderBaseUrl(endpointInput, kind); + state.endpointUrl = localInference.rewriteHostLoopbackForSandbox( + normalizeProviderBaseUrl(endpointInput, kind), + ); if (!state.endpointUrl) { console.error( selected.key === "custom" diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index 076973113a9..0fd817e7209 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -115,6 +115,7 @@ export interface OnboardDashboardHelpers { provider: string, nimContainer?: string | null, agent?: AgentDefinition | null, + ready?: boolean, ): void; stopAllDashboardForwards(): void; } @@ -439,6 +440,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa provider: string, nimContainer: string | null = null, agent: AgentDefinition | null = null, + ready = true, ): void { const nimStatus = deps.nimStatus ?? nim.nimStatus; const nimStatusByName = deps.nimStatusByName ?? nim.nimStatusByName; @@ -471,7 +473,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa console.log(""); console.log(` ${"─".repeat(50)}`); - console.log(` ${deps.agentProductName()} is ready`); + console.log(` ${deps.agentProductName()} is ${ready ? "ready" : "not ready"}`); console.log(""); console.log(` Sandbox: ${sandboxName}`); console.log(` Model: ${model} (${providerLabel})`); diff --git a/src/lib/onboard/finalization-deps.test.ts b/src/lib/onboard/finalization-deps.test.ts new file mode 100644 index 00000000000..c9bcffa9f5a --- /dev/null +++ b/src/lib/onboard/finalization-deps.test.ts @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it } from "vitest"; + +import type { VerifyDeploymentResult } from "../verify-deployment"; +import { finalizationHandlerDeps } from "./finalization-deps"; + +describe("finalizationHandlerDeps.reportDeploymentReadiness", () => { + const originalExitCode = process.exitCode; + afterEach(() => { + process.exitCode = originalExitCode; + }); + + it("sets a non-zero exit code when the deployment is not ready", () => { + process.exitCode = 0; + finalizationHandlerDeps.reportDeploymentReadiness(false); + expect(process.exitCode).toBe(1); + }); + + it("leaves the exit code unchanged when the deployment is ready", () => { + process.exitCode = 0; + finalizationHandlerDeps.reportDeploymentReadiness(true); + expect(process.exitCode).toBe(0); + }); +}); + +describe("finalizationHandlerDeps.isDeploymentHealthy", () => { + it("reports the verification healthy flag", () => { + const healthy = { healthy: true } as unknown as VerifyDeploymentResult; + const unhealthy = { healthy: false } as unknown as VerifyDeploymentResult; + expect(finalizationHandlerDeps.isDeploymentHealthy(healthy)).toBe(true); + expect(finalizationHandlerDeps.isDeploymentHealthy(unhealthy)).toBe(false); + }); +}); diff --git a/src/lib/onboard/finalization-deps.ts b/src/lib/onboard/finalization-deps.ts index c504aa54fff..d74d03ca123 100644 --- a/src/lib/onboard/finalization-deps.ts +++ b/src/lib/onboard/finalization-deps.ts @@ -29,4 +29,10 @@ export const finalizationHandlerDeps = { require("../actions/sandbox/auto-pair-warmup"); warmup.runSandboxScopeWarmupRun(name); }, + isDeploymentHealthy(result: import("../verify-deployment").VerifyDeploymentResult): boolean { + return result.healthy; + }, + reportDeploymentReadiness(healthy: boolean): void { + if (!healthy) process.exitCode = 1; + }, }; diff --git a/src/lib/onboard/machine/handlers/finalization.test.ts b/src/lib/onboard/machine/handlers/finalization.test.ts index 9fea41cd87c..218395a777e 100644 --- a/src/lib/onboard/machine/handlers/finalization.test.ts +++ b/src/lib/onboard/machine/handlers/finalization.test.ts @@ -42,6 +42,8 @@ function createDeps( diagnostics: vi.fn(() => [" ✓ verified"]), verifyWebSearch: vi.fn(), dashboard: vi.fn(), + isHealthy: vi.fn(() => true), + reportReadiness: vi.fn(), error: vi.fn(), log: vi.fn(), }; @@ -63,6 +65,8 @@ function createDeps( formatVerificationDiagnostics: calls.diagnostics, verifyWebSearchInsideSandbox: calls.verifyWebSearch, printDashboard: calls.dashboard, + isDeploymentHealthy: calls.isHealthy, + reportDeploymentReadiness: calls.reportReadiness, error: calls.error, log: calls.log, ...overrides, @@ -104,7 +108,14 @@ describe("handleFinalizationState", () => { expect(calls.buildChain).toHaveBeenCalledWith("http://127.0.0.1:18789"); expect(calls.verify).toHaveBeenCalledWith("my-assistant", { port: 18789 }); expect(calls.log).toHaveBeenCalledWith(" ✓ verified"); - expect(calls.dashboard).toHaveBeenCalledWith("my-assistant", "model", "provider", null, null); + expect(calls.dashboard).toHaveBeenCalledWith( + "my-assistant", + "model", + "provider", + null, + null, + true, + ); expect(calls.postVerify).toHaveBeenCalledOnce(); expect(result.stateResult).toEqual({ type: "complete", @@ -120,6 +131,23 @@ describe("handleFinalizationState", () => { expect(result.verificationDiagnostics).toEqual([" ✓ verified"]); }); + it("prints a not-ready dashboard and signals not-ready when verification is unhealthy", async () => { + const { deps, calls } = createDeps({ isDeploymentHealthy: vi.fn(() => false) }); + + const result = await handleFinalizationState(baseOptions(deps)); + + expect(calls.dashboard).toHaveBeenCalledWith( + "my-assistant", + "model", + "provider", + null, + null, + false, + ); + expect(calls.reportReadiness).toHaveBeenCalledWith(false); + expect(result.deploymentHealthy).toBe(false); + }); + it("ensures agent dashboard forwarding before completion for non-OpenClaw agents", async () => { const { deps, calls } = createDeps(); const agent = { name: "hermes" }; @@ -130,7 +158,14 @@ describe("handleFinalizationState", () => { expect(calls.ensureAgentDashboard.mock.invocationCallOrder[0]).toBeLessThan( calls.dashboard.mock.invocationCallOrder[0], ); - expect(calls.dashboard).toHaveBeenCalledWith("my-assistant", "model", "provider", null, agent); + expect(calls.dashboard).toHaveBeenCalledWith( + "my-assistant", + "model", + "provider", + null, + agent, + true, + ); }); it("skips dashboard and gateway verification for terminal agents without forwards", async () => { diff --git a/src/lib/onboard/machine/handlers/finalization.ts b/src/lib/onboard/machine/handlers/finalization.ts index afd14d7f9d1..fb6d3fb95d7 100644 --- a/src/lib/onboard/machine/handlers/finalization.ts +++ b/src/lib/onboard/machine/handlers/finalization.ts @@ -55,6 +55,8 @@ export interface FinalizationStateOptions; formatVerificationDiagnostics(result: VerificationResult): string[]; + isDeploymentHealthy(result: VerificationResult): boolean; + reportDeploymentReadiness(healthy: boolean): void; /** * Best-effort probe that confirms the agent runtime actually accepted the * web-search config and (for Brave) that the L7 proxy rewrites the @@ -68,6 +70,7 @@ export interface FinalizationStateOptions { expect(infDiag?.status).toBe("warn"); }); + it("reports unhealthy when the inference route is reachable but returns HTTP 5xx", async () => { + const deps = makeDeps({ + executeSandboxCommand: (_name: string, script: string) => { + if (script.includes("inference.local")) { + return { status: 0, stdout: "503", stderr: "" }; + } + return { status: 0, stdout: "200", stderr: "" }; + }, + }); + const result = await verifyDeployment("my-sandbox", chain, deps, NO_RETRY); + expect(result.healthy).toBe(false); + expect(result.verification.inferenceRouteWorking).toBe(false); + const infDiag = result.diagnostics.find((d) => d.link === "inference"); + expect(infDiag?.status).toBe("fail"); + expect(infDiag?.detail).toContain("503"); + }); + it("messaging failure is a warning, not a blocker", async () => { const deps = makeDeps({ getMessagingChannels: () => ["slack", "discord"], diff --git a/src/lib/verify-deployment.ts b/src/lib/verify-deployment.ts index c8f359bac68..aad1f5c24dd 100644 --- a/src/lib/verify-deployment.ts +++ b/src/lib/verify-deployment.ts @@ -213,30 +213,49 @@ function fetchGatewayVersion(sandboxName: string, deps: VerifyDeploymentDeps): s return version && version !== "" ? version : null; } -/** - * Probe the inference route from inside the sandbox. - * Sends a minimal request to inference.local to verify the proxy is working. - */ -function verifyInferenceRoute( +type InferenceRouteStatus = "ok" | "unreachable" | "unhealthy"; + +function probeInferenceRouteOnce( sandboxName: string, deps: VerifyDeploymentDeps, -): { working: boolean; detail: string } { - // Just check that inference.local resolves and the proxy responds. - // We don't send a real completion request — just hit /v1/models to confirm routing. +): { status: InferenceRouteStatus; detail: string } { const script = `HTTP_CODE=$(curl -so /dev/null -w '%{http_code}' --max-time 5 ` + `https://inference.local/v1/models 2>/dev/null || echo 000); echo $HTTP_CODE`; const result = deps.executeSandboxCommand(sandboxName, script); if (!result) { - return { working: false, detail: "sandbox unreachable" }; + return { status: "unreachable", detail: "sandbox unreachable" }; } const code = parseInt(result.stdout.trim(), 10) || 0; - // Any HTTP response (even 401/403) means the proxy is routing. - // 000 means DNS failed or connection refused. - if (code > 0) { - return { working: true, detail: `inference.local responded HTTP ${code}` }; + if (code === 0) { + return { + status: "unreachable", + detail: "inference.local unreachable (DNS or proxy not running)", + }; } - return { working: false, detail: "inference.local unreachable (DNS or proxy not running)" }; + if (code >= 500) { + return { + status: "unhealthy", + detail: `inference.local returned HTTP ${code} (route reachable but endpoint unhealthy)`, + }; + } + return { status: "ok", detail: `inference.local responded HTTP ${code}` }; +} + +async function verifyInferenceRoute( + sandboxName: string, + deps: VerifyDeploymentDeps, + retryDelaysMs: readonly number[], + sleep: (ms: number) => Promise, +): Promise<{ status: InferenceRouteStatus; detail: string }> { + let last = probeInferenceRouteOnce(sandboxName, deps); + if (last.status === "ok") return last; + for (const delayMs of retryDelaysMs) { + await sleep(delayMs); + last = probeInferenceRouteOnce(sandboxName, deps); + if (last.status === "ok") return last; + } + return last; } /** @@ -515,14 +534,23 @@ export async function verifyDeployment( }); // 4. Inference route - const inference = verifyInferenceRoute(sandboxName, deps); + const inference = await verifyInferenceRoute( + sandboxName, + deps, + gateway.reachable ? retryDelaysMs : [], + sleep, + ); + const inferenceRouteWorking = inference.status === "ok"; diagnostics.push({ link: "inference", - status: inference.working ? "ok" : "warn", + status: inference.status === "ok" ? "ok" : inference.status === "unhealthy" ? "fail" : "warn", detail: inference.detail, - hint: inference.working - ? "" - : "The inference proxy may not be ready yet. Try: nemoclaw status (it may take a few seconds after creation).", + hint: + inference.status === "ok" + ? "" + : inference.status === "unhealthy" + ? "The inference route is reachable but the endpoint returned a server error (HTTP 5xx). If the endpoint runs on the host, confirm it is reachable from the sandbox — a loopback 127.0.0.1/localhost bind is not; bind it to 0.0.0.0 or use host.openshell.internal — then re-run: nemoclaw status." + : "The inference proxy may not be ready yet. Try: nemoclaw status (it may take a few seconds after creation).", }); // 5. Messaging bridges (providers attached AND runtime config exposes @@ -542,7 +570,7 @@ export async function verifyDeployment( const verification: DeploymentVerification = { gatewayReachable: gateway.reachable, gatewayVersion, - inferenceRouteWorking: inference.working, + inferenceRouteWorking, dashboardReachable: dashboard.reachable, messagingBridgesHealthy: messaging.healthy, messagingRuntimeChannelsMissing: messaging.runtimeMissing, @@ -550,9 +578,7 @@ export async function verifyDeployment( accessMethod, }; - // Healthy = gateway reachable AND dashboard reachable from host. - // Inference and messaging are warn-level (non-blocking). - const healthy = gateway.reachable && dashboard.reachable; + const healthy = gateway.reachable && dashboard.reachable && inference.status !== "unhealthy"; return { healthy, verification, diagnostics }; } diff --git a/test/helpers/onboard-final-flow-phases.ts b/test/helpers/onboard-final-flow-phases.ts index 72c394745df..b7d6f16260f 100644 --- a/test/helpers/onboard-final-flow-phases.ts +++ b/test/helpers/onboard-final-flow-phases.ts @@ -250,6 +250,8 @@ export function createPhases( checkAndRecoverSandboxProcesses: vi.fn(), warmupScopeUpgrade: vi.fn(), autoPairScopeApproval: vi.fn(), + isDeploymentHealthy: () => true, + reportDeploymentReadiness: vi.fn(), getChatUiUrl: () => "http://127.0.0.1:45123", buildVerifyChain: (): DashboardDeliveryChain => ({ accessUrl: "http://127.0.0.1:45123", From e5216a0223a6f4275ebe7f49e12de9ebad914ddf Mon Sep 17 00:00:00 2001 From: Tinson Lai Date: Tue, 14 Jul 2026 16:44:24 +0000 Subject: [PATCH 02/12] fix(inference): match loopback hostnames exactly and stop double-rewriting compatible endpoints rewriteHostLoopbackForSandbox tested the raw endpoint string, so a remote URL merely containing "localhost" or "127.0.0.1" in its host, path, query, or fragment was wrongly rewritten. Parse the URL first and rewrite only an exact localhost/127.0.0.1 hostname. The custom and Anthropic-compatible endpoint branch in handleRemoteProviderSelection also wrapped its endpoint in this rewrite before gatewayReachableCompatibleEndpointUrl ran its own exact-authority, bundled-port gateway rewrite downstream. Rewriting twice replaced the raw loopback URL needed for host-side credential validation and for that gateway check, breaking both. Stop rewriting in handleRemoteProviderSelection and let the existing gateway-route check handle it, as it already did before this change. Restructure the growth-guardrail-triggering conditional mock in the new verify-deployment.ts unhealthy-inference-route test into a flat response. Signed-off-by: Tinson Lai --- src/lib/inference/local.test.ts | 12 ++++++++++++ src/lib/inference/local.ts | 4 +++- src/lib/onboard.ts | 4 +--- src/lib/verify-deployment.test.ts | 7 +------ 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/lib/inference/local.test.ts b/src/lib/inference/local.test.ts index 8343b86f55c..2e5b414e74c 100644 --- a/src/lib/inference/local.test.ts +++ b/src/lib/inference/local.test.ts @@ -52,12 +52,24 @@ describe("rewriteHostLoopbackForSandbox", () => { expect(rewriteHostLoopbackForSandbox("http://localhost/v1?x=1#frag")).toBe( "http://host.openshell.internal/v1?x=1#frag", ); + expect(rewriteHostLoopbackForSandbox("http://LOCALHOST:8000/v1")).toBe( + "http://host.openshell.internal:8000/v1", + ); }); it("leaves a non-loopback or empty endpoint unchanged", () => { expect(rewriteHostLoopbackForSandbox("http://host.openshell.internal:8000/v1")).toBe( "http://host.openshell.internal:8000/v1", ); + expect(rewriteHostLoopbackForSandbox("https://notlocalhost.example/v1")).toBe( + "https://notlocalhost.example/v1", + ); + expect(rewriteHostLoopbackForSandbox("https://api.example.com/v1?target=localhost")).toBe( + "https://api.example.com/v1?target=localhost", + ); + expect(rewriteHostLoopbackForSandbox("https://api.example.com/127.0.0.1")).toBe( + "https://api.example.com/127.0.0.1", + ); expect(rewriteHostLoopbackForSandbox("https://api.example.com/v1")).toBe( "https://api.example.com/v1", ); diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index 3c4af5a3495..983ad3a60ae 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -62,9 +62,11 @@ export function resetOllamaContainerPortCache(): void { export const HOST_GATEWAY_URL = "http://host.openshell.internal"; export function rewriteHostLoopbackForSandbox(url: string): string { - if (!url || !/localhost|127\.0\.0\.1/.test(url)) return url; + if (!url) return url; try { const parsed = new URL(url); + const hostname = parsed.hostname.toLowerCase(); + if (hostname !== "localhost" && hostname !== "127.0.0.1") return url; const port = parsed.port ? `:${parsed.port}` : ""; return `${HOST_GATEWAY_URL}${port}${parsed.pathname}${parsed.search}${parsed.hash}`; } catch { diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index af2a6acf81e..63e15e88fa1 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -3325,9 +3325,7 @@ async function handleRemoteProviderSelection(args: RemoteProviderSelectionArgs, if (navigation === "exit") { exitOnboardFromPrompt(); } - state.endpointUrl = localInference.rewriteHostLoopbackForSandbox( - normalizeProviderBaseUrl(endpointInput, kind), - ); + state.endpointUrl = normalizeProviderBaseUrl(endpointInput, kind); if (!state.endpointUrl) { console.error( selected.key === "custom" diff --git a/src/lib/verify-deployment.test.ts b/src/lib/verify-deployment.test.ts index 4c8ba6ea03c..37a2833a76c 100644 --- a/src/lib/verify-deployment.test.ts +++ b/src/lib/verify-deployment.test.ts @@ -198,12 +198,7 @@ describe("verifyDeployment", () => { it("reports unhealthy when the inference route is reachable but returns HTTP 5xx", async () => { const deps = makeDeps({ - executeSandboxCommand: (_name: string, script: string) => { - if (script.includes("inference.local")) { - return { status: 0, stdout: "503", stderr: "" }; - } - return { status: 0, stdout: "200", stderr: "" }; - }, + executeSandboxCommand: () => ({ status: 0, stdout: "503", stderr: "" }), }); const result = await verifyDeployment("my-sandbox", chain, deps, NO_RETRY); expect(result.healthy).toBe(false); From 07cca790d1f723258d140f5c10a1c496f68c2e38 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 14 Jul 2026 13:43:24 -0400 Subject: [PATCH 03/12] test(e2e): allow combined coverage validation budget Signed-off-by: Julie Yaunches --- .../support/jetson-workflow-boundary.test.ts | 91 ++++++++++--------- 1 file changed, 48 insertions(+), 43 deletions(-) diff --git a/test/e2e/support/jetson-workflow-boundary.test.ts b/test/e2e/support/jetson-workflow-boundary.test.ts index 31f539b1eef..6b703e0f004 100644 --- a/test/e2e/support/jetson-workflow-boundary.test.ts +++ b/test/e2e/support/jetson-workflow-boundary.test.ts @@ -9,6 +9,7 @@ import { describe, expect, it } from "vitest"; import YAML from "yaml"; import { validateE2eWorkflowBoundary } from "../../../tools/e2e/workflow-boundary.mts"; import { readWorkflow } from "../../helpers/e2e-workflow-contract.ts"; +import { testTimeoutOptions } from "../../helpers/timeouts"; function validateWorkflowMutation( mutate: (workflow: ReturnType) => void, @@ -26,51 +27,55 @@ function validateWorkflowMutation( } describe("Jetson nvmap GPU E2E workflow boundary", () => { - it("rejects unsafe runner opt-in, routing, and guard ordering drift (#6430)", () => { - const inputErrors = validateWorkflowMutation((workflow) => { - const triggers = (workflow.on ?? workflow[true as unknown as string]) as { - workflow_dispatch?: { - inputs?: Record; + it( + "rejects unsafe runner opt-in, routing, and guard ordering drift (#6430)", + testTimeoutOptions(15_000), + () => { + const inputErrors = validateWorkflowMutation((workflow) => { + const triggers = (workflow.on ?? workflow[true as unknown as string]) as { + workflow_dispatch?: { + inputs?: Record; + }; }; - }; - const input = triggers.workflow_dispatch!.inputs!.allow_jetson_runner_queue; - input.type = "string"; - input.default = true; - input.description = "Queue the runner"; - }); - expect(inputErrors).toEqual( - expect.arrayContaining([ - "workflow_dispatch allow_jetson_runner_queue input must be boolean", - "workflow_dispatch allow_jetson_runner_queue input must default to false", - "workflow_dispatch allow_jetson_runner_queue input must identify repository administrators and NVIDIA/NemoClaw Settings -> Actions -> Runners as the authoritative runner inventory, and document queued timeout behavior", - ]), - ); + const input = triggers.workflow_dispatch!.inputs!.allow_jetson_runner_queue; + input.type = "string"; + input.default = true; + input.description = "Queue the runner"; + }); + expect(inputErrors).toEqual( + expect.arrayContaining([ + "workflow_dispatch allow_jetson_runner_queue input must be boolean", + "workflow_dispatch allow_jetson_runner_queue input must default to false", + "workflow_dispatch allow_jetson_runner_queue input must identify repository administrators and NVIDIA/NemoClaw Settings -> Actions -> Runners as the authoritative runner inventory, and document queued timeout behavior", + ]), + ); - const guardErrors = validateWorkflowMutation((workflow) => { - const job = (workflow.jobs as Record)["jetson-nvmap-gpu"] as { - "runs-on"?: string; - steps?: Array<{ if?: string; name?: string; uses?: string }>; - }; - job["runs-on"] = "self-hosted"; - const steps = job.steps!; - const guardIndex = steps.findIndex((step) => step.name === "Guard Jetson runner dispatch"); - const [guard] = steps.splice(guardIndex, 1); - guard!.if = "always()"; - const authIndex = steps.findIndex((step) => step.name === "Authenticate to Docker Hub"); - steps.splice(authIndex + 1, 0, guard!); - steps.find((step) => step.name === "Upload Jetson nvmap GPU artifacts")!.if = "success()"; - steps.find((step) => step.name === "Clean up Docker auth")!.if = "success()"; - }); - expect(guardErrors).toEqual( - expect.arrayContaining([ - "jetson-nvmap-gpu job must use ubuntu-latest unless allow_jetson_runner_queue is true", - "jetson-nvmap-gpu dispatch guard must run before Docker Hub auth", - "jetson-nvmap-gpu dispatch guard must run unless allow_jetson_runner_queue is true", - "jetson-nvmap-gpu upload-e2e-artifacts invocation must run with always()", - "jetson-nvmap-gpu Docker Hub cleanup step must always run", - ]), - ); - }); + const guardErrors = validateWorkflowMutation((workflow) => { + const job = (workflow.jobs as Record)["jetson-nvmap-gpu"] as { + "runs-on"?: string; + steps?: Array<{ if?: string; name?: string; uses?: string }>; + }; + job["runs-on"] = "self-hosted"; + const steps = job.steps!; + const guardIndex = steps.findIndex((step) => step.name === "Guard Jetson runner dispatch"); + const [guard] = steps.splice(guardIndex, 1); + guard!.if = "always()"; + const authIndex = steps.findIndex((step) => step.name === "Authenticate to Docker Hub"); + steps.splice(authIndex + 1, 0, guard!); + steps.find((step) => step.name === "Upload Jetson nvmap GPU artifacts")!.if = "success()"; + steps.find((step) => step.name === "Clean up Docker auth")!.if = "success()"; + }); + expect(guardErrors).toEqual( + expect.arrayContaining([ + "jetson-nvmap-gpu job must use ubuntu-latest unless allow_jetson_runner_queue is true", + "jetson-nvmap-gpu dispatch guard must run before Docker Hub auth", + "jetson-nvmap-gpu dispatch guard must run unless allow_jetson_runner_queue is true", + "jetson-nvmap-gpu upload-e2e-artifacts invocation must run with always()", + "jetson-nvmap-gpu Docker Hub cleanup step must always run", + ]), + ); + }, + ); it("rejects a Jetson guard that only prints the fallback runner label (#6430)", () => { const errors = validateWorkflowMutation((workflow) => { From 59329e2f568dafa40bc7cef88ef1eaddd5aff193 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 14 Jul 2026 14:31:31 -0400 Subject: [PATCH 04/12] fix(onboard): block unreachable inference readiness Signed-off-by: Julie Yaunches --- src/lib/verify-deployment.test.ts | 7 ++++--- src/lib/verify-deployment.ts | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/lib/verify-deployment.test.ts b/src/lib/verify-deployment.test.ts index 37a2833a76c..0c200d0526d 100644 --- a/src/lib/verify-deployment.test.ts +++ b/src/lib/verify-deployment.test.ts @@ -179,7 +179,7 @@ describe("verifyDeployment", () => { expect(dashDiag?.hint).toContain("forward"); }); - it("inference failure is a warning, not a blocker", async () => { + it("reports unhealthy when the inference route is unreachable (#6849)", async () => { const deps = makeDeps({ executeSandboxCommand: (_name: string, script: string) => { if (script.includes("inference.local")) { @@ -190,10 +190,11 @@ describe("verifyDeployment", () => { }, }); const result = await verifyDeployment("my-sandbox", chain, deps, NO_RETRY); - expect(result.healthy).toBe(true); // inference is non-blocking + expect(result.healthy).toBe(false); expect(result.verification.inferenceRouteWorking).toBe(false); const infDiag = result.diagnostics.find((d) => d.link === "inference"); - expect(infDiag?.status).toBe("warn"); + expect(infDiag?.status).toBe("fail"); + expect(infDiag?.hint).toContain("unreachable"); }); it("reports unhealthy when the inference route is reachable but returns HTTP 5xx", async () => { diff --git a/src/lib/verify-deployment.ts b/src/lib/verify-deployment.ts index aad1f5c24dd..2be7b1f6796 100644 --- a/src/lib/verify-deployment.ts +++ b/src/lib/verify-deployment.ts @@ -543,14 +543,14 @@ export async function verifyDeployment( const inferenceRouteWorking = inference.status === "ok"; diagnostics.push({ link: "inference", - status: inference.status === "ok" ? "ok" : inference.status === "unhealthy" ? "fail" : "warn", + status: inference.status === "ok" ? "ok" : "fail", detail: inference.detail, hint: inference.status === "ok" ? "" : inference.status === "unhealthy" ? "The inference route is reachable but the endpoint returned a server error (HTTP 5xx). If the endpoint runs on the host, confirm it is reachable from the sandbox — a loopback 127.0.0.1/localhost bind is not; bind it to 0.0.0.0 or use host.openshell.internal — then re-run: nemoclaw status." - : "The inference proxy may not be ready yet. Try: nemoclaw status (it may take a few seconds after creation).", + : "The inference proxy is unreachable. Confirm the configured endpoint is running and reachable from the sandbox, then re-run: nemoclaw status.", }); // 5. Messaging bridges (providers attached AND runtime config exposes @@ -578,7 +578,7 @@ export async function verifyDeployment( accessMethod, }; - const healthy = gateway.reachable && dashboard.reachable && inference.status !== "unhealthy"; + const healthy = gateway.reachable && dashboard.reachable && inference.status === "ok"; return { healthy, verification, diagnostics }; } From 64d582c4f9af2c4f86686df61ac8a050563397d9 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 14 Jul 2026 14:36:58 -0400 Subject: [PATCH 05/12] docs(onboard): explain inference readiness failure Signed-off-by: Julie Yaunches --- docs/get-started/quickstart.mdx | 6 ++++-- docs/inference/verify-inference-route.mdx | 3 +++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index 14cf91c3667..02dca1912b4 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -322,9 +322,11 @@ Use these details when your first-run path needs more control. The selector can include destinations such as GitHub, Jira, Slack, Telegram, or local inference. Press `r` to switch a selected preset between read-only and read-write when it supports both modes. - Before it prints the ready summary, NemoClaw checks that the sandbox gateway and dashboard port forward are reachable. + Before it prints the final summary, NemoClaw checks that the sandbox gateway, dashboard port forward, and `inference.local` route are reachable. When web search is enabled, it also checks the selected provider configuration and sends a real search request through sandbox egress. - Web search, inference-route, and messaging-bridge checks report warnings instead of aborting onboarding when they need more time or configuration. + An inference route that is unreachable or returns HTTP 5xx marks the sandbox not ready and makes onboarding exit non-zero. + Restore the configured endpoint or proxy, then rerun `nemoclaw status` to verify the route. + Web search and messaging-bridge checks remain warnings when they need more time or configuration. ```text ────────────────────────────────────────────────── diff --git a/docs/inference/verify-inference-route.mdx b/docs/inference/verify-inference-route.mdx index 159c832bb2f..52824324e4f 100644 --- a/docs/inference/verify-inference-route.mdx +++ b/docs/inference/verify-inference-route.mdx @@ -34,6 +34,9 @@ $$nemoclaw status The `Inference` row checks the sandbox's `inference.local` path and reports the provider, model, and endpoint with the rest of the sandbox state. This path includes the OpenShell proxy and its authentication rewrite. +Before dashboard-based onboarding prints its final summary, NemoClaw runs the same route-reachability probe from inside the sandbox. +An unreachable route or an HTTP 5xx response marks the sandbox not ready and makes onboarding exit non-zero. +Restore the configured endpoint or proxy, then rerun the status command. ## Understand Post-Ready Checks From aeb55f0521203d900928e0df634f30b931fa1b23 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 14 Jul 2026 14:53:21 -0400 Subject: [PATCH 06/12] docs(onboard): use direct readiness guidance Signed-off-by: Julie Yaunches --- docs/get-started/quickstart.mdx | 4 ++-- docs/inference/verify-inference-route.mdx | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index 02dca1912b4..caa3f2014f8 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -322,9 +322,9 @@ Use these details when your first-run path needs more control. The selector can include destinations such as GitHub, Jira, Slack, Telegram, or local inference. Press `r` to switch a selected preset between read-only and read-write when it supports both modes. - Before it prints the final summary, NemoClaw checks that the sandbox gateway, dashboard port forward, and `inference.local` route are reachable. + Use the final onboarding summary to verify that the sandbox gateway, dashboard port forward, and `inference.local` route are reachable. When web search is enabled, it also checks the selected provider configuration and sends a real search request through sandbox egress. - An inference route that is unreachable or returns HTTP 5xx marks the sandbox not ready and makes onboarding exit non-zero. + Treat an unreachable route or HTTP 5xx response as a failed readiness check: onboarding marks the sandbox not ready and exits non-zero. Restore the configured endpoint or proxy, then rerun `nemoclaw status` to verify the route. Web search and messaging-bridge checks remain warnings when they need more time or configuration. diff --git a/docs/inference/verify-inference-route.mdx b/docs/inference/verify-inference-route.mdx index 52824324e4f..6dc67aced91 100644 --- a/docs/inference/verify-inference-route.mdx +++ b/docs/inference/verify-inference-route.mdx @@ -34,8 +34,8 @@ $$nemoclaw status The `Inference` row checks the sandbox's `inference.local` path and reports the provider, model, and endpoint with the rest of the sandbox state. This path includes the OpenShell proxy and its authentication rewrite. -Before dashboard-based onboarding prints its final summary, NemoClaw runs the same route-reachability probe from inside the sandbox. -An unreachable route or an HTTP 5xx response marks the sandbox not ready and makes onboarding exit non-zero. +Use the final dashboard-based onboarding summary to verify that NemoClaw ran the same route-reachability probe from inside the sandbox. +Treat an unreachable route or HTTP 5xx response as a failed readiness check: onboarding marks the sandbox not ready and exits non-zero. Restore the configured endpoint or proxy, then rerun the status command. ## Understand Post-Ready Checks From b152e600f66e6d6c65308751f3eadd35b25e9ce2 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 14 Jul 2026 15:14:03 -0400 Subject: [PATCH 07/12] fix(onboard): harden inference recovery guidance Signed-off-by: Julie Yaunches --- src/lib/verify-deployment.test.ts | 55 +++++++++++++++++++++++++++++++ src/lib/verify-deployment.ts | 2 +- 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/src/lib/verify-deployment.test.ts b/src/lib/verify-deployment.test.ts index 0c200d0526d..4d902f82bba 100644 --- a/src/lib/verify-deployment.test.ts +++ b/src/lib/verify-deployment.test.ts @@ -207,6 +207,9 @@ describe("verifyDeployment", () => { const infDiag = result.diagnostics.find((d) => d.link === "inference"); expect(infDiag?.status).toBe("fail"); expect(infDiag?.detail).toContain("503"); + expect(infDiag?.hint).toContain("host.openshell.internal"); + expect(infDiag?.hint).toContain("firewall"); + expect(infDiag?.hint).not.toContain("0.0.0.0"); }); it("messaging failure is a warning, not a blocker", async () => { @@ -515,6 +518,58 @@ describe("verifyDeployment", () => { expect(dashboardCalls).toBe(2); }); + it("retries the inference probe and recovers when the route comes up late (#6849)", async () => { + let inferenceCalls = 0; + const deps = makeDeps({ + executeSandboxCommand: (_name: string, script: string) => { + if (script.includes("inference.local")) { + inferenceCalls += 1; + return { status: 0, stdout: inferenceCalls === 1 ? "000" : "200", stderr: "" }; + } + return { status: 0, stdout: "200", stderr: "" }; + }, + }); + const sleepCalls: number[] = []; + const result = await verifyDeployment("my-sandbox", chain, deps, { + retryDelaysMs: [10, 20], + sleep: async (ms: number) => { + sleepCalls.push(ms); + }, + }); + expect(result.healthy).toBe(true); + expect(result.verification.inferenceRouteWorking).toBe(true); + expect(inferenceCalls).toBe(2); + expect(sleepCalls).toEqual([10]); + }); + + it("does not retry inference after the gateway retry budget is exhausted (#6849)", async () => { + let gatewayCalls = 0; + let inferenceCalls = 0; + const deps = makeDeps({ + executeSandboxCommand: (_name: string, script: string) => { + if (script.includes("inference.local")) { + inferenceCalls += 1; + } else if (!script.includes("openclaw --version")) { + gatewayCalls += 1; + } + return { status: 0, stdout: "000", stderr: "" }; + }, + }); + const sleepCalls: number[] = []; + const result = await verifyDeployment("my-sandbox", chain, deps, { + retryDelaysMs: [10, 20], + sleep: async (ms: number) => { + sleepCalls.push(ms); + }, + }); + expect(result.healthy).toBe(false); + expect(result.verification.gatewayReachable).toBe(false); + expect(result.verification.inferenceRouteWorking).toBe(false); + expect(gatewayCalls).toBe(3); + expect(inferenceCalls).toBe(1); + expect(sleepCalls).toEqual([10, 20]); + }); + it("gives up after retry budget is exhausted and surfaces the last failure detail", async () => { const deps = makeDeps({ executeSandboxCommand: () => ({ status: 0, stdout: "000", stderr: "" }), diff --git a/src/lib/verify-deployment.ts b/src/lib/verify-deployment.ts index 2be7b1f6796..c4838712568 100644 --- a/src/lib/verify-deployment.ts +++ b/src/lib/verify-deployment.ts @@ -549,7 +549,7 @@ export async function verifyDeployment( inference.status === "ok" ? "" : inference.status === "unhealthy" - ? "The inference route is reachable but the endpoint returned a server error (HTTP 5xx). If the endpoint runs on the host, confirm it is reachable from the sandbox — a loopback 127.0.0.1/localhost bind is not; bind it to 0.0.0.0 or use host.openshell.internal — then re-run: nemoclaw status." + ? "The inference route is reachable but the endpoint returned a server error (HTTP 5xx). If the endpoint runs on the host, configure it to listen on a host address reachable through host.openshell.internal and restrict access with the host firewall or equivalent controls; a 127.0.0.1/localhost-only bind is not reachable from the sandbox. Then re-run: nemoclaw status." : "The inference proxy is unreachable. Confirm the configured endpoint is running and reachable from the sandbox, then re-run: nemoclaw status.", }); From c437dec84396011e99f5a6fa2535ccb4615bff87 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Tue, 14 Jul 2026 15:42:44 -0400 Subject: [PATCH 08/12] test(onboard): keep retry fixtures linear Signed-off-by: Julie Yaunches --- src/lib/verify-deployment.test.ts | 37 +++++++++++++++---------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/src/lib/verify-deployment.test.ts b/src/lib/verify-deployment.test.ts index 4d902f82bba..755de632ad6 100644 --- a/src/lib/verify-deployment.test.ts +++ b/src/lib/verify-deployment.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { buildChain } from "./dashboard/contract.js"; import { formatVerificationDiagnostics, verifyDeployment } from "./verify-deployment.js"; @@ -519,15 +519,15 @@ describe("verifyDeployment", () => { }); it("retries the inference probe and recovers when the route comes up late (#6849)", async () => { - let inferenceCalls = 0; + const probeInference = vi + .fn() + .mockReturnValueOnce({ status: 0, stdout: "000", stderr: "" }) + .mockReturnValue({ status: 0, stdout: "200", stderr: "" }); const deps = makeDeps({ - executeSandboxCommand: (_name: string, script: string) => { - if (script.includes("inference.local")) { - inferenceCalls += 1; - return { status: 0, stdout: inferenceCalls === 1 ? "000" : "200", stderr: "" }; - } - return { status: 0, stdout: "200", stderr: "" }; - }, + executeSandboxCommand: (_name: string, script: string) => + script.includes("inference.local") + ? probeInference() + : { status: 0, stdout: "200", stderr: "" }, }); const sleepCalls: number[] = []; const result = await verifyDeployment("my-sandbox", chain, deps, { @@ -538,20 +538,15 @@ describe("verifyDeployment", () => { }); expect(result.healthy).toBe(true); expect(result.verification.inferenceRouteWorking).toBe(true); - expect(inferenceCalls).toBe(2); + expect(probeInference).toHaveBeenCalledTimes(2); expect(sleepCalls).toEqual([10]); }); it("does not retry inference after the gateway retry budget is exhausted (#6849)", async () => { - let gatewayCalls = 0; - let inferenceCalls = 0; + const scripts: string[] = []; const deps = makeDeps({ executeSandboxCommand: (_name: string, script: string) => { - if (script.includes("inference.local")) { - inferenceCalls += 1; - } else if (!script.includes("openclaw --version")) { - gatewayCalls += 1; - } + scripts.push(script); return { status: 0, stdout: "000", stderr: "" }; }, }); @@ -565,8 +560,12 @@ describe("verifyDeployment", () => { expect(result.healthy).toBe(false); expect(result.verification.gatewayReachable).toBe(false); expect(result.verification.inferenceRouteWorking).toBe(false); - expect(gatewayCalls).toBe(3); - expect(inferenceCalls).toBe(1); + expect( + scripts.filter( + (script) => !script.includes("inference.local") && !script.includes("openclaw --version"), + ), + ).toHaveLength(3); + expect(scripts.filter((script) => script.includes("inference.local"))).toHaveLength(1); expect(sleepCalls).toEqual([10, 20]); }); From 9661f7a333a1b11f0da88127a67c0d2c4073b7b8 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 15 Jul 2026 11:48:16 -0400 Subject: [PATCH 09/12] fix(onboard): keep failed readiness retryable Signed-off-by: Julie Yaunches --- docs/get-started/quickstart.mdx | 2 +- docs/inference/verify-inference-route.mdx | 16 ++-- src/lib/inference/local.test.ts | 34 -------- src/lib/inference/local.ts | 13 ---- src/lib/onboard.ts | 12 ++- src/lib/onboard/lifecycle-contracts.md | 3 +- src/lib/onboard/machine/README.md | 7 +- .../machine/final-flow-phases.runtime.test.ts | 77 +++++++++++++++++++ .../onboard/machine/final-flow-phases.test.ts | 6 +- src/lib/onboard/machine/final-flow-phases.ts | 4 +- .../machine/handlers/finalization.test.ts | 11 +++ .../onboard/machine/handlers/finalization.ts | 26 +++++-- src/lib/onboard/machine/result.test.ts | 8 +- src/lib/onboard/machine/result.ts | 14 ++++ src/lib/onboard/machine/runner.test.ts | 38 ++++++++- src/lib/onboard/machine/runner.ts | 3 + src/lib/onboard/machine/runtime.test.ts | 37 ++++++++- src/lib/onboard/machine/runtime.ts | 13 ++++ src/lib/onboard/runtime-boundary.test.ts | 2 + src/lib/onboard/runtime-boundary.ts | 11 +++ src/lib/verify-deployment.test.ts | 7 +- src/lib/verify-deployment.ts | 4 +- test/e2e/live/onboard-resume.test.ts | 76 +++++++++++++++++- test/helpers/onboard-final-flow-phases.ts | 2 +- test/onboard-exit-handler.test.ts | 1 + 25 files changed, 343 insertions(+), 84 deletions(-) diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index caa3f2014f8..981d6c393b5 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -325,7 +325,7 @@ Use these details when your first-run path needs more control. Use the final onboarding summary to verify that the sandbox gateway, dashboard port forward, and `inference.local` route are reachable. When web search is enabled, it also checks the selected provider configuration and sends a real search request through sandbox egress. Treat an unreachable route or HTTP 5xx response as a failed readiness check: onboarding marks the sandbox not ready and exits non-zero. - Restore the configured endpoint or proxy, then rerun `nemoclaw status` to verify the route. + Restore the configured endpoint or proxy, run `nemoclaw onboard --resume` to complete onboarding, then run `nemoclaw status` to verify the route. Web search and messaging-bridge checks remain warnings when they need more time or configuration. ```text diff --git a/docs/inference/verify-inference-route.mdx b/docs/inference/verify-inference-route.mdx index 6dc67aced91..328c574490a 100644 --- a/docs/inference/verify-inference-route.mdx +++ b/docs/inference/verify-inference-route.mdx @@ -34,18 +34,18 @@ $$nemoclaw status The `Inference` row checks the sandbox's `inference.local` path and reports the provider, model, and endpoint with the rest of the sandbox state. This path includes the OpenShell proxy and its authentication rewrite. -Use the final dashboard-based onboarding summary to verify that NemoClaw ran the same route-reachability probe from inside the sandbox. +When onboarding prints a dashboard summary, use it to verify that NemoClaw ran the same route-reachability probe from inside the sandbox. Treat an unreachable route or HTTP 5xx response as a failed readiness check: onboarding marks the sandbox not ready and exits non-zero. -Restore the configured endpoint or proxy, then rerun the status command. +Restore the configured endpoint or proxy, then rerun `$$nemoclaw onboard --resume` to complete onboarding. -## Understand Post-Ready Checks +## Understand Final Route Checks -For local Ollama and vLLM, onboarding performs an additional check after the sandbox becomes ready. -It requests `https://inference.local/v1/models` from inside the sandbox and accepts only a 2xx response. -When this check fails, onboarding reports the endpoint and recovery steps before the first agent prompt. +When onboarding prints a dashboard summary, it first requests `https://inference.local/v1/models` from inside the sandbox after policy and process recovery. +A transport failure or HTTP 5xx response leaves the onboarding session retryable at final verification instead of completing it. +After restoring the route, resume onboarding to run the check again without rebuilding a healthy sandbox. -NVIDIA NIM and other compatible endpoints receive their provider validation during onboarding but do not receive this post-ready sandbox-route check. -For those routes, use the status command and a short agent request after onboarding. +Provider setup still performs its own model, credential, and endpoint validation before this final route check. +Use the status command and a short agent request after onboarding to verify ongoing availability and model responses. ## Send a Short Agent Request diff --git a/src/lib/inference/local.test.ts b/src/lib/inference/local.test.ts index 2e5b414e74c..74d472c70fb 100644 --- a/src/lib/inference/local.test.ts +++ b/src/lib/inference/local.test.ts @@ -39,44 +39,10 @@ import { probeOllamaAuthProxyHealth, QWEN3_6_OLLAMA_MODEL, resetOllamaContainerPortCache, - rewriteHostLoopbackForSandbox, validateLocalProvider, validateOllamaModel, } from "./local"; -describe("rewriteHostLoopbackForSandbox", () => { - it("rewrites a loopback host to the sandbox-facing gateway alias, preserving port, path, query, and fragment", () => { - expect(rewriteHostLoopbackForSandbox("http://127.0.0.1:8000/v1")).toBe( - "http://host.openshell.internal:8000/v1", - ); - expect(rewriteHostLoopbackForSandbox("http://localhost/v1?x=1#frag")).toBe( - "http://host.openshell.internal/v1?x=1#frag", - ); - expect(rewriteHostLoopbackForSandbox("http://LOCALHOST:8000/v1")).toBe( - "http://host.openshell.internal:8000/v1", - ); - }); - - it("leaves a non-loopback or empty endpoint unchanged", () => { - expect(rewriteHostLoopbackForSandbox("http://host.openshell.internal:8000/v1")).toBe( - "http://host.openshell.internal:8000/v1", - ); - expect(rewriteHostLoopbackForSandbox("https://notlocalhost.example/v1")).toBe( - "https://notlocalhost.example/v1", - ); - expect(rewriteHostLoopbackForSandbox("https://api.example.com/v1?target=localhost")).toBe( - "https://api.example.com/v1?target=localhost", - ); - expect(rewriteHostLoopbackForSandbox("https://api.example.com/127.0.0.1")).toBe( - "https://api.example.com/127.0.0.1", - ); - expect(rewriteHostLoopbackForSandbox("https://api.example.com/v1")).toBe( - "https://api.example.com/v1", - ); - expect(rewriteHostLoopbackForSandbox("")).toBe(""); - }); -}); - describe("local inference helpers", () => { const originalSandboxHostUrl = process.env[LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV]; const originalPath = process.env.PATH; diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index 983ad3a60ae..3f839927005 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -60,19 +60,6 @@ export function resetOllamaContainerPortCache(): void { } export const HOST_GATEWAY_URL = "http://host.openshell.internal"; - -export function rewriteHostLoopbackForSandbox(url: string): string { - if (!url) return url; - try { - const parsed = new URL(url); - const hostname = parsed.hostname.toLowerCase(); - if (hostname !== "localhost" && hostname !== "127.0.0.1") return url; - const port = parsed.port ? `:${parsed.port}` : ""; - return `${HOST_GATEWAY_URL}${port}${parsed.pathname}${parsed.search}${parsed.hash}`; - } catch { - return url; - } -} export const LOCAL_INFERENCE_SANDBOX_HOST_URL_ENV = "NEMOCLAW_LOCAL_INFERENCE_SANDBOX_HOST_URL"; export const CONTAINER_REACHABILITY_IMAGE = "curlimages/curl:8.10.1"; // These tags are convenience aliases for callers that want to refer to a diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 26b0d2273a0..eec2b6b3c42 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -3116,7 +3116,13 @@ async function handleRoutedSelection( state.provider = bp.provider_name || "nvidia-router"; state.model = bp.model; - state.endpointUrl = localInference.rewriteHostLoopbackForSandbox(bp.endpoint || ""); + const { HOST_GATEWAY_URL } = require("./inference/local"); + const routerEndpointUrl = bp.endpoint || ""; + state.endpointUrl = routerEndpointUrl; + if (routerEndpointUrl.match(/localhost|127\.0\.0\.1/)) { + const u = new URL(routerEndpointUrl); + state.endpointUrl = `${HOST_GATEWAY_URL}:${u.port}${u.pathname}`; + } state.preferredInferenceApi = "openai-completions"; state.assertRouteCompatible?.(); @@ -4662,7 +4668,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { }, }); - await runFinalOnboardFlowSlice({ + const finalFlowResult = await runFinalOnboardFlowSlice({ context: finalFlowContext, runtime: onboardRuntimeBoundary.getRuntime(), phases: [branchSetupPhase, policiesPhase, finalizationPhase], @@ -4677,7 +4683,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { }, }); completed = true; - traceCompleted = true; + traceCompleted = finalFlowResult.session.machine.state === "complete"; } finally { releaseOnboardLock(); onboardRuntimeBoundary.clear(); diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index 4d2542ae680..0ef82ddaa86 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -15,7 +15,7 @@ Related guides: [`README.md`](README.md) describes package placement, [`machine/ | **plan** | Intent plus observed state, ready to apply | `MessagingWorkflowPlanner.buildPlan`; `materializeSandboxCreatePlan` | | **apply** | Effectful phase that binds credentials and live capabilities | `bindMessagingTokenDefs`; create, rebuild, and mutation executors | | **checkpoint** | Durable, secret-minimized state from which a later process can continue | onboard session and machine snapshot; registry; backup/recovery manifests | -| **result** | Handler outcome: advance, retry, branch, complete, or fail | `OnboardStateResult`, applied by `OnboardRuntime` through `OnboardRuntimeBoundary` | +| **result** | Handler outcome: advance, retry, branch, pause, complete, or fail | `OnboardStateResult`, applied by `OnboardRuntime` through `OnboardRuntimeBoundary` | | **compensation** | Effect that undoes or limits a partial apply | failed-create deletion, `cancel-rollback.ts`, `rollbackChannelAdd`, recovery-registry restore | | **reconcile** | Align recorded and live state without replaying the full journey | sandbox drift checks, `reconcileSandboxMessaging`, `mergeOpenClawRestoredConfig` | @@ -31,6 +31,7 @@ inference --retry--> provider_selection inference --advance--> sandbox sandbox --branch--> openclaw -> policies -> finalizing -> post_verify -> complete sandbox --branch--> agent_setup -> policies -> finalizing -> post_verify -> complete +post_verify --pause--> post_verify (retryable handoff without a state transition) each nonterminal state --failure--> failed ``` diff --git a/src/lib/onboard/machine/README.md b/src/lib/onboard/machine/README.md index 1b11fb02765..bc5960fd7cf 100644 --- a/src/lib/onboard/machine/README.md +++ b/src/lib/onboard/machine/README.md @@ -13,9 +13,9 @@ The target shape is a machine-driven onboarding runner: 2. Build an onboarding context that contains sanitized operator choices, runtime dependencies, and mutable values returned by states. 3. Enter `runOnboardMachine(context)`. 4. Dispatch the current machine state to a handler. -5. Let the handler return an explicit state result such as advance, retry, branch, complete, or failed. +5. Let the handler return an explicit state result such as advance, retry, branch, pause, complete, or failed. 6. Apply the result through `OnboardRuntime`, which validates the transition, updates the persisted session snapshot, and emits redacted machine events. -7. Continue until the machine reaches `complete` or `failed`. +7. Continue until the machine reaches `complete` or `failed`, or a handler pauses at a retryable non-terminal state. In that final shape, `src/lib/onboard.ts` should be a thin entrypoint. State handlers should own state-specific prompts, resume validation, repair decisions, and side effects. @@ -80,7 +80,8 @@ sequence must declare its source state in `metadata.state`, and that source must machine's current state when the result is applied. The runner also checks the handler's sequence ownership allowlist; add a new entry in `DEFAULT_SEQUENCE_OWNERSHIP` before introducing another composite handler that crosses into a later state. Terminal results (`complete` or `failed`) end -the sequence immediately. +the sequence immediately. A `pause` result persists any supplied safe context and returns control +without a state transition so a later process can resume the same non-terminal state. ## Runtime responsibilities diff --git a/src/lib/onboard/machine/final-flow-phases.runtime.test.ts b/src/lib/onboard/machine/final-flow-phases.runtime.test.ts index 1bcd7703711..445a7fa3428 100644 --- a/src/lib/onboard/machine/final-flow-phases.runtime.test.ts +++ b/src/lib/onboard/machine/final-flow-phases.runtime.test.ts @@ -285,4 +285,81 @@ describe("final onboard flow runtime boundary", () => { machine: { state: "post_verify" }, }); }); + + it("keeps an unhealthy final verification retryable and completes after a later resume", async () => { + const order: string[] = []; + const harness = createRuntimeHarness(sessionAt("openclaw")); + const recorders = harness.boundary.recorders(); + const unhealthy = { + healthy: false, + verification: { + gatewayReachable: true, + gatewayVersion: "test", + inferenceRouteWorking: false, + dashboardReachable: true, + messagingBridgesHealthy: true, + messagingRuntimeChannelsMissing: null, + messagingConfigChannelsMissing: null, + accessMethod: "localhost" as const, + }, + diagnostics: [], + }; + const healthy = { + ...unhealthy, + healthy: true, + verification: { ...unhealthy.verification, inferenceRouteWorking: true }, + }; + const verifyDeployment = vi + .fn() + .mockResolvedValueOnce(unhealthy) + .mockResolvedValueOnce(healthy); + const phases = createPhases("openclaw", order, { + loadSession: harness.getSession, + recordStepSkipped: recorders.recordStepSkipped, + recordStateSkipped: recorders.recordStateSkipped, + startRecordedStep: recorders.startRecordedStep, + recordStepComplete: recorders.recordStepComplete, + recordPostVerifyStarted: recorders.recordPostVerifyStarted, + verifyDeployment, + }); + + const first = await runFinalOnboardFlowSlice({ + context: context({ session: harness.getSession() }), + runtime: harness.boundary.getRuntime(), + phases, + resume: false, + recordStateResult: harness.boundary.recordStateResultWithStepCompatibility.bind( + harness.boundary, + ), + recordInvalidatedStateResult: harness.boundary.recordInvalidatedStateResult.bind( + harness.boundary, + ), + }); + + expect(first.session).toMatchObject({ + status: "in_progress", + resumable: true, + machine: { state: "post_verify" }, + }); + + const resumed = await runFinalOnboardFlowSlice({ + context: context({ resume: true, session: harness.getSession() }), + runtime: harness.boundary.getRuntime(), + phases, + resume: true, + recordStateResult: harness.boundary.recordStateResultWithStepCompatibility.bind( + harness.boundary, + ), + recordInvalidatedStateResult: harness.boundary.recordInvalidatedStateResult.bind( + harness.boundary, + ), + }); + + expect(verifyDeployment).toHaveBeenCalledTimes(2); + expect(resumed.session).toMatchObject({ + status: "complete", + resumable: false, + machine: { state: "complete" }, + }); + }); }); diff --git a/src/lib/onboard/machine/final-flow-phases.test.ts b/src/lib/onboard/machine/final-flow-phases.test.ts index 59238bb4ce0..bc00f35fb97 100644 --- a/src/lib/onboard/machine/final-flow-phases.test.ts +++ b/src/lib/onboard/machine/final-flow-phases.test.ts @@ -71,10 +71,10 @@ describe("final onboard flow phases", () => { phases, resume: true, recordStateResult: async (result) => { - if (result.type === "complete" || result.type === "failed") { - recorded.push(result.type); - } else { + if (result.type === "transition") { recorded.push(result.next); + } else { + recorded.push(result.type); } }, recordInvalidatedStateResult: async (result) => { diff --git a/src/lib/onboard/machine/final-flow-phases.ts b/src/lib/onboard/machine/final-flow-phases.ts index b2c91d4ccd1..3e388dd2828 100644 --- a/src/lib/onboard/machine/final-flow-phases.ts +++ b/src/lib/onboard/machine/final-flow-phases.ts @@ -165,7 +165,7 @@ export async function runFinalOnboardFlowSlice { +}) { // Recompute plan for live resume repair when durable machine snapshots // are already downstream of this slice even though branch setup/readiness, // policy reconciliation, and final verification must still re-run. Those @@ -180,7 +180,7 @@ export async function runFinalOnboardFlowSlice { ); expect(calls.reportReadiness).toHaveBeenCalledWith(false); expect(result.deploymentHealthy).toBe(false); + expect(result.stateResult).toEqual({ + type: "pause", + updates: { + sandboxName: "my-assistant", + provider: "provider", + model: "model", + hermesAuthMethod: null, + hermesToolGateways: [], + }, + metadata: { state: "finalizing", reason: "deployment_not_ready" }, + }); }); it("ensures agent dashboard forwarding before completion for non-OpenClaw agents", async () => { diff --git a/src/lib/onboard/machine/handlers/finalization.ts b/src/lib/onboard/machine/handlers/finalization.ts index fb6d3fb95d7..1350e28758d 100644 --- a/src/lib/onboard/machine/handlers/finalization.ts +++ b/src/lib/onboard/machine/handlers/finalization.ts @@ -3,7 +3,12 @@ import type { Session } from "../../../state/onboard-session"; import { type DashboardRuntimeAgent, shouldManageDashboardForAgent } from "../../dashboard-runtime"; -import { completeOnboardMachine, type OnboardStateCompleteResult } from "../result"; +import { + completeOnboardMachine, + type OnboardStateCompleteResult, + type OnboardStatePauseResult, + pauseOnboardMachine, +} from "../result"; export interface FinalizationStateOptions { sandboxName: string; @@ -78,7 +83,7 @@ export interface FinalizationStateOptions { branchTo("agent_setup", { metadata: "bad" }); }); - it("builds terminal completion and failure results", () => { + it("builds pause, terminal completion, and failure results", () => { + expect(pauseOnboardMachine({ sandboxName: "my-assistant" }, { reason: "not-ready" })).toEqual({ + type: "pause", + updates: { sandboxName: "my-assistant" }, + metadata: { reason: "not-ready" }, + }); expect(completeOnboardMachine({ sandboxName: "my-assistant" }, { verified: true })).toEqual({ type: "complete", updates: { sandboxName: "my-assistant" }, diff --git a/src/lib/onboard/machine/result.ts b/src/lib/onboard/machine/result.ts index 895640709e6..4f2134b43fc 100644 --- a/src/lib/onboard/machine/result.ts +++ b/src/lib/onboard/machine/result.ts @@ -31,6 +31,12 @@ export interface OnboardStateCompleteResult { metadata?: Record | null; } +export interface OnboardStatePauseResult { + type: "pause"; + updates?: SessionUpdates; + metadata?: Record | null; +} + export interface OnboardStateFailedResult { type: "failed"; error: string | null; @@ -40,6 +46,7 @@ export interface OnboardStateFailedResult { export type OnboardStateResult = | OnboardStateTransitionResult + | OnboardStatePauseResult | OnboardStateCompleteResult | OnboardStateFailedResult; @@ -84,6 +91,13 @@ export function completeOnboardMachine( return { type: "complete", updates, metadata }; } +export function pauseOnboardMachine( + updates: SessionUpdates = {}, + metadata: Record | null = null, +): OnboardStatePauseResult { + return { type: "pause", updates, metadata }; +} + export function failOnboardMachine( error: string | null, options: { step?: string | null; metadata?: Record | null } = {}, diff --git a/src/lib/onboard/machine/runner.test.ts b/src/lib/onboard/machine/runner.test.ts index 3fd9db63f2d..d749b0cc678 100644 --- a/src/lib/onboard/machine/runner.test.ts +++ b/src/lib/onboard/machine/runner.test.ts @@ -12,7 +12,14 @@ import { type SessionUpdates, sanitizeFailure, } from "../../state/onboard-session"; -import { advanceTo, branchTo, completeOnboardMachine, failOnboardMachine, retryTo } from "./result"; +import { + advanceTo, + branchTo, + completeOnboardMachine, + failOnboardMachine, + pauseOnboardMachine, + retryTo, +} from "./result"; import { MissingOnboardStateHandlerError, OnboardMachineTransitionLimitError, @@ -152,6 +159,35 @@ describe("runOnboardMachine", () => { expect(policies).not.toHaveBeenCalled(); }); + it("stops on a pause result without leaving the current retryable state", async () => { + const session = createSession({ + machine: { + version: MACHINE_SNAPSHOT_VERSION, + state: "post_verify", + stateEnteredAt: "2026-05-28T00:00:00.000Z", + revision: 4, + }, + }); + const runtime = createRuntime(session); + const postVerify = vi.fn(() => + pauseOnboardMachine({ sandboxName: "my-assistant" }, { reason: "not-ready" }), + ); + + const result = await runOnboardMachine({ + context: { attempts: 0, visited: [] } as RunnerContext, + runtime, + handlers: { post_verify: postVerify }, + }); + + expect(postVerify).toHaveBeenCalledOnce(); + expect(result.session).toMatchObject({ + status: "in_progress", + resumable: true, + sandboxName: "my-assistant", + machine: { state: "post_verify", revision: 4 }, + }); + }); + it("returns immediately for terminal sessions", async () => { const startedAt = "2026-05-28T00:00:00.000Z"; const completeSession = createSession({ diff --git a/src/lib/onboard/machine/runner.ts b/src/lib/onboard/machine/runner.ts index 8c18794c413..4c1dea3070e 100644 --- a/src/lib/onboard/machine/runner.ts +++ b/src/lib/onboard/machine/runner.ts @@ -263,6 +263,9 @@ export async function runOnboardMachine({ context = updateContext ? await updateContext({ context, state: resultState, result, session }) : context; + if (result.type === "pause") { + return { context, session }; + } if ( isTerminalOnboardMachineState(session.machine.state) || stopStates.includes(session.machine.state) diff --git a/src/lib/onboard/machine/runtime.test.ts b/src/lib/onboard/machine/runtime.test.ts index fb0a8dc2604..332ddebe356 100644 --- a/src/lib/onboard/machine/runtime.test.ts +++ b/src/lib/onboard/machine/runtime.test.ts @@ -13,7 +13,14 @@ import { } from "../../state/onboard-session"; import type { StepMutationOptions } from "../../state/onboard-step-mutation"; import type { OnboardMachineEvent } from "./events"; -import { advanceTo, branchTo, completeOnboardMachine, failOnboardMachine, retryTo } from "./result"; +import { + advanceTo, + branchTo, + completeOnboardMachine, + failOnboardMachine, + pauseOnboardMachine, + retryTo, +} from "./result"; import { OnboardRuntime, type OnboardRuntimeDeps } from "./runtime"; import { InvalidOnboardMachineTransitionError } from "./transitions"; @@ -297,6 +304,31 @@ describe("OnboardRuntime", () => { }); }); + it("persists safe context while leaving a paused non-terminal session retryable", async () => { + const harness = createHarness(sessionInState("post_verify")); + + await harness.runtime.applyResult( + pauseOnboardMachine( + { sandboxName: "my-assistant", provider: "compatible-endpoint" }, + { reason: "deployment_not_ready" }, + ), + ); + + expect(harness.getSession()).toMatchObject({ + status: "in_progress", + resumable: true, + sandboxName: "my-assistant", + provider: "compatible-endpoint", + machine: { state: "post_verify", revision: 7 }, + }); + expect(harness.events).toHaveLength(1); + expect(harness.events[0]).toMatchObject({ + type: "context.updated", + state: "post_verify", + metadata: { reason: "deployment_not_ready" }, + }); + }); + it("rejects invalid explicit transition kinds before mutating context", async () => { const { runtime, getSession } = createHarness(sessionInState("inference")); @@ -325,6 +357,9 @@ describe("OnboardRuntime", () => { it("rejects terminal-state failure and invalid completion transitions", async () => { const completeHarness = createHarness(sessionInState("complete")); await expect(completeHarness.runtime.fail("boom")).rejects.toThrow("complete -> failed"); + await expect(completeHarness.runtime.applyResult(pauseOnboardMachine())).rejects.toThrow( + "Cannot pause terminal onboarding state: complete", + ); expect(completeHarness.getSession().machine.state).toBe("complete"); const policiesHarness = createHarness(sessionInState("policies")); diff --git a/src/lib/onboard/machine/runtime.ts b/src/lib/onboard/machine/runtime.ts index f069c5ec579..a4f6dd6aadd 100644 --- a/src/lib/onboard/machine/runtime.ts +++ b/src/lib/onboard/machine/runtime.ts @@ -287,6 +287,19 @@ export class OnboardRuntime { } async applyResult(result: OnboardStateResult): Promise { + if (result.type === "pause") { + const current = this.ensureSession(); + if (isTerminalOnboardMachineState(current.machine.state)) { + throw new Error(`Cannot pause terminal onboarding state: ${current.machine.state}`); + } + if (result.updates && Object.keys(this.deps.filterSafeUpdates(result.updates)).length > 0) { + return this.updateContext(result.updates, { + state: current.machine.state, + metadata: result.metadata, + }); + } + return current; + } if (result.type === "complete") { return this.complete(result.updates ?? {}, { metadata: result.metadata }); } diff --git a/src/lib/onboard/runtime-boundary.test.ts b/src/lib/onboard/runtime-boundary.test.ts index bbf27a6b4a8..899dbbaa471 100644 --- a/src/lib/onboard/runtime-boundary.test.ts +++ b/src/lib/onboard/runtime-boundary.test.ts @@ -17,6 +17,7 @@ import { branchTo, completeOnboardMachine, failOnboardMachine, + pauseOnboardMachine, retryTo, } from "./machine/result"; import { OnboardRuntime, type OnboardRuntimeDeps } from "./machine/runtime"; @@ -346,6 +347,7 @@ describe("OnboardRuntimeBoundary", () => { }); it.each([ + { label: "pause", result: () => pauseOnboardMachine() }, { label: "complete", result: () => completeOnboardMachine() }, { label: "failed", result: () => failOnboardMachine("boom") }, ] as const)("rejects non-transition $label results before emitting invalidation (#6227)", async ({ diff --git a/src/lib/onboard/runtime-boundary.ts b/src/lib/onboard/runtime-boundary.ts index a1462ada49a..6543b2420b0 100644 --- a/src/lib/onboard/runtime-boundary.ts +++ b/src/lib/onboard/runtime-boundary.ts @@ -139,6 +139,17 @@ export class OnboardRuntimeBoundary { return; } + if (result.type === "pause") { + const sourceState = + result.metadata && typeof result.metadata.state === "string" ? result.metadata.state : null; + if (sourceState && current.machine.state !== sourceState) { + throw new Error( + `Paused onboarding state result source mismatch: ${sourceState} != ${current.machine.state}`, + ); + } + return; + } + const sourceState = result.metadata && typeof result.metadata.state === "string" ? result.metadata.state : null; if (current.machine.state === result.next) { diff --git a/src/lib/verify-deployment.test.ts b/src/lib/verify-deployment.test.ts index 755de632ad6..5a88c2ecfe6 100644 --- a/src/lib/verify-deployment.test.ts +++ b/src/lib/verify-deployment.test.ts @@ -199,10 +199,15 @@ describe("verifyDeployment", () => { it("reports unhealthy when the inference route is reachable but returns HTTP 5xx", async () => { const deps = makeDeps({ - executeSandboxCommand: () => ({ status: 0, stdout: "503", stderr: "" }), + executeSandboxCommand: (_name: string, script: string) => ({ + status: 0, + stdout: script.includes("inference.local") ? "503" : "200", + stderr: "", + }), }); const result = await verifyDeployment("my-sandbox", chain, deps, NO_RETRY); expect(result.healthy).toBe(false); + expect(result.verification.gatewayReachable).toBe(true); expect(result.verification.inferenceRouteWorking).toBe(false); const infDiag = result.diagnostics.find((d) => d.link === "inference"); expect(infDiag?.status).toBe("fail"); diff --git a/src/lib/verify-deployment.ts b/src/lib/verify-deployment.ts index c4838712568..310bef40dd1 100644 --- a/src/lib/verify-deployment.ts +++ b/src/lib/verify-deployment.ts @@ -598,7 +598,9 @@ export function formatVerificationDiagnostics(result: VerifyDeploymentResult): s const RESET = "\x1b[0m"; if (result.healthy) { - lines.push(` ${G}✓${RESET} Deployment verified — gateway and dashboard are healthy.`); + lines.push( + ` ${G}✓${RESET} Deployment verified — gateway, dashboard, and inference route are healthy.`, + ); if (result.verification.gatewayVersion) { lines.push(` OpenClaw version: ${result.verification.gatewayVersion}`); } diff --git a/test/e2e/live/onboard-resume.test.ts b/test/e2e/live/onboard-resume.test.ts index 632a4da6449..219b139a163 100644 --- a/test/e2e/live/onboard-resume.test.ts +++ b/test/e2e/live/onboard-resume.test.ts @@ -82,6 +82,12 @@ interface SessionStateComplete { >; } +interface SessionStatePostVerify { + status: "in_progress"; + resumable: true; + machine: { state: "post_verify" }; +} + interface MutableSessionState extends Record { status?: string; resumable?: boolean; @@ -171,6 +177,7 @@ test("onboard-resume: interrupted onboard then --resume can recreate with cached "resume sandbox recreation filters stale extra providers while preserving live attachments", "resume proves recreated sandbox provider attachments are selectively reconciled", "host trust-store anchor corporate CA source is baked and merged after resume", + "an unreachable committed route pauses at final verification and completes after repair", "implicit resume is detected and --fresh suppresses that auto-resume", ], }); @@ -212,7 +219,7 @@ test("onboard-resume: interrupted onboard then --resume can recreate with cached // fake OpenAI-compatible endpoint at a host address the OpenShell gateway and // sandbox can route to, matching test/e2e/lib/hermetic-compatible-inference.sh. const fakePublicHost = "host.openshell.internal"; - const fake = await startFakeOpenAiCompatibleServer({ + let fake = await startFakeOpenAiCompatibleServer({ apiKey: FAKE_COMPATIBLE_AUTH_VALUE, host: "0.0.0.0", model: FAKE_COMPATIBLE_MODEL, @@ -230,6 +237,7 @@ test("onboard-resume: interrupted onboard then --resume can recreate with cached publicHost: fakePublicHost, }); const localModelsUrl = new URL(`${fake.baseUrl}/models`); + const fakePort = Number(localModelsUrl.port); localModelsUrl.hostname = "127.0.0.1"; const modelsResponse = await fetch(localModelsUrl, { headers: { Authorization: `Bearer ${FAKE_COMPATIBLE_AUTH_VALUE}` }, @@ -541,7 +549,67 @@ test("onboard-resume: interrupted onboard then --resume can recreate with cached expect(containsExactJsonToken(registry, SANDBOX_NAME)).toBe(true); // ────────────────────────────────────────────────────────────────── - // Phase 3.5: implicit resume — a plain `onboard` auto-detects an + // Phase 3.5: a committed route that goes offline leaves final + // verification retryable; restoring the same endpoint lets a later resume + // re-probe and complete without recreating the sandbox. + // ────────────────────────────────────────────────────────────────── + markSessionInProgress(SESSION_FILE); + await fake.close(); + + const unavailableResumeRun = await host.command( + "node", + [CLI_ENTRYPOINT, "onboard", "--resume", "--non-interactive"], + { + artifactName: "phase-3-5-onboard-resume-route-unavailable", + env: resumeEnv, + redactionValues: [FAKE_COMPATIBLE_AUTH_VALUE], + timeoutMs: ONBOARD_TIMEOUT_MS, + }, + ); + const unavailableResumeText = `${unavailableResumeRun.stdout}\n${unavailableResumeRun.stderr}`; + expect(unavailableResumeRun.exitCode, unavailableResumeText).not.toBe(0); + expect(unavailableResumeText).toContain("is not ready"); + expect(unavailableResumeText).toContain("inference"); + + const paused = readSession(SESSION_FILE); + await artifacts.writeJson("phase-3-5-session-route-unavailable.json", { + status: paused.status, + resumable: paused.resumable, + machineState: paused.machine.state, + }); + expect(paused.status).toBe("in_progress"); + expect(paused.resumable).toBe(true); + expect(paused.machine.state).toBe("post_verify"); + + fake = await startFakeOpenAiCompatibleServer({ + apiKey: FAKE_COMPATIBLE_AUTH_VALUE, + host: "0.0.0.0", + model: FAKE_COMPATIBLE_MODEL, + port: fakePort, + publicHost: fakePublicHost, + requireAuth: true, + requireAuthModels: true, + }); + expect(fake.baseUrl).toBe(`http://${fakePublicHost}:${String(fakePort)}/v1`); + + const repairedResumeRun = await host.command( + "node", + [CLI_ENTRYPOINT, "onboard", "--resume", "--non-interactive"], + { + artifactName: "phase-3-5-onboard-resume-route-restored", + env: resumeEnv, + redactionValues: [FAKE_COMPATIBLE_AUTH_VALUE], + timeoutMs: ONBOARD_TIMEOUT_MS, + }, + ); + const repairedResumeText = `${repairedResumeRun.stdout}\n${repairedResumeRun.stderr}`; + expect(repairedResumeRun.exitCode, repairedResumeText).toBe(0); + expect(repairedResumeText).toContain("is ready"); + const repaired = readSession(SESSION_FILE); + expect(repaired.status).toBe("complete"); + + // ────────────────────────────────────────────────────────────────── + // Phase 4: implicit resume — a plain `onboard` auto-detects an // in_progress session, and `--fresh` suppresses that auto-resume. // ────────────────────────────────────────────────────────────────── markSessionInProgress(SESSION_FILE); @@ -549,7 +617,7 @@ test("onboard-resume: interrupted onboard then --resume can recreate with cached "node", [CLI_ENTRYPOINT, "onboard", "--non-interactive"], { - artifactName: "phase-3-5-onboard-implicit-resume", + artifactName: "phase-4-onboard-implicit-resume", env: { ...buildAvailabilityProbeEnv(), NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, @@ -574,7 +642,7 @@ test("onboard-resume: interrupted onboard then --resume can recreate with cached "node", [CLI_ENTRYPOINT, "onboard", "--fresh", "--non-interactive"], { - artifactName: "phase-3-5-onboard-fresh-suppresses-resume", + artifactName: "phase-4-onboard-fresh-suppresses-resume", env: { ...buildAvailabilityProbeEnv(), NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, diff --git a/test/helpers/onboard-final-flow-phases.ts b/test/helpers/onboard-final-flow-phases.ts index b7d6f16260f..8550d4f9d1b 100644 --- a/test/helpers/onboard-final-flow-phases.ts +++ b/test/helpers/onboard-final-flow-phases.ts @@ -250,7 +250,7 @@ export function createPhases( checkAndRecoverSandboxProcesses: vi.fn(), warmupScopeUpgrade: vi.fn(), autoPairScopeApproval: vi.fn(), - isDeploymentHealthy: () => true, + isDeploymentHealthy: (result) => result.healthy, reportDeploymentReadiness: vi.fn(), getChatUiUrl: () => "http://127.0.0.1:45123", buildVerifyChain: (): DashboardDeliveryChain => ({ diff --git a/test/onboard-exit-handler.test.ts b/test/onboard-exit-handler.test.ts index c3ebfd536a3..0a0478664d4 100644 --- a/test/onboard-exit-handler.test.ts +++ b/test/onboard-exit-handler.test.ts @@ -294,6 +294,7 @@ finalPhases.runFinalOnboardFlowSlice = async ({ runtime }) => { { sandboxName: "complete-seam", provider: "nvidia", model: "nemotron-test" }, { state: "post_verify" }, )); + return { context: null, session: await runtime.session() }; }; const { onboard } = require(${onboardPath}); From 1ee8088e99356653f8d6de72d4916dcbecef4f1b Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Wed, 15 Jul 2026 09:32:33 -0700 Subject: [PATCH 10/12] fix(onboard): keep unhealthy finalization resumable Signed-off-by: Carlos Villela --- docs/get-started/quickstart.mdx | 2 +- docs/inference/verify-inference-route.mdx | 2 +- src/lib/onboard.ts | 4 +- .../machine/final-flow-phases.runtime.test.ts | 86 +++++++++++++++++++ .../machine/handlers/finalization.test.ts | 9 +- .../onboard/machine/handlers/finalization.ts | 27 ++++-- src/lib/verify-deployment.test.ts | 9 +- test/helpers/onboard-final-flow-phases.ts | 4 +- 8 files changed, 129 insertions(+), 14 deletions(-) diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index caa3f2014f8..57c133d9b2f 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -325,7 +325,7 @@ Use these details when your first-run path needs more control. Use the final onboarding summary to verify that the sandbox gateway, dashboard port forward, and `inference.local` route are reachable. When web search is enabled, it also checks the selected provider configuration and sends a real search request through sandbox egress. Treat an unreachable route or HTTP 5xx response as a failed readiness check: onboarding marks the sandbox not ready and exits non-zero. - Restore the configured endpoint or proxy, then rerun `nemoclaw status` to verify the route. + Restore the configured endpoint or proxy, run `nemoclaw onboard --resume` to complete the retained onboarding session, then rerun `nemoclaw status` to verify the route. Web search and messaging-bridge checks remain warnings when they need more time or configuration. ```text diff --git a/docs/inference/verify-inference-route.mdx b/docs/inference/verify-inference-route.mdx index 6dc67aced91..6fb8c76e0a4 100644 --- a/docs/inference/verify-inference-route.mdx +++ b/docs/inference/verify-inference-route.mdx @@ -36,7 +36,7 @@ The `Inference` row checks the sandbox's `inference.local` path and reports the This path includes the OpenShell proxy and its authentication rewrite. Use the final dashboard-based onboarding summary to verify that NemoClaw ran the same route-reachability probe from inside the sandbox. Treat an unreachable route or HTTP 5xx response as a failed readiness check: onboarding marks the sandbox not ready and exits non-zero. -Restore the configured endpoint or proxy, then rerun the status command. +Restore the configured endpoint or proxy, run `$$nemoclaw onboard --resume` to complete the retained onboarding session, then rerun the status command. ## Understand Post-Ready Checks diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 26b0d2273a0..e20430e3640 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -4676,8 +4676,8 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { liveFinalFlowContext = context; }, }); - completed = true; - traceCompleted = true; + completed = (await onboardRuntimeBoundary.getRuntime().session()).machine.state === "complete"; + traceCompleted = completed; } finally { releaseOnboardLock(); onboardRuntimeBoundary.clear(); diff --git a/src/lib/onboard/machine/final-flow-phases.runtime.test.ts b/src/lib/onboard/machine/final-flow-phases.runtime.test.ts index 1bcd7703711..cad0c23ae72 100644 --- a/src/lib/onboard/machine/final-flow-phases.runtime.test.ts +++ b/src/lib/onboard/machine/final-flow-phases.runtime.test.ts @@ -9,8 +9,27 @@ import { sessionAt, } from "../../../../test/helpers/onboard-final-flow-phases"; import { createSession } from "../../state/onboard-session"; +import type { VerifyDeploymentResult } from "../../verify-deployment"; +import { applySessionRecovery } from "../session-recovery"; import { runFinalOnboardFlowSlice } from "./final-flow-phases"; +function deploymentResult(healthy: boolean): VerifyDeploymentResult { + return { + healthy, + verification: { + gatewayReachable: true, + gatewayVersion: "test", + inferenceRouteWorking: healthy, + dashboardReachable: true, + messagingBridgesHealthy: true, + messagingRuntimeChannelsMissing: null, + messagingConfigChannelsMissing: null, + accessMethod: "localhost", + }, + diagnostics: [], + }; +} + describe("final onboard flow runtime boundary", () => { it("uses the strict final runner for fresh OpenClaw sessions with a real runtime boundary", async () => { const order: string[] = []; @@ -285,4 +304,71 @@ describe("final onboard flow runtime boundary", () => { machine: { state: "post_verify" }, }); }); + + it("retries an unhealthy deployment after session recovery and completes when repaired (#6849)", async () => { + const verifyDeployment = vi + .fn() + .mockResolvedValueOnce(deploymentResult(false)) + .mockResolvedValueOnce(deploymentResult(true)); + + async function run( + harness: ReturnType, + resume: boolean, + ): Promise { + const recorders = harness.boundary.recorders(); + const phases = createPhases("openclaw", [], { + loadSession: harness.getSession, + recordStepSkipped: recorders.recordStepSkipped, + recordStateSkipped: recorders.recordStateSkipped, + startRecordedStep: recorders.startRecordedStep, + recordStepComplete: recorders.recordStepComplete, + recordPostVerifyStarted: recorders.recordPostVerifyStarted, + verifyDeployment, + }); + await runFinalOnboardFlowSlice({ + context: context({ resume, session: harness.getSession() }), + runtime: harness.boundary.getRuntime(), + phases, + resume, + recordStateResult: harness.boundary.recordStateResultWithStepCompatibility.bind( + harness.boundary, + ), + recordInvalidatedStateResult: harness.boundary.recordInvalidatedStateResult.bind( + harness.boundary, + ), + }); + } + + const firstHarness = createRuntimeHarness(sessionAt("openclaw")); + await run(firstHarness, false); + + expect(firstHarness.getSession()).toMatchObject({ + status: "failed", + resumable: true, + failure: { message: "Sandbox 'my-sandbox' failed deployment verification." }, + machine: { state: "failed" }, + }); + + const recovered = firstHarness.getSession(); + // Production step compatibility records the completed policy phase; the + // focused FSM harness intentionally tracks only machine transitions. + recovered.steps.policies.status = "complete"; + recovered.lastCompletedStep = "policies"; + expect(applySessionRecovery(recovered, "2026-06-10T00:01:00.000Z")).toMatchObject({ + action: "recover", + entry: "finalizing", + }); + expect(recovered.machine.state).toBe("finalizing"); + + const resumedHarness = createRuntimeHarness(recovered); + await run(resumedHarness, true); + + expect(verifyDeployment).toHaveBeenCalledTimes(2); + expect(resumedHarness.getSession()).toMatchObject({ + status: "complete", + resumable: false, + failure: null, + machine: { state: "complete" }, + }); + }); }); diff --git a/src/lib/onboard/machine/handlers/finalization.test.ts b/src/lib/onboard/machine/handlers/finalization.test.ts index 218395a777e..7b9be061ab7 100644 --- a/src/lib/onboard/machine/handlers/finalization.test.ts +++ b/src/lib/onboard/machine/handlers/finalization.test.ts @@ -131,7 +131,7 @@ describe("handleFinalizationState", () => { expect(result.verificationDiagnostics).toEqual([" ✓ verified"]); }); - it("prints a not-ready dashboard and signals not-ready when verification is unhealthy", async () => { + it("prints a not-ready dashboard and returns a resumable failure when verification is unhealthy", async () => { const { deps, calls } = createDeps({ isDeploymentHealthy: vi.fn(() => false) }); const result = await handleFinalizationState(baseOptions(deps)); @@ -145,7 +145,14 @@ describe("handleFinalizationState", () => { false, ); expect(calls.reportReadiness).toHaveBeenCalledWith(false); + expect(calls.postVerify).toHaveBeenCalledOnce(); expect(result.deploymentHealthy).toBe(false); + expect(result.stateResult).toEqual({ + type: "failed", + error: "Sandbox 'my-assistant' failed deployment verification.", + step: undefined, + metadata: { state: "finalizing" }, + }); }); it("ensures agent dashboard forwarding before completion for non-OpenClaw agents", async () => { diff --git a/src/lib/onboard/machine/handlers/finalization.ts b/src/lib/onboard/machine/handlers/finalization.ts index fb6d3fb95d7..ca0ad8b8fe7 100644 --- a/src/lib/onboard/machine/handlers/finalization.ts +++ b/src/lib/onboard/machine/handlers/finalization.ts @@ -3,7 +3,12 @@ import type { Session } from "../../../state/onboard-session"; import { type DashboardRuntimeAgent, shouldManageDashboardForAgent } from "../../dashboard-runtime"; -import { completeOnboardMachine, type OnboardStateCompleteResult } from "../result"; +import { + completeOnboardMachine, + failOnboardMachine, + type OnboardStateCompleteResult, + type OnboardStateResult, +} from "../result"; export interface FinalizationStateOptions { sandboxName: string; @@ -78,7 +83,7 @@ export interface FinalizationStateOptions { expect(infDiag?.hint).toContain("unreachable"); }); - it("reports unhealthy when the inference route is reachable but returns HTTP 5xx", async () => { + it("reports unhealthy when only the inference route returns HTTP 5xx (#6849)", async () => { const deps = makeDeps({ - executeSandboxCommand: () => ({ status: 0, stdout: "503", stderr: "" }), + executeSandboxCommand: (_name: string, script: string) => ({ + status: 0, + stdout: script.includes("inference.local") ? "503" : "200", + stderr: "", + }), }); const result = await verifyDeployment("my-sandbox", chain, deps, NO_RETRY); expect(result.healthy).toBe(false); + expect(result.verification.gatewayReachable).toBe(true); expect(result.verification.inferenceRouteWorking).toBe(false); const infDiag = result.diagnostics.find((d) => d.link === "inference"); expect(infDiag?.status).toBe("fail"); diff --git a/test/helpers/onboard-final-flow-phases.ts b/test/helpers/onboard-final-flow-phases.ts index b7d6f16260f..1517587a3a2 100644 --- a/test/helpers/onboard-final-flow-phases.ts +++ b/test/helpers/onboard-final-flow-phases.ts @@ -50,6 +50,7 @@ export type RecorderOverrides = { sandboxName: string, chain: DashboardDeliveryChain, ) => Promise; + isDeploymentHealthy?: (result: VerifyDeploymentResult) => boolean; printDashboard?: ( sandboxName: string, model: string, @@ -250,7 +251,8 @@ export function createPhases( checkAndRecoverSandboxProcesses: vi.fn(), warmupScopeUpgrade: vi.fn(), autoPairScopeApproval: vi.fn(), - isDeploymentHealthy: () => true, + isDeploymentHealthy: + recorders.isDeploymentHealthy ?? ((result: VerifyDeploymentResult) => result.healthy), reportDeploymentReadiness: vi.fn(), getChatUiUrl: () => "http://127.0.0.1:45123", buildVerifyChain: (): DashboardDeliveryChain => ({ From e0f34e78be25e8c510ed17670afd46c2bad33b5c Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Wed, 15 Jul 2026 12:51:36 -0400 Subject: [PATCH 11/12] test(docs): align final route verification contract Signed-off-by: Julie Yaunches --- test/inference-options-docs.test.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/inference-options-docs.test.ts b/test/inference-options-docs.test.ts index 4fa4761a319..26219374823 100644 --- a/test/inference-options-docs.test.ts +++ b/test/inference-options-docs.test.ts @@ -347,9 +347,9 @@ describe("inference setup navigation", () => { ); }); - it("scopes post-ready sandbox route verification to local inference providers", () => { + it("documents universal final route verification separately from provider-specific warmup", () => { const markdown = fs.readFileSync(verifyInferenceRoutePath, "utf8"); - const start = markdown.indexOf("## Understand Post-Ready Checks"); + const start = markdown.indexOf("## Understand Final Route Checks"); const end = markdown.indexOf("## Send a Short Agent Request", start); expect(start).toBeGreaterThanOrEqual(0); expect(end).toBeGreaterThan(start); @@ -363,8 +363,9 @@ describe("inference setup navigation", () => { ); expect(getSandboxRuntimeInferenceEndpoint("nvidia-nim")).toBeNull(); expect(getSandboxRuntimeInferenceEndpoint("compatible-endpoint")).toBeNull(); - expect(section).toContain("For local Ollama and vLLM"); - expect(section).toContain("NVIDIA NIM and other compatible endpoints"); + expect(section).toContain("`https://inference.local/v1/models`"); + expect(section).toContain("retryable at final verification"); + expect(section).toContain("Provider setup still performs its own"); }); it("explains the host-side validation limit of the containerized gateway alias", () => { From adc90c09ed80b982580cbbfc93f9b34b68274d47 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Wed, 15 Jul 2026 09:58:40 -0700 Subject: [PATCH 12/12] docs(inference): preserve local post-ready checks Signed-off-by: Carlos Villela --- docs/inference/verify-inference-route.mdx | 9 +++++++++ test/inference-options-docs.test.ts | 20 +++++++++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/docs/inference/verify-inference-route.mdx b/docs/inference/verify-inference-route.mdx index af442d5d2f7..2f9d8c1d8ce 100644 --- a/docs/inference/verify-inference-route.mdx +++ b/docs/inference/verify-inference-route.mdx @@ -38,6 +38,15 @@ When onboarding prints a dashboard summary, use it to verify that NemoClaw ran t Treat an unreachable route or HTTP 5xx response as a failed readiness check: onboarding marks the sandbox not ready and exits non-zero. Restore the configured endpoint or proxy, run `$$nemoclaw onboard --resume` to complete the retained onboarding session, then rerun the status command. +## Understand Local Provider Post-Ready Checks + +For local Ollama and vLLM on Docker GPU sandboxes using the compatibility route, onboarding performs an additional check after the sandbox becomes ready. +It requests `https://inference.local/v1/models` from inside the sandbox and accepts only a 2xx response. +When this check fails, onboarding reports the endpoint and local-provider recovery steps before the first agent prompt. + +NVIDIA NIM and other compatible endpoints receive their provider validation during onboarding but do not receive this local-provider post-ready check. +For those routes, continue to the final route check, then use the status command and a short agent request after onboarding. + ## Understand Final Route Checks When onboarding prints a dashboard summary, it first requests `https://inference.local/v1/models` from inside the sandbox after policy and process recovery. diff --git a/test/inference-options-docs.test.ts b/test/inference-options-docs.test.ts index 26219374823..bddf4aef7ca 100644 --- a/test/inference-options-docs.test.ts +++ b/test/inference-options-docs.test.ts @@ -347,10 +347,10 @@ describe("inference setup navigation", () => { ); }); - it("documents universal final route verification separately from provider-specific warmup", () => { + it("scopes post-ready sandbox route verification to local inference providers", () => { const markdown = fs.readFileSync(verifyInferenceRoutePath, "utf8"); - const start = markdown.indexOf("## Understand Final Route Checks"); - const end = markdown.indexOf("## Send a Short Agent Request", start); + const start = markdown.indexOf("## Understand Local Provider Post-Ready Checks"); + const end = markdown.indexOf("## Understand Final Route Checks", start); expect(start).toBeGreaterThanOrEqual(0); expect(end).toBeGreaterThan(start); const section = markdown.slice(start, end); @@ -363,6 +363,20 @@ describe("inference setup navigation", () => { ); expect(getSandboxRuntimeInferenceEndpoint("nvidia-nim")).toBeNull(); expect(getSandboxRuntimeInferenceEndpoint("compatible-endpoint")).toBeNull(); + expect(section).toContain( + "For local Ollama and vLLM on Docker GPU sandboxes using the compatibility route", + ); + expect(section).toContain("NVIDIA NIM and other compatible endpoints"); + }); + + it("documents universal final route verification separately from local warmup", () => { + const markdown = fs.readFileSync(verifyInferenceRoutePath, "utf8"); + const start = markdown.indexOf("## Understand Final Route Checks"); + const end = markdown.indexOf("## Send a Short Agent Request", start); + expect(start).toBeGreaterThanOrEqual(0); + expect(end).toBeGreaterThan(start); + const section = markdown.slice(start, end); + expect(section).toContain("`https://inference.local/v1/models`"); expect(section).toContain("retryable at final verification"); expect(section).toContain("Provider setup still performs its own");