diff --git a/docs/get-started/quickstart.mdx b/docs/get-started/quickstart.mdx index 62e3dcadcb0..d53c36aac8f 100644 --- a/docs/get-started/quickstart.mdx +++ b/docs/get-started/quickstart.mdx @@ -317,9 +317,11 @@ Use these details when your first-run path needs more control. The selector can include destinations such as GitHub, Jira, Slack, Telegram, or local inference. Press `r` to switch a selected preset between read-only and read-write when it supports both modes. - Before it prints the ready summary, NemoClaw checks that the sandbox gateway and dashboard port forward are reachable. + Use the final onboarding summary to verify that the sandbox gateway, dashboard port forward, and `inference.local` route are reachable. When web search is enabled, it also checks the selected provider configuration and sends a real search request through sandbox egress. - Web search, inference-route, and messaging-bridge checks report warnings instead of aborting onboarding when they need more time or configuration. + Treat an unreachable route or HTTP 5xx response as a failed readiness check: onboarding marks the sandbox not ready and exits non-zero. + Restore the configured endpoint or proxy, run `nemoclaw onboard --resume` to complete the retained onboarding session, then rerun `nemoclaw status` to verify the route. + Web search and messaging-bridge checks remain warnings when they need more time or configuration. ```text ────────────────────────────────────────────────── diff --git a/docs/inference/verify-inference-route.mdx b/docs/inference/verify-inference-route.mdx index 159c832bb2f..2f9d8c1d8ce 100644 --- a/docs/inference/verify-inference-route.mdx +++ b/docs/inference/verify-inference-route.mdx @@ -34,15 +34,27 @@ $$nemoclaw status The `Inference` row checks the sandbox's `inference.local` path and reports the provider, model, and endpoint with the rest of the sandbox state. This path includes the OpenShell proxy and its authentication rewrite. +When onboarding prints a dashboard summary, use it to verify that NemoClaw ran the same route-reachability probe from inside the sandbox. +Treat an unreachable route or HTTP 5xx response as a failed readiness check: onboarding marks the sandbox not ready and exits non-zero. +Restore the configured endpoint or proxy, run `$$nemoclaw onboard --resume` to complete the retained onboarding session, then rerun the status command. -## Understand Post-Ready Checks +## Understand Local Provider Post-Ready Checks -For local Ollama and vLLM, onboarding performs an additional check after the sandbox becomes ready. +For local Ollama and vLLM on Docker GPU sandboxes using the compatibility route, onboarding performs an additional check after the sandbox becomes ready. It requests `https://inference.local/v1/models` from inside the sandbox and accepts only a 2xx response. -When this check fails, onboarding reports the endpoint and recovery steps before the first agent prompt. +When this check fails, onboarding reports the endpoint and local-provider recovery steps before the first agent prompt. -NVIDIA NIM and other compatible endpoints receive their provider validation during onboarding but do not receive this post-ready sandbox-route check. -For those routes, use the status command and a short agent request after onboarding. +NVIDIA NIM and other compatible endpoints receive their provider validation during onboarding but do not receive this local-provider post-ready check. +For those routes, continue to the final route check, then use the status command and a short agent request after onboarding. + +## Understand Final Route Checks + +When onboarding prints a dashboard summary, it first requests `https://inference.local/v1/models` from inside the sandbox after policy and process recovery. +A transport failure or HTTP 5xx response leaves the onboarding session retryable at final verification instead of completing it. +After restoring the route, resume onboarding to run the check again without rebuilding a healthy sandbox. + +Provider setup still performs its own model, credential, and endpoint validation before this final route check. +Use the status command and a short agent request after onboarding to verify ongoing availability and model responses. ## Send a Short Agent Request diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index df6934a4fdb..e59a2a6e8cd 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -4666,7 +4666,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { }, }); - await runFinalOnboardFlowSlice({ + const finalFlowResult = await runFinalOnboardFlowSlice({ context: finalFlowContext, runtime: onboardRuntimeBoundary.getRuntime(), phases: [branchSetupPhase, policiesPhase, finalizationPhase], @@ -4681,7 +4681,7 @@ async function runOnboard(opts: OnboardOptions = {}): Promise { }, }); completed = true; - traceCompleted = true; + traceCompleted = finalFlowResult.session.machine.state === "complete"; } finally { releaseOnboardLock(); onboardRuntimeBoundary.clear(); diff --git a/src/lib/onboard/dashboard.ts b/src/lib/onboard/dashboard.ts index 076973113a9..0fd817e7209 100644 --- a/src/lib/onboard/dashboard.ts +++ b/src/lib/onboard/dashboard.ts @@ -115,6 +115,7 @@ export interface OnboardDashboardHelpers { provider: string, nimContainer?: string | null, agent?: AgentDefinition | null, + ready?: boolean, ): void; stopAllDashboardForwards(): void; } @@ -439,6 +440,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa provider: string, nimContainer: string | null = null, agent: AgentDefinition | null = null, + ready = true, ): void { const nimStatus = deps.nimStatus ?? nim.nimStatus; const nimStatusByName = deps.nimStatusByName ?? nim.nimStatusByName; @@ -471,7 +473,7 @@ export function createOnboardDashboardHelpers(deps: OnboardDashboardDeps): Onboa console.log(""); console.log(` ${"─".repeat(50)}`); - console.log(` ${deps.agentProductName()} is ready`); + console.log(` ${deps.agentProductName()} is ${ready ? "ready" : "not ready"}`); console.log(""); console.log(` Sandbox: ${sandboxName}`); console.log(` Model: ${model} (${providerLabel})`); diff --git a/src/lib/onboard/finalization-deps.test.ts b/src/lib/onboard/finalization-deps.test.ts new file mode 100644 index 00000000000..c9bcffa9f5a --- /dev/null +++ b/src/lib/onboard/finalization-deps.test.ts @@ -0,0 +1,35 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it } from "vitest"; + +import type { VerifyDeploymentResult } from "../verify-deployment"; +import { finalizationHandlerDeps } from "./finalization-deps"; + +describe("finalizationHandlerDeps.reportDeploymentReadiness", () => { + const originalExitCode = process.exitCode; + afterEach(() => { + process.exitCode = originalExitCode; + }); + + it("sets a non-zero exit code when the deployment is not ready", () => { + process.exitCode = 0; + finalizationHandlerDeps.reportDeploymentReadiness(false); + expect(process.exitCode).toBe(1); + }); + + it("leaves the exit code unchanged when the deployment is ready", () => { + process.exitCode = 0; + finalizationHandlerDeps.reportDeploymentReadiness(true); + expect(process.exitCode).toBe(0); + }); +}); + +describe("finalizationHandlerDeps.isDeploymentHealthy", () => { + it("reports the verification healthy flag", () => { + const healthy = { healthy: true } as unknown as VerifyDeploymentResult; + const unhealthy = { healthy: false } as unknown as VerifyDeploymentResult; + expect(finalizationHandlerDeps.isDeploymentHealthy(healthy)).toBe(true); + expect(finalizationHandlerDeps.isDeploymentHealthy(unhealthy)).toBe(false); + }); +}); diff --git a/src/lib/onboard/finalization-deps.ts b/src/lib/onboard/finalization-deps.ts index c504aa54fff..d74d03ca123 100644 --- a/src/lib/onboard/finalization-deps.ts +++ b/src/lib/onboard/finalization-deps.ts @@ -29,4 +29,10 @@ export const finalizationHandlerDeps = { require("../actions/sandbox/auto-pair-warmup"); warmup.runSandboxScopeWarmupRun(name); }, + isDeploymentHealthy(result: import("../verify-deployment").VerifyDeploymentResult): boolean { + return result.healthy; + }, + reportDeploymentReadiness(healthy: boolean): void { + if (!healthy) process.exitCode = 1; + }, }; diff --git a/src/lib/onboard/lifecycle-contracts.md b/src/lib/onboard/lifecycle-contracts.md index 4d2542ae680..0ef82ddaa86 100644 --- a/src/lib/onboard/lifecycle-contracts.md +++ b/src/lib/onboard/lifecycle-contracts.md @@ -15,7 +15,7 @@ Related guides: [`README.md`](README.md) describes package placement, [`machine/ | **plan** | Intent plus observed state, ready to apply | `MessagingWorkflowPlanner.buildPlan`; `materializeSandboxCreatePlan` | | **apply** | Effectful phase that binds credentials and live capabilities | `bindMessagingTokenDefs`; create, rebuild, and mutation executors | | **checkpoint** | Durable, secret-minimized state from which a later process can continue | onboard session and machine snapshot; registry; backup/recovery manifests | -| **result** | Handler outcome: advance, retry, branch, complete, or fail | `OnboardStateResult`, applied by `OnboardRuntime` through `OnboardRuntimeBoundary` | +| **result** | Handler outcome: advance, retry, branch, pause, complete, or fail | `OnboardStateResult`, applied by `OnboardRuntime` through `OnboardRuntimeBoundary` | | **compensation** | Effect that undoes or limits a partial apply | failed-create deletion, `cancel-rollback.ts`, `rollbackChannelAdd`, recovery-registry restore | | **reconcile** | Align recorded and live state without replaying the full journey | sandbox drift checks, `reconcileSandboxMessaging`, `mergeOpenClawRestoredConfig` | @@ -31,6 +31,7 @@ inference --retry--> provider_selection inference --advance--> sandbox sandbox --branch--> openclaw -> policies -> finalizing -> post_verify -> complete sandbox --branch--> agent_setup -> policies -> finalizing -> post_verify -> complete +post_verify --pause--> post_verify (retryable handoff without a state transition) each nonterminal state --failure--> failed ``` diff --git a/src/lib/onboard/machine/README.md b/src/lib/onboard/machine/README.md index 1b11fb02765..bc5960fd7cf 100644 --- a/src/lib/onboard/machine/README.md +++ b/src/lib/onboard/machine/README.md @@ -13,9 +13,9 @@ The target shape is a machine-driven onboarding runner: 2. Build an onboarding context that contains sanitized operator choices, runtime dependencies, and mutable values returned by states. 3. Enter `runOnboardMachine(context)`. 4. Dispatch the current machine state to a handler. -5. Let the handler return an explicit state result such as advance, retry, branch, complete, or failed. +5. Let the handler return an explicit state result such as advance, retry, branch, pause, complete, or failed. 6. Apply the result through `OnboardRuntime`, which validates the transition, updates the persisted session snapshot, and emits redacted machine events. -7. Continue until the machine reaches `complete` or `failed`. +7. Continue until the machine reaches `complete` or `failed`, or a handler pauses at a retryable non-terminal state. In that final shape, `src/lib/onboard.ts` should be a thin entrypoint. State handlers should own state-specific prompts, resume validation, repair decisions, and side effects. @@ -80,7 +80,8 @@ sequence must declare its source state in `metadata.state`, and that source must machine's current state when the result is applied. The runner also checks the handler's sequence ownership allowlist; add a new entry in `DEFAULT_SEQUENCE_OWNERSHIP` before introducing another composite handler that crosses into a later state. Terminal results (`complete` or `failed`) end -the sequence immediately. +the sequence immediately. A `pause` result persists any supplied safe context and returns control +without a state transition so a later process can resume the same non-terminal state. ## Runtime responsibilities diff --git a/src/lib/onboard/machine/final-flow-phases.runtime.test.ts b/src/lib/onboard/machine/final-flow-phases.runtime.test.ts index 1bcd7703711..6bab3532b33 100644 --- a/src/lib/onboard/machine/final-flow-phases.runtime.test.ts +++ b/src/lib/onboard/machine/final-flow-phases.runtime.test.ts @@ -9,8 +9,26 @@ import { sessionAt, } from "../../../../test/helpers/onboard-final-flow-phases"; import { createSession } from "../../state/onboard-session"; +import type { VerifyDeploymentResult } from "../../verify-deployment"; import { runFinalOnboardFlowSlice } from "./final-flow-phases"; +function deploymentResult(healthy: boolean): VerifyDeploymentResult { + return { + healthy, + verification: { + gatewayReachable: true, + gatewayVersion: "test", + inferenceRouteWorking: healthy, + dashboardReachable: true, + messagingBridgesHealthy: true, + messagingRuntimeChannelsMissing: null, + messagingConfigChannelsMissing: null, + accessMethod: "localhost", + }, + diagnostics: [], + }; +} + describe("final onboard flow runtime boundary", () => { it("uses the strict final runner for fresh OpenClaw sessions with a real runtime boundary", async () => { const order: string[] = []; @@ -285,4 +303,62 @@ describe("final onboard flow runtime boundary", () => { machine: { state: "post_verify" }, }); }); + + it("keeps an unhealthy final verification retryable and completes after a later resume (#6849)", async () => { + const order: string[] = []; + const harness = createRuntimeHarness(sessionAt("openclaw")); + const recorders = harness.boundary.recorders(); + const verifyDeployment = vi + .fn() + .mockResolvedValueOnce(deploymentResult(false)) + .mockResolvedValueOnce(deploymentResult(true)); + const phases = createPhases("openclaw", order, { + loadSession: harness.getSession, + recordStepSkipped: recorders.recordStepSkipped, + recordStateSkipped: recorders.recordStateSkipped, + startRecordedStep: recorders.startRecordedStep, + recordStepComplete: recorders.recordStepComplete, + recordPostVerifyStarted: recorders.recordPostVerifyStarted, + verifyDeployment, + }); + + const first = await runFinalOnboardFlowSlice({ + context: context({ session: harness.getSession() }), + runtime: harness.boundary.getRuntime(), + phases, + resume: false, + recordStateResult: harness.boundary.recordStateResultWithStepCompatibility.bind( + harness.boundary, + ), + recordInvalidatedStateResult: harness.boundary.recordInvalidatedStateResult.bind( + harness.boundary, + ), + }); + + expect(first.session).toMatchObject({ + status: "in_progress", + resumable: true, + machine: { state: "post_verify" }, + }); + + const resumed = await runFinalOnboardFlowSlice({ + context: context({ resume: true, session: harness.getSession() }), + runtime: harness.boundary.getRuntime(), + phases, + resume: true, + recordStateResult: harness.boundary.recordStateResultWithStepCompatibility.bind( + harness.boundary, + ), + recordInvalidatedStateResult: harness.boundary.recordInvalidatedStateResult.bind( + harness.boundary, + ), + }); + + expect(verifyDeployment).toHaveBeenCalledTimes(2); + expect(resumed.session).toMatchObject({ + status: "complete", + resumable: false, + machine: { state: "complete" }, + }); + }); }); diff --git a/src/lib/onboard/machine/final-flow-phases.test.ts b/src/lib/onboard/machine/final-flow-phases.test.ts index 59238bb4ce0..bc00f35fb97 100644 --- a/src/lib/onboard/machine/final-flow-phases.test.ts +++ b/src/lib/onboard/machine/final-flow-phases.test.ts @@ -71,10 +71,10 @@ describe("final onboard flow phases", () => { phases, resume: true, recordStateResult: async (result) => { - if (result.type === "complete" || result.type === "failed") { - recorded.push(result.type); - } else { + if (result.type === "transition") { recorded.push(result.next); + } else { + recorded.push(result.type); } }, recordInvalidatedStateResult: async (result) => { diff --git a/src/lib/onboard/machine/final-flow-phases.ts b/src/lib/onboard/machine/final-flow-phases.ts index b2c91d4ccd1..3e388dd2828 100644 --- a/src/lib/onboard/machine/final-flow-phases.ts +++ b/src/lib/onboard/machine/final-flow-phases.ts @@ -165,7 +165,7 @@ export async function runFinalOnboardFlowSlice { +}) { // Recompute plan for live resume repair when durable machine snapshots // are already downstream of this slice even though branch setup/readiness, // policy reconciliation, and final verification must still re-run. Those @@ -180,7 +180,7 @@ export async function runFinalOnboardFlowSlice [" ✓ verified"]), verifyWebSearch: vi.fn(), dashboard: vi.fn(), + isHealthy: vi.fn(() => true), + reportReadiness: vi.fn(), error: vi.fn(), log: vi.fn(), }; @@ -63,6 +65,8 @@ function createDeps( formatVerificationDiagnostics: calls.diagnostics, verifyWebSearchInsideSandbox: calls.verifyWebSearch, printDashboard: calls.dashboard, + isDeploymentHealthy: calls.isHealthy, + reportDeploymentReadiness: calls.reportReadiness, error: calls.error, log: calls.log, ...overrides, @@ -104,7 +108,14 @@ describe("handleFinalizationState", () => { expect(calls.buildChain).toHaveBeenCalledWith("http://127.0.0.1:18789"); expect(calls.verify).toHaveBeenCalledWith("my-assistant", { port: 18789 }); expect(calls.log).toHaveBeenCalledWith(" ✓ verified"); - expect(calls.dashboard).toHaveBeenCalledWith("my-assistant", "model", "provider", null, null); + expect(calls.dashboard).toHaveBeenCalledWith( + "my-assistant", + "model", + "provider", + null, + null, + true, + ); expect(calls.postVerify).toHaveBeenCalledOnce(); expect(result.stateResult).toEqual({ type: "complete", @@ -120,6 +131,35 @@ describe("handleFinalizationState", () => { expect(result.verificationDiagnostics).toEqual([" ✓ verified"]); }); + it("prints a not-ready dashboard and returns a resumable failure when verification is unhealthy", async () => { + const { deps, calls } = createDeps({ isDeploymentHealthy: vi.fn(() => false) }); + + const result = await handleFinalizationState(baseOptions(deps)); + + expect(calls.dashboard).toHaveBeenCalledWith( + "my-assistant", + "model", + "provider", + null, + null, + false, + ); + expect(calls.reportReadiness).toHaveBeenCalledWith(false); + expect(calls.postVerify).toHaveBeenCalledOnce(); + expect(result.deploymentHealthy).toBe(false); + expect(result.stateResult).toEqual({ + type: "pause", + updates: { + sandboxName: "my-assistant", + provider: "provider", + model: "model", + hermesAuthMethod: null, + hermesToolGateways: [], + }, + metadata: { state: "finalizing", reason: "deployment_not_ready" }, + }); + }); + it("ensures agent dashboard forwarding before completion for non-OpenClaw agents", async () => { const { deps, calls } = createDeps(); const agent = { name: "hermes" }; @@ -130,7 +170,14 @@ describe("handleFinalizationState", () => { expect(calls.ensureAgentDashboard.mock.invocationCallOrder[0]).toBeLessThan( calls.dashboard.mock.invocationCallOrder[0], ); - expect(calls.dashboard).toHaveBeenCalledWith("my-assistant", "model", "provider", null, agent); + expect(calls.dashboard).toHaveBeenCalledWith( + "my-assistant", + "model", + "provider", + null, + agent, + true, + ); }); it("skips dashboard and gateway verification for terminal agents without forwards", async () => { diff --git a/src/lib/onboard/machine/handlers/finalization.ts b/src/lib/onboard/machine/handlers/finalization.ts index afd14d7f9d1..1350e28758d 100644 --- a/src/lib/onboard/machine/handlers/finalization.ts +++ b/src/lib/onboard/machine/handlers/finalization.ts @@ -3,7 +3,12 @@ import type { Session } from "../../../state/onboard-session"; import { type DashboardRuntimeAgent, shouldManageDashboardForAgent } from "../../dashboard-runtime"; -import { completeOnboardMachine, type OnboardStateCompleteResult } from "../result"; +import { + completeOnboardMachine, + type OnboardStateCompleteResult, + type OnboardStatePauseResult, + pauseOnboardMachine, +} from "../result"; export interface FinalizationStateOptions { sandboxName: string; @@ -55,6 +60,8 @@ export interface FinalizationStateOptions; formatVerificationDiagnostics(result: VerificationResult): string[]; + isDeploymentHealthy(result: VerificationResult): boolean; + reportDeploymentReadiness(healthy: boolean): void; /** * Best-effort probe that confirms the agent runtime actually accepted the * web-search config and (for Brave) that the L7 proxy rewrites the @@ -68,6 +75,7 @@ export interface FinalizationStateOptions { branchTo("agent_setup", { metadata: "bad" }); }); - it("builds terminal completion and failure results", () => { + it("builds pause, terminal completion, and failure results", () => { + expect(pauseOnboardMachine({ sandboxName: "my-assistant" }, { reason: "not-ready" })).toEqual({ + type: "pause", + updates: { sandboxName: "my-assistant" }, + metadata: { reason: "not-ready" }, + }); expect(completeOnboardMachine({ sandboxName: "my-assistant" }, { verified: true })).toEqual({ type: "complete", updates: { sandboxName: "my-assistant" }, diff --git a/src/lib/onboard/machine/result.ts b/src/lib/onboard/machine/result.ts index 895640709e6..4f2134b43fc 100644 --- a/src/lib/onboard/machine/result.ts +++ b/src/lib/onboard/machine/result.ts @@ -31,6 +31,12 @@ export interface OnboardStateCompleteResult { metadata?: Record | null; } +export interface OnboardStatePauseResult { + type: "pause"; + updates?: SessionUpdates; + metadata?: Record | null; +} + export interface OnboardStateFailedResult { type: "failed"; error: string | null; @@ -40,6 +46,7 @@ export interface OnboardStateFailedResult { export type OnboardStateResult = | OnboardStateTransitionResult + | OnboardStatePauseResult | OnboardStateCompleteResult | OnboardStateFailedResult; @@ -84,6 +91,13 @@ export function completeOnboardMachine( return { type: "complete", updates, metadata }; } +export function pauseOnboardMachine( + updates: SessionUpdates = {}, + metadata: Record | null = null, +): OnboardStatePauseResult { + return { type: "pause", updates, metadata }; +} + export function failOnboardMachine( error: string | null, options: { step?: string | null; metadata?: Record | null } = {}, diff --git a/src/lib/onboard/machine/runner.test.ts b/src/lib/onboard/machine/runner.test.ts index 3fd9db63f2d..d749b0cc678 100644 --- a/src/lib/onboard/machine/runner.test.ts +++ b/src/lib/onboard/machine/runner.test.ts @@ -12,7 +12,14 @@ import { type SessionUpdates, sanitizeFailure, } from "../../state/onboard-session"; -import { advanceTo, branchTo, completeOnboardMachine, failOnboardMachine, retryTo } from "./result"; +import { + advanceTo, + branchTo, + completeOnboardMachine, + failOnboardMachine, + pauseOnboardMachine, + retryTo, +} from "./result"; import { MissingOnboardStateHandlerError, OnboardMachineTransitionLimitError, @@ -152,6 +159,35 @@ describe("runOnboardMachine", () => { expect(policies).not.toHaveBeenCalled(); }); + it("stops on a pause result without leaving the current retryable state", async () => { + const session = createSession({ + machine: { + version: MACHINE_SNAPSHOT_VERSION, + state: "post_verify", + stateEnteredAt: "2026-05-28T00:00:00.000Z", + revision: 4, + }, + }); + const runtime = createRuntime(session); + const postVerify = vi.fn(() => + pauseOnboardMachine({ sandboxName: "my-assistant" }, { reason: "not-ready" }), + ); + + const result = await runOnboardMachine({ + context: { attempts: 0, visited: [] } as RunnerContext, + runtime, + handlers: { post_verify: postVerify }, + }); + + expect(postVerify).toHaveBeenCalledOnce(); + expect(result.session).toMatchObject({ + status: "in_progress", + resumable: true, + sandboxName: "my-assistant", + machine: { state: "post_verify", revision: 4 }, + }); + }); + it("returns immediately for terminal sessions", async () => { const startedAt = "2026-05-28T00:00:00.000Z"; const completeSession = createSession({ diff --git a/src/lib/onboard/machine/runner.ts b/src/lib/onboard/machine/runner.ts index 8c18794c413..4c1dea3070e 100644 --- a/src/lib/onboard/machine/runner.ts +++ b/src/lib/onboard/machine/runner.ts @@ -263,6 +263,9 @@ export async function runOnboardMachine({ context = updateContext ? await updateContext({ context, state: resultState, result, session }) : context; + if (result.type === "pause") { + return { context, session }; + } if ( isTerminalOnboardMachineState(session.machine.state) || stopStates.includes(session.machine.state) diff --git a/src/lib/onboard/machine/runtime.test.ts b/src/lib/onboard/machine/runtime.test.ts index fb0a8dc2604..332ddebe356 100644 --- a/src/lib/onboard/machine/runtime.test.ts +++ b/src/lib/onboard/machine/runtime.test.ts @@ -13,7 +13,14 @@ import { } from "../../state/onboard-session"; import type { StepMutationOptions } from "../../state/onboard-step-mutation"; import type { OnboardMachineEvent } from "./events"; -import { advanceTo, branchTo, completeOnboardMachine, failOnboardMachine, retryTo } from "./result"; +import { + advanceTo, + branchTo, + completeOnboardMachine, + failOnboardMachine, + pauseOnboardMachine, + retryTo, +} from "./result"; import { OnboardRuntime, type OnboardRuntimeDeps } from "./runtime"; import { InvalidOnboardMachineTransitionError } from "./transitions"; @@ -297,6 +304,31 @@ describe("OnboardRuntime", () => { }); }); + it("persists safe context while leaving a paused non-terminal session retryable", async () => { + const harness = createHarness(sessionInState("post_verify")); + + await harness.runtime.applyResult( + pauseOnboardMachine( + { sandboxName: "my-assistant", provider: "compatible-endpoint" }, + { reason: "deployment_not_ready" }, + ), + ); + + expect(harness.getSession()).toMatchObject({ + status: "in_progress", + resumable: true, + sandboxName: "my-assistant", + provider: "compatible-endpoint", + machine: { state: "post_verify", revision: 7 }, + }); + expect(harness.events).toHaveLength(1); + expect(harness.events[0]).toMatchObject({ + type: "context.updated", + state: "post_verify", + metadata: { reason: "deployment_not_ready" }, + }); + }); + it("rejects invalid explicit transition kinds before mutating context", async () => { const { runtime, getSession } = createHarness(sessionInState("inference")); @@ -325,6 +357,9 @@ describe("OnboardRuntime", () => { it("rejects terminal-state failure and invalid completion transitions", async () => { const completeHarness = createHarness(sessionInState("complete")); await expect(completeHarness.runtime.fail("boom")).rejects.toThrow("complete -> failed"); + await expect(completeHarness.runtime.applyResult(pauseOnboardMachine())).rejects.toThrow( + "Cannot pause terminal onboarding state: complete", + ); expect(completeHarness.getSession().machine.state).toBe("complete"); const policiesHarness = createHarness(sessionInState("policies")); diff --git a/src/lib/onboard/machine/runtime.ts b/src/lib/onboard/machine/runtime.ts index f069c5ec579..a4f6dd6aadd 100644 --- a/src/lib/onboard/machine/runtime.ts +++ b/src/lib/onboard/machine/runtime.ts @@ -287,6 +287,19 @@ export class OnboardRuntime { } async applyResult(result: OnboardStateResult): Promise { + if (result.type === "pause") { + const current = this.ensureSession(); + if (isTerminalOnboardMachineState(current.machine.state)) { + throw new Error(`Cannot pause terminal onboarding state: ${current.machine.state}`); + } + if (result.updates && Object.keys(this.deps.filterSafeUpdates(result.updates)).length > 0) { + return this.updateContext(result.updates, { + state: current.machine.state, + metadata: result.metadata, + }); + } + return current; + } if (result.type === "complete") { return this.complete(result.updates ?? {}, { metadata: result.metadata }); } diff --git a/src/lib/onboard/runtime-boundary.test.ts b/src/lib/onboard/runtime-boundary.test.ts index bbf27a6b4a8..899dbbaa471 100644 --- a/src/lib/onboard/runtime-boundary.test.ts +++ b/src/lib/onboard/runtime-boundary.test.ts @@ -17,6 +17,7 @@ import { branchTo, completeOnboardMachine, failOnboardMachine, + pauseOnboardMachine, retryTo, } from "./machine/result"; import { OnboardRuntime, type OnboardRuntimeDeps } from "./machine/runtime"; @@ -346,6 +347,7 @@ describe("OnboardRuntimeBoundary", () => { }); it.each([ + { label: "pause", result: () => pauseOnboardMachine() }, { label: "complete", result: () => completeOnboardMachine() }, { label: "failed", result: () => failOnboardMachine("boom") }, ] as const)("rejects non-transition $label results before emitting invalidation (#6227)", async ({ diff --git a/src/lib/onboard/runtime-boundary.ts b/src/lib/onboard/runtime-boundary.ts index a1462ada49a..6543b2420b0 100644 --- a/src/lib/onboard/runtime-boundary.ts +++ b/src/lib/onboard/runtime-boundary.ts @@ -139,6 +139,17 @@ export class OnboardRuntimeBoundary { return; } + if (result.type === "pause") { + const sourceState = + result.metadata && typeof result.metadata.state === "string" ? result.metadata.state : null; + if (sourceState && current.machine.state !== sourceState) { + throw new Error( + `Paused onboarding state result source mismatch: ${sourceState} != ${current.machine.state}`, + ); + } + return; + } + const sourceState = result.metadata && typeof result.metadata.state === "string" ? result.metadata.state : null; if (current.machine.state === result.next) { diff --git a/src/lib/verify-deployment.test.ts b/src/lib/verify-deployment.test.ts index 77540fb2656..eff4f9d566f 100644 --- a/src/lib/verify-deployment.test.ts +++ b/src/lib/verify-deployment.test.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { buildChain } from "./dashboard/contract.js"; import { formatVerificationDiagnostics, verifyDeployment } from "./verify-deployment.js"; @@ -179,7 +179,7 @@ describe("verifyDeployment", () => { expect(dashDiag?.hint).toContain("forward"); }); - it("inference failure is a warning, not a blocker", async () => { + it("reports unhealthy when the inference route is unreachable (#6849)", async () => { const deps = makeDeps({ executeSandboxCommand: (_name: string, script: string) => { if (script.includes("inference.local")) { @@ -190,10 +190,31 @@ describe("verifyDeployment", () => { }, }); const result = await verifyDeployment("my-sandbox", chain, deps, NO_RETRY); - expect(result.healthy).toBe(true); // inference is non-blocking + expect(result.healthy).toBe(false); + expect(result.verification.inferenceRouteWorking).toBe(false); + const infDiag = result.diagnostics.find((d) => d.link === "inference"); + expect(infDiag?.status).toBe("fail"); + expect(infDiag?.hint).toContain("unreachable"); + }); + + it("reports unhealthy when only the inference route returns HTTP 5xx (#6849)", async () => { + const deps = makeDeps({ + executeSandboxCommand: (_name: string, script: string) => ({ + status: 0, + stdout: script.includes("inference.local") ? "503" : "200", + stderr: "", + }), + }); + const result = await verifyDeployment("my-sandbox", chain, deps, NO_RETRY); + expect(result.healthy).toBe(false); + expect(result.verification.gatewayReachable).toBe(true); expect(result.verification.inferenceRouteWorking).toBe(false); const infDiag = result.diagnostics.find((d) => d.link === "inference"); - expect(infDiag?.status).toBe("warn"); + expect(infDiag?.status).toBe("fail"); + expect(infDiag?.detail).toContain("503"); + expect(infDiag?.hint).toContain("host.openshell.internal"); + expect(infDiag?.hint).toContain("firewall"); + expect(infDiag?.hint).not.toContain("0.0.0.0"); }); it("messaging failure is a warning, not a blocker", async () => { @@ -502,6 +523,57 @@ describe("verifyDeployment", () => { expect(dashboardCalls).toBe(2); }); + it("retries the inference probe and recovers when the route comes up late (#6849)", async () => { + const probeInference = vi + .fn() + .mockReturnValueOnce({ status: 0, stdout: "000", stderr: "" }) + .mockReturnValue({ status: 0, stdout: "200", stderr: "" }); + const deps = makeDeps({ + executeSandboxCommand: (_name: string, script: string) => + script.includes("inference.local") + ? probeInference() + : { status: 0, stdout: "200", stderr: "" }, + }); + const sleepCalls: number[] = []; + const result = await verifyDeployment("my-sandbox", chain, deps, { + retryDelaysMs: [10, 20], + sleep: async (ms: number) => { + sleepCalls.push(ms); + }, + }); + expect(result.healthy).toBe(true); + expect(result.verification.inferenceRouteWorking).toBe(true); + expect(probeInference).toHaveBeenCalledTimes(2); + expect(sleepCalls).toEqual([10]); + }); + + it("does not retry inference after the gateway retry budget is exhausted (#6849)", async () => { + const scripts: string[] = []; + const deps = makeDeps({ + executeSandboxCommand: (_name: string, script: string) => { + scripts.push(script); + return { status: 0, stdout: "000", stderr: "" }; + }, + }); + const sleepCalls: number[] = []; + const result = await verifyDeployment("my-sandbox", chain, deps, { + retryDelaysMs: [10, 20], + sleep: async (ms: number) => { + sleepCalls.push(ms); + }, + }); + expect(result.healthy).toBe(false); + expect(result.verification.gatewayReachable).toBe(false); + expect(result.verification.inferenceRouteWorking).toBe(false); + expect( + scripts.filter( + (script) => !script.includes("inference.local") && !script.includes("openclaw --version"), + ), + ).toHaveLength(3); + expect(scripts.filter((script) => script.includes("inference.local"))).toHaveLength(1); + expect(sleepCalls).toEqual([10, 20]); + }); + it("gives up after retry budget is exhausted and surfaces the last failure detail", async () => { const deps = makeDeps({ executeSandboxCommand: () => ({ status: 0, stdout: "000", stderr: "" }), diff --git a/src/lib/verify-deployment.ts b/src/lib/verify-deployment.ts index c8f359bac68..310bef40dd1 100644 --- a/src/lib/verify-deployment.ts +++ b/src/lib/verify-deployment.ts @@ -213,30 +213,49 @@ function fetchGatewayVersion(sandboxName: string, deps: VerifyDeploymentDeps): s return version && version !== "" ? version : null; } -/** - * Probe the inference route from inside the sandbox. - * Sends a minimal request to inference.local to verify the proxy is working. - */ -function verifyInferenceRoute( +type InferenceRouteStatus = "ok" | "unreachable" | "unhealthy"; + +function probeInferenceRouteOnce( sandboxName: string, deps: VerifyDeploymentDeps, -): { working: boolean; detail: string } { - // Just check that inference.local resolves and the proxy responds. - // We don't send a real completion request — just hit /v1/models to confirm routing. +): { status: InferenceRouteStatus; detail: string } { const script = `HTTP_CODE=$(curl -so /dev/null -w '%{http_code}' --max-time 5 ` + `https://inference.local/v1/models 2>/dev/null || echo 000); echo $HTTP_CODE`; const result = deps.executeSandboxCommand(sandboxName, script); if (!result) { - return { working: false, detail: "sandbox unreachable" }; + return { status: "unreachable", detail: "sandbox unreachable" }; } const code = parseInt(result.stdout.trim(), 10) || 0; - // Any HTTP response (even 401/403) means the proxy is routing. - // 000 means DNS failed or connection refused. - if (code > 0) { - return { working: true, detail: `inference.local responded HTTP ${code}` }; + if (code === 0) { + return { + status: "unreachable", + detail: "inference.local unreachable (DNS or proxy not running)", + }; + } + if (code >= 500) { + return { + status: "unhealthy", + detail: `inference.local returned HTTP ${code} (route reachable but endpoint unhealthy)`, + }; + } + return { status: "ok", detail: `inference.local responded HTTP ${code}` }; +} + +async function verifyInferenceRoute( + sandboxName: string, + deps: VerifyDeploymentDeps, + retryDelaysMs: readonly number[], + sleep: (ms: number) => Promise, +): Promise<{ status: InferenceRouteStatus; detail: string }> { + let last = probeInferenceRouteOnce(sandboxName, deps); + if (last.status === "ok") return last; + for (const delayMs of retryDelaysMs) { + await sleep(delayMs); + last = probeInferenceRouteOnce(sandboxName, deps); + if (last.status === "ok") return last; } - return { working: false, detail: "inference.local unreachable (DNS or proxy not running)" }; + return last; } /** @@ -515,14 +534,23 @@ export async function verifyDeployment( }); // 4. Inference route - const inference = verifyInferenceRoute(sandboxName, deps); + const inference = await verifyInferenceRoute( + sandboxName, + deps, + gateway.reachable ? retryDelaysMs : [], + sleep, + ); + const inferenceRouteWorking = inference.status === "ok"; diagnostics.push({ link: "inference", - status: inference.working ? "ok" : "warn", + status: inference.status === "ok" ? "ok" : "fail", detail: inference.detail, - hint: inference.working - ? "" - : "The inference proxy may not be ready yet. Try: nemoclaw status (it may take a few seconds after creation).", + hint: + inference.status === "ok" + ? "" + : inference.status === "unhealthy" + ? "The inference route is reachable but the endpoint returned a server error (HTTP 5xx). If the endpoint runs on the host, configure it to listen on a host address reachable through host.openshell.internal and restrict access with the host firewall or equivalent controls; a 127.0.0.1/localhost-only bind is not reachable from the sandbox. Then re-run: nemoclaw status." + : "The inference proxy is unreachable. Confirm the configured endpoint is running and reachable from the sandbox, then re-run: nemoclaw status.", }); // 5. Messaging bridges (providers attached AND runtime config exposes @@ -542,7 +570,7 @@ export async function verifyDeployment( const verification: DeploymentVerification = { gatewayReachable: gateway.reachable, gatewayVersion, - inferenceRouteWorking: inference.working, + inferenceRouteWorking, dashboardReachable: dashboard.reachable, messagingBridgesHealthy: messaging.healthy, messagingRuntimeChannelsMissing: messaging.runtimeMissing, @@ -550,9 +578,7 @@ export async function verifyDeployment( accessMethod, }; - // Healthy = gateway reachable AND dashboard reachable from host. - // Inference and messaging are warn-level (non-blocking). - const healthy = gateway.reachable && dashboard.reachable; + const healthy = gateway.reachable && dashboard.reachable && inference.status === "ok"; return { healthy, verification, diagnostics }; } @@ -572,7 +598,9 @@ export function formatVerificationDiagnostics(result: VerifyDeploymentResult): s const RESET = "\x1b[0m"; if (result.healthy) { - lines.push(` ${G}✓${RESET} Deployment verified — gateway and dashboard are healthy.`); + lines.push( + ` ${G}✓${RESET} Deployment verified — gateway, dashboard, and inference route are healthy.`, + ); if (result.verification.gatewayVersion) { lines.push(` OpenClaw version: ${result.verification.gatewayVersion}`); } diff --git a/test/e2e/live/onboard-resume.test.ts b/test/e2e/live/onboard-resume.test.ts index 632a4da6449..219b139a163 100644 --- a/test/e2e/live/onboard-resume.test.ts +++ b/test/e2e/live/onboard-resume.test.ts @@ -82,6 +82,12 @@ interface SessionStateComplete { >; } +interface SessionStatePostVerify { + status: "in_progress"; + resumable: true; + machine: { state: "post_verify" }; +} + interface MutableSessionState extends Record { status?: string; resumable?: boolean; @@ -171,6 +177,7 @@ test("onboard-resume: interrupted onboard then --resume can recreate with cached "resume sandbox recreation filters stale extra providers while preserving live attachments", "resume proves recreated sandbox provider attachments are selectively reconciled", "host trust-store anchor corporate CA source is baked and merged after resume", + "an unreachable committed route pauses at final verification and completes after repair", "implicit resume is detected and --fresh suppresses that auto-resume", ], }); @@ -212,7 +219,7 @@ test("onboard-resume: interrupted onboard then --resume can recreate with cached // fake OpenAI-compatible endpoint at a host address the OpenShell gateway and // sandbox can route to, matching test/e2e/lib/hermetic-compatible-inference.sh. const fakePublicHost = "host.openshell.internal"; - const fake = await startFakeOpenAiCompatibleServer({ + let fake = await startFakeOpenAiCompatibleServer({ apiKey: FAKE_COMPATIBLE_AUTH_VALUE, host: "0.0.0.0", model: FAKE_COMPATIBLE_MODEL, @@ -230,6 +237,7 @@ test("onboard-resume: interrupted onboard then --resume can recreate with cached publicHost: fakePublicHost, }); const localModelsUrl = new URL(`${fake.baseUrl}/models`); + const fakePort = Number(localModelsUrl.port); localModelsUrl.hostname = "127.0.0.1"; const modelsResponse = await fetch(localModelsUrl, { headers: { Authorization: `Bearer ${FAKE_COMPATIBLE_AUTH_VALUE}` }, @@ -541,7 +549,67 @@ test("onboard-resume: interrupted onboard then --resume can recreate with cached expect(containsExactJsonToken(registry, SANDBOX_NAME)).toBe(true); // ────────────────────────────────────────────────────────────────── - // Phase 3.5: implicit resume — a plain `onboard` auto-detects an + // Phase 3.5: a committed route that goes offline leaves final + // verification retryable; restoring the same endpoint lets a later resume + // re-probe and complete without recreating the sandbox. + // ────────────────────────────────────────────────────────────────── + markSessionInProgress(SESSION_FILE); + await fake.close(); + + const unavailableResumeRun = await host.command( + "node", + [CLI_ENTRYPOINT, "onboard", "--resume", "--non-interactive"], + { + artifactName: "phase-3-5-onboard-resume-route-unavailable", + env: resumeEnv, + redactionValues: [FAKE_COMPATIBLE_AUTH_VALUE], + timeoutMs: ONBOARD_TIMEOUT_MS, + }, + ); + const unavailableResumeText = `${unavailableResumeRun.stdout}\n${unavailableResumeRun.stderr}`; + expect(unavailableResumeRun.exitCode, unavailableResumeText).not.toBe(0); + expect(unavailableResumeText).toContain("is not ready"); + expect(unavailableResumeText).toContain("inference"); + + const paused = readSession(SESSION_FILE); + await artifacts.writeJson("phase-3-5-session-route-unavailable.json", { + status: paused.status, + resumable: paused.resumable, + machineState: paused.machine.state, + }); + expect(paused.status).toBe("in_progress"); + expect(paused.resumable).toBe(true); + expect(paused.machine.state).toBe("post_verify"); + + fake = await startFakeOpenAiCompatibleServer({ + apiKey: FAKE_COMPATIBLE_AUTH_VALUE, + host: "0.0.0.0", + model: FAKE_COMPATIBLE_MODEL, + port: fakePort, + publicHost: fakePublicHost, + requireAuth: true, + requireAuthModels: true, + }); + expect(fake.baseUrl).toBe(`http://${fakePublicHost}:${String(fakePort)}/v1`); + + const repairedResumeRun = await host.command( + "node", + [CLI_ENTRYPOINT, "onboard", "--resume", "--non-interactive"], + { + artifactName: "phase-3-5-onboard-resume-route-restored", + env: resumeEnv, + redactionValues: [FAKE_COMPATIBLE_AUTH_VALUE], + timeoutMs: ONBOARD_TIMEOUT_MS, + }, + ); + const repairedResumeText = `${repairedResumeRun.stdout}\n${repairedResumeRun.stderr}`; + expect(repairedResumeRun.exitCode, repairedResumeText).toBe(0); + expect(repairedResumeText).toContain("is ready"); + const repaired = readSession(SESSION_FILE); + expect(repaired.status).toBe("complete"); + + // ────────────────────────────────────────────────────────────────── + // Phase 4: implicit resume — a plain `onboard` auto-detects an // in_progress session, and `--fresh` suppresses that auto-resume. // ────────────────────────────────────────────────────────────────── markSessionInProgress(SESSION_FILE); @@ -549,7 +617,7 @@ test("onboard-resume: interrupted onboard then --resume can recreate with cached "node", [CLI_ENTRYPOINT, "onboard", "--non-interactive"], { - artifactName: "phase-3-5-onboard-implicit-resume", + artifactName: "phase-4-onboard-implicit-resume", env: { ...buildAvailabilityProbeEnv(), NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, @@ -574,7 +642,7 @@ test("onboard-resume: interrupted onboard then --resume can recreate with cached "node", [CLI_ENTRYPOINT, "onboard", "--fresh", "--non-interactive"], { - artifactName: "phase-3-5-onboard-fresh-suppresses-resume", + artifactName: "phase-4-onboard-fresh-suppresses-resume", env: { ...buildAvailabilityProbeEnv(), NEMOCLAW_SANDBOX_NAME: SANDBOX_NAME, diff --git a/test/helpers/onboard-final-flow-phases.ts b/test/helpers/onboard-final-flow-phases.ts index 72c394745df..1517587a3a2 100644 --- a/test/helpers/onboard-final-flow-phases.ts +++ b/test/helpers/onboard-final-flow-phases.ts @@ -50,6 +50,7 @@ export type RecorderOverrides = { sandboxName: string, chain: DashboardDeliveryChain, ) => Promise; + isDeploymentHealthy?: (result: VerifyDeploymentResult) => boolean; printDashboard?: ( sandboxName: string, model: string, @@ -250,6 +251,9 @@ export function createPhases( checkAndRecoverSandboxProcesses: vi.fn(), warmupScopeUpgrade: vi.fn(), autoPairScopeApproval: vi.fn(), + isDeploymentHealthy: + recorders.isDeploymentHealthy ?? ((result: VerifyDeploymentResult) => result.healthy), + reportDeploymentReadiness: vi.fn(), getChatUiUrl: () => "http://127.0.0.1:45123", buildVerifyChain: (): DashboardDeliveryChain => ({ accessUrl: "http://127.0.0.1:45123", diff --git a/test/inference-options-docs.test.ts b/test/inference-options-docs.test.ts index 4fa4761a319..bddf4aef7ca 100644 --- a/test/inference-options-docs.test.ts +++ b/test/inference-options-docs.test.ts @@ -349,8 +349,8 @@ describe("inference setup navigation", () => { it("scopes post-ready sandbox route verification to local inference providers", () => { const markdown = fs.readFileSync(verifyInferenceRoutePath, "utf8"); - const start = markdown.indexOf("## Understand Post-Ready Checks"); - const end = markdown.indexOf("## Send a Short Agent Request", start); + const start = markdown.indexOf("## Understand Local Provider Post-Ready Checks"); + const end = markdown.indexOf("## Understand Final Route Checks", start); expect(start).toBeGreaterThanOrEqual(0); expect(end).toBeGreaterThan(start); const section = markdown.slice(start, end); @@ -363,10 +363,25 @@ describe("inference setup navigation", () => { ); expect(getSandboxRuntimeInferenceEndpoint("nvidia-nim")).toBeNull(); expect(getSandboxRuntimeInferenceEndpoint("compatible-endpoint")).toBeNull(); - expect(section).toContain("For local Ollama and vLLM"); + expect(section).toContain( + "For local Ollama and vLLM on Docker GPU sandboxes using the compatibility route", + ); expect(section).toContain("NVIDIA NIM and other compatible endpoints"); }); + it("documents universal final route verification separately from local warmup", () => { + const markdown = fs.readFileSync(verifyInferenceRoutePath, "utf8"); + const start = markdown.indexOf("## Understand Final Route Checks"); + const end = markdown.indexOf("## Send a Short Agent Request", start); + expect(start).toBeGreaterThanOrEqual(0); + expect(end).toBeGreaterThan(start); + const section = markdown.slice(start, end); + + expect(section).toContain("`https://inference.local/v1/models`"); + expect(section).toContain("retryable at final verification"); + expect(section).toContain("Provider setup still performs its own"); + }); + it("explains the host-side validation limit of the containerized gateway alias", () => { const markdown = fs.readFileSync(compatibleEndpointPath, "utf8"); const result = probeOpenAiLikeEndpoint( diff --git a/test/onboard-exit-handler.test.ts b/test/onboard-exit-handler.test.ts index c3ebfd536a3..0a0478664d4 100644 --- a/test/onboard-exit-handler.test.ts +++ b/test/onboard-exit-handler.test.ts @@ -294,6 +294,7 @@ finalPhases.runFinalOnboardFlowSlice = async ({ runtime }) => { { sandboxName: "complete-seam", provider: "nvidia", model: "nemotron-test" }, { state: "post_verify" }, )); + return { context: null, session: await runtime.session() }; }; const { onboard } = require(${onboardPath});