diff --git a/src/lib/actions/sandbox/doctor-flow.test.ts b/src/lib/actions/sandbox/doctor-flow.test.ts index 86f20420a23..e15703cb93b 100644 --- a/src/lib/actions/sandbox/doctor-flow.test.ts +++ b/src/lib/actions/sandbox/doctor-flow.test.ts @@ -433,6 +433,61 @@ describe("runSandboxDoctor flow", () => { ); }); + it("probes the inference.local route for cloud providers, not just local ones (#6192)", async () => { + const harness = createDoctorHarness(); + // Cloud provider selected: registry + live `inference get` both report it. + harness.getSandboxSpy.mockReturnValue({ + name: "alpha", + agent: "openclaw", + model: "registry-model", + provider: "nvidia-prod", + openshellDriver: "docker", + gatewayName: "nemoclaw-19080", + gatewayPort: 19080, + }); + const openShellResponses: Record = { + "sandbox list": { status: 0, output: "alpha Ready" }, + "inference get": { status: 0, output: "Provider: nvidia-prod\nModel: nemotron\n" }, + }; + harness.captureOpenShellSpy.mockImplementation((args: unknown) => { + const argv = Array.isArray(args) ? args : []; + return openShellResponses[`${argv[0]} ${argv[1]}`] ?? { status: 0, output: "" }; + }); + // The upstream provider endpoint is reachable... + harness.healthProbeSpy.mockReturnValue({ + ok: true, + probed: true, + providerLabel: "NVIDIA Cloud", + endpoint: "https://integrate.api.nvidia.com/v1/models", + detail: "reachable", + }); + // ...but the in-sandbox inference.local route the agent actually calls is + // broken (harness default: gateway probe resolves ok: false). + + const report = await harness.runSandboxDoctor("alpha", ["--json"], { quietJson: true }); + + // Before #6192 this probe was gated to ollama/vllm-local; a cloud route + // never had inference.local checked, so a broken route read as healthy. + expect(harness.probeSandboxInferenceGatewayHealthSpy).toHaveBeenCalledWith("alpha"); + expect(report?.checks).toEqual( + expect.arrayContaining([ + // Upstream reachability stays green on its own... + expect.objectContaining({ + group: "Inference", + label: "Provider health", + status: "ok", + }), + // ...but the broken inference.local route now surfaces as a failure. + expect.objectContaining({ + group: "Inference", + label: "Provider health (gateway)", + status: "fail", + }), + ]), + ); + expect(report?.status).toBe("fail"); + }); + it("reports agent definition failures instead of hiding the runtime channel check", async () => { const harness = createDoctorHarness(); harness.configuredMessagingChannelsSpy.mockReturnValue(["telegram"]); diff --git a/src/lib/actions/sandbox/doctor.ts b/src/lib/actions/sandbox/doctor.ts index 5e8b244f054..0e17ea304bc 100644 --- a/src/lib/actions/sandbox/doctor.ts +++ b/src/lib/actions/sandbox/doctor.ts @@ -15,7 +15,11 @@ import { recoverNamedGatewayRuntime, } from "../../gateway-runtime-action"; import { parseGatewayInference } from "../../inference/config"; -import { type ProviderHealthStatus, probeProviderHealth } from "../../inference/health"; +import { + buildInferenceGatewaySubprobe, + type ProviderHealthStatus, + probeProviderHealth, +} from "../../inference/health"; import { resolveGatewayName, resolveSandboxGatewayName } from "../../onboard/gateway-binding"; import { executeSandboxCommandForVerification } from "../../onboard/sandbox-verification-exec"; import { ROOT } from "../../runner"; @@ -326,10 +330,6 @@ function inferenceRouteCheck(sandboxName: string, route: InferenceRoute): Doctor }; } -function isLocalInferenceProvider(provider: string): boolean { - return provider === "ollama-local" || provider === "vllm-local"; -} - function skippedInferenceGatewayProbe(): ProviderHealthStatus { return { ok: false, @@ -341,28 +341,22 @@ function skippedInferenceGatewayProbe(): ProviderHealthStatus { }; } +// Probe the `inference.local` route the agent actually calls, regardless of +// provider. Cloud providers (nvidia-prod, etc.) route through the same +// in-sandbox openclaw gateway / auth proxy as local ones, so a reachable +// upstream endpoint does not prove the agent's route works. #3265 added this +// hop for ollama/vllm-local only; #6192 extends it to every provider so +// status/doctor stop reporting "healthy" when `inference.local` is broken. +// Unknown providers never reach here (collectInferenceChecks returns early). async function collectInferenceSubprobes( sandboxName: string, - provider: string, sandboxReachable: boolean, existing: ProviderHealthStatus[], ): Promise { - if (!isLocalInferenceProvider(provider)) return existing; if (!sandboxReachable) return [...existing, skippedInferenceGatewayProbe()]; const gateway = await probeSandboxInferenceGatewayHealth(sandboxName); if (!gateway) return existing; - return [ - ...existing, - { - ok: gateway.ok, - probed: true, - providerLabel: "Inference gateway chain", - endpoint: gateway.endpoint, - detail: gateway.detail, - probeLabel: "gateway", - ...(gateway.ok ? {} : { failureLabel: "unreachable" as const }), - }, - ]; + return [...existing, buildInferenceGatewaySubprobe(gateway)]; } async function collectInferenceChecks( @@ -385,7 +379,6 @@ async function collectInferenceChecks( const subprobes = await collectInferenceSubprobes( sandboxName, - route.provider, sandboxReachable, health.subprobes ?? [], ); diff --git a/src/lib/actions/sandbox/status-flow.test.ts b/src/lib/actions/sandbox/status-flow.test.ts index 5d0b663d2d9..ee4b3e3674e 100644 --- a/src/lib/actions/sandbox/status-flow.test.ts +++ b/src/lib/actions/sandbox/status-flow.test.ts @@ -138,6 +138,46 @@ describe("showSandboxStatus flow", () => { expect(output).toContain("http://127.0.0.1:11434/api/tags"); }); + it("reports inference unhealthy in text when the inference.local route is broken (#6192)", async () => { + // Mirrors what collectSandboxStatusSnapshot now produces for a cloud + // provider whose upstream endpoint is reachable but whose inference.local + // route is broken: the aggregate is degraded so the top-level line does not + // render a bare "healthy" that contradicts `connect`. + const harness = createStatusFlowHarness({ + currentProvider: "nvidia-prod", + inferenceHealth: { + ok: false, + probed: true, + providerLabel: "NVIDIA Cloud", + endpoint: "https://integrate.api.nvidia.com/v1/models", + detail: + "Provider endpoint is reachable, but the inference.local route the agent uses is broken. " + + "Inference gateway unreachable on https://inference.local/v1/models", + failureLabel: "unreachable", + subprobes: [ + { + ok: false, + probed: true, + providerLabel: "Inference gateway chain", + endpoint: "https://inference.local/v1/models", + detail: "Inference gateway unreachable on https://inference.local/v1/models", + probeLabel: "gateway", + failureLabel: "unreachable", + }, + ], + }, + sandboxEntry: { provider: "nvidia-prod" }, + }); + + await expect(harness.showSandboxStatus("alpha")).resolves.toBeUndefined(); + + const output = harness.logSpy.mock.calls.flat().join("\n"); + expect(output).not.toContain("Inference: healthy"); + expect(output).toContain("Inference: unreachable"); + expect(output).toContain("Inference (gateway):"); + expect(output).toContain("the inference.local route the agent uses is broken"); + }); + it("renders fresh shields posture as not configured rather than down", async () => { const harness = createStatusFlowHarness({ shieldsPosture: { diff --git a/src/lib/actions/sandbox/status-snapshot.test.ts b/src/lib/actions/sandbox/status-snapshot.test.ts new file mode 100644 index 00000000000..f797494647b --- /dev/null +++ b/src/lib/actions/sandbox/status-snapshot.test.ts @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import type { ProviderHealthStatus } from "../../inference/health"; +import type { SandboxGatewayState } from "./gateway-state"; +import { collectSandboxStatusSnapshot } from "./status-snapshot"; + +const presentLookup: SandboxGatewayState = { + state: "present", + output: "Name: alpha\nPhase: Ready\n", +}; + +// Upstream provider endpoint is reachable on its own; a broken in-sandbox +// route must still be surfaced. +const reachableProviderHealth: ProviderHealthStatus = { + ok: true, + probed: true, + providerLabel: "NVIDIA Cloud", + endpoint: "https://integrate.api.nvidia.com/v1/models", + detail: "reachable", +}; + +function runSnapshot(options: { provider: string | null; gatewayOk: boolean }) { + const sandboxEntry = { + name: "alpha", + agent: "openclaw", + model: "registry-model", + ...(options.provider ? { provider: options.provider } : {}), + openshellDriver: "docker", + }; + const probeInferenceGateway = vi.fn(async () => + options.gatewayOk + ? { + ok: true, + endpoint: "https://inference.local/v1/models", + httpStatus: 200, + detail: "Inference gateway responded HTTP 200 (full chain reachable).", + } + : { + ok: false, + endpoint: "https://inference.local/v1/models", + httpStatus: 0, + detail: "Inference gateway unreachable on https://inference.local/v1/models", + }, + ); + const snapshot = collectSandboxStatusSnapshot("alpha", { + deps: { + getSandbox: (() => sandboxEntry) as never, + reconcile: async () => presentLookup, + // Live `inference get` reports the active provider so the snapshot picks + // the cloud route without spawning openshell. + captureLiveInference: (async () => ({ + status: 0, + output: options.provider ? `Provider: ${options.provider}\nModel: nemotron\n` : "", + })) as never, + probeInferenceGateway: probeInferenceGateway as never, + // A fresh clone per call: the snapshot mutates `subprobes`/`ok` in place. + probeProviderHealthImpl: (provider: string) => + provider === "unknown" ? null : { ...reachableProviderHealth }, + }, + }); + return { probeInferenceGateway, snapshot }; +} + +describe("collectSandboxStatusSnapshot inference.local subprobe (#6192)", () => { + it("degrades the aggregate to unhealthy when the inference.local route is broken", async () => { + const { probeInferenceGateway, snapshot } = runSnapshot({ + provider: "nvidia-prod", + gatewayOk: false, + }); + const result = await snapshot; + + // Cloud providers now have the route the agent actually uses probed. + expect(probeInferenceGateway).toHaveBeenCalledWith("alpha"); + expect(result.currentProvider).toBe("nvidia-prod"); + // The aggregate is surfaced verbatim as `report.inferenceHealth` in --json + // and drives the text summary. It must NOT stay healthy when the real route + // is broken, even though the upstream provider endpoint is reachable. + expect(result.inferenceHealth?.ok).toBe(false); + expect(result.inferenceHealth?.failureLabel).toBe("unreachable"); + expect(result.inferenceHealth?.subprobes).toEqual([ + expect.objectContaining({ + providerLabel: "Inference gateway chain", + probeLabel: "gateway", + ok: false, + }), + ]); + }); + + it("keeps the aggregate healthy when the inference.local route is reachable", async () => { + const { snapshot } = runSnapshot({ provider: "nvidia-prod", gatewayOk: true }); + const result = await snapshot; + + expect(result.inferenceHealth?.ok).toBe(true); + expect(result.inferenceHealth?.failureLabel).toBeUndefined(); + expect(result.inferenceHealth?.subprobes?.[0]?.ok).toBe(true); + }); + + it("skips the inference.local subprobe when the provider is unknown", async () => { + const { probeInferenceGateway, snapshot } = runSnapshot({ provider: null, gatewayOk: false }); + const result = await snapshot; + + expect(result.currentProvider).toBe("unknown"); + expect(probeInferenceGateway).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/actions/sandbox/status-snapshot.ts b/src/lib/actions/sandbox/status-snapshot.ts index dbef4db2fb4..3393dff5dd6 100644 --- a/src/lib/actions/sandbox/status-snapshot.ts +++ b/src/lib/actions/sandbox/status-snapshot.ts @@ -10,6 +10,7 @@ import { type AgentDefinition, getAgentRuntimeKind, loadAgent } from "../../agen import { withStdoutRedirectedToStderr } from "../../cli/stdout-guard"; import { parseGatewayInference } from "../../inference/config"; import { + buildInferenceGatewaySubprobe, type ProviderHealthProbeOptions, type ProviderHealthStatus, probeProviderHealth, @@ -156,6 +157,8 @@ interface CollectSandboxStatusSnapshotDeps { probeProviderHealthImpl?: ProbeProviderHealth; probeTerminalRuntimeHealth?: ProbeTerminalRuntimeHealth; reconcile?: ReconcileSandboxGatewayState; + captureLiveInference?: typeof captureOpenshellForStatus; + probeInferenceGateway?: typeof probeSandboxInferenceGatewayHealth; } export async function collectSandboxStatusSnapshot( @@ -172,6 +175,9 @@ export async function collectSandboxStatusSnapshot( getState: getSandboxGatewayStateForStatus, })); const getSandbox = opts.deps?.getSandbox ?? registry.getSandbox; + const captureLiveInference = opts.deps?.captureLiveInference ?? captureOpenshellForStatus; + const probeInferenceGateway = + opts.deps?.probeInferenceGateway ?? probeSandboxInferenceGatewayHealth; const sb = getSandbox(sandboxName); let lookup: SandboxGatewayState; try { @@ -186,7 +192,7 @@ export async function collectSandboxStatusSnapshot( let liveResult: Awaited> | null = null; if (lookup.state === "present") { try { - liveResult = await captureOpenshellForStatus(["inference", "get"]); + liveResult = await captureLiveInference(["inference", "get"]); } catch { liveResult = null; } @@ -220,23 +226,28 @@ export async function collectSandboxStatusSnapshot( currentModel, opts.deps?.probeProviderHealthImpl, ); - if ( - inferenceHealth && - lookup.state === "present" && - (currentProvider === "ollama-local" || currentProvider === "vllm-local") - ) { - const gatewayChain = await probeSandboxInferenceGatewayHealth(sandboxName); + // Probe the `inference.local` route the agent actually calls for every known + // provider, not just local ones. Cloud providers route through the same + // in-sandbox gateway / auth proxy, so upstream reachability alone must not + // report the route as healthy when `inference.local` is broken (#6192). + if (inferenceHealth && lookup.state === "present" && currentProvider !== "unknown") { + const gatewayChain = await probeInferenceGateway(sandboxName); if (gatewayChain) { - const gatewaySubprobe: ProviderHealthStatus = { - ok: gatewayChain.ok, - probed: true, - providerLabel: "Inference gateway chain", - endpoint: gatewayChain.endpoint, - detail: gatewayChain.detail, - probeLabel: "gateway", - ...(gatewayChain.ok ? {} : { failureLabel: "unreachable" as const }), - }; + const gatewaySubprobe = buildInferenceGatewaySubprobe(gatewayChain); inferenceHealth.subprobes = [...(inferenceHealth.subprobes ?? []), gatewaySubprobe]; + if (!gatewaySubprobe.ok) { + // The agent's real route (`inference.local`) is broken even though the + // upstream provider endpoint is reachable. The aggregate must not stay + // `ok: true`, or `status` (text and --json) would report healthy while + // `connect` reports the route BROKEN — the exact contradiction #6192 + // set out to fix. Degrade the top-level result and explain the cause. + inferenceHealth.ok = false; + inferenceHealth.failureLabel = + inferenceHealth.failureLabel ?? gatewaySubprobe.failureLabel ?? "unreachable"; + inferenceHealth.detail = + "Provider endpoint is reachable, but the inference.local route the agent " + + `uses is broken. ${gatewaySubprobe.detail}`; + } } } const statusAgent = resolveSandboxStatusAgent(sb?.agent || "openclaw"); diff --git a/src/lib/inference/health.ts b/src/lib/inference/health.ts index 4d05cc3f92c..dce5a4a9b0b 100644 --- a/src/lib/inference/health.ts +++ b/src/lib/inference/health.ts @@ -38,6 +38,27 @@ export interface ProviderHealthStatus { subprobes?: ProviderHealthStatus[]; } +/** + * Build the `inference.local` gateway-chain subprobe rendered alongside the + * provider health probe. Shared by `status` and `doctor` so the subprobe shape + * stays in sync across both call sites (#3265, #6192). + */ +export function buildInferenceGatewaySubprobe(gateway: { + ok: boolean; + endpoint: string; + detail: string; +}): ProviderHealthStatus { + return { + ok: gateway.ok, + probed: true, + providerLabel: "Inference gateway chain", + endpoint: gateway.endpoint, + detail: gateway.detail, + probeLabel: "gateway", + ...(gateway.ok ? {} : { failureLabel: "unreachable" as const }), + }; +} + export interface ProviderHealthProbeOptions { runCurlProbeImpl?: (argv: string[], opts?: CurlProbeOptions) => CurlProbeResult; model?: string | null;