diff --git a/docs/reference/commands.mdx b/docs/reference/commands.mdx index b1162dbba2c..cdbdc7d9aef 100644 --- a/docs/reference/commands.mdx +++ b/docs/reference/commands.mdx @@ -2025,6 +2025,15 @@ It uses a 3-second connection timeout, a 5-second total timeout, and an 8-token If the request reaches the time limit, NemoClaw reports the provider as `not probed` and leaves model health unverified instead of reporting it as unhealthy. These checks are diagnostic only and do not override the authoritative `inference.local` result or determine the command exit status. +The `Inference (upstream)` check authenticates with the host credential that NemoClaw resolves for the provider, such as `NVIDIA_INFERENCE_API_KEY`. +The gateway stores the provider credential that the sandbox route uses. +The CLI cannot read the stored value back, so the two credentials can hold different secrets. +When the `inference.local` route has already served the inference request, an `unauthorized` result on `Inference (upstream)` describes the host credential. +NemoClaw then reports that check as `not probed` and names both credential sources. +An `Inference (upstream)` check that fails for another reason, such as `unreachable`, still reports its own state. +Local backend and auth proxy checks, such as `Inference (auth proxy)`, always report their own state and their own repair step. +`$$nemoclaw doctor` sends no inference request, so it always reports the `Inference (upstream)` state that it measured. + Local providers add host-side backend diagnostics. For Local Ollama, the command can also print an `Inference (auth proxy)` diagnostic when a proxy token is available. Use these diagnostics to identify a failing auxiliary hop after checking the main `Inference` line. diff --git a/src/lib/actions/sandbox/inference-route-health.ts b/src/lib/actions/sandbox/inference-route-health.ts index e0e1de332f6..8761226fab2 100644 --- a/src/lib/actions/sandbox/inference-route-health.ts +++ b/src/lib/actions/sandbox/inference-route-health.ts @@ -102,13 +102,43 @@ export async function probeSandboxInferenceGatewayHealth( }; } +/** + * The upstream probe authenticates with the host credential this command + * resolves. The gateway stores the provider credential the sandbox route uses + * and does not return its value, so the two can hold different secrets. Once + * the route has served an inference request, a provider rejection of the host + * credential reports nothing about the sandbox route. Local backend and auth + * proxy hops carry their own probeLabel and keep their own remediation. + */ +function unattributedUpstreamProbe(probe: ProviderHealthStatus): ProviderHealthStatus { + const { failureLabel: _failureLabel, ...rest } = probe; + return { + ...rest, + ok: true, + probed: false, + detail: + `${probe.detail} The sandbox ` + + "route served an inference request with the provider credential stored in the gateway, so " + + "NemoClaw does not attribute this result to the sandbox route.", + }; +} + function providerHealthDiagnostics( providerHealth: ProviderHealthStatus | null, + routeServedRequest: boolean, ): ProviderHealthStatus[] { if (!providerHealth) return []; const { subprobes = [], ...primary } = providerHealth; const labeledPrimary = primary.probeLabel ? primary : { ...primary, probeLabel: "upstream" }; - return [labeledPrimary, ...subprobes]; + return [labeledPrimary, ...subprobes].map((probe) => + routeServedRequest && + probe.probeLabel === "upstream" && + probe.probed && + !probe.ok && + probe.failureLabel === "unauthorized" + ? unattributedUpstreamProbe(probe) + : probe, + ); } function classifyInferenceInvocationFailureLabel( @@ -176,7 +206,7 @@ export function buildSandboxInferenceRouteHealth( invocation: SandboxInferenceInvocationResult | null, ): ProviderHealthStatus { const endpoint = gateway?.endpoint ?? "https://inference.local/v1/models"; - const diagnostics = providerHealthDiagnostics(providerHealth); + const diagnostics = providerHealthDiagnostics(providerHealth, Boolean(invocation?.ok)); let routeHealth: ProviderHealthStatus; if (gateway?.ok && invocation) { routeHealth = buildInvokedRouteHealth(gateway, endpoint, invocation); diff --git a/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts b/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts index ef2e5a8a4b9..6edc989fb72 100644 --- a/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts +++ b/src/lib/actions/sandbox/status-snapshot-inference-health.test.ts @@ -371,7 +371,7 @@ describe("collectSandboxStatusSnapshot inference route health", () => { expect(snapshot.inferenceHealth?.failureLabel).toBeUndefined(); }); - it("keeps an upstream subprobe failure out of the served-route verdict (#6846)", async () => { + it("keeps an unreachable upstream subprobe failure out of the served-route verdict (#6846)", async () => { const gateway: SandboxInferenceRouteHealth = { ok: true, endpoint: "https://inference.local/v1/models", @@ -385,7 +385,7 @@ describe("collectSandboxStatusSnapshot inference route health", () => { providerLabel: "NVIDIA Endpoints", endpoint: "https://integrate.api.nvidia.com/v1/chat/completions", detail: "model invocation probe failed", - failureLabel: "unauthorized", + failureLabel: "unreachable", }; const snapshot = await collectSandboxStatusSnapshot( @@ -430,6 +430,140 @@ describe("collectSandboxStatusSnapshot inference route health", () => { ); }); + it("reports the upstream check as not probed after the provider rejected the host credential and the route served a request (#9595)", async () => { + const gateway: SandboxInferenceRouteHealth = { + ok: true, + endpoint: "https://inference.local/v1/models", + httpStatus: 200, + detail: + "Inference gateway responded HTTP 200 on https://inference.local/v1/models (full chain reachable).", + }; + const providerHealth: ProviderHealthStatus = { + ok: false, + probed: true, + providerLabel: "NVIDIA Endpoints", + endpoint: "https://integrate.api.nvidia.com/v1/chat/completions", + detail: + "NVIDIA Endpoints rejected the host credential in NVIDIA_INFERENCE_API_KEY. " + + "Check NVIDIA_INFERENCE_API_KEY where you run this command.", + failureLabel: "unauthorized", + }; + + const snapshot = await collectSandboxStatusSnapshot( + "alpha", + snapshotDeps(gateway, providerHealth), + ); + + expect(snapshot.inferenceHealth).toMatchObject({ ok: true }); + const upstream = snapshot.inferenceHealth?.subprobes?.find( + (subprobe) => subprobe.probeLabel === "upstream", + ); + expect(upstream).toMatchObject({ ok: true, probed: false }); + expect(upstream?.failureLabel).toBeUndefined(); + expect(upstream?.detail).toContain("rejected the host credential"); + expect(upstream?.detail).toContain("NVIDIA_INFERENCE_API_KEY"); + expect(upstream?.detail).toContain("provider credential stored in the gateway"); + expect(upstream?.detail).toContain("does not attribute this result to the sandbox route"); + }); + + it("keeps an unreachable upstream check after the route served a request (#9595)", async () => { + const gateway: SandboxInferenceRouteHealth = { + ok: true, + endpoint: "https://inference.local/v1/models", + httpStatus: 200, + detail: + "Inference gateway responded HTTP 200 on https://inference.local/v1/models (full chain reachable).", + }; + const providerHealth: ProviderHealthStatus = { + ok: false, + probed: true, + providerLabel: "NVIDIA Endpoints", + endpoint: "https://integrate.api.nvidia.com/v1/chat/completions", + detail: "model invocation probe could not reach the endpoint", + failureLabel: "unreachable", + }; + + const snapshot = await collectSandboxStatusSnapshot( + "alpha", + snapshotDeps(gateway, providerHealth), + ); + + expect(snapshot.inferenceHealth?.subprobes).toContainEqual({ + ...providerHealth, + probeLabel: "upstream", + }); + }); + + it("keeps an unauthorized auth proxy subprobe after the route served a request (#9595)", async () => { + const gateway: SandboxInferenceRouteHealth = { + ok: true, + endpoint: "https://inference.local/v1/models", + httpStatus: 200, + detail: + "Inference gateway responded HTTP 200 on https://inference.local/v1/models (full chain reachable).", + }; + const authProxy: ProviderHealthStatus = { + ok: false, + probed: true, + providerLabel: "Local Ollama", + probeLabel: "auth proxy", + endpoint: "http://127.0.0.1:11435/api/tags", + detail: + "Ollama auth proxy returned 401 — the persisted token is no longer accepted. " + + "Re-run `nemoclaw onboard` (Ollama path) to rotate the proxy token.", + failureLabel: "unauthorized", + }; + const providerHealth: ProviderHealthStatus = { + ok: true, + probed: true, + providerLabel: "Local Ollama", + probeLabel: "ollama backend", + endpoint: "http://127.0.0.1:11434/api/tags", + detail: "Local Ollama is reachable.", + subprobes: [authProxy], + }; + + const snapshot = await collectSandboxStatusSnapshot( + "alpha", + snapshotDeps(gateway, providerHealth), + ); + + expect(snapshot.inferenceHealth?.subprobes).toContainEqual(authProxy); + }); + + it("keeps an unauthorized upstream check when the route did not serve a request (#9595)", async () => { + const gateway: SandboxInferenceRouteHealth = { + ok: true, + endpoint: "https://inference.local/v1/models", + httpStatus: 401, + detail: + "Inference gateway responded HTTP 401 on https://inference.local/v1/models (full chain reachable).", + }; + const providerHealth: ProviderHealthStatus = { + ok: false, + probed: true, + providerLabel: "NVIDIA Endpoints", + endpoint: "https://integrate.api.nvidia.com/v1/chat/completions", + detail: "model invocation probe rejected the credential", + failureLabel: "unauthorized", + }; + + const snapshot = await collectSandboxStatusSnapshot( + "alpha", + snapshotDeps(gateway, providerHealth, { + ok: false, + detail: "sandbox inference invocation probe returned HTTP 401", + httpStatus: 401, + }), + ); + + expect(snapshot.inferenceHealth).toMatchObject({ ok: false, failureLabel: "unauthorized" }); + expect(snapshot.inferenceHealth?.subprobes).toContainEqual({ + ...providerHealth, + probeLabel: "upstream", + }); + }); + it("does not send an agent request when the route probe already failed", async () => { const gateway: SandboxInferenceRouteHealth = { ok: false, diff --git a/src/lib/inference/health.test.ts b/src/lib/inference/health.test.ts index 27d6e701c9c..f3922086197 100644 --- a/src/lib/inference/health.test.ts +++ b/src/lib/inference/health.test.ts @@ -444,6 +444,20 @@ describe("inference health", () => { expect(result?.failureLabel).toBe("unauthorized"); }); + it("names the host credential environment variable when the provider rejects the request (#9595)", () => { + const result = probeRemoteProviderHealth("nvidia-prod", { + model: "meta/llama-3.3-70b-instruct", + getCredentialImpl: () => "nvapi-stale", + runCurlProbeImpl: () => httpUnauthorized(), + }); + + expect(result?.failureLabel).toBe("unauthorized"); + expect(result?.detail).toContain("rejected the"); + expect(result?.detail).toContain("host credential in NVIDIA_INFERENCE_API_KEY"); + expect(result?.detail).toContain("not the provider credential stored in the gateway"); + expect(result?.detail).not.toContain("Check your network connection"); + }); + it("reports unhealthy on a non-auth HTTP failure", () => { const result = probeRemoteProviderHealth("openai-api", { model: "gpt-4o-mini", diff --git a/src/lib/inference/health.ts b/src/lib/inference/health.ts index 7fff8d55881..3a9addcdfcf 100644 --- a/src/lib/inference/health.ts +++ b/src/lib/inference/health.ts @@ -420,6 +420,14 @@ function buildInvocationProbeDetail( `as proof that the model ran. (${result.message})` ); } + if (classifyHealthProbeFailureLabel(result) === "unauthorized") { + return ( + `${endpoint} rejected the ${route} request. ` + + `This probe authenticates with the host credential in ${credentialEnv}, not the provider ` + + `credential stored in the gateway. Check ${credentialEnv} where you run this command. ` + + `(${result.message})` + ); + } return ( `${route} at ${endpoint} did not succeed. ` + `Check your network connection or ${credentialEnv}. (${result.message})`