diff --git a/ci/source-architecture-budget.json b/ci/source-architecture-budget.json index 8b64b6d931e..0e0f98da7c2 100644 --- a/ci/source-architecture-budget.json +++ b/ci/source-architecture-budget.json @@ -31,7 +31,7 @@ "src/lib/state/registry.ts": 96, "src/lib/state/state-root.ts": 21, "src/lib/subprocess-env.ts": 22, - "src/lib/validation.ts": 24 + "src/lib/validation.ts": 23 } }, "fanOut": { @@ -64,7 +64,7 @@ "src/lib/actions": 18, "src/lib/actions/sandbox": 179, "src/lib/state": 38, - "src/lib/inference": 63, + "src/lib/inference": 64, "scripts": 42 } } diff --git a/docs/inference/understand-provider-validation.mdx b/docs/inference/understand-provider-validation.mdx index f5a6c49c220..cd05f33d80a 100644 --- a/docs/inference/understand-provider-validation.mdx +++ b/docs/inference/understand-provider-validation.mdx @@ -31,7 +31,7 @@ NemoClaw sends a provider-specific request that exercises the API surface intend | Provider | Validation request | |---|---| | OpenAI | Tries `/responses`, then `/chat/completions`. | -| NVIDIA Endpoints | Uses `/v1/chat/completions` and skips `/v1/responses`. | +| NVIDIA Endpoints | Uses `/v1/chat/completions` for model and smoke validation and skips `/v1/responses`. | | OpenRouter | Uses `/v1/chat/completions` for catalog, model, and smoke validation. | | Google Gemini | Uses the OpenAI-compatible chat-completions path and skips `/v1/responses`. | | Other OpenAI-compatible endpoint | Tries `/v1/responses` with tool-calling and streaming checks, then falls back to `/v1/chat/completions`. | diff --git a/docs/inference/verify-inference-route.mdx b/docs/inference/verify-inference-route.mdx index 52e1bd3f157..e92aca796cf 100644 --- a/docs/inference/verify-inference-route.mdx +++ b/docs/inference/verify-inference-route.mdx @@ -37,9 +37,14 @@ When the live provider differs, `status` does not carry the recorded API family The row reports `healthy` only when the route returns a structurally valid result for the selected API family. An empty body, malformed JSON, provider-error envelope, or wrong response shape reports `unhealthy`, even with a 2xx status. Status diagnostics do not include the response body. +When the models route responds and `status` sends an inference request, a failing row names the endpoint that request used. +When the models route itself does not respond, the row names the models route. An HTTP `401` or `403` response reports `unauthorized`. Correct the stored provider credential. +An HTTP `404` for a model that is in the NVIDIA Build catalog but is not deployed for your account explains that cause and prompts you to select a different model. The route-reachability and upstream provider subprobes remain available to identify the failing hop. +The route-reachability subprobe accompanies a row that sent an inference request. +It reads `reachable` for a 2xx models route and reports the status the models route returned for any other answer. The provider, model, and endpoint appear with the rest of the sandbox state. This path includes the OpenShell proxy and its authentication rewrite. When onboarding prints a dashboard summary, use it to verify that NemoClaw ran the same route-reachability probe from inside the sandbox. diff --git a/src/lib/actions/sandbox/inference-invocation-probe.test.ts b/src/lib/actions/sandbox/inference-invocation-probe.test.ts index 31f0773e13a..cf458a34bda 100644 --- a/src/lib/actions/sandbox/inference-invocation-probe.test.ts +++ b/src/lib/actions/sandbox/inference-invocation-probe.test.ts @@ -1,7 +1,12 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; +import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; import { describe, expect, it, vi } from "vitest"; +import { isNvcfFunctionNotFoundForAccount } from "../../inference/nvcf-model-access"; import { buildDcodeSandboxInferenceInvocationArgs, @@ -16,6 +21,14 @@ const input = { preferredInferenceApi: "openai-completions", }; +// Each API family posts to its own path, so a failure must name the request it +// actually made rather than the models route (#10879). +const INVOCATION_ENDPOINTS: Record = { + "openai-completions": "https://inference.local/v1/chat/completions", + "openai-responses": "https://inference.local/v1/responses", + "anthropic-messages": "https://inference.local/v1/messages", +}; + function openshellResult(status: number, stdout: string, stderr: string) { return { pid: 1, @@ -27,6 +40,40 @@ function openshellResult(status: number, stdout: string, stderr: string) { }; } +/** + * Run the generated probe command under a real shell with a stub curl that + * serves `body` at `code`, so the in-sandbox classification is exercised rather + * than simulated. Returns the probe's stdout. + */ +function runProbeCommandWithBody(code: string, body: string): string { + const dir = mkdtempSync(path.join(tmpdir(), "nemoclaw-probe-parity-")); + const bin = path.join(dir, "bin"); + mkdirSync(bin); + writeFileSync(path.join(dir, "body.txt"), body); + writeFileSync( + path.join(bin, "curl"), + [ + "#!/bin/sh", + 'out=""; prev=""', + 'for a in "$@"; do [ "$prev" = "-o" ] && out="$a"; prev="$a"; done', + `cat ${JSON.stringify(path.join(dir, "body.txt"))} > "$out"`, + `printf '%s' ${JSON.stringify(code)}`, + ].join("\n"), + { mode: 0o755 }, + ); + const run = spawnSync("/bin/sh", ["-c", buildSandboxInferenceInvocationCommand(input)], { + encoding: "utf8", + env: { ...process.env, PATH: `${bin}:${process.env.PATH || ""}` }, + }); + return run.stdout || ""; +} + +const NVCF_BODY_VARIANTS = [ + ["canonical", `{"status":404,"detail":"Function 'abc-123': Not found for account 'acct-42'"}`], + ["case variant", `{"status":404,"detail":"Function 'abc-123': not FOUND for ACCOUNT 'acct-42'"}`], + ["extra whitespace", `{"status":404,"detail":"Function 'abc-123': Not found for account"}`], +] as const; + describe("sandbox inference invocation probe", () => { it("probes the recorded model through inference.local without embedding a credential (#6195)", () => { const command = buildSandboxInferenceInvocationCommand(input); @@ -55,6 +102,7 @@ describe("sandbox inference invocation probe", () => { ok: false, detail: "sandbox inference invocation probe returned HTTP 401", httpStatus: 401, + endpoint: "https://inference.local/v1/chat/completions", }); expect(JSON.stringify(result)).not.toContain("sk-secret-value-that-is-long-enough"); }); @@ -72,8 +120,102 @@ describe("sandbox inference invocation probe", () => { ok: false, detail: "sandbox inference invocation probe returned HTTP 500", httpStatus: 500, + endpoint: "https://inference.local/v1/chat/completions", + }); + expect(JSON.stringify(result)).not.toContain("canary-replay-marker"); + }); + + it("classifies an NVCF account 404 inside the sandbox without echoing the body (#10879)", () => { + const command = buildSandboxInferenceInvocationCommand(input); + + // Only the fixed marker may cross the sandbox boundary, never the body it + // was matched against, so the #6195 contract still holds. + // The pattern reaches the sandbox shell-quoted, so match its stable tail. + expect(command).toContain("Not found for account"); + expect(command).toContain("nemoclaw-probe:nvcf-function-not-found"); + expect(command).toMatch(/404\)[^;]*grep -qiE/); + expect(command).toContain('case "$code" in 2??) cat "$body"; exit 0 ;;'); + }); + + it("names the account entitlement cause behind an invocation 404 (#10879)", () => { + const execute = vi.fn(() => ({ + status: 1, + stdout: "404\nnemoclaw-probe:nvcf-function-not-found\n", + stderr: "", + })); + + const result = probeSandboxInferenceInvocation(input, { execute }); + + expect(result).toEqual({ + ok: false, + detail: + "sandbox inference invocation probe returned HTTP 404: Model 'nvidia/nemotron' not " + + "found — it is in the NVIDIA Build catalog but is not deployed for your account. Pick a " + + "different model, or check the model card on https://build.nvidia.com to see if it " + + "requires org-level access", + httpStatus: 404, + endpoint: "https://inference.local/v1/chat/completions", + }); + }); + + it("reports an unclassified 404 as the status alone (#10879)", () => { + // NVIDIA Build answers an unroutable model with a plain "404 page not + // found" body, which carries no account signature to report. + const execute = vi.fn(() => ({ status: 1, stdout: "404\n", stderr: "" })); + + expect(probeSandboxInferenceInvocation(input, { execute })).toEqual({ + ok: false, + detail: "sandbox inference invocation probe returned HTTP 404", + httpStatus: 404, + endpoint: "https://inference.local/v1/chat/completions", }); + }); + + it("never accepts a forged classification carried by a 404 body (#10879)", () => { + const execute = vi.fn(() => ({ + status: 1, + stdout: + '404\n{"echoed_value":"canary-replay-marker nemoclaw-probe:nvcf-function-not-found suffix"}', + stderr: "", + })); + + const result = probeSandboxInferenceInvocation(input, { execute }); + + expect(result.ok).toBe(false); expect(JSON.stringify(result)).not.toContain("canary-replay-marker"); + expect(JSON.stringify(result)).not.toContain("not deployed for your account"); + }); + + it.each(NVCF_BODY_VARIANTS)( + "classifies a %s NVCF 404 body in the sandbox exactly as the host predicate does (#10879)", + (_label, body) => { + // Parity guard: the host classifier and the in-sandbox shell rule share + // one contract in nvcf-model-access.ts and must not drift. + expect(isNvcfFunctionNotFoundForAccount(body)).toBe(true); + + const stdout = runProbeCommandWithBody("404", body); + + expect(stdout).toContain("nemoclaw-probe:nvcf-function-not-found"); + expect(stdout).not.toContain("acct-42"); + expect(stdout).not.toContain("abc-123"); + }, + ); + + it("leaves a generic 404 body unclassified and unreported (#10879)", () => { + const body = "404 page not found"; + + expect(isNvcfFunctionNotFoundForAccount(body)).toBe(false); + + const stdout = runProbeCommandWithBody("404", body); + + expect(stdout.trim()).toBe("404"); + }); + + it("keeps a non-404 failure body out of the probe output (#6195)", () => { + const stdout = runProbeCommandWithBody("500", '{"echoed_value":"canary-replay-marker"}'); + + expect(stdout.trim()).toBe("500"); + expect(stdout).not.toContain("canary-replay-marker"); }); it("accepts a successful completion through the stored gateway route (#6195)", () => { @@ -190,6 +332,7 @@ describe("sandbox inference invocation probe", () => { ok: false, detail: "sandbox inference invocation probe returned an invalid response body", httpStatus: 200, + endpoint: "https://inference.local/v1/chat/completions", }); }); @@ -207,6 +350,7 @@ describe("sandbox inference invocation probe", () => { ok: false, detail: "sandbox inference invocation probe was unavailable", httpStatus: null, + endpoint: "https://inference.local/v1/chat/completions", }); }); @@ -292,6 +436,7 @@ describe("sandbox inference invocation probe", () => { ok: false, detail: "sandbox inference invocation probe returned an invalid response body", httpStatus: Number.parseInt(stdout.slice(0, 3), 10), + endpoint: INVOCATION_ENDPOINTS[preferredInferenceApi], }); }); diff --git a/src/lib/actions/sandbox/inference-invocation-probe.ts b/src/lib/actions/sandbox/inference-invocation-probe.ts index 28507ab8372..b310bcbb10d 100644 --- a/src/lib/actions/sandbox/inference-invocation-probe.ts +++ b/src/lib/actions/sandbox/inference-invocation-probe.ts @@ -6,6 +6,12 @@ import type { OpenShellRuntimeSelection } from "../../adapters/openshell/runtime import { getSandboxInferenceConfig } from "../../inference/config"; import { validateInferenceResponseBody } from "../../inference/health"; import { MIN_PROBE_REPLY_TOKENS, resolveMaxTokensField } from "../../inference/max-tokens-field"; +import { + NVCF_FUNCTION_NOT_FOUND_MARKER, + NVCF_FUNCTION_NOT_FOUND_SHELL_ERE, + NVCF_FUNCTION_NOT_FOUND_SHELL_MATCH_ARGS, + nvcfFunctionNotFoundMessage, +} from "../../inference/nvcf-model-access"; import { shellQuote } from "../../runner"; import { DCODE_MANAGED_EXEC_LAUNCHER } from "./connect-inference-route-probe"; import { @@ -27,7 +33,7 @@ export type SandboxInferenceInvocationInput = { export type SandboxInferenceInvocationResult = | { ok: true } - | { ok: false; detail: string; httpStatus: number | null }; + | { ok: false; detail: string; httpStatus: number | null; endpoint?: string }; export type SandboxInferenceInvocationDeps = { runOpenshell?: typeof runOpenshellProviderCommand; @@ -91,6 +97,13 @@ function buildProbeRequest(input: SandboxInferenceInvocationInput): { }; } +/** The endpoint this input's API family posts to, for callers that report a hop. */ +export function resolveSandboxInferenceInvocationEndpoint( + input: SandboxInferenceInvocationInput, +): string { + return buildProbeRequest(input).endpoint; +} + export function buildSandboxInferenceInvocationCommand( input: SandboxInferenceInvocationInput, ): string { @@ -106,7 +119,10 @@ export function buildSandboxInferenceInvocationCommand( "trap 'rm -f \"$body\"' EXIT HUP INT TERM", `code=$(curl -sS --connect-timeout 5 --max-time 90 --max-filesize ${INFERENCE_INVOCATION_MAX_RESPONSE_BYTES} -o "$body" -w '%{http_code}' ${headerArgs} --data-binary ${payload} ${endpoint}) || { rc=$?; printf 'curl-error:%s\\n' "$rc"; exit "$rc"; }`, "printf '%s\\n' \"$code\"", - 'case "$code" in 2??) cat "$body"; exit 0 ;; *) exit 1 ;; esac', + // A non-2xx body never leaves the sandbox (#6195). A 404 is classified + // here instead, so status can name the cause the onboarding probe already + // recognises without carrying the body that proved it (#10879). + `case "$code" in 2??) cat "$body"; exit 0 ;; 404) grep ${NVCF_FUNCTION_NOT_FOUND_SHELL_MATCH_ARGS} ${shellQuote(NVCF_FUNCTION_NOT_FOUND_SHELL_ERE)} "$body" && printf '%s\\n' ${shellQuote(NVCF_FUNCTION_NOT_FOUND_MARKER)}; exit 1 ;; *) exit 1 ;; esac`, ].join("; "); } @@ -198,6 +214,7 @@ export function probeSandboxInferenceInvocation( ok: false, detail: "sandbox inference invocation probe was unavailable", httpStatus: null, + endpoint: resolveSandboxInferenceInvocationEndpoint(input), }; } if (result.status === 0) { @@ -217,14 +234,25 @@ export function probeSandboxInferenceInvocation( ok: false, detail: "sandbox inference invocation probe returned an invalid response body", httpStatus, + endpoint: resolveSandboxInferenceInvocationEndpoint(input), }; } const httpStatus = result.stdout.match(/(?:^|\n)([1-5]\d\d)(?:\n|$)/)?.[1]; + // Only the fixed marker is read back, never the line that carried it, so an + // upstream body can still not reach diagnostics (#6195). + const nvcfFunctionNotFound = result.stdout + .split("\n") + .some((line) => line.trim() === NVCF_FUNCTION_NOT_FOUND_MARKER); + const detail = httpStatus + ? `sandbox inference invocation probe returned HTTP ${httpStatus}` + : `sandbox inference invocation probe exited with status ${result.status}`; return { ok: false, - detail: httpStatus - ? `sandbox inference invocation probe returned HTTP ${httpStatus}` - : `sandbox inference invocation probe exited with status ${result.status}`, + detail: + httpStatus === "404" && nvcfFunctionNotFound + ? `${detail}: ${nvcfFunctionNotFoundMessage(input.model).replace(/\.$/, "")}` + : detail, httpStatus: httpStatus ? Number.parseInt(httpStatus, 10) : null, + endpoint: resolveSandboxInferenceInvocationEndpoint(input), }; } diff --git a/src/lib/actions/sandbox/inference-route-health.test.ts b/src/lib/actions/sandbox/inference-route-health.test.ts index 0fa665fd55c..6d1063d5249 100644 --- a/src/lib/actions/sandbox/inference-route-health.test.ts +++ b/src/lib/actions/sandbox/inference-route-health.test.ts @@ -9,6 +9,7 @@ import { import { buildSandboxInferenceRouteHealth, probeSandboxInferenceGatewayHealth, + runSandboxInferenceInvocationProbe, type SandboxInferenceRouteHealth, } from "./inference-route-health"; @@ -127,6 +128,98 @@ describe("buildSandboxInferenceRouteHealth (#10080)", () => { detail: `probe returned ${httpStatus}`, }); + it.each([ + ["openai-completions", "https://inference.local/v1/chat/completions"], + ["openai-responses", "https://inference.local/v1/responses"], + ["anthropic-messages", "https://inference.local/v1/messages"], + ])("names the %s endpoint when the probe itself throws (#10879)", (api, endpoint) => { + const invocation = runSandboxInferenceInvocationProbe( + { + sandboxName: "alpha", + provider: "compatible-endpoint", + model: "nvidia/nemotron", + preferredInferenceApi: api, + }, + () => { + throw new Error("openshell exec exploded"); + }, + ); + + expect(invocation).toMatchObject({ ok: false, httpStatus: null, endpoint }); + expect( + buildSandboxInferenceRouteHealth(gateway(200), null, invocation, { + agentName: "openclaw", + provider: "compatible-endpoint", + }).endpoint, + ).toBe(endpoint); + }); + + it("names the request that failed, not the models route (#10879)", () => { + const result = buildSandboxInferenceRouteHealth( + gateway(200), + null, + { + ok: false, + detail: "sandbox inference invocation probe returned HTTP 404", + httpStatus: 404, + endpoint: "https://inference.local/v1/chat/completions", + }, + { agentName: "openclaw", provider: "nvidia-prod" }, + ); + + expect(result.ok).toBe(false); + expect(result.endpoint).toBe("https://inference.local/v1/chat/completions"); + expect(result.subprobes?.[0]).toMatchObject({ + probeLabel: "route reachability", + endpoint: "https://inference.local/v1/models", + }); + }); + + it("falls back to the models route when the invocation reports no endpoint", () => { + const result = buildSandboxInferenceRouteHealth( + gateway(200), + null, + { ok: false, detail: "probe was unavailable", httpStatus: null }, + { agentName: "openclaw", provider: "nvidia-prod" }, + ); + + expect(result.endpoint).toBe("https://inference.local/v1/models"); + }); + + it.each([404, 401, 403])( + "carries the models route status into the reachability hop for HTTP %s (#10879)", + (httpStatus) => { + const result = buildSandboxInferenceRouteHealth( + gateway(httpStatus), + null, + { + ok: false, + detail: "sandbox inference invocation probe returned HTTP 404", + httpStatus: 404, + endpoint: "https://inference.local/v1/chat/completions", + }, + { agentName: "openclaw", provider: "nvidia-prod" }, + ); + + expect(result.subprobes?.[0]).toMatchObject({ + probeLabel: "route reachability", + ok: true, + okLabel: `reachable (HTTP ${httpStatus})`, + }); + }, + ); + + it("keeps the plain reachable label for a 2xx models route (#6846)", () => { + const result = buildSandboxInferenceRouteHealth( + gateway(200), + null, + { ok: true }, + { agentName: "openclaw", provider: "nvidia-prod" }, + ); + + expect(result.subprobes?.[0]).toMatchObject({ ok: true, okLabel: "reachable" }); + }); + it("fails closed for a non-DCode agent when the route 404s, even if invocation succeeds", () => { const result = buildSandboxInferenceRouteHealth( gateway(404), diff --git a/src/lib/actions/sandbox/inference-route-health.ts b/src/lib/actions/sandbox/inference-route-health.ts index ac59437c078..4d98e2d6894 100644 --- a/src/lib/actions/sandbox/inference-route-health.ts +++ b/src/lib/actions/sandbox/inference-route-health.ts @@ -14,6 +14,7 @@ import { import { probeSandboxInferenceInvocation, READINESS_INFERENCE_INVOCATION_TIMEOUT_MS, + resolveSandboxInferenceInvocationEndpoint, type SandboxInferenceInvocationInput, type SandboxInferenceInvocationResult, } from "./inference-invocation-probe"; @@ -162,6 +163,12 @@ function reachableRouteSubprobe( gateway: SandboxInferenceRouteHealth, endpoint: string, ): ProviderHealthStatus { + // The probe grades any final HTTP 200-499 as reachable, and the renderer + // prints an ok probe's label without its detail, so a bare "reachable" hid + // the status the models route actually returned — including a 404 catalog + // that validated nothing (#10879). Keep the hop green, because the route did + // answer, but carry the code in the label for any non-2xx answer. + const answered2xx = gateway.httpStatus >= 200 && gateway.httpStatus < 300; return { ok: true, probed: true, @@ -169,7 +176,7 @@ function reachableRouteSubprobe( probeLabel: "route reachability", endpoint, detail: gateway.detail, - okLabel: "reachable", + okLabel: answered2xx ? "reachable" : `reachable (HTTP ${gateway.httpStatus})`, }; } @@ -198,7 +205,10 @@ function buildInvokedRouteHealth( ok: false, probed: true, providerLabel: "Inference route", - endpoint, + // The invocation is a POST to the selected API family's path, not the + // models route. Reporting the models endpoint here told operators the + // wrong request had failed (#10879). + endpoint: invocation.endpoint ?? endpoint, detail: `Inference gateway did not serve an inference request: ${invocation.detail}.`, failureLabel: classifyInferenceInvocationFailureLabel(invocation.httpStatus), subprobes: [reachableRouteSubprobe(gateway, endpoint)], @@ -327,6 +337,10 @@ export function runSandboxInferenceInvocationProbe( ok: false, detail: "sandbox inference invocation probe could not run", httpStatus: null, + // An abnormal probe still failed against the selected API family's path. + // Without this the row falls back to the models route and misdirects + // recovery to a request that never ran (#10879). + endpoint: resolveSandboxInferenceInvocationEndpoint(input), }; } } diff --git a/src/lib/inference/nvcf-model-access.ts b/src/lib/inference/nvcf-model-access.ts new file mode 100644 index 00000000000..9324ec5cea5 --- /dev/null +++ b/src/lib/inference/nvcf-model-access.ts @@ -0,0 +1,64 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * NVIDIA Cloud Functions model-access classification, shared by the host-side + * onboarding probe and the in-sandbox status invocation probe so both name the + * same cause for the same HTTP 404. + */ + +/** + * Detect NVIDIA Cloud Functions "Function not found for account" errors. + * + * NVIDIA Build (integrate.api.nvidia.com) returns this when a model is in the + * public catalog but is not deployed for the caller's account/org. The raw + * body looks like: + * + * {"status":404,"title":"Not Found", + * "detail":"Function '': Not found for account ''"} + * + * Detecting this lets the wizard surface an actionable error instead of the + * raw NVCF body. See issue #1601. + */ +export function isNvcfFunctionNotFoundForAccount(message: string): boolean { + return /Function\s+'[^']+':\s*Not found for account/i.test(String(message || "")); +} + +/** + * The same condition as `isNvcfFunctionNotFoundForAccount`, expressed as a + * POSIX extended regular expression for the in-sandbox status probe, which must + * classify the body where it is and forward only a verdict (#10879). + * + * Both forms live here so the two cannot drift: the host and the sandbox must + * agree on what an account-entitlement 404 is, or the same provider response + * yields a remediation during onboarding and a bare HTTP 404 at `status`. + * `NVCF_FUNCTION_NOT_FOUND_SHELL_MATCH_ARGS` carries the case-insensitive flag + * that `/i` supplies on the TypeScript side; `[[:space:]]` mirrors `\s`. + */ +export const NVCF_FUNCTION_NOT_FOUND_SHELL_ERE = + "Function[[:space:]]+'[^']+':[[:space:]]*Not found for account"; +export const NVCF_FUNCTION_NOT_FOUND_SHELL_MATCH_ARGS = "-qiE"; + +/** + * Verdict token the in-sandbox probe prints when the body matches. Only this + * constant crosses the sandbox boundary, never the body it was matched against, + * so status diagnostics still carry no response body (#6195). + */ +export const NVCF_FUNCTION_NOT_FOUND_MARKER = "nemoclaw-probe:nvcf-function-not-found"; + +/** + * Build the user-facing message for an NVCF "Function not found for account" + * failure. The model is in the catalog but cannot be invoked from this key. + * + * The wording deliberately starts with "Model '' not found" so that + * `classifyValidationFailure()` matches its `model.+not found` regex and + * routes the user into the model-selection recovery path instead of + * collapsing to the generic `unknown`/`selection` branch. + */ +export function nvcfFunctionNotFoundMessage(model: string): string { + return ( + `Model '${model}' not found — it is in the NVIDIA Build catalog but is not deployed ` + + "for your account. Pick a different model, or check the model card on " + + "https://build.nvidia.com to see if it requires org-level access." + ); +} diff --git a/src/lib/inference/onboard-probes.ts b/src/lib/inference/onboard-probes.ts index cc1a798df67..03b9dc806fd 100644 --- a/src/lib/inference/onboard-probes.ts +++ b/src/lib/inference/onboard-probes.ts @@ -35,7 +35,10 @@ const { getHostDockerInternalProbeFailure, isHijackedDockerInternalUrl, } = require("./onboard-host-docker-internal"); -const { isNvcfFunctionNotFoundForAccount, nvcfFunctionNotFoundMessage } = require("../validation"); +const { + isNvcfFunctionNotFoundForAccount, + nvcfFunctionNotFoundMessage, +} = require("./nvcf-model-access"); const { isPrivateHostname, isPrivateIp, isLoopbackHostname } = require("../private-networks"); const { buildResolvePinArgs, isOperatorTrustablePrivateIp } = require("./endpoint-ssrf-preflight"); const { @@ -506,9 +509,7 @@ function probeChatCompletionsToolCalling(endpointUrl, model, apiKey, options = { // ── OpenAI-like probe ──────────────────────────────────────────── function needsExtendedNvidiaEndpointValidationBudget(model) { - return ( - vllmProbePolicyForModel(String(model || "")) === EXTENDED_NVIDIA_ENDPOINT_PROBE_POLICY - ); + return vllmProbePolicyForModel(String(model || "")) === EXTENDED_NVIDIA_ENDPOINT_PROBE_POLICY; } function getChatCompletionsProbeTimingArgs(model, opts) { @@ -1154,7 +1155,15 @@ export function shouldSmokeOpenAiLikeOnboardRoute( return false; } const { REMOTE_PROVIDER_CONFIG } = require("../onboard/providers"); - if (provider === "nvidia-nim" || provider === "nvidia-router") return true; + // NVIDIA Endpoints registers as OpenShell provider type "nvidia", which the + // providerType test below does not match, so nvidia-prod was the only + // OpenAI-completions remote provider whose onboarding never sent a Chat + // Completions request. A model that is in the NVIDIA Build catalog but not + // deployed for the account then onboarded clean and first failed at + // `status` with a bare HTTP 404 (#10879). Smoke it like its siblings. + if (provider === "nvidia-prod" || provider === "nvidia-nim" || provider === "nvidia-router") { + return true; + } return Object.values(REMOTE_PROVIDER_CONFIG).some( (entry) => entry.providerName === provider && @@ -1222,9 +1231,8 @@ export async function verifyOnboardInferenceSmoke(options: any, dependencies: an try { const teardownOrphanManagedGatewayOnAbort = dependencies.teardownOrphanManagedGatewayOnAbort ?? - ( - require("../onboard/gateway-destroy") as typeof import("../onboard/gateway-destroy") - ).teardownOrphanManagedGatewayOnAbort; + (require("../onboard/gateway-destroy") as typeof import("../onboard/gateway-destroy")) + .teardownOrphanManagedGatewayOnAbort; teardownOrphanManagedGatewayOnAbort(); } catch (error) { // Helper never throws; this covers require/load failures only. diff --git a/src/lib/validation.ts b/src/lib/validation.ts index 47263e2844f..0a2a30eb26a 100644 --- a/src/lib/validation.ts +++ b/src/lib/validation.ts @@ -284,39 +284,13 @@ export function isSafeModelId(value: string): boolean { return /^[A-Za-z0-9._:/-]+$/.test(value); } -/** - * Detect NVIDIA Cloud Functions "Function not found for account" errors. - * - * NVIDIA Build (integrate.api.nvidia.com) returns this when a model is in the - * public catalog but is not deployed for the caller's account/org. The raw - * body looks like: - * - * {"status":404,"title":"Not Found", - * "detail":"Function '': Not found for account ''"} - * - * Detecting this lets the wizard surface an actionable error instead of the - * raw NVCF body. See issue #1601. - */ -export function isNvcfFunctionNotFoundForAccount(message: string): boolean { - return /Function\s+'[^']+':\s*Not found for account/i.test(String(message || "")); -} - -/** - * Build the user-facing message for an NVCF "Function not found for account" - * failure. The model is in the catalog but cannot be invoked from this key. - * - * The wording deliberately starts with "Model '' not found" so that - * `classifyValidationFailure()` matches its `model.+not found` regex and - * routes the user into the model-selection recovery path instead of - * collapsing to the generic `unknown`/`selection` branch. - */ -export function nvcfFunctionNotFoundMessage(model: string): string { - return ( - `Model '${model}' not found — it is in the NVIDIA Build catalog but is not deployed ` + - "for your account. Pick a different model, or check the model card on " + - "https://build.nvidia.com to see if it requires org-level access." - ); -} +// Re-exported so existing importers keep one validation entry point while the +// NVIDIA Cloud Functions classification lives with the inference layer that +// owns both of its callers. +export { + isNvcfFunctionNotFoundForAccount, + nvcfFunctionNotFoundMessage, +} from "./inference/nvcf-model-access"; /** * Whether the wizard should skip probing the OpenAI Responses API entirely diff --git a/test/cli/helpers.ts b/test/cli/helpers.ts index 4b6a07d749e..0b4138eec93 100644 --- a/test/cli/helpers.ts +++ b/test/cli/helpers.ts @@ -319,7 +319,12 @@ export function writeHealthyDockerStub(localBin: string): void { * stub. The general sandbox transport writes an exec marker before the HTTP * response. The managed DCode launcher returns the HTTP response directly. */ -export function inferenceInvocationStubLines(httpStatus = "200", exitCode = 0): string[] { +export function inferenceInvocationStubLines( + httpStatus = "200", + exitCode = 0, + /** Extra probe stdout after the status line, e.g. a failure classification token. */ + extraStdout: readonly string[] = [], +): string[] { const bodyLines = new Map([ [ @@ -350,6 +355,7 @@ export function inferenceInvocationStubLines(httpStatus = "200", exitCode = 0): " esac", ` printf '%s\\n' ${JSON.stringify(httpStatus)}`, ...bodyLines, + ...extraStdout.map((line) => ` printf '%s\\n' ${JSON.stringify(line)}`), ` exit ${String(exitCode)}`, " ;;", " esac", @@ -515,7 +521,11 @@ export function createDebugCommandTestEnv( fs.mkdirSync(localBin, { recursive: true }); // Register the env-sourced sandbox plus any extra names supplied via the // --sandbox flag so the validation gate accepts them. - writeSandboxRegistry(home, sandboxName, options.gatewayPort ? { gatewayPort: options.gatewayPort } : {}); + writeSandboxRegistry( + home, + sandboxName, + options.gatewayPort ? { gatewayPort: options.gatewayPort } : {}, + ); if (options.extraSandboxNames && options.extraSandboxNames.length > 0) { const registryPath = path.join(home, ".nemoclaw", "sandboxes.json"); const current = JSON.parse(fs.readFileSync(registryPath, "utf-8")) as { diff --git a/test/cli/sandbox-status-json.test.ts b/test/cli/sandbox-status-json.test.ts index a6ba1b0e408..1a355b31639 100644 --- a/test/cli/sandbox-status-json.test.ts +++ b/test/cli/sandbox-status-json.test.ts @@ -22,6 +22,7 @@ function createInferenceRouteStatusSetup(options: { upstreamExit?: number; invocationHttpStatus?: string; invocationExit?: number; + invocationClassification?: string; }) { const home = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-cli-status-route-")); const localBin = path.join(home, "bin"); @@ -72,7 +73,11 @@ function createInferenceRouteStatusSetup(options: { " exit 0", "fi", 'if [ "$1" = "sandbox" ] && [ "$2" = "exec" ]; then', - ...inferenceInvocationStubLines(options.invocationHttpStatus, options.invocationExit), + ...inferenceInvocationStubLines( + options.invocationHttpStatus, + options.invocationExit, + options.invocationClassification ? [options.invocationClassification] : [], + ), ...(options.executeRouteCommand ? [ ' while [ "$#" -gt 0 ] && [ "$1" != "--" ]; do shift; done', @@ -279,6 +284,32 @@ describe("CLI sandbox status JSON output", testTimeoutOptions(20_000), () => { ); }); + it("sandbox status --json names the NVIDIA Build account entitlement behind a 404 (#10879)", () => { + const { home, localBin, sandboxName } = createInferenceRouteStatusSetup({ + routeOutput: "OK 200", + invocationHttpStatus: "404", + invocationExit: 1, + invocationClassification: "nemoclaw-probe:nvcf-function-not-found", + }); + + const result = runWithEnv(`${sandboxName} status --json`, { + HOME: home, + PATH: `${localBin}:${process.env.PATH || ""}`, + }); + + expect(result.code).toBe(1); + const parsed = JSON.parse(result.out); + expect(parsed.inferenceHealth).toMatchObject({ + ok: false, + probed: true, + failureLabel: "unhealthy", + endpoint: "https://inference.local/v1/chat/completions", + }); + expect(parsed.inferenceHealth.detail).toContain("not deployed for your account"); + expect(parsed.inferenceHealth.detail).toContain("nvidia/nemotron"); + expect(parsed.inferenceHealth.detail).not.toContain(".."); + }); + it.each([401, 403])( "sandbox status --json fails an inference.local HTTP %s that rejects an agent request", (httpStatus) => { @@ -299,11 +330,18 @@ describe("CLI sandbox status JSON output", testTimeoutOptions(20_000), () => { ok: false, probed: true, failureLabel: "unauthorized", - endpoint: "https://inference.local/v1/models", + // The rejected request is the invocation, so the row names its path + // rather than the models route it did not use (#10879). + endpoint: "https://inference.local/v1/chat/completions", }); expect(parsed.inferenceHealth.detail).toContain(String(httpStatus)); expect(parsed.inferenceHealth.subprobes).toContainEqual( - expect.objectContaining({ ok: true, probeLabel: "route reachability" }), + expect.objectContaining({ + ok: true, + probeLabel: "route reachability", + endpoint: "https://inference.local/v1/models", + okLabel: `reachable (HTTP ${httpStatus})`, + }), ); }, ); diff --git a/test/onboarding/onboard-smoke-verifier.test.ts b/test/onboarding/onboard-smoke-verifier.test.ts index 213857a086d..01509a07e14 100644 --- a/test/onboarding/onboard-smoke-verifier.test.ts +++ b/test/onboarding/onboard-smoke-verifier.test.ts @@ -12,6 +12,36 @@ describe("Hermes onboard smoke verification", () => { expect(shouldSmokeOpenAiLikeOnboardRoute("openai-api")).toBe(true); }); + it("host-smokes every NVIDIA Endpoints route like its siblings (#10879)", () => { + // nvidia-prod registers as OpenShell provider type "nvidia", which the + // providerType allowlist does not match, so it used to onboard without a + // single Chat Completions request and first failed at `status`. + expect(shouldSmokeOpenAiLikeOnboardRoute("nvidia-prod")).toBe(true); + expect(shouldSmokeOpenAiLikeOnboardRoute("nvidia-nim")).toBe(true); + expect(shouldSmokeOpenAiLikeOnboardRoute("nvidia-router")).toBe(true); + }); + + it("sends the NVIDIA Endpoints host smoke through Chat Completions (#10879)", async () => { + const calls = await runVerifyOnboardSmokeHarness([ + { + provider: "nvidia-prod", + credentialEnv: "NVIDIA_INFERENCE_API_KEY", + endpointUrl: "https://integrate.api.nvidia.com/v1", + model: "nvidia/nemotron-3-super-120b-a12b", + }, + ]); + + // Before #10879 the runtime verifier returned before any probe, so a model + // the credential cannot invoke onboarded clean and first failed at `status`. + expect(calls.filter((call) => call[0] === "runCurlProbe")).toEqual([ + [ + "runCurlProbe", + "https://integrate.api.nvidia.com/v1/chat/completions", + "Authorization: Bearer resolved-NVIDIA_INFERENCE_API_KEY", + ], + ]); + }); + it("skips only the Hermes OAuth smoke path in the runtime verifier", async () => { const calls = await runVerifyOnboardSmokeHarness([ { credentialEnv: "OPENAI_API_KEY" },