fix(inference): validate the NVIDIA Endpoints route and name its 404 cause

ci/source-architecture-budget.json

@@ -31,7 +31,7 @@
       "src/lib/state/registry.ts": 96,
       "src/lib/state/state-root.ts": 21,
       "src/lib/subprocess-env.ts": 22,
-      "src/lib/validation.ts": 24
+      "src/lib/validation.ts": 23
     }
   },
   "fanOut": {
@@ -64,7 +64,7 @@
     "src/lib/actions": 18,
     "src/lib/actions/sandbox": 179,
     "src/lib/state": 38,
-    "src/lib/inference": 63,
+    "src/lib/inference": 64,
     "scripts": 42
   }
 }

docs/inference/understand-provider-validation.mdx

@@ -31,7 +31,7 @@ NemoClaw sends a provider-specific request that exercises the API surface intend
 | Provider | Validation request |
 |---|---|
 | OpenAI | Tries `/responses`, then `/chat/completions`. |
-| NVIDIA Endpoints | Uses `/v1/chat/completions` and skips `/v1/responses`. |
+| NVIDIA Endpoints | Uses `/v1/chat/completions` for model and smoke validation and skips `/v1/responses`. |
 | OpenRouter | Uses `/v1/chat/completions` for catalog, model, and smoke validation. |
 | Google Gemini | Uses the OpenAI-compatible chat-completions path and skips `/v1/responses`. |
 | Other OpenAI-compatible endpoint | Tries `/v1/responses` with tool-calling and streaming checks, then falls back to `/v1/chat/completions`. |

docs/inference/verify-inference-route.mdx

@@ -37,9 +37,14 @@ When the live provider differs, `status` does not carry the recorded API family
 The row reports `healthy` only when the route returns a structurally valid result for the selected API family.
 An empty body, malformed JSON, provider-error envelope, or wrong response shape reports `unhealthy`, even with a 2xx status.
 Status diagnostics do not include the response body.
+When the models route responds and `status` sends an inference request, a failing row names the endpoint that request used.
+When the models route itself does not respond, the row names the models route.
 An HTTP `401` or `403` response reports `unauthorized`.
 Correct the stored provider credential.
+An HTTP `404` for a model that is in the NVIDIA Build catalog but is not deployed for your account explains that cause and prompts you to select a different model.
 The route-reachability and upstream provider subprobes remain available to identify the failing hop.
+The route-reachability subprobe accompanies a row that sent an inference request.
+It reads `reachable` for a 2xx models route and reports the status the models route returned for any other answer.
 The provider, model, and endpoint appear with the rest of the sandbox state.
 This path includes the OpenShell proxy and its authentication rewrite.
 When onboarding prints a dashboard summary, use it to verify that NemoClaw ran the same route-reachability probe from inside the sandbox.

src/lib/actions/sandbox/inference-invocation-probe.test.ts

@@ -1,7 +1,12 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+import { spawnSync } from "node:child_process";
+import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import path from "node:path";
 import { describe, expect, it, vi } from "vitest";
+import { isNvcfFunctionNotFoundForAccount } from "../../inference/nvcf-model-access";
 
 import {
   buildDcodeSandboxInferenceInvocationArgs,
@@ -16,6 +21,14 @@ const input = {
   preferredInferenceApi: "openai-completions",
 };
 
+// Each API family posts to its own path, so a failure must name the request it
+// actually made rather than the models route (#10879).
+const INVOCATION_ENDPOINTS: Record<string, string> = {
+  "openai-completions": "https://inference.local/v1/chat/completions",
+  "openai-responses": "https://inference.local/v1/responses",
+  "anthropic-messages": "https://inference.local/v1/messages",
+};
+
 function openshellResult(status: number, stdout: string, stderr: string) {
   return {
     pid: 1,
@@ -27,6 +40,40 @@ function openshellResult(status: number, stdout: string, stderr: string) {
   };
 }
 
+/**
+ * Run the generated probe command under a real shell with a stub curl that
+ * serves `body` at `code`, so the in-sandbox classification is exercised rather
+ * than simulated. Returns the probe's stdout.
+ */
+function runProbeCommandWithBody(code: string, body: string): string {
+  const dir = mkdtempSync(path.join(tmpdir(), "nemoclaw-probe-parity-"));
+  const bin = path.join(dir, "bin");
+  mkdirSync(bin);
+  writeFileSync(path.join(dir, "body.txt"), body);
+  writeFileSync(
+    path.join(bin, "curl"),
+    [
+      "#!/bin/sh",
+      'out=""; prev=""',
+      'for a in "$@"; do [ "$prev" = "-o" ] && out="$a"; prev="$a"; done',
+      `cat ${JSON.stringify(path.join(dir, "body.txt"))} > "$out"`,
+      `printf '%s' ${JSON.stringify(code)}`,
+    ].join("\n"),
+    { mode: 0o755 },
+  );
+  const run = spawnSync("/bin/sh", ["-c", buildSandboxInferenceInvocationCommand(input)], {
+    encoding: "utf8",
+    env: { ...process.env, PATH: `${bin}:${process.env.PATH || ""}` },
+  });
+  return run.stdout || "";
+}
+
+const NVCF_BODY_VARIANTS = [
+  ["canonical", `{"status":404,"detail":"Function 'abc-123': Not found for account 'acct-42'"}`],
+  ["case variant", `{"status":404,"detail":"Function 'abc-123': not FOUND for ACCOUNT 'acct-42'"}`],
+  ["extra whitespace", `{"status":404,"detail":"Function  'abc-123':   Not found for account"}`],
+] as const;
+
 describe("sandbox inference invocation probe", () => {
   it("probes the recorded model through inference.local without embedding a credential (#6195)", () => {
     const command = buildSandboxInferenceInvocationCommand(input);
@@ -55,6 +102,7 @@ describe("sandbox inference invocation probe", () => {
       ok: false,
       detail: "sandbox inference invocation probe returned HTTP 401",
       httpStatus: 401,
+      endpoint: "https://inference.local/v1/chat/completions",
     });
     expect(JSON.stringify(result)).not.toContain("sk-secret-value-that-is-long-enough");
   });
@@ -72,8 +120,102 @@ describe("sandbox inference invocation probe", () => {
       ok: false,
       detail: "sandbox inference invocation probe returned HTTP 500",
       httpStatus: 500,
+      endpoint: "https://inference.local/v1/chat/completions",
+    });
+    expect(JSON.stringify(result)).not.toContain("canary-replay-marker");
+  });
+
+  it("classifies an NVCF account 404 inside the sandbox without echoing the body (#10879)", () => {
+    const command = buildSandboxInferenceInvocationCommand(input);
+
+    // Only the fixed marker may cross the sandbox boundary, never the body it
+    // was matched against, so the #6195 contract still holds.
+    // The pattern reaches the sandbox shell-quoted, so match its stable tail.
+    expect(command).toContain("Not found for account");
+    expect(command).toContain("nemoclaw-probe:nvcf-function-not-found");
+    expect(command).toMatch(/404\)[^;]*grep -qiE/);
+    expect(command).toContain('case "$code" in 2??) cat "$body"; exit 0 ;;');
+  });
+
+  it("names the account entitlement cause behind an invocation 404 (#10879)", () => {
+    const execute = vi.fn(() => ({
+      status: 1,
+      stdout: "404\nnemoclaw-probe:nvcf-function-not-found\n",
+      stderr: "",
+    }));
+
+    const result = probeSandboxInferenceInvocation(input, { execute });
+
+    expect(result).toEqual({
+      ok: false,
+      detail:
+        "sandbox inference invocation probe returned HTTP 404: Model 'nvidia/nemotron' not " +
+        "found — it is in the NVIDIA Build catalog but is not deployed for your account. Pick a " +
+        "different model, or check the model card on https://build.nvidia.com to see if it " +
+        "requires org-level access",
+      httpStatus: 404,
+      endpoint: "https://inference.local/v1/chat/completions",
+    });
+  });
+
+  it("reports an unclassified 404 as the status alone (#10879)", () => {
+    // NVIDIA Build answers an unroutable model with a plain "404 page not
+    // found" body, which carries no account signature to report.
+    const execute = vi.fn(() => ({ status: 1, stdout: "404\n", stderr: "" }));
+
+    expect(probeSandboxInferenceInvocation(input, { execute })).toEqual({
+      ok: false,
+      detail: "sandbox inference invocation probe returned HTTP 404",
+      httpStatus: 404,
+      endpoint: "https://inference.local/v1/chat/completions",
     });
+  });
+
+  it("never accepts a forged classification carried by a 404 body (#10879)", () => {
+    const execute = vi.fn(() => ({
+      status: 1,
+      stdout:
+        '404\n{"echoed_value":"canary-replay-marker nemoclaw-probe:nvcf-function-not-found suffix"}',
+      stderr: "",
+    }));
+
+    const result = probeSandboxInferenceInvocation(input, { execute });
+
+    expect(result.ok).toBe(false);
     expect(JSON.stringify(result)).not.toContain("canary-replay-marker");
+    expect(JSON.stringify(result)).not.toContain("not deployed for your account");
+  });
+
+  it.each(NVCF_BODY_VARIANTS)(
+    "classifies a %s NVCF 404 body in the sandbox exactly as the host predicate does (#10879)",
+    (_label, body) => {
+      // Parity guard: the host classifier and the in-sandbox shell rule share
+      // one contract in nvcf-model-access.ts and must not drift.
+      expect(isNvcfFunctionNotFoundForAccount(body)).toBe(true);
+
+      const stdout = runProbeCommandWithBody("404", body);
+
+      expect(stdout).toContain("nemoclaw-probe:nvcf-function-not-found");
+      expect(stdout).not.toContain("acct-42");
+      expect(stdout).not.toContain("abc-123");
+    },
+  );
+
+  it("leaves a generic 404 body unclassified and unreported (#10879)", () => {
+    const body = "404 page not found";
+
+    expect(isNvcfFunctionNotFoundForAccount(body)).toBe(false);
+
+    const stdout = runProbeCommandWithBody("404", body);
+
+    expect(stdout.trim()).toBe("404");
+  });
+
+  it("keeps a non-404 failure body out of the probe output (#6195)", () => {
+    const stdout = runProbeCommandWithBody("500", '{"echoed_value":"canary-replay-marker"}');
+
+    expect(stdout.trim()).toBe("500");
+    expect(stdout).not.toContain("canary-replay-marker");
   });
 
   it("accepts a successful completion through the stored gateway route (#6195)", () => {
@@ -190,6 +332,7 @@ describe("sandbox inference invocation probe", () => {
       ok: false,
       detail: "sandbox inference invocation probe returned an invalid response body",
       httpStatus: 200,
+      endpoint: "https://inference.local/v1/chat/completions",
     });
   });
 
@@ -207,6 +350,7 @@ describe("sandbox inference invocation probe", () => {
       ok: false,
       detail: "sandbox inference invocation probe was unavailable",
       httpStatus: null,
+      endpoint: "https://inference.local/v1/chat/completions",
     });
   });
 
@@ -292,6 +436,7 @@ describe("sandbox inference invocation probe", () => {
       ok: false,
       detail: "sandbox inference invocation probe returned an invalid response body",
       httpStatus: Number.parseInt(stdout.slice(0, 3), 10),
+      endpoint: INVOCATION_ENDPOINTS[preferredInferenceApi],
     });
   });
 

src/lib/actions/sandbox/inference-invocation-probe.ts

@@ -6,6 +6,12 @@ import type { OpenShellRuntimeSelection } from "../../adapters/openshell/runtime
 import { getSandboxInferenceConfig } from "../../inference/config";
 import { validateInferenceResponseBody } from "../../inference/health";
 import { MIN_PROBE_REPLY_TOKENS, resolveMaxTokensField } from "../../inference/max-tokens-field";
+import {
+  NVCF_FUNCTION_NOT_FOUND_MARKER,
+  NVCF_FUNCTION_NOT_FOUND_SHELL_ERE,
+  NVCF_FUNCTION_NOT_FOUND_SHELL_MATCH_ARGS,
+  nvcfFunctionNotFoundMessage,
+} from "../../inference/nvcf-model-access";
 import { shellQuote } from "../../runner";
 import { DCODE_MANAGED_EXEC_LAUNCHER } from "./connect-inference-route-probe";
 import {
@@ -27,7 +33,7 @@ export type SandboxInferenceInvocationInput = {
 
 export type SandboxInferenceInvocationResult =
   | { ok: true }
-  | { ok: false; detail: string; httpStatus: number | null };
+  | { ok: false; detail: string; httpStatus: number | null; endpoint?: string };
 
 export type SandboxInferenceInvocationDeps = {
   runOpenshell?: typeof runOpenshellProviderCommand;
@@ -91,6 +97,13 @@ function buildProbeRequest(input: SandboxInferenceInvocationInput): {
   };
 }
 
+/** The endpoint this input's API family posts to, for callers that report a hop. */
+export function resolveSandboxInferenceInvocationEndpoint(
+  input: SandboxInferenceInvocationInput,
+): string {
+  return buildProbeRequest(input).endpoint;
+}
+
 export function buildSandboxInferenceInvocationCommand(
   input: SandboxInferenceInvocationInput,
 ): string {
@@ -106,7 +119,10 @@ export function buildSandboxInferenceInvocationCommand(
     "trap 'rm -f \"$body\"' EXIT HUP INT TERM",
     `code=$(curl -sS --connect-timeout 5 --max-time 90 --max-filesize ${INFERENCE_INVOCATION_MAX_RESPONSE_BYTES} -o "$body" -w '%{http_code}' ${headerArgs} --data-binary ${payload} ${endpoint}) || { rc=$?; printf 'curl-error:%s\\n' "$rc"; exit "$rc"; }`,
     "printf '%s\\n' \"$code\"",
-    'case "$code" in 2??) cat "$body"; exit 0 ;; *) exit 1 ;; esac',
+    // A non-2xx body never leaves the sandbox (#6195). A 404 is classified
+    // here instead, so status can name the cause the onboarding probe already
+    // recognises without carrying the body that proved it (#10879).
+    `case "$code" in 2??) cat "$body"; exit 0 ;; 404) grep ${NVCF_FUNCTION_NOT_FOUND_SHELL_MATCH_ARGS} ${shellQuote(NVCF_FUNCTION_NOT_FOUND_SHELL_ERE)} "$body" && printf '%s\\n' ${shellQuote(NVCF_FUNCTION_NOT_FOUND_MARKER)}; exit 1 ;; *) exit 1 ;; esac`,
   ].join("; ");
 }
 
@@ -198,6 +214,7 @@ export function probeSandboxInferenceInvocation(
       ok: false,
       detail: "sandbox inference invocation probe was unavailable",
       httpStatus: null,
+      endpoint: resolveSandboxInferenceInvocationEndpoint(input),
     };
   }
   if (result.status === 0) {
@@ -217,14 +234,25 @@ export function probeSandboxInferenceInvocation(
       ok: false,
       detail: "sandbox inference invocation probe returned an invalid response body",
       httpStatus,
+      endpoint: resolveSandboxInferenceInvocationEndpoint(input),
     };
   }
   const httpStatus = result.stdout.match(/(?:^|\n)([1-5]\d\d)(?:\n|$)/)?.[1];
+  // Only the fixed marker is read back, never the line that carried it, so an
+  // upstream body can still not reach diagnostics (#6195).
+  const nvcfFunctionNotFound = result.stdout
+    .split("\n")
+    .some((line) => line.trim() === NVCF_FUNCTION_NOT_FOUND_MARKER);
+  const detail = httpStatus
+    ? `sandbox inference invocation probe returned HTTP ${httpStatus}`
+    : `sandbox inference invocation probe exited with status ${result.status}`;
   return {
     ok: false,
-    detail: httpStatus
-      ? `sandbox inference invocation probe returned HTTP ${httpStatus}`
-      : `sandbox inference invocation probe exited with status ${result.status}`,
+    detail:
+      httpStatus === "404" && nvcfFunctionNotFound
+        ? `${detail}: ${nvcfFunctionNotFoundMessage(input.model).replace(/\.$/, "")}`
+        : detail,
     httpStatus: httpStatus ? Number.parseInt(httpStatus, 10) : null,
+    endpoint: resolveSandboxInferenceInvocationEndpoint(input),
   };
 }