fix(status): stop attributing an upstream credential rejection to the sandbox route

docs/reference/commands.mdx

@@ -2025,6 +2025,15 @@ It uses a 3-second connection timeout, a 5-second total timeout, and an 8-token
 If the request reaches the time limit, NemoClaw reports the provider as `not probed` and leaves model health unverified instead of reporting it as unhealthy.
 These checks are diagnostic only and do not override the authoritative `inference.local` result or determine the command exit status.
 
+The `Inference (upstream)` check authenticates with the host credential that NemoClaw resolves for the provider, such as `NVIDIA_INFERENCE_API_KEY`.
+The gateway stores the provider credential that the sandbox route uses.
+The CLI cannot read the stored value back, so the two credentials can hold different secrets.
+When the `inference.local` route has already served the inference request, an `unauthorized` result on `Inference (upstream)` describes the host credential.
+NemoClaw then reports that check as `not probed` and names both credential sources.
+An `Inference (upstream)` check that fails for another reason, such as `unreachable`, still reports its own state.
+Local backend and auth proxy checks, such as `Inference (auth proxy)`, always report their own state and their own repair step.
+`$$nemoclaw <name> doctor` sends no inference request, so it always reports the `Inference (upstream)` state that it measured.
+
 Local providers add host-side backend diagnostics.
 For Local Ollama, the command can also print an `Inference (auth proxy)` diagnostic when a proxy token is available.
 Use these diagnostics to identify a failing auxiliary hop after checking the main `Inference` line.

src/lib/actions/sandbox/inference-route-health.ts

@@ -102,13 +102,43 @@ export async function probeSandboxInferenceGatewayHealth(
   };
 }
 
+/**
+ * The upstream probe authenticates with the host credential this command
+ * resolves. The gateway stores the provider credential the sandbox route uses
+ * and does not return its value, so the two can hold different secrets. Once
+ * the route has served an inference request, a provider rejection of the host
+ * credential reports nothing about the sandbox route. Local backend and auth
+ * proxy hops carry their own probeLabel and keep their own remediation.
+ */
+function unattributedUpstreamProbe(probe: ProviderHealthStatus): ProviderHealthStatus {
+  const { failureLabel: _failureLabel, ...rest } = probe;
+  return {
+    ...rest,
+    ok: true,
+    probed: false,
+    detail:
+      `${probe.detail} The sandbox ` +
+      "route served an inference request with the provider credential stored in the gateway, so " +
+      "NemoClaw does not attribute this result to the sandbox route.",
+  };
+}
+
 function providerHealthDiagnostics(
   providerHealth: ProviderHealthStatus | null,
+  routeServedRequest: boolean,
 ): ProviderHealthStatus[] {
   if (!providerHealth) return [];
   const { subprobes = [], ...primary } = providerHealth;
   const labeledPrimary = primary.probeLabel ? primary : { ...primary, probeLabel: "upstream" };
-  return [labeledPrimary, ...subprobes];
+  return [labeledPrimary, ...subprobes].map((probe) =>
+    routeServedRequest &&
+    probe.probeLabel === "upstream" &&
+    probe.probed &&
+    !probe.ok &&
+    probe.failureLabel === "unauthorized"
+      ? unattributedUpstreamProbe(probe)
+      : probe,
+  );
 }
 
 function classifyInferenceInvocationFailureLabel(
@@ -176,7 +206,7 @@ export function buildSandboxInferenceRouteHealth(
   invocation: SandboxInferenceInvocationResult | null,
 ): ProviderHealthStatus {
   const endpoint = gateway?.endpoint ?? "https://inference.local/v1/models";
-  const diagnostics = providerHealthDiagnostics(providerHealth);
+  const diagnostics = providerHealthDiagnostics(providerHealth, Boolean(invocation?.ok));
   let routeHealth: ProviderHealthStatus;
   if (gateway?.ok && invocation) {
     routeHealth = buildInvokedRouteHealth(gateway, endpoint, invocation);

src/lib/actions/sandbox/status-snapshot-inference-health.test.ts

@@ -371,7 +371,7 @@ describe("collectSandboxStatusSnapshot inference route health", () => {
     expect(snapshot.inferenceHealth?.failureLabel).toBeUndefined();
   });
 
-  it("keeps an upstream subprobe failure out of the served-route verdict (#6846)", async () => {
+  it("keeps an unreachable upstream subprobe failure out of the served-route verdict (#6846)", async () => {
     const gateway: SandboxInferenceRouteHealth = {
       ok: true,
       endpoint: "https://inference.local/v1/models",
@@ -385,7 +385,7 @@ describe("collectSandboxStatusSnapshot inference route health", () => {
       providerLabel: "NVIDIA Endpoints",
       endpoint: "https://integrate.api.nvidia.com/v1/chat/completions",
       detail: "model invocation probe failed",
-      failureLabel: "unauthorized",
+      failureLabel: "unreachable",
     };
 
     const snapshot = await collectSandboxStatusSnapshot(
@@ -430,6 +430,140 @@ describe("collectSandboxStatusSnapshot inference route health", () => {
     );
   });
 
+  it("reports the upstream check as not probed after the provider rejected the host credential and the route served a request (#9595)", async () => {
+    const gateway: SandboxInferenceRouteHealth = {
+      ok: true,
+      endpoint: "https://inference.local/v1/models",
+      httpStatus: 200,
+      detail:
+        "Inference gateway responded HTTP 200 on https://inference.local/v1/models (full chain reachable).",
+    };
+    const providerHealth: ProviderHealthStatus = {
+      ok: false,
+      probed: true,
+      providerLabel: "NVIDIA Endpoints",
+      endpoint: "https://integrate.api.nvidia.com/v1/chat/completions",
+      detail:
+        "NVIDIA Endpoints rejected the host credential in NVIDIA_INFERENCE_API_KEY. " +
+        "Check NVIDIA_INFERENCE_API_KEY where you run this command.",
+      failureLabel: "unauthorized",
+    };
+
+    const snapshot = await collectSandboxStatusSnapshot(
+      "alpha",
+      snapshotDeps(gateway, providerHealth),
+    );
+
+    expect(snapshot.inferenceHealth).toMatchObject({ ok: true });
+    const upstream = snapshot.inferenceHealth?.subprobes?.find(
+      (subprobe) => subprobe.probeLabel === "upstream",
+    );
+    expect(upstream).toMatchObject({ ok: true, probed: false });
+    expect(upstream?.failureLabel).toBeUndefined();
+    expect(upstream?.detail).toContain("rejected the host credential");
+    expect(upstream?.detail).toContain("NVIDIA_INFERENCE_API_KEY");
+    expect(upstream?.detail).toContain("provider credential stored in the gateway");
+    expect(upstream?.detail).toContain("does not attribute this result to the sandbox route");
+  });
+
+  it("keeps an unreachable upstream check after the route served a request (#9595)", async () => {
+    const gateway: SandboxInferenceRouteHealth = {
+      ok: true,
+      endpoint: "https://inference.local/v1/models",
+      httpStatus: 200,
+      detail:
+        "Inference gateway responded HTTP 200 on https://inference.local/v1/models (full chain reachable).",
+    };
+    const providerHealth: ProviderHealthStatus = {
+      ok: false,
+      probed: true,
+      providerLabel: "NVIDIA Endpoints",
+      endpoint: "https://integrate.api.nvidia.com/v1/chat/completions",
+      detail: "model invocation probe could not reach the endpoint",
+      failureLabel: "unreachable",
+    };
+
+    const snapshot = await collectSandboxStatusSnapshot(
+      "alpha",
+      snapshotDeps(gateway, providerHealth),
+    );
+
+    expect(snapshot.inferenceHealth?.subprobes).toContainEqual({
+      ...providerHealth,
+      probeLabel: "upstream",
+    });
+  });
+
+  it("keeps an unauthorized auth proxy subprobe after the route served a request (#9595)", async () => {
+    const gateway: SandboxInferenceRouteHealth = {
+      ok: true,
+      endpoint: "https://inference.local/v1/models",
+      httpStatus: 200,
+      detail:
+        "Inference gateway responded HTTP 200 on https://inference.local/v1/models (full chain reachable).",
+    };
+    const authProxy: ProviderHealthStatus = {
+      ok: false,
+      probed: true,
+      providerLabel: "Local Ollama",
+      probeLabel: "auth proxy",
+      endpoint: "http://127.0.0.1:11435/api/tags",
+      detail:
+        "Ollama auth proxy returned 401 — the persisted token is no longer accepted. " +
+        "Re-run `nemoclaw onboard` (Ollama path) to rotate the proxy token.",
+      failureLabel: "unauthorized",
+    };
+    const providerHealth: ProviderHealthStatus = {
+      ok: true,
+      probed: true,
+      providerLabel: "Local Ollama",
+      probeLabel: "ollama backend",
+      endpoint: "http://127.0.0.1:11434/api/tags",
+      detail: "Local Ollama is reachable.",
+      subprobes: [authProxy],
+    };
+
+    const snapshot = await collectSandboxStatusSnapshot(
+      "alpha",
+      snapshotDeps(gateway, providerHealth),
+    );
+
+    expect(snapshot.inferenceHealth?.subprobes).toContainEqual(authProxy);
+  });
+
+  it("keeps an unauthorized upstream check when the route did not serve a request (#9595)", async () => {
+    const gateway: SandboxInferenceRouteHealth = {
+      ok: true,
+      endpoint: "https://inference.local/v1/models",
+      httpStatus: 401,
+      detail:
+        "Inference gateway responded HTTP 401 on https://inference.local/v1/models (full chain reachable).",
+    };
+    const providerHealth: ProviderHealthStatus = {
+      ok: false,
+      probed: true,
+      providerLabel: "NVIDIA Endpoints",
+      endpoint: "https://integrate.api.nvidia.com/v1/chat/completions",
+      detail: "model invocation probe rejected the credential",
+      failureLabel: "unauthorized",
+    };
+
+    const snapshot = await collectSandboxStatusSnapshot(
+      "alpha",
+      snapshotDeps(gateway, providerHealth, {
+        ok: false,
+        detail: "sandbox inference invocation probe returned HTTP 401",
+        httpStatus: 401,
+      }),
+    );
+
+    expect(snapshot.inferenceHealth).toMatchObject({ ok: false, failureLabel: "unauthorized" });
+    expect(snapshot.inferenceHealth?.subprobes).toContainEqual({
+      ...providerHealth,
+      probeLabel: "upstream",
+    });
+  });
+
   it("does not send an agent request when the route probe already failed", async () => {
     const gateway: SandboxInferenceRouteHealth = {
       ok: false,

src/lib/inference/health.test.ts

@@ -444,6 +444,20 @@ describe("inference health", () => {
       expect(result?.failureLabel).toBe("unauthorized");
     });
 
+    it("names the host credential environment variable when the provider rejects the request (#9595)", () => {
+      const result = probeRemoteProviderHealth("nvidia-prod", {
+        model: "meta/llama-3.3-70b-instruct",
+        getCredentialImpl: () => "nvapi-stale",
+        runCurlProbeImpl: () => httpUnauthorized(),
+      });
+
+      expect(result?.failureLabel).toBe("unauthorized");
+      expect(result?.detail).toContain("rejected the");
+      expect(result?.detail).toContain("host credential in NVIDIA_INFERENCE_API_KEY");
+      expect(result?.detail).toContain("not the provider credential stored in the gateway");
+      expect(result?.detail).not.toContain("Check your network connection");
+    });
+
     it("reports unhealthy on a non-auth HTTP failure", () => {
       const result = probeRemoteProviderHealth("openai-api", {
         model: "gpt-4o-mini",

src/lib/inference/health.ts

@@ -420,6 +420,14 @@ function buildInvocationProbeDetail(
       `as proof that the model ran. (${result.message})`
     );
   }
+  if (classifyHealthProbeFailureLabel(result) === "unauthorized") {
+    return (
+      `${endpoint} rejected the ${route} request. ` +
+      `This probe authenticates with the host credential in ${credentialEnv}, not the provider ` +
+      `credential stored in the gateway. Check ${credentialEnv} where you run this command. ` +
+      `(${result.message})`
+    );
+  }
   return (
     `${route} at ${endpoint} did not succeed. ` +
     `Check your network connection or ${credentialEnv}. (${result.message})`