From 17765d1b126034656af61e728c4147ec7bc562a4 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 14 Jun 2026 09:37:47 -0700 Subject: [PATCH 1/2] fix(onboard): summarize inference validation failures Signed-off-by: Carlos Villela --- .../onboard/inference-selection-validation.ts | 17 +++--- src/lib/onboard/probe-diagnostics.test.ts | 53 +++++++++++++++++++ src/lib/onboard/probe-diagnostics.ts | 30 +++++++++++ 3 files changed, 94 insertions(+), 6 deletions(-) create mode 100644 src/lib/onboard/probe-diagnostics.test.ts create mode 100644 src/lib/onboard/probe-diagnostics.ts diff --git a/src/lib/onboard/inference-selection-validation.ts b/src/lib/onboard/inference-selection-validation.ts index 424049951af..e97373285fe 100644 --- a/src/lib/onboard/inference-selection-validation.ts +++ b/src/lib/onboard/inference-selection-validation.ts @@ -20,6 +20,7 @@ const { probeAnthropicEndpoint, probeOpenAiLikeEndpoint } = import { shouldForceCompletionsApi } from "../validation"; import { getProbeRecovery } from "../validation-recovery"; +import { summarizeProbeForDisplay } from "./probe-diagnostics"; export type EndpointValidationResult = | { ok: true; api: string | null; retry?: undefined } @@ -80,9 +81,13 @@ export interface InferenceSelectionValidationHelpers { export function createInferenceSelectionValidationHelpers( deps: InferenceSelectionValidationDeps, ): InferenceSelectionValidationHelpers { - function printValidationFailure(label: string): void { + function printValidationFailure( + label: string, + probe?: { failures?: unknown[]; message?: unknown }, + ): void { console.error(` ${label} endpoint validation failed.`); - console.error(" Validation details were omitted to avoid exposing credentials."); + if (probe) console.error(` Validation probe summary: ${summarizeProbeForDisplay(probe)}.`); + console.error(" Detailed response bodies were omitted to avoid exposing credentials."); } async function validateOpenAiLikeSelection( @@ -104,7 +109,7 @@ export function createInferenceSelectionValidationHelpers( const apiKey = credentialEnv ? getCredential(credentialEnv) : ""; const probe = probeOpenAiLikeEndpoint(endpointUrl, model, apiKey, options); if (!probe.ok) { - printValidationFailure(label); + printValidationFailure(label, probe); if (deps.isNonInteractive()) { process.exit(1); } @@ -139,7 +144,7 @@ export function createInferenceSelectionValidationHelpers( const apiKey = getCredential(credentialEnv); const probe = probeAnthropicEndpoint(endpointUrl, model, apiKey); if (!probe.ok) { - printValidationFailure(label); + printValidationFailure(label, probe); if (deps.isNonInteractive()) { process.exit(1); } @@ -182,7 +187,7 @@ export function createInferenceSelectionValidationHelpers( } return { ok: true, api: probe.api ?? "openai-completions" }; } - printValidationFailure(label); + printValidationFailure(label, probe); if (deps.isNonInteractive()) { process.exit(1); } @@ -212,7 +217,7 @@ export function createInferenceSelectionValidationHelpers( console.log(` ${probe.label} available — ${deps.agentProductName()} will use ${probe.api}.`); return { ok: true, api: probe.api }; } - printValidationFailure(label); + printValidationFailure(label, probe); if (deps.isNonInteractive()) { process.exit(1); } diff --git a/src/lib/onboard/probe-diagnostics.test.ts b/src/lib/onboard/probe-diagnostics.test.ts new file mode 100644 index 00000000000..974884afc93 --- /dev/null +++ b/src/lib/onboard/probe-diagnostics.test.ts @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { summarizeProbeForDisplay } from "./probe-diagnostics"; + +describe("summarizeProbeForDisplay", () => { + it("summarizes HTTP statuses without raw response bodies", () => { + const summary = summarizeProbeForDisplay({ + message: "Chat Completions API: HTTP 429: raw provider body with secret-key", + failures: [ + { + name: "Chat Completions API", + httpStatus: 429, + curlStatus: 0, + message: "HTTP 429: raw provider body with secret-key", + body: "raw provider body with secret-key", + }, + ], + }); + + expect(summary).toBe("Chat Completions API: HTTP 429"); + expect(summary).not.toContain("secret-key"); + expect(summary).not.toContain("raw provider body"); + }); + + it("summarizes curl/timeout failures without raw stderr", () => { + const summary = summarizeProbeForDisplay({ + message: "curl failed (exit 28): operation timed out with token secret-key", + failures: [ + { + name: "Chat Completions API", + httpStatus: 0, + curlStatus: 28, + message: "curl failed (exit 28): operation timed out with token secret-key", + }, + ], + }); + + expect(summary).toBe("Chat Completions API: curl exit 28"); + expect(summary).not.toContain("secret-key"); + expect(summary).not.toContain("operation timed out with token"); + }); + + it("falls back to coarse message classification", () => { + expect(summarizeProbeForDisplay({ message: "HTTP 404: not found for secret-key" })).toBe( + "HTTP 404", + ); + expect(summarizeProbeForDisplay({ message: "request timed out with secret-key" })).toBe( + "timeout", + ); + }); +}); diff --git a/src/lib/onboard/probe-diagnostics.ts b/src/lib/onboard/probe-diagnostics.ts new file mode 100644 index 00000000000..6759f266221 --- /dev/null +++ b/src/lib/onboard/probe-diagnostics.ts @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +function summarizeProbeFailureForDisplay(failure: Record): string { + const name = typeof failure.name === "string" ? failure.name : "probe"; + const httpStatus = typeof failure.httpStatus === "number" ? failure.httpStatus : 0; + const curlStatus = typeof failure.curlStatus === "number" ? failure.curlStatus : 0; + if (httpStatus > 0) return `${name}: HTTP ${httpStatus}`; + if (curlStatus !== 0) return `${name}: curl exit ${curlStatus}`; + return `${name}: no HTTP response`; +} + +export function summarizeProbeForDisplay(probe: { + failures?: unknown[]; + message?: unknown; +}): string { + const failures = Array.isArray(probe.failures) + ? probe.failures.filter((failure): failure is Record => { + return Boolean(failure) && typeof failure === "object"; + }) + : []; + if (failures.length > 0) return failures.map(summarizeProbeFailureForDisplay).join("; "); + const message = typeof probe.message === "string" ? probe.message : "no probe details available"; + const httpMatch = message.match(/\bHTTP\s+(\d{3})\b/i); + if (httpMatch) return `HTTP ${httpMatch[1]}`; + const curlMatch = message.match(/curl failed \(exit (-?\d+)\)/i); + if (curlMatch) return `curl exit ${curlMatch[1]}`; + if (/timed? out|timeout/i.test(message)) return "timeout"; + return "probe failed"; +} From f002451b0163feaad0b2c58d17bbf0c7c0ee0444 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sun, 14 Jun 2026 09:47:20 -0700 Subject: [PATCH 2/2] Update src/lib/onboard/inference-selection-validation.ts Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- src/lib/onboard/inference-selection-validation.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/onboard/inference-selection-validation.ts b/src/lib/onboard/inference-selection-validation.ts index e97373285fe..4775eaac767 100644 --- a/src/lib/onboard/inference-selection-validation.ts +++ b/src/lib/onboard/inference-selection-validation.ts @@ -87,7 +87,7 @@ export function createInferenceSelectionValidationHelpers( ): void { console.error(` ${label} endpoint validation failed.`); if (probe) console.error(` Validation probe summary: ${summarizeProbeForDisplay(probe)}.`); - console.error(" Detailed response bodies were omitted to avoid exposing credentials."); + console.error(" Validation details were omitted to avoid exposing credentials."); } async function validateOpenAiLikeSelection(