From c108c07b143fe3862721b43e3407a5aff54f63c2 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 15 May 2026 12:46:13 -0400 Subject: [PATCH 1/6] fix(onboard): smoke test inference before success Add a post-route inference smoke probe so onboard fails before reporting success when the configured provider/model cannot serve chat completions. Fixes #3253 --- src/lib/onboard.ts | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 60c5f32ec0..28bd183ff4 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2041,6 +2041,46 @@ function isInferenceRouteReady(provider: string, model: string): boolean { return Boolean(live && live.provider === provider && live.model === model); } +function shouldSmokeOpenAiLikeOnboardRoute(provider: string): boolean { + if (provider === "nvidia-nim" || provider === "nvidia-router") return true; + return Object.values(REMOTE_PROVIDER_CONFIG).some( + (entry) => entry.providerName === provider && entry.providerType === "openai", + ); +} + +function verifyOnboardInferenceSmoke(options: { + provider: string; + model: string; + endpointUrl?: string | null; + credentialEnv?: string | null; +}): void { + if (!shouldSmokeOpenAiLikeOnboardRoute(options.provider)) return; + if (process.env.VITEST === "true") return; + + const endpointUrl = options.endpointUrl || INFERENCE_ROUTE_URL; + const credentialEnv = options.credentialEnv || null; + const apiKey = credentialEnv + ? hydrateCredentialEnv(credentialEnv) || getCredential(credentialEnv) || "" + : ""; + const probe = probeOpenAiLikeEndpoint(endpointUrl, options.model, apiKey, { + authMode: getProbeAuthMode(options.provider), + skipResponsesProbe: true, + }); + + if (probe.ok) { + console.log(` ✓ Inference smoke passed: ${options.provider} / ${options.model}`); + return; + } + + console.error(" Onboard inference smoke check failed."); + console.error(` Provider: ${options.provider}`); + console.error(` Model: ${options.model}`); + console.error(` API base: ${endpointUrl}`); + if (credentialEnv) console.error(` Credential env: ${credentialEnv}`); + console.error(` Upstream error: ${compactText(redact(probe.message || "unknown inference failure"))}`); + process.exit(1); +} + function verifyCompatibleEndpointSandboxSmoke(options: { sandboxName: string; provider: string; @@ -7708,6 +7748,7 @@ async function setupInference( } verifyInferenceRoute(provider, model); + verifyOnboardInferenceSmoke({ provider, model, endpointUrl, credentialEnv }); if (sandboxName) { registry.updateSandbox(sandboxName, { model, provider }); } @@ -7983,6 +8024,7 @@ async function setupInference( } verifyInferenceRoute(provider, model); + verifyOnboardInferenceSmoke({ provider, model, endpointUrl, credentialEnv }); if (sandboxName) { registry.updateSandbox(sandboxName, { model, provider }); } @@ -10878,4 +10920,6 @@ module.exports = { checkTelegramReachability, TELEGRAM_NETWORK_CURL_CODES, verifyCompatibleEndpointSandboxSmoke, + verifyOnboardInferenceSmoke, + shouldSmokeOpenAiLikeOnboardRoute, }; From 57769e4001a23a02de87249d224015a5bb77bcd3 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 15 May 2026 13:11:21 -0400 Subject: [PATCH 2/6] fix(onboard): extract inference smoke helper --- src/lib/onboard.ts | 36 +++------------------ src/lib/onboard/inference-smoke.ts | 51 ++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 31 deletions(-) create mode 100644 src/lib/onboard/inference-smoke.ts diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 28bd183ff4..70f210ffd1 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2041,12 +2041,10 @@ function isInferenceRouteReady(provider: string, model: string): boolean { return Boolean(live && live.provider === provider && live.model === model); } -function shouldSmokeOpenAiLikeOnboardRoute(provider: string): boolean { - if (provider === "nvidia-nim" || provider === "nvidia-router") return true; - return Object.values(REMOTE_PROVIDER_CONFIG).some( - (entry) => entry.providerName === provider && entry.providerType === "openai", - ); -} +const { + verifyOnboardInferenceSmoke: verifyOnboardInferenceSmokeWithDeps, + shouldSmokeOpenAiLikeOnboardRoute, +} = require("./onboard/inference-smoke"); function verifyOnboardInferenceSmoke(options: { provider: string; @@ -2054,31 +2052,7 @@ function verifyOnboardInferenceSmoke(options: { endpointUrl?: string | null; credentialEnv?: string | null; }): void { - if (!shouldSmokeOpenAiLikeOnboardRoute(options.provider)) return; - if (process.env.VITEST === "true") return; - - const endpointUrl = options.endpointUrl || INFERENCE_ROUTE_URL; - const credentialEnv = options.credentialEnv || null; - const apiKey = credentialEnv - ? hydrateCredentialEnv(credentialEnv) || getCredential(credentialEnv) || "" - : ""; - const probe = probeOpenAiLikeEndpoint(endpointUrl, options.model, apiKey, { - authMode: getProbeAuthMode(options.provider), - skipResponsesProbe: true, - }); - - if (probe.ok) { - console.log(` ✓ Inference smoke passed: ${options.provider} / ${options.model}`); - return; - } - - console.error(" Onboard inference smoke check failed."); - console.error(` Provider: ${options.provider}`); - console.error(` Model: ${options.model}`); - console.error(` API base: ${endpointUrl}`); - if (credentialEnv) console.error(` Credential env: ${credentialEnv}`); - console.error(` Upstream error: ${compactText(redact(probe.message || "unknown inference failure"))}`); - process.exit(1); + verifyOnboardInferenceSmokeWithDeps(options, { hydrateCredentialEnv }); } function verifyCompatibleEndpointSandboxSmoke(options: { diff --git a/src/lib/onboard/inference-smoke.ts b/src/lib/onboard/inference-smoke.ts new file mode 100644 index 0000000000..f07d936598 --- /dev/null +++ b/src/lib/onboard/inference-smoke.ts @@ -0,0 +1,51 @@ +// @ts-nocheck +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const { getCredential } = require("../credentials/store"); +const { compactText } = require("../core/url-utils"); +const { INFERENCE_ROUTE_URL } = require("../inference/config"); +const { getProbeAuthMode, probeOpenAiLikeEndpoint } = require("../inference/onboard-probes"); +const { redact } = require("../runner"); +const { REMOTE_PROVIDER_CONFIG } = require("./providers"); + +function shouldSmokeOpenAiLikeOnboardRoute(provider) { + if (provider === "nvidia-nim" || provider === "nvidia-router") return true; + return Object.values(REMOTE_PROVIDER_CONFIG).some( + (entry) => entry.providerName === provider && entry.providerType === "openai", + ); +} + +function verifyOnboardInferenceSmoke(options, deps = {}) { + if (!shouldSmokeOpenAiLikeOnboardRoute(options.provider)) return; + if (process.env.VITEST === "true") return; + + const hydrateCredentialEnv = deps.hydrateCredentialEnv || (() => null); + const endpointUrl = options.endpointUrl || INFERENCE_ROUTE_URL; + const credentialEnv = options.credentialEnv || null; + const apiKey = credentialEnv + ? hydrateCredentialEnv(credentialEnv) || getCredential(credentialEnv) || "" + : ""; + const probe = probeOpenAiLikeEndpoint(endpointUrl, options.model, apiKey, { + authMode: getProbeAuthMode(options.provider), + skipResponsesProbe: true, + }); + + if (probe.ok) { + console.log(` ✓ Inference smoke passed: ${options.provider} / ${options.model}`); + return; + } + + console.error(" Onboard inference smoke check failed."); + console.error(` Provider: ${options.provider}`); + console.error(` Model: ${options.model}`); + console.error(` API base: ${endpointUrl}`); + if (credentialEnv) console.error(` Credential env: ${credentialEnv}`); + console.error(` Upstream error: ${compactText(redact(probe.message || "unknown inference failure"))}`); + process.exit(1); +} + +module.exports = { + shouldSmokeOpenAiLikeOnboardRoute, + verifyOnboardInferenceSmoke, +}; From b91f668457ff1a7d769c4e2f33a152a7128d7a53 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 15 May 2026 13:15:25 -0400 Subject: [PATCH 3/6] fix(onboard): keep inference smoke out of entrypoint --- src/lib/onboard.ts | 18 +++--------------- src/lib/onboard/inference-smoke.ts | 3 ++- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 70f210ffd1..48787a2bea 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -1498,6 +1498,8 @@ function hydrateCredentialEnv(envName: string | null | undefined): string | null return resolveProviderCredential(envName); } +globalThis.__nemoclawHydrateCredentialEnv = hydrateCredentialEnv; + const { getCurlTimingArgs, summarizeCurlFailure, @@ -2041,19 +2043,6 @@ function isInferenceRouteReady(provider: string, model: string): boolean { return Boolean(live && live.provider === provider && live.model === model); } -const { - verifyOnboardInferenceSmoke: verifyOnboardInferenceSmokeWithDeps, - shouldSmokeOpenAiLikeOnboardRoute, -} = require("./onboard/inference-smoke"); - -function verifyOnboardInferenceSmoke(options: { - provider: string; - model: string; - endpointUrl?: string | null; - credentialEnv?: string | null; -}): void { - verifyOnboardInferenceSmokeWithDeps(options, { hydrateCredentialEnv }); -} function verifyCompatibleEndpointSandboxSmoke(options: { sandboxName: string; @@ -10874,6 +10863,7 @@ module.exports = { hasResponsesToolCall, hasChatCompletionsToolCall, hasChatCompletionsToolCallLeak, + verifyOnboardInferenceSmoke, upsertProvider, normalizeHermesAuthMethod, hashCredential, @@ -10894,6 +10884,4 @@ module.exports = { checkTelegramReachability, TELEGRAM_NETWORK_CURL_CODES, verifyCompatibleEndpointSandboxSmoke, - verifyOnboardInferenceSmoke, - shouldSmokeOpenAiLikeOnboardRoute, }; diff --git a/src/lib/onboard/inference-smoke.ts b/src/lib/onboard/inference-smoke.ts index f07d936598..e2be40558a 100644 --- a/src/lib/onboard/inference-smoke.ts +++ b/src/lib/onboard/inference-smoke.ts @@ -20,7 +20,8 @@ function verifyOnboardInferenceSmoke(options, deps = {}) { if (!shouldSmokeOpenAiLikeOnboardRoute(options.provider)) return; if (process.env.VITEST === "true") return; - const hydrateCredentialEnv = deps.hydrateCredentialEnv || (() => null); + const hydrateCredentialEnv = + deps.hydrateCredentialEnv || globalThis.__nemoclawHydrateCredentialEnv || (() => null); const endpointUrl = options.endpointUrl || INFERENCE_ROUTE_URL; const credentialEnv = options.credentialEnv || null; const apiKey = credentialEnv From 6ee069f873b6a0425c38d227c09ae29de090a8b4 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 15 May 2026 13:16:17 -0400 Subject: [PATCH 4/6] fix(onboard): move smoke probe into inference probes --- src/lib/inference/onboard-probes.ts | 43 +++++++++++++++++++++++- src/lib/onboard.ts | 4 +-- src/lib/onboard/inference-smoke.ts | 52 ----------------------------- 3 files changed, 43 insertions(+), 56 deletions(-) delete mode 100644 src/lib/onboard/inference-smoke.ts diff --git a/src/lib/inference/onboard-probes.ts b/src/lib/inference/onboard-probes.ts index 38e7fa9dde..37f07c94cc 100644 --- a/src/lib/inference/onboard-probes.ts +++ b/src/lib/inference/onboard-probes.ts @@ -5,7 +5,7 @@ // Inference endpoint probes — validate that a provider's API responds // before committing the onboard wizard to a model selection. -const { normalizeCredentialValue } = require("../credentials/store"); +const { getCredential, normalizeCredentialValue, resolveProviderCredential } = require("../credentials/store"); const { isWsl } = require("../platform"); const httpProbe = require("../adapters/http/probe"); const { @@ -804,3 +804,44 @@ module.exports = { probeAnthropicEndpoint, RETRIABLE_HTTP_PROBE_STATUSES, }; + +function shouldSmokeOpenAiLikeOnboardRoute(provider) { + const { REMOTE_PROVIDER_CONFIG } = require("../onboard/providers"); + if (provider === "nvidia-nim" || provider === "nvidia-router") return true; + return Object.values(REMOTE_PROVIDER_CONFIG).some( + (entry) => entry.providerName === provider && entry.providerType === "openai", + ); +} + +function verifyOnboardInferenceSmoke(options) { + if (!shouldSmokeOpenAiLikeOnboardRoute(options.provider)) return; + if (process.env.VITEST === "true") return; + + const endpointUrl = options.endpointUrl || require("./config").INFERENCE_ROUTE_URL; + const credentialEnv = options.credentialEnv || null; + const apiKey = credentialEnv + ? resolveProviderCredential(credentialEnv) || getCredential(credentialEnv) || "" + : ""; + const probe = probeOpenAiLikeEndpoint(endpointUrl, options.model, apiKey, { + authMode: getProbeAuthMode(options.provider), + skipResponsesProbe: true, + }); + + if (probe.ok) { + console.log(` ✓ Inference smoke passed: ${options.provider} / ${options.model}`); + return; + } + + const { compactText } = require("../core/url-utils"); + const { redact } = require("../runner"); + console.error(" Onboard inference smoke check failed."); + console.error(` Provider: ${options.provider}`); + console.error(` Model: ${options.model}`); + console.error(` API base: ${endpointUrl}`); + if (credentialEnv) console.error(` Credential env: ${credentialEnv}`); + console.error(` Upstream error: ${compactText(redact(probe.message || "unknown inference failure"))}`); + process.exit(1); +} + +module.exports.shouldSmokeOpenAiLikeOnboardRoute = shouldSmokeOpenAiLikeOnboardRoute; +module.exports.verifyOnboardInferenceSmoke = verifyOnboardInferenceSmoke; diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 48787a2bea..16ae5dc94e 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -1498,8 +1498,6 @@ function hydrateCredentialEnv(envName: string | null | undefined): string | null return resolveProviderCredential(envName); } -globalThis.__nemoclawHydrateCredentialEnv = hydrateCredentialEnv; - const { getCurlTimingArgs, summarizeCurlFailure, @@ -2370,6 +2368,7 @@ const { hasChatCompletionsToolCall, hasChatCompletionsToolCallLeak, shouldRequireResponsesToolCalling, + verifyOnboardInferenceSmoke, getProbeAuthMode, getValidationProbeCurlArgs, probeOpenAiLikeEndpoint, @@ -10863,7 +10862,6 @@ module.exports = { hasResponsesToolCall, hasChatCompletionsToolCall, hasChatCompletionsToolCallLeak, - verifyOnboardInferenceSmoke, upsertProvider, normalizeHermesAuthMethod, hashCredential, diff --git a/src/lib/onboard/inference-smoke.ts b/src/lib/onboard/inference-smoke.ts deleted file mode 100644 index e2be40558a..0000000000 --- a/src/lib/onboard/inference-smoke.ts +++ /dev/null @@ -1,52 +0,0 @@ -// @ts-nocheck -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -const { getCredential } = require("../credentials/store"); -const { compactText } = require("../core/url-utils"); -const { INFERENCE_ROUTE_URL } = require("../inference/config"); -const { getProbeAuthMode, probeOpenAiLikeEndpoint } = require("../inference/onboard-probes"); -const { redact } = require("../runner"); -const { REMOTE_PROVIDER_CONFIG } = require("./providers"); - -function shouldSmokeOpenAiLikeOnboardRoute(provider) { - if (provider === "nvidia-nim" || provider === "nvidia-router") return true; - return Object.values(REMOTE_PROVIDER_CONFIG).some( - (entry) => entry.providerName === provider && entry.providerType === "openai", - ); -} - -function verifyOnboardInferenceSmoke(options, deps = {}) { - if (!shouldSmokeOpenAiLikeOnboardRoute(options.provider)) return; - if (process.env.VITEST === "true") return; - - const hydrateCredentialEnv = - deps.hydrateCredentialEnv || globalThis.__nemoclawHydrateCredentialEnv || (() => null); - const endpointUrl = options.endpointUrl || INFERENCE_ROUTE_URL; - const credentialEnv = options.credentialEnv || null; - const apiKey = credentialEnv - ? hydrateCredentialEnv(credentialEnv) || getCredential(credentialEnv) || "" - : ""; - const probe = probeOpenAiLikeEndpoint(endpointUrl, options.model, apiKey, { - authMode: getProbeAuthMode(options.provider), - skipResponsesProbe: true, - }); - - if (probe.ok) { - console.log(` ✓ Inference smoke passed: ${options.provider} / ${options.model}`); - return; - } - - console.error(" Onboard inference smoke check failed."); - console.error(` Provider: ${options.provider}`); - console.error(` Model: ${options.model}`); - console.error(` API base: ${endpointUrl}`); - if (credentialEnv) console.error(` Credential env: ${credentialEnv}`); - console.error(` Upstream error: ${compactText(redact(probe.message || "unknown inference failure"))}`); - process.exit(1); -} - -module.exports = { - shouldSmokeOpenAiLikeOnboardRoute, - verifyOnboardInferenceSmoke, -}; From 4c3bd0ff1ebe9b624d137ec015e603fe4c63aa39 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 15 May 2026 13:20:17 -0400 Subject: [PATCH 5/6] fix(onboard): keep entrypoint budget neutral --- src/lib/onboard.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 16ae5dc94e..f4584f1266 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -2041,7 +2041,6 @@ function isInferenceRouteReady(provider: string, model: string): boolean { return Boolean(live && live.provider === provider && live.model === model); } - function verifyCompatibleEndpointSandboxSmoke(options: { sandboxName: string; provider: string; @@ -2375,10 +2374,6 @@ const { probeAnthropicEndpoint, } = require("./inference/onboard-probes"); -// shouldSkipResponsesProbe and isNvcfFunctionNotFoundForAccount / -// nvcfFunctionNotFoundMessage — see validation import above. They live in -// src/lib/validation.ts so they can be unit-tested independently. - async function validateOpenAiLikeSelection( label: string, endpointUrl: string, From 68d9be996f5630a64bf1f8e1078a9a1d69448960 Mon Sep 17 00:00:00 2001 From: Julie Yaunches Date: Fri, 15 May 2026 13:36:03 -0400 Subject: [PATCH 6/6] fix(onboard): avoid logging credential env name --- src/lib/inference/onboard-probes.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/inference/onboard-probes.ts b/src/lib/inference/onboard-probes.ts index 37f07c94cc..9276ee7f21 100644 --- a/src/lib/inference/onboard-probes.ts +++ b/src/lib/inference/onboard-probes.ts @@ -838,7 +838,7 @@ function verifyOnboardInferenceSmoke(options) { console.error(` Provider: ${options.provider}`); console.error(` Model: ${options.model}`); console.error(` API base: ${endpointUrl}`); - if (credentialEnv) console.error(` Credential env: ${credentialEnv}`); + if (credentialEnv) console.error(" Credential env: configured"); console.error(` Upstream error: ${compactText(redact(probe.message || "unknown inference failure"))}`); process.exit(1); }