diff --git a/src/lib/inference/onboard-probes.ts b/src/lib/inference/onboard-probes.ts index 37d1797fc87..9ec1f76c022 100644 --- a/src/lib/inference/onboard-probes.ts +++ b/src/lib/inference/onboard-probes.ts @@ -1051,6 +1051,20 @@ export async function verifyOnboardInferenceSmoke(options: any) { if (process.env.VITEST === "true") return; const endpointUrl = options.endpointUrl || require("./config").INFERENCE_ROUTE_URL; + if ( + options.capabilityCache?.takeCompletedOpenAiChat({ + endpointUrl, + model: options.model, + authMode: getProbeAuthMode(options.provider), + extraHeaders: getProbeExtraHeaders(options.provider), + pinnedAddresses: options.pinnedAddresses, + }) + ) { + console.log( + ` ✓ Reusing selected Chat Completions validation: ${options.provider} / ${options.model}`, + ); + return; + } const credentialEnv = options.credentialEnv || null; const apiKey = credentialEnv ? resolveProviderCredential(credentialEnv) || getCredential(credentialEnv) || "" @@ -1067,6 +1081,8 @@ export async function verifyOnboardInferenceSmoke(options: any) { return; } + options.capabilityCache?.invalidate(); + const { compactText } = require("../core/url-utils"); const { redact } = require("../runner"); console.error(" Onboard inference smoke check failed."); diff --git a/src/lib/onboard/inference-capability-cache.test.ts b/src/lib/onboard/inference-capability-cache.test.ts new file mode 100644 index 00000000000..517d395caaf --- /dev/null +++ b/src/lib/onboard/inference-capability-cache.test.ts @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { OnboardInferenceCapabilityCache } from "./inference-capability-cache"; + +describe("OnboardInferenceCapabilityCache", () => { + it("reuses one selected Chat Completions validation for one matching smoke request", () => { + const cache = new OnboardInferenceCapabilityCache(); + const input = { + endpointUrl: "https://api.example.test/v1/", + model: "model-a", + authMode: "bearer" as const, + }; + + expect(cache.rememberCompletedOpenAiChat(input)).toBe(true); + expect( + cache.takeCompletedOpenAiChat({ ...input, endpointUrl: "https://api.example.test/v1" }), + ).toBe(true); + expect(cache.takeCompletedOpenAiChat(input)).toBe(false); + }); + + it("does not reuse mismatched or security-sensitive validation", () => { + const cache = new OnboardInferenceCapabilityCache(); + const input = { + endpointUrl: "https://api.example.test/v1", + model: "model-a", + authMode: "query-param" as const, + }; + + expect(cache.rememberCompletedOpenAiChat(input)).toBe(true); + expect(cache.takeCompletedOpenAiChat({ ...input, model: "model-b" })).toBe(false); + expect(cache.takeCompletedOpenAiChat({ ...input, authMode: "bearer" })).toBe(false); + expect( + cache.rememberCompletedOpenAiChat({ + ...input, + endpointUrl: "https://api.example.test/v1?key=x", + }), + ).toBe(false); + expect( + cache.rememberCompletedOpenAiChat({ ...input, pinnedAddresses: ["93.184.216.34"] }), + ).toBe(false); + + cache.invalidate(); + expect(cache.takeCompletedOpenAiChat(input)).toBe(false); + }); +}); diff --git a/src/lib/onboard/inference-capability-cache.ts b/src/lib/onboard/inference-capability-cache.ts new file mode 100644 index 00000000000..4dea2129f14 --- /dev/null +++ b/src/lib/onboard/inference-capability-cache.ts @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +type OpenAiChatCapabilityInput = { + endpointUrl: string; + model: string; + authMode?: "bearer" | "query-param"; + requireChatCompletionsToolCalling?: boolean; + extraHeaders?: readonly string[]; + pinnedAddresses?: readonly string[]; +}; + +function capabilityKey(input: OpenAiChatCapabilityInput): string | null { + // Query strings, embedded URL credentials, and custom headers can carry + // credentials. Do not retain either those values or a derived identifier. + if (input.extraHeaders?.length || input.pinnedAddresses?.length) return null; + + let endpoint: URL; + try { + endpoint = new URL(input.endpointUrl); + } catch { + return null; + } + if ( + (endpoint.protocol !== "http:" && endpoint.protocol !== "https:") || + endpoint.username || + endpoint.password || + endpoint.search || + endpoint.hash + ) { + return null; + } + + const model = input.model.trim(); + if (!model || model !== input.model) return null; + endpoint.pathname = endpoint.pathname.replace(/\/+$/, "") || "/"; + return JSON.stringify({ + endpoint: endpoint.toString(), + authMode: input.authMode ?? "bearer", + model, + requireChatCompletionsToolCalling: input.requireChatCompletionsToolCalling === true, + }); +} + +/** + * One onboarding invocation may validate a selected Chat Completions route and + * then immediately run the same host-side smoke check. This cache is strictly + * in-memory, one-shot, and refuses credential-bearing or pinned paths. + */ +export class OnboardInferenceCapabilityCache { + readonly #entries = new Set(); + + rememberCompletedOpenAiChat(input: OpenAiChatCapabilityInput): boolean { + const key = capabilityKey(input); + if (!key) return false; + this.#entries.add(key); + return true; + } + + takeCompletedOpenAiChat(input: OpenAiChatCapabilityInput): boolean { + const key = capabilityKey(input); + if (!key || !this.#entries.has(key)) return false; + this.#entries.delete(key); + return true; + } + + invalidate(): void { + this.#entries.clear(); + } +} diff --git a/src/lib/onboard/inference-providers/remote.ts b/src/lib/onboard/inference-providers/remote.ts index 0e9c4a71ea7..4c3b0d8e38a 100644 --- a/src/lib/onboard/inference-providers/remote.ts +++ b/src/lib/onboard/inference-providers/remote.ts @@ -129,6 +129,7 @@ export async function setupRemoteProviderInference( skipHostInferenceSmoke?: boolean; preferredInferenceApi?: string | null; pinnedAddresses?: readonly string[]; + capabilityCache?: import("../inference-capability-cache").OnboardInferenceCapabilityCache; }, deps: RemoteProviderDeps, ): Promise<{ done: true; result: SetupInferenceResult } | { done: false }> { @@ -142,6 +143,7 @@ export async function setupRemoteProviderInference( skipHostInferenceSmoke, preferredInferenceApi, pinnedAddresses, + capabilityCache, } = args; const { runOpenshell, @@ -321,6 +323,7 @@ export async function setupRemoteProviderInference( } } if (!providerResult.ok) { + capabilityCache?.invalidate(); error(` ${providerResult.message}`); if (isNonInteractive()) { return exitProcess(providerResult.status || 1); @@ -355,6 +358,7 @@ export async function setupRemoteProviderInference( const message = compactText(redact(`${applyResult.stderr || ""} ${applyResult.stdout || ""}`)) || `Failed to configure inference provider '${provider}'.`; + capabilityCache?.invalidate(); error(` ${message}`); if (isNonInteractive()) { return exitProcess(applyResult.status || 1); diff --git a/src/lib/onboard/inference-providers/types.ts b/src/lib/onboard/inference-providers/types.ts index 28fa3f8e509..a4465f311c4 100644 --- a/src/lib/onboard/inference-providers/types.ts +++ b/src/lib/onboard/inference-providers/types.ts @@ -16,6 +16,7 @@ // duplicate every helper's exact signature. import type { HermesAuthMethod } from "../hermes-auth"; +import type { OnboardInferenceCapabilityCache } from "../inference-capability-cache"; export type SetupInferenceResult = { ok: true; retry?: undefined } | { retry: "selection" }; @@ -66,6 +67,7 @@ export type VerifyOnboardInferenceSmoke = (input: { credentialEnv?: string | null; forceOpenAiLike?: boolean; pinnedAddresses?: readonly string[]; + capabilityCache?: OnboardInferenceCapabilityCache; }) => void | Promise; export type PromptValidationRecovery = ( diff --git a/src/lib/onboard/inference-selection-validation.test.ts b/src/lib/onboard/inference-selection-validation.test.ts index c7777507f91..c488463e863 100644 --- a/src/lib/onboard/inference-selection-validation.test.ts +++ b/src/lib/onboard/inference-selection-validation.test.ts @@ -6,9 +6,48 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; +import { OnboardInferenceCapabilityCache } from "./inference-capability-cache"; import { createInferenceSelectionValidationHelpers } from "./inference-selection-validation"; describe("inference selection validation", () => { + it("records a completed Chat Completions selection for the matching smoke check", async () => { + const capabilityCache = new OnboardInferenceCapabilityCache(); + const helpers = createInferenceSelectionValidationHelpers({ + isNonInteractive: () => false, + agentProductName: () => "OpenClaw", + getCredential: () => "test-key", + probeOpenAiLikeEndpoint: vi.fn(() => ({ + ok: true, + api: "openai-completions", + label: "Chat Completions API", + })), + promptValidationRecovery: vi.fn(async () => "selection" as const), + }); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + + try { + await expect( + helpers.validateOpenAiLikeSelection( + "OpenAI", + "https://api.example.test/v1/", + "model-a", + "OPENAI_API_KEY", + undefined, + undefined, + { capabilityCache }, + ), + ).resolves.toEqual({ ok: true, api: "openai-completions" }); + expect( + capabilityCache.takeCompletedOpenAiChat({ + endpointUrl: "https://api.example.test/v1", + model: "model-a", + }), + ).toBe(true); + } finally { + log.mockRestore(); + } + }); + it("preserves non-zero exit signaling when non-interactive endpoint validation fails (#5721)", async () => { const originalExitCode = process.exitCode; const error = vi.spyOn(console, "error").mockImplementation(() => {}); diff --git a/src/lib/onboard/inference-selection-validation.ts b/src/lib/onboard/inference-selection-validation.ts index 6badc85f4b4..2adeb7160c8 100644 --- a/src/lib/onboard/inference-selection-validation.ts +++ b/src/lib/onboard/inference-selection-validation.ts @@ -3,6 +3,7 @@ import { getCredential } from "../credentials/store"; import { getCompatibleAnthropicOpenAiSurfaceBaseUrl } from "../inference/config"; +import type { OnboardInferenceCapabilityCache } from "./inference-capability-cache"; const { probeAnthropicEndpoint, probeOpenAiLikeEndpointOptimized } = require("../inference/onboard-probes") as { @@ -78,6 +79,7 @@ export interface InferenceSelectionValidationHelpers { skipResponsesProbe?: boolean; probeStreaming?: boolean; allowHostDockerInternal?: boolean; + capabilityCache?: OnboardInferenceCapabilityCache; }, ): Promise; validateAnthropicSelectionWithRetryMessage( @@ -196,6 +198,7 @@ export function createInferenceSelectionValidationHelpers( skipResponsesProbe?: boolean; probeStreaming?: boolean; allowHostDockerInternal?: boolean; + capabilityCache?: OnboardInferenceCapabilityCache; } = {}, ): Promise { const apiKey = credentialEnv ? resolveCredential(credentialEnv) : ""; @@ -204,6 +207,7 @@ export function createInferenceSelectionValidationHelpers( calibrateTimeouts: true, }); if (!probe.ok) { + options.capabilityCache?.invalidate(); printValidationFailure(label, probe); if (deps.isNonInteractive()) { exitNonInteractiveValidationFailure(); @@ -225,7 +229,17 @@ export function createInferenceSelectionValidationHelpers( } else { console.log(` ${probe.label} available — ${deps.agentProductName()} will use ${probe.api}.`); } - return { ok: true, api: probe.api ?? "openai-completions" }; + const api = probe.api ?? "openai-completions"; + if (api === "openai-completions" && probe.validated !== false) { + options.capabilityCache?.rememberCompletedOpenAiChat({ + endpointUrl, + model, + authMode: options.authMode, + requireChatCompletionsToolCalling: options.requireChatCompletionsToolCalling, + extraHeaders: options.extraHeaders, + }); + } + return { ok: true, api }; } async function validateAnthropicSelectionWithRetryMessage( diff --git a/src/lib/onboard/machine/handlers/provider-inference.ts b/src/lib/onboard/machine/handlers/provider-inference.ts index 689fda5c192..cc78ae9fbd9 100644 --- a/src/lib/onboard/machine/handlers/provider-inference.ts +++ b/src/lib/onboard/machine/handlers/provider-inference.ts @@ -10,6 +10,7 @@ import type { } from "../../../inference/gateway-route-compatibility"; import type { WebSearchConfig } from "../../../inference/web-search"; import type { HermesAuthMethod, Session, SessionUpdates } from "../../../state/onboard-session"; +import type { OnboardInferenceCapabilityCache } from "../../inference-capability-cache"; import type { createProviderRecoveryReceiptLedger, ProviderRecoveryReceipt, @@ -39,6 +40,8 @@ export interface ProviderInferenceSetupOptions { preferredInferenceApi?: string | null; /** Public addresses approved for custom endpoint host probes. */ endpointPinnedAddresses?: readonly string[]; + /** One-shot host capability cache carried only through this onboarding run. */ + inferenceCapabilityCache?: OnboardInferenceCapabilityCache; /** Onboard session that owns the route reservation this setup creates. */ reservationSessionId?: string; /** Recheck recorded-route ownership after acquiring route mutation locks. */ @@ -60,6 +63,7 @@ export interface ProviderSelectionResult { reuseGatewayCredentialWithoutLocalKey?: boolean; recoveredFromSandbox?: boolean; endpointPinnedAddresses?: string[]; + inferenceCapabilityCache?: OnboardInferenceCapabilityCache; } export interface ProviderInferenceStateOptions { @@ -339,6 +343,7 @@ export async function handleProviderInferenceState({ let skipHostInferenceSmoke = false; let reuseGatewayCredentialWithoutLocalKey = false; let endpointPinnedAddresses: string[] | undefined; + let inferenceCapabilityCache: OnboardInferenceCapabilityCache | undefined; const effectiveResume = resume && !fresh; const stateResults: OnboardStateTransitionResult[] = []; const retryStateResults: OnboardStateTransitionResult[] = []; @@ -509,6 +514,7 @@ export async function handleProviderInferenceState({ recoveredRecordedProvider = selection.recoveredFromSandbox === true; forceInferenceSetup ||= recoveredRecordedProvider; endpointPinnedAddresses = selection.endpointPinnedAddresses; + inferenceCapabilityCache = selection.inferenceCapabilityCache; shouldRecordProviderSelection = true; } @@ -584,6 +590,7 @@ export async function handleProviderInferenceState({ : {}), ...(preferredInferenceApi ? { preferredInferenceApi } : {}), ...(endpointPinnedAddresses ? { endpointPinnedAddresses } : {}), + ...(inferenceCapabilityCache ? { inferenceCapabilityCache } : {}), reservationSessionId: session?.sessionId, }; await deps.startRecordedStep("inference", { provider, model }); @@ -772,6 +779,7 @@ export async function handleProviderInferenceState({ ...(reuseGatewayCredentialWithoutLocalKey ? { reuseGatewayCredentialWithoutLocalKey } : {}), ...(preferredInferenceApi ? { preferredInferenceApi } : {}), ...(endpointPinnedAddresses ? { endpointPinnedAddresses } : {}), + ...(inferenceCapabilityCache ? { inferenceCapabilityCache } : {}), ...providerRecovery.setupOptions( recoveredRecordedProvider, confirmedSandboxName, diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index e8038322985..0f52e471573 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -311,6 +311,7 @@ export function createSetupInference( deps.verifyOnboardInferenceSmoke({ ...input, pinnedAddresses: endpointPinnedAddresses, + capabilityCache: options.inferenceCapabilityCache, }), isNonInteractive: deps.isNonInteractive, registry: { @@ -364,6 +365,7 @@ export function createSetupInference( skipHostInferenceSmoke: options.skipHostInferenceSmoke === true, preferredInferenceApi: options.preferredInferenceApi ?? null, pinnedAddresses: endpointPinnedAddresses, + capabilityCache: options.inferenceCapabilityCache, }, { ...commonDeps, @@ -449,6 +451,7 @@ export function createSetupInference( endpointUrl, credentialEnv, pinnedAddresses: endpointPinnedAddresses, + capabilityCache: options.inferenceCapabilityCache, }); if (sandboxName) { commonDeps.registry.updateSandbox(sandboxName); diff --git a/src/lib/onboard/setup-nim-flow.test.ts b/src/lib/onboard/setup-nim-flow.test.ts index b6c0961237d..ea1a5a4cca9 100644 --- a/src/lib/onboard/setup-nim-flow.test.ts +++ b/src/lib/onboard/setup-nim-flow.test.ts @@ -5,6 +5,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { AgentDefinition } from "../agent/defs"; import type { VllmProfile } from "../inference/vllm"; +import { OnboardInferenceCapabilityCache } from "./inference-capability-cache"; import { getWindowsHostOllamaDockerRequirement } from "./local-inference-topology"; import type { InferenceProviderHostState } from "./provider-host-state"; import { createSetupNim, type SetupNimFlowDeps } from "./setup-nim-flow"; @@ -280,7 +281,9 @@ describe("createSetupNim", () => { expect(maybePromptForInferenceInputCapability).toHaveBeenCalledWith( "nvidia/nemotron-3-super-120b-a12b", ); - expect(result).toEqual({ + const { inferenceCapabilityCache, ...resultWithoutCache } = result; + expect(inferenceCapabilityCache).toBeInstanceOf(OnboardInferenceCapabilityCache); + expect(resultWithoutCache).toEqual({ model: "nvidia/nemotron-3-super-120b-a12b", provider: "nvidia-prod", endpointUrl: "https://integrate.api.nvidia.com/v1", diff --git a/src/lib/onboard/setup-nim-flow.ts b/src/lib/onboard/setup-nim-flow.ts index 9003aa77a79..93d28c4b7c4 100644 --- a/src/lib/onboard/setup-nim-flow.ts +++ b/src/lib/onboard/setup-nim-flow.ts @@ -10,6 +10,7 @@ import type { GatewayRouteDiscoveryConstraints } from "../inference/gateway-rout import type { VllmProfile } from "../inference/vllm"; import { isBackToSelection } from "../navigation"; import type { HermesAuthMethod } from "./hermes-auth"; +import { OnboardInferenceCapabilityCache } from "./inference-capability-cache"; import type { ProviderSelectionResult } from "./machine/handlers/provider-inference"; import type { ProviderInferenceProbeRoute } from "./machine/handlers/provider-inference-route-containment"; import type { @@ -249,6 +250,7 @@ export function createSetupNim( let allowToolsIncompatible = false; let reuseGatewayCredential = false; let endpointPinnedAddresses: string[] | undefined; + const inferenceCapabilityCache = new OnboardInferenceCapabilityCache(); const nvidiaFeaturedModels = deps.createNvidiaFeaturedModelSession({ defaultModel: resolveAgentDefaultCloudModel(agent), writeLine: deps.log, @@ -267,6 +269,7 @@ export function createSetupNim( nimContainer, allowToolsIncompatible, ...(endpointPinnedAddresses ? { endpointPinnedAddresses } : {}), + inferenceCapabilityCache, nvidiaFeaturedModels, openRouterFeaturedModels, }; @@ -635,6 +638,7 @@ export function createSetupNim( reuseGatewayCredentialWithoutLocalKey: reuseGatewayCredential, ...(recoveredFromSandbox ? { recoveredFromSandbox: true } : {}), ...(endpointPinnedAddresses ? { endpointPinnedAddresses } : {}), + inferenceCapabilityCache, }; }; } diff --git a/src/lib/onboard/setup-nim-selection.ts b/src/lib/onboard/setup-nim-selection.ts index 03bdf35bd76..9d761cac95f 100644 --- a/src/lib/onboard/setup-nim-selection.ts +++ b/src/lib/onboard/setup-nim-selection.ts @@ -4,6 +4,7 @@ import { applyCompatibleEndpointContextWindow } from "../inference/compatible-endpoint-context"; import type { GatewayRouteDiscoveryConstraints } from "../inference/gateway-route-compatibility"; import { getProbeExtraHeaders } from "../inference/onboard-probes"; +import type { OnboardInferenceCapabilityCache } from "./inference-capability-cache"; import type { NvidiaFeaturedModelSession } from "./nvidia-featured-model-selection"; export { createNvidiaFeaturedModelSession } from "./nvidia-featured-model-selection"; @@ -25,6 +26,8 @@ export type SetupNimSelectionState = { /** Public addresses approved for the selected custom endpoint. */ endpointPinnedAddresses?: string[]; reuseGatewayCredentialWithoutLocalKey?: boolean; + /** Ephemeral selection-to-smoke validation cache; never written to session state. */ + inferenceCapabilityCache?: OnboardInferenceCapabilityCache; nvidiaFeaturedModels?: NvidiaFeaturedModelSession; openRouterFeaturedModels?: NvidiaFeaturedModelSession; /** Attempt-wide shared-gateway guard, invoked after identity selection and before probes. */ @@ -110,6 +113,7 @@ type ProbeOptions = { skipResponsesProbe?: boolean; authMode?: ProbeAuthMode; extraHeaders?: readonly string[]; + capabilityCache?: OnboardInferenceCapabilityCache; }; type ValidationResult = @@ -316,6 +320,7 @@ export function createRemoteModelValidator(deps: RemoteModelValidatorDeps): { authMode: deps.getProbeAuthMode(state.provider), extraHeaders: deps.getProbeExtraHeaders?.(state.provider) ?? getProbeExtraHeaders(state.provider), + capabilityCache: state.inferenceCapabilityCache, }, ); if (validation.ok) { diff --git a/test/helpers/onboard-smoke-verifier-harness.ts b/test/helpers/onboard-smoke-verifier-harness.ts index 78ef18d0143..604653a937d 100644 --- a/test/helpers/onboard-smoke-verifier-harness.ts +++ b/test/helpers/onboard-smoke-verifier-harness.ts @@ -7,6 +7,7 @@ import path from "node:path"; export type SmokeVerifierHarnessCall = [string, ...unknown[]]; type VerifyOnboardSmokeInvocation = { + selectedChatCapability?: boolean; credentialEnv?: string; endpointUrl?: string; forceOpenAiLike?: boolean; @@ -83,17 +84,37 @@ Module._load = function patchedLoad(request, _parent, _isMain) { return originalLoad.apply(this, arguments); }; -const { verifyOnboardInferenceSmoke } = require(process.env.PROBES_MODULE); +const { + getProbeAuthMode, + getProbeExtraHeaders, + verifyOnboardInferenceSmoke, +} = require(process.env.PROBES_MODULE); +const { OnboardInferenceCapabilityCache } = require(process.env.CAPABILITY_CACHE_MODULE); const invocations = JSON.parse(process.env.SMOKE_INVOCATIONS || "[]"); console.log = (...args) => calls.push(["log", args.join(" ")]); (async () => { for (const invocation of invocations) { - await verifyOnboardInferenceSmoke({ + const { selectedChatCapability, ...input } = invocation; + const capabilityCache = selectedChatCapability ? new OnboardInferenceCapabilityCache() : undefined; + const effectiveInvocation = { endpointUrl: "https://api.example.com/v1", model: "nous/test-model", provider: "hermes-provider", - ...invocation, + ...input, + }; + if (capabilityCache) { + const primed = capabilityCache.rememberCompletedOpenAiChat({ + endpointUrl: effectiveInvocation.endpointUrl, + model: effectiveInvocation.model, + authMode: getProbeAuthMode(effectiveInvocation.provider), + extraHeaders: getProbeExtraHeaders(effectiveInvocation.provider), + }); + if (!primed) throw new Error("failed to prime selected Chat Completions capability"); + } + await verifyOnboardInferenceSmoke({ + ...effectiveInvocation, + capabilityCache, }); } process.stdout.write(JSON.stringify(calls)); @@ -108,6 +129,10 @@ console.log = (...args) => calls.push(["log", args.join(" ")]); env: { ...process.env, PROBES_MODULE: path.join(process.cwd(), "src/lib/inference/onboard-probes.ts"), + CAPABILITY_CACHE_MODULE: path.join( + process.cwd(), + "src/lib/onboard/inference-capability-cache.ts", + ), SMOKE_INVOCATIONS: JSON.stringify(invocations), VITEST: "false", }, diff --git a/test/onboard-smoke-verifier.test.ts b/test/onboard-smoke-verifier.test.ts index 8fc8d4ca98f..a0bba40369c 100644 --- a/test/onboard-smoke-verifier.test.ts +++ b/test/onboard-smoke-verifier.test.ts @@ -37,4 +37,34 @@ describe("Hermes onboard smoke verification", () => { ], ]); }); + + it("does not send a duplicate smoke request for a matching selected Chat Completions capability", async () => { + const calls = await runVerifyOnboardSmokeHarness([ + { + credentialEnv: "NOUS_API_KEY", + endpointUrl: "https://override.example/v1", + model: "override/model", + provider: "hermes-provider", + selectedChatCapability: true, + }, + ]); + + expect(calls.filter((call) => call[0] === "runCurlProbe")).toHaveLength(0); + expect(calls).toContainEqual([ + "log", + " ✓ Reusing selected Chat Completions validation: hermes-provider / override/model", + ]); + }); + + it("fails when the selected capability cannot be safely cached", async () => { + await expect( + runVerifyOnboardSmokeHarness([ + { + credentialEnv: "NOUS_API_KEY", + endpointUrl: "https://api.example.com/v1?credential-bearing=true", + selectedChatCapability: true, + }, + ]), + ).rejects.toThrow("failed to prime selected Chat Completions capability"); + }); });