From dbf6f0f64700573f86f3da5ce21db35ef50e1a22 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 17:53:27 -0700 Subject: [PATCH 1/7] fix(inference): skip host smoke for Hermes OAuth provider Replay the Hermes Provider onboarding fix from NVIDIA/NemoClaw#4385 so OAuth-backed agent-key storage does not get blocked by an unrelated ambient host OPENAI_API_KEY. Co-authored-by: shannonsands <7897813+shannonsands@users.noreply.github.com> Signed-off-by: Aaron Erickson --- src/lib/inference/onboard-probes.test.ts | 8 +++++++- src/lib/inference/onboard-probes.ts | 5 +++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/lib/inference/onboard-probes.test.ts b/src/lib/inference/onboard-probes.test.ts index 20a4a0cf3e0..0300fdd67a6 100644 --- a/src/lib/inference/onboard-probes.test.ts +++ b/src/lib/inference/onboard-probes.test.ts @@ -1,10 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { describe, expect, it } from "vitest"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; +import { describe, expect, it } from "vitest"; const { getChatCompletionsProbeCurlArgs, @@ -17,9 +17,15 @@ const { isSandboxInternalUrl, probeOpenAiLikeEndpoint, RETRIABLE_HTTP_PROBE_STATUSES, + shouldSmokeOpenAiLikeOnboardRoute, } = require("../../../dist/lib/inference/onboard-probes"); describe("OpenAI-compatible inference probe response parsing", () => { + it("does not host-smoke Hermes Provider with the ambient OPENAI_API_KEY", () => { + expect(shouldSmokeOpenAiLikeOnboardRoute("hermes-provider")).toBe(false); + expect(shouldSmokeOpenAiLikeOnboardRoute("openai-api")).toBe(true); + }); + it("detects tool-calling responses payloads conservatively", () => { expect( hasResponsesToolCall( diff --git a/src/lib/inference/onboard-probes.ts b/src/lib/inference/onboard-probes.ts index a86a1afb957..ac84fbe3e06 100644 --- a/src/lib/inference/onboard-probes.ts +++ b/src/lib/inference/onboard-probes.ts @@ -835,6 +835,11 @@ module.exports = { }; function shouldSmokeOpenAiLikeOnboardRoute(provider) { + // Hermes Provider OAuth mints a short-lived agent key and stores it with + // OpenShell provider storage. A host-side direct probe would resolve the + // ambient OPENAI_API_KEY instead, which can falsely fail after successful + // OAuth if the user's shell has a different OpenAI key staged. + if (provider === "hermes-provider") return false; const { REMOTE_PROVIDER_CONFIG } = require("../onboard/providers"); if (provider === "nvidia-nim" || provider === "nvidia-router") return true; return Object.values(REMOTE_PROVIDER_CONFIG).some( From 69a1d8ab24167d7da7669d4d1e188bc2b9131d12 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 17:54:26 -0700 Subject: [PATCH 2/7] fix(inference): keep Hermes API key smoke validation Limit the Hermes Provider host-smoke skip to OAuth-backed OPENAI_API_KEY storage. Nous API key onboarding still has a host credential, so keep direct validation for bad keys, base URLs, or models. Signed-off-by: Aaron Erickson --- src/lib/inference/onboard-probes.test.ts | 3 ++- src/lib/inference/onboard-probes.ts | 14 ++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/lib/inference/onboard-probes.test.ts b/src/lib/inference/onboard-probes.test.ts index 0300fdd67a6..6d230792d31 100644 --- a/src/lib/inference/onboard-probes.test.ts +++ b/src/lib/inference/onboard-probes.test.ts @@ -22,7 +22,8 @@ const { describe("OpenAI-compatible inference probe response parsing", () => { it("does not host-smoke Hermes Provider with the ambient OPENAI_API_KEY", () => { - expect(shouldSmokeOpenAiLikeOnboardRoute("hermes-provider")).toBe(false); + expect(shouldSmokeOpenAiLikeOnboardRoute("hermes-provider", "OPENAI_API_KEY")).toBe(false); + expect(shouldSmokeOpenAiLikeOnboardRoute("hermes-provider", "NOUS_API_KEY")).toBe(true); expect(shouldSmokeOpenAiLikeOnboardRoute("openai-api")).toBe(true); }); diff --git a/src/lib/inference/onboard-probes.ts b/src/lib/inference/onboard-probes.ts index ac84fbe3e06..4dec83a1d8f 100644 --- a/src/lib/inference/onboard-probes.ts +++ b/src/lib/inference/onboard-probes.ts @@ -834,12 +834,13 @@ module.exports = { RETRIABLE_HTTP_PROBE_STATUSES, }; -function shouldSmokeOpenAiLikeOnboardRoute(provider) { +function shouldSmokeOpenAiLikeOnboardRoute(provider, credentialEnv = null) { // Hermes Provider OAuth mints a short-lived agent key and stores it with // OpenShell provider storage. A host-side direct probe would resolve the // ambient OPENAI_API_KEY instead, which can falsely fail after successful - // OAuth if the user's shell has a different OpenAI key staged. - if (provider === "hermes-provider") return false; + // OAuth if the user's shell has a different OpenAI key staged. The Nous API + // key path still has a host credential and should keep the direct smoke. + if (provider === "hermes-provider" && credentialEnv === "OPENAI_API_KEY") return false; const { REMOTE_PROVIDER_CONFIG } = require("../onboard/providers"); if (provider === "nvidia-nim" || provider === "nvidia-router") return true; return Object.values(REMOTE_PROVIDER_CONFIG).some( @@ -848,7 +849,12 @@ function shouldSmokeOpenAiLikeOnboardRoute(provider) { } function verifyOnboardInferenceSmoke(options) { - if (!options.forceOpenAiLike && !shouldSmokeOpenAiLikeOnboardRoute(options.provider)) return; + if ( + !options.forceOpenAiLike && + !shouldSmokeOpenAiLikeOnboardRoute(options.provider, options.credentialEnv) + ) { + return; + } if (process.env.VITEST === "true") return; const endpointUrl = options.endpointUrl || require("./config").INFERENCE_ROUTE_URL; From 8a54352e67b1757874c00c9244cdbceeb560eeb4 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 18:20:40 -0700 Subject: [PATCH 3/7] test(inference): cover Hermes smoke verifier gate Signed-off-by: Aaron Erickson --- src/lib/inference/onboard-probes.test.ts | 105 +++++++++++++++++++++++ 1 file changed, 105 insertions(+) diff --git a/src/lib/inference/onboard-probes.test.ts b/src/lib/inference/onboard-probes.test.ts index 6d230792d31..5536af75a5f 100644 --- a/src/lib/inference/onboard-probes.test.ts +++ b/src/lib/inference/onboard-probes.test.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -27,6 +28,110 @@ describe("OpenAI-compatible inference probe response parsing", () => { expect(shouldSmokeOpenAiLikeOnboardRoute("openai-api")).toBe(true); }); + it("skips only the Hermes OAuth smoke path in the runtime verifier", () => { + const harness = String.raw` +const Module = require("node:module"); +const originalLoad = Module._load; +const calls = []; + +process.env.VITEST = "false"; + +Module._load = function patchedLoad(request, parent, isMain) { + if (request === "../credentials/store") { + return { + getCredential(name) { + calls.push(["getCredential", name]); + return "stored-" + name; + }, + normalizeCredentialValue(value) { + calls.push(["normalizeCredentialValue", value]); + return value; + }, + resolveProviderCredential(name) { + calls.push(["resolveProviderCredential", name]); + return "resolved-" + name; + }, + }; + } + if (request === "../adapters/http/probe") { + return { + getCurlTimingArgs() { + return []; + }, + runChatCompletionsStreamingProbe() { + throw new Error("unexpected streaming probe"); + }, + runCurlProbe(args) { + const authHeader = + args.find((arg) => String(arg).startsWith("Authorization: Bearer ")) || "no-auth"; + calls.push(["runCurlProbe", args[args.length - 1], authHeader]); + return { + ok: true, + httpStatus: 200, + curlStatus: 0, + message: "OK", + body: '{"choices":[{"message":{"content":"OK"}}]}', + }; + }, + runStreamingEventProbe() { + throw new Error("unexpected streaming event probe"); + }, + }; + } + return originalLoad.apply(this, arguments); +}; + +const { verifyOnboardInferenceSmoke } = require(process.env.PROBES_MODULE); +console.log = (...args) => calls.push(["log", args.join(" ")]); + +const baseOptions = { + endpointUrl: "https://api.example.com/v1", + model: "nous/test-model", + provider: "hermes-provider", +}; + +verifyOnboardInferenceSmoke({ ...baseOptions, credentialEnv: "OPENAI_API_KEY" }); +verifyOnboardInferenceSmoke({ ...baseOptions, credentialEnv: "NOUS_API_KEY" }); +verifyOnboardInferenceSmoke({ + ...baseOptions, + credentialEnv: "OPENAI_API_KEY", + forceOpenAiLike: true, +}); + +process.stdout.write(JSON.stringify(calls)); +`; + const result = spawnSync(process.execPath, ["-e", harness], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + PROBES_MODULE: path.join(process.cwd(), "dist/lib/inference/onboard-probes.js"), + VITEST: "false", + }, + }); + + expect(result.status, result.stderr).toBe(0); + const calls = JSON.parse(result.stdout) as [string, ...unknown[]][]; + expect( + calls.filter((call) => + ["resolveProviderCredential", "getCredential", "runCurlProbe"].includes(call[0]), + ), + ).toEqual([ + ["resolveProviderCredential", "NOUS_API_KEY"], + [ + "runCurlProbe", + "https://api.example.com/v1/chat/completions", + "Authorization: Bearer resolved-NOUS_API_KEY", + ], + ["resolveProviderCredential", "OPENAI_API_KEY"], + [ + "runCurlProbe", + "https://api.example.com/v1/chat/completions", + "Authorization: Bearer resolved-OPENAI_API_KEY", + ], + ]); + }); + it("detects tool-calling responses payloads conservatively", () => { expect( hasResponsesToolCall( From e41c7b2c3aca05c2a1fc0fb1b99586276c69007f Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 18:29:41 -0700 Subject: [PATCH 4/7] test(inference): extract smoke verifier harness Signed-off-by: Aaron Erickson --- src/lib/inference/onboard-probes.test.ts | 90 +-------------- src/lib/inference/onboard-probes.ts | 10 +- .../helpers/onboard-smoke-verifier-harness.ts | 106 ++++++++++++++++++ 3 files changed, 121 insertions(+), 85 deletions(-) create mode 100644 test/helpers/onboard-smoke-verifier-harness.ts diff --git a/src/lib/inference/onboard-probes.test.ts b/src/lib/inference/onboard-probes.test.ts index 5536af75a5f..91a96287660 100644 --- a/src/lib/inference/onboard-probes.test.ts +++ b/src/lib/inference/onboard-probes.test.ts @@ -1,11 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; +import { runVerifyOnboardSmokeHarness } from "../../../test/helpers/onboard-smoke-verifier-harness"; const { getChatCompletionsProbeCurlArgs, @@ -29,89 +29,11 @@ describe("OpenAI-compatible inference probe response parsing", () => { }); it("skips only the Hermes OAuth smoke path in the runtime verifier", () => { - const harness = String.raw` -const Module = require("node:module"); -const originalLoad = Module._load; -const calls = []; - -process.env.VITEST = "false"; - -Module._load = function patchedLoad(request, parent, isMain) { - if (request === "../credentials/store") { - return { - getCredential(name) { - calls.push(["getCredential", name]); - return "stored-" + name; - }, - normalizeCredentialValue(value) { - calls.push(["normalizeCredentialValue", value]); - return value; - }, - resolveProviderCredential(name) { - calls.push(["resolveProviderCredential", name]); - return "resolved-" + name; - }, - }; - } - if (request === "../adapters/http/probe") { - return { - getCurlTimingArgs() { - return []; - }, - runChatCompletionsStreamingProbe() { - throw new Error("unexpected streaming probe"); - }, - runCurlProbe(args) { - const authHeader = - args.find((arg) => String(arg).startsWith("Authorization: Bearer ")) || "no-auth"; - calls.push(["runCurlProbe", args[args.length - 1], authHeader]); - return { - ok: true, - httpStatus: 200, - curlStatus: 0, - message: "OK", - body: '{"choices":[{"message":{"content":"OK"}}]}', - }; - }, - runStreamingEventProbe() { - throw new Error("unexpected streaming event probe"); - }, - }; - } - return originalLoad.apply(this, arguments); -}; - -const { verifyOnboardInferenceSmoke } = require(process.env.PROBES_MODULE); -console.log = (...args) => calls.push(["log", args.join(" ")]); - -const baseOptions = { - endpointUrl: "https://api.example.com/v1", - model: "nous/test-model", - provider: "hermes-provider", -}; - -verifyOnboardInferenceSmoke({ ...baseOptions, credentialEnv: "OPENAI_API_KEY" }); -verifyOnboardInferenceSmoke({ ...baseOptions, credentialEnv: "NOUS_API_KEY" }); -verifyOnboardInferenceSmoke({ - ...baseOptions, - credentialEnv: "OPENAI_API_KEY", - forceOpenAiLike: true, -}); - -process.stdout.write(JSON.stringify(calls)); -`; - const result = spawnSync(process.execPath, ["-e", harness], { - cwd: process.cwd(), - encoding: "utf8", - env: { - ...process.env, - PROBES_MODULE: path.join(process.cwd(), "dist/lib/inference/onboard-probes.js"), - VITEST: "false", - }, - }); - - expect(result.status, result.stderr).toBe(0); - const calls = JSON.parse(result.stdout) as [string, ...unknown[]][]; + const calls = runVerifyOnboardSmokeHarness([ + { credentialEnv: "OPENAI_API_KEY" }, + { credentialEnv: "NOUS_API_KEY" }, + { credentialEnv: "OPENAI_API_KEY", forceOpenAiLike: true }, + ]); expect( calls.filter((call) => ["resolveProviderCredential", "getCredential", "runCurlProbe"].includes(call[0]), diff --git a/src/lib/inference/onboard-probes.ts b/src/lib/inference/onboard-probes.ts index 4dec83a1d8f..3a873faa10f 100644 --- a/src/lib/inference/onboard-probes.ts +++ b/src/lib/inference/onboard-probes.ts @@ -835,12 +835,20 @@ module.exports = { }; function shouldSmokeOpenAiLikeOnboardRoute(provider, credentialEnv = null) { + const { + HERMES_INFERENCE_CREDENTIAL_ENV, + HERMES_PROVIDER_NAME, + } = require("../hermes-provider-auth"); // Hermes Provider OAuth mints a short-lived agent key and stores it with // OpenShell provider storage. A host-side direct probe would resolve the // ambient OPENAI_API_KEY instead, which can falsely fail after successful // OAuth if the user's shell has a different OpenAI key staged. The Nous API // key path still has a host credential and should keep the direct smoke. - if (provider === "hermes-provider" && credentialEnv === "OPENAI_API_KEY") return false; + // Remove this exception once the host smoke can resolve the actual Hermes + // OAuth agent key from OpenShell provider storage. + if (provider === HERMES_PROVIDER_NAME && credentialEnv === HERMES_INFERENCE_CREDENTIAL_ENV) { + return false; + } const { REMOTE_PROVIDER_CONFIG } = require("../onboard/providers"); if (provider === "nvidia-nim" || provider === "nvidia-router") return true; return Object.values(REMOTE_PROVIDER_CONFIG).some( diff --git a/test/helpers/onboard-smoke-verifier-harness.ts b/test/helpers/onboard-smoke-verifier-harness.ts new file mode 100644 index 00000000000..cad2b1e07c4 --- /dev/null +++ b/test/helpers/onboard-smoke-verifier-harness.ts @@ -0,0 +1,106 @@ +import { spawnSync } from "node:child_process"; +import path from "node:path"; + +export type SmokeVerifierHarnessCall = [string, ...unknown[]]; + +type VerifyOnboardSmokeInvocation = { + credentialEnv?: string; + endpointUrl?: string; + forceOpenAiLike?: boolean; + model?: string; + provider?: string; +}; + +export function runVerifyOnboardSmokeHarness( + invocations: VerifyOnboardSmokeInvocation[], +): SmokeVerifierHarnessCall[] { + const harness = String.raw` +const Module = require("node:module"); +const originalLoad = Module._load; +const calls = []; + +process.env.VITEST = "false"; + +Module._load = function patchedLoad(request, parent, isMain) { + if (request === "../credentials/store") { + return { + getCredential(name) { + calls.push(["getCredential", name]); + return "stored-" + name; + }, + normalizeCredentialValue(value) { + calls.push(["normalizeCredentialValue", value]); + return value; + }, + resolveProviderCredential(name) { + calls.push(["resolveProviderCredential", name]); + return "resolved-" + name; + }, + }; + } + if (request === "../hermes-provider-auth") { + return { + HERMES_PROVIDER_NAME: "hermes-provider", + HERMES_INFERENCE_CREDENTIAL_ENV: "OPENAI_API_KEY", + HERMES_NOUS_API_KEY_CREDENTIAL_ENV: "NOUS_API_KEY", + }; + } + if (request === "../adapters/http/probe") { + return { + getCurlTimingArgs() { + return []; + }, + runChatCompletionsStreamingProbe() { + throw new Error("unexpected streaming probe"); + }, + runCurlProbe(args) { + const authHeader = + args.find((arg) => String(arg).startsWith("Authorization: Bearer ")) || "no-auth"; + calls.push(["runCurlProbe", args[args.length - 1], authHeader]); + return { + ok: true, + httpStatus: 200, + curlStatus: 0, + message: "OK", + body: '{"choices":[{"message":{"content":"OK"}}]}', + }; + }, + runStreamingEventProbe() { + throw new Error("unexpected streaming event probe"); + }, + }; + } + return originalLoad.apply(this, arguments); +}; + +const { verifyOnboardInferenceSmoke } = require(process.env.PROBES_MODULE); +const invocations = JSON.parse(process.env.SMOKE_INVOCATIONS || "[]"); +console.log = (...args) => calls.push(["log", args.join(" ")]); + +for (const invocation of invocations) { + verifyOnboardInferenceSmoke({ + endpointUrl: "https://api.example.com/v1", + model: "nous/test-model", + provider: "hermes-provider", + ...invocation, + }); +} + +process.stdout.write(JSON.stringify(calls)); +`; + const result = spawnSync(process.execPath, ["-e", harness], { + cwd: process.cwd(), + encoding: "utf8", + env: { + ...process.env, + PROBES_MODULE: path.join(process.cwd(), "dist/lib/inference/onboard-probes.js"), + SMOKE_INVOCATIONS: JSON.stringify(invocations), + VITEST: "false", + }, + }); + + if (result.status !== 0) { + throw new Error(result.stderr || result.stdout || "smoke verifier harness failed"); + } + return JSON.parse(result.stdout) as SmokeVerifierHarnessCall[]; +} From 1f8165dfccd834a41d1c0e2d0518226b9d4c2ae0 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 18:39:35 -0700 Subject: [PATCH 5/7] test(inference): add smoke harness hygiene Signed-off-by: Aaron Erickson --- test/helpers/onboard-smoke-verifier-harness.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/helpers/onboard-smoke-verifier-harness.ts b/test/helpers/onboard-smoke-verifier-harness.ts index cad2b1e07c4..6b0b7238cf3 100644 --- a/test/helpers/onboard-smoke-verifier-harness.ts +++ b/test/helpers/onboard-smoke-verifier-harness.ts @@ -1,3 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + import { spawnSync } from "node:child_process"; import path from "node:path"; @@ -21,7 +24,7 @@ const calls = []; process.env.VITEST = "false"; -Module._load = function patchedLoad(request, parent, isMain) { +Module._load = function patchedLoad(request, _parent, _isMain) { if (request === "../credentials/store") { return { getCredential(name) { From 9a19be501c360d91ff9abbbc6072e22f903b6255 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 18:48:59 -0700 Subject: [PATCH 6/7] test(inference): split Hermes smoke verifier spec Signed-off-by: Aaron Erickson --- src/lib/inference/onboard-probes.test.ts | 34 -------------------- test/onboard-smoke-verifier.test.ts | 41 ++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 34 deletions(-) create mode 100644 test/onboard-smoke-verifier.test.ts diff --git a/src/lib/inference/onboard-probes.test.ts b/src/lib/inference/onboard-probes.test.ts index 91a96287660..2dbe76a6bb1 100644 --- a/src/lib/inference/onboard-probes.test.ts +++ b/src/lib/inference/onboard-probes.test.ts @@ -5,7 +5,6 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { describe, expect, it } from "vitest"; -import { runVerifyOnboardSmokeHarness } from "../../../test/helpers/onboard-smoke-verifier-harness"; const { getChatCompletionsProbeCurlArgs, @@ -18,42 +17,9 @@ const { isSandboxInternalUrl, probeOpenAiLikeEndpoint, RETRIABLE_HTTP_PROBE_STATUSES, - shouldSmokeOpenAiLikeOnboardRoute, } = require("../../../dist/lib/inference/onboard-probes"); describe("OpenAI-compatible inference probe response parsing", () => { - it("does not host-smoke Hermes Provider with the ambient OPENAI_API_KEY", () => { - expect(shouldSmokeOpenAiLikeOnboardRoute("hermes-provider", "OPENAI_API_KEY")).toBe(false); - expect(shouldSmokeOpenAiLikeOnboardRoute("hermes-provider", "NOUS_API_KEY")).toBe(true); - expect(shouldSmokeOpenAiLikeOnboardRoute("openai-api")).toBe(true); - }); - - it("skips only the Hermes OAuth smoke path in the runtime verifier", () => { - const calls = runVerifyOnboardSmokeHarness([ - { credentialEnv: "OPENAI_API_KEY" }, - { credentialEnv: "NOUS_API_KEY" }, - { credentialEnv: "OPENAI_API_KEY", forceOpenAiLike: true }, - ]); - expect( - calls.filter((call) => - ["resolveProviderCredential", "getCredential", "runCurlProbe"].includes(call[0]), - ), - ).toEqual([ - ["resolveProviderCredential", "NOUS_API_KEY"], - [ - "runCurlProbe", - "https://api.example.com/v1/chat/completions", - "Authorization: Bearer resolved-NOUS_API_KEY", - ], - ["resolveProviderCredential", "OPENAI_API_KEY"], - [ - "runCurlProbe", - "https://api.example.com/v1/chat/completions", - "Authorization: Bearer resolved-OPENAI_API_KEY", - ], - ]); - }); - it("detects tool-calling responses payloads conservatively", () => { expect( hasResponsesToolCall( diff --git a/test/onboard-smoke-verifier.test.ts b/test/onboard-smoke-verifier.test.ts new file mode 100644 index 00000000000..b1ebc30f93f --- /dev/null +++ b/test/onboard-smoke-verifier.test.ts @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; +import { runVerifyOnboardSmokeHarness } from "./helpers/onboard-smoke-verifier-harness"; + +const { shouldSmokeOpenAiLikeOnboardRoute } = require("../dist/lib/inference/onboard-probes"); + +describe("Hermes onboard smoke verification", () => { + it("does not host-smoke Hermes Provider with the ambient OPENAI_API_KEY", () => { + expect(shouldSmokeOpenAiLikeOnboardRoute("hermes-provider", "OPENAI_API_KEY")).toBe(false); + expect(shouldSmokeOpenAiLikeOnboardRoute("hermes-provider", "NOUS_API_KEY")).toBe(true); + expect(shouldSmokeOpenAiLikeOnboardRoute("openai-api")).toBe(true); + }); + + it("skips only the Hermes OAuth smoke path in the runtime verifier", () => { + const calls = runVerifyOnboardSmokeHarness([ + { credentialEnv: "OPENAI_API_KEY" }, + { credentialEnv: "NOUS_API_KEY" }, + { credentialEnv: "OPENAI_API_KEY", forceOpenAiLike: true }, + ]); + expect( + calls.filter((call) => + ["resolveProviderCredential", "getCredential", "runCurlProbe"].includes(call[0]), + ), + ).toEqual([ + ["resolveProviderCredential", "NOUS_API_KEY"], + [ + "runCurlProbe", + "https://api.example.com/v1/chat/completions", + "Authorization: Bearer resolved-NOUS_API_KEY", + ], + ["resolveProviderCredential", "OPENAI_API_KEY"], + [ + "runCurlProbe", + "https://api.example.com/v1/chat/completions", + "Authorization: Bearer resolved-OPENAI_API_KEY", + ], + ]); + }); +}); From cae9cadbd07bc48e215031442fb3f6508246c86e Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Thu, 28 May 2026 18:55:20 -0700 Subject: [PATCH 7/7] test(inference): use esm smoke verifier import --- src/lib/inference/onboard-probes.ts | 4 ++-- test/onboard-smoke-verifier.test.ts | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/lib/inference/onboard-probes.ts b/src/lib/inference/onboard-probes.ts index 3a873faa10f..de949e16125 100644 --- a/src/lib/inference/onboard-probes.ts +++ b/src/lib/inference/onboard-probes.ts @@ -834,7 +834,7 @@ module.exports = { RETRIABLE_HTTP_PROBE_STATUSES, }; -function shouldSmokeOpenAiLikeOnboardRoute(provider, credentialEnv = null) { +export function shouldSmokeOpenAiLikeOnboardRoute(provider: string, credentialEnv: string | null = null) { const { HERMES_INFERENCE_CREDENTIAL_ENV, HERMES_PROVIDER_NAME, @@ -856,7 +856,7 @@ function shouldSmokeOpenAiLikeOnboardRoute(provider, credentialEnv = null) { ); } -function verifyOnboardInferenceSmoke(options) { +export function verifyOnboardInferenceSmoke(options: any) { if ( !options.forceOpenAiLike && !shouldSmokeOpenAiLikeOnboardRoute(options.provider, options.credentialEnv) diff --git a/test/onboard-smoke-verifier.test.ts b/test/onboard-smoke-verifier.test.ts index b1ebc30f93f..4e0db3357ff 100644 --- a/test/onboard-smoke-verifier.test.ts +++ b/test/onboard-smoke-verifier.test.ts @@ -2,10 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 import { describe, expect, it } from "vitest"; +import { shouldSmokeOpenAiLikeOnboardRoute } from "../dist/lib/inference/onboard-probes"; import { runVerifyOnboardSmokeHarness } from "./helpers/onboard-smoke-verifier-harness"; -const { shouldSmokeOpenAiLikeOnboardRoute } = require("../dist/lib/inference/onboard-probes"); - describe("Hermes onboard smoke verification", () => { it("does not host-smoke Hermes Provider with the ambient OPENAI_API_KEY", () => { expect(shouldSmokeOpenAiLikeOnboardRoute("hermes-provider", "OPENAI_API_KEY")).toBe(false);