From 80f467fb7a8b03bf871890ac47f37d11b1b840cd Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 4 Jul 2026 09:23:38 -0700 Subject: [PATCH 01/16] perf(test): reduce onboarding subprocess isolation Signed-off-by: Carlos Villela --- ci/test-file-size-budget.json | 4 +- src/lib/onboard.ts | 318 +-- src/lib/onboard/bedrock-runtime.ts | 9 +- src/lib/onboard/local-inference-route.ts | 10 +- src/lib/onboard/windows-host-ollama.ts | 43 +- test/onboard-selection.test.ts | 1210 +++++------- test/onboard.test.ts | 2230 ++++++++-------------- 7 files changed, 1547 insertions(+), 2277 deletions(-) diff --git a/ci/test-file-size-budget.json b/ci/test-file-size-budget.json index 9d7cbc501df..3754a828c30 100644 --- a/ci/test-file-size-budget.json +++ b/ci/test-file-size-budget.json @@ -10,8 +10,8 @@ "test/install-preflight.test.ts": 3934, "test/nemoclaw-start.test.ts": 4827, "test/onboard-messaging.test.ts": 2062, - "test/onboard-selection.test.ts": 6867, - "test/onboard.test.ts": 4774, + "test/onboard-selection.test.ts": 6603, + "test/onboard.test.ts": 4202, "test/policies.test.ts": 2332 } } diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 78529a3bfa6..b701e965fba 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -4162,137 +4162,216 @@ async function setupNim(gpu: ReturnType, sandboxName: stri // ── Step 4: Inference provider ─────────────────────────────────── -async function setupInference( - sandboxName: string | null, - model: string, - provider: string, - endpointUrl: string | null = null, - credentialEnv: string | null = null, - hermesAuthMethod: HermesAuthMethod | string | null = null, - hermesToolGateways: string[] = [], - options: import("./onboard/machine/handlers/provider-inference").ProviderInferenceSetupOptions = {}, -): Promise<{ ok: true; retry?: undefined } | { retry: "selection" }> { - step(4, 8, "Setting up inference provider"); - runOpenshell(["gateway", "select", GATEWAY_NAME], { ignoreError: true }); - - const commonDeps = { +function getSetupInferenceDeps() { + return { + step, + getGatewayName: () => GATEWAY_NAME, runOpenshell, upsertProvider, verifyInferenceRoute, verifyOnboardInferenceSmoke, isNonInteractive, - registry, + updateSandbox: registry.updateSandbox, + hermesProviderAuth, + getHermesToolGatewayBroker, + providerExistsInGateway, + normalizeHermesAuthMethod, + resolveHermesNousApiKey, + checkHermesProviderStoreReachable, + hermesAuthMethodLabel, + hermesConstants: { + HERMES_NOUS_API_KEY_CREDENTIAL_ENV, + HERMES_AUTH_METHOD_API_KEY, + HERMES_AUTH_METHOD_OAUTH, + }, + requireValue, + redact, + compactText, + REMOTE_PROVIDER_CONFIG, + hydrateCredentialEnv, + promptValidationRecovery, + classifyApplyFailure, + localInferenceTimeoutSecs: LOCAL_INFERENCE_TIMEOUT_SECS, + bedrockRuntimeOnboard, + validateLocalProvider, + getLocalProviderHealthCheck, + getLocalProviderBaseUrl, + applyLocalInferenceRoute, + run, + vllmLocalCredentialEnv: VLLM_LOCAL_CREDENTIAL_ENV, + getOllamaWarmupCommand, + shouldFrontOllamaWithProxy, + ensureOllamaAuthProxy, + isProxyHealthy, + getOllamaProxyToken, + persistAndProbeOllamaProxy, + localInference, + ollamaProxyCredentialEnv: OLLAMA_PROXY_CREDENTIAL_ENV, + isRoutedInferenceProvider, + reconcileModelRouter, + routedInference, + log: (message: string) => console.log(message), + error: (message: string) => console.error(message), + exitProcess: (code: number): never => process.exit(code), }; +} - if (provider === hermesProviderAuth.HERMES_PROVIDER_NAME) { - return inferenceProviders.setupHermesProviderInference( - { - sandboxName, - model, - provider, - endpointUrl, - credentialEnv, - hermesAuthMethod, - hermesToolGateways, - }, - { - ...commonDeps, - hermesProviderAuth, - getHermesToolGatewayBroker, - providerExistsInGateway, - normalizeHermesAuthMethod, - resolveHermesNousApiKey, - checkHermesProviderStoreReachable, - hermesAuthMethodLabel, - hermesConstants: { - HERMES_NOUS_API_KEY_CREDENTIAL_ENV, - HERMES_AUTH_METHOD_API_KEY, - HERMES_AUTH_METHOD_OAUTH, +export type SetupInferenceDeps = ReturnType; + +type ProviderInferenceSetupOptions = + import("./onboard/machine/handlers/provider-inference").ProviderInferenceSetupOptions; + +export type SetupInference = ( + sandboxName: string | null, + model: string, + provider: string, + endpointUrl?: string | null, + credentialEnv?: string | null, + hermesAuthMethod?: HermesAuthMethod | string | null, + hermesToolGateways?: string[], + options?: ProviderInferenceSetupOptions, +) => Promise<{ ok: true; retry?: undefined } | { retry: "selection" }>; + +function createSetupInference(overrides: Partial = {}): SetupInference { + const deps: SetupInferenceDeps = { ...getSetupInferenceDeps(), ...overrides }; + + return async function setupInferenceWithDeps( + sandboxName: string | null, + model: string, + provider: string, + endpointUrl: string | null = null, + credentialEnv: string | null = null, + hermesAuthMethod: HermesAuthMethod | string | null = null, + hermesToolGateways: string[] = [], + options: ProviderInferenceSetupOptions = {}, + ): Promise<{ ok: true; retry?: undefined } | { retry: "selection" }> { + deps.step(4, 8, "Setting up inference provider"); + deps.runOpenshell(["gateway", "select", deps.getGatewayName()], { ignoreError: true }); + + const commonDeps = { + runOpenshell: deps.runOpenshell, + upsertProvider: deps.upsertProvider, + verifyInferenceRoute: deps.verifyInferenceRoute, + verifyOnboardInferenceSmoke: deps.verifyOnboardInferenceSmoke, + isNonInteractive: deps.isNonInteractive, + registry: { updateSandbox: deps.updateSandbox }, + }; + + if (provider === deps.hermesProviderAuth.HERMES_PROVIDER_NAME) { + return inferenceProviders.setupHermesProviderInference( + { + sandboxName, + model, + provider, + endpointUrl, + credentialEnv, + hermesAuthMethod, + hermesToolGateways, }, - requireValue, - redact, - compactText, - }, - ); - } + { + ...commonDeps, + hermesProviderAuth: deps.hermesProviderAuth, + getHermesToolGatewayBroker: deps.getHermesToolGatewayBroker, + providerExistsInGateway: deps.providerExistsInGateway, + normalizeHermesAuthMethod: deps.normalizeHermesAuthMethod, + resolveHermesNousApiKey: deps.resolveHermesNousApiKey, + checkHermesProviderStoreReachable: deps.checkHermesProviderStoreReachable, + hermesAuthMethodLabel: deps.hermesAuthMethodLabel, + hermesConstants: deps.hermesConstants, + requireValue: deps.requireValue, + redact: deps.redact, + compactText: deps.compactText, + }, + ); + } - if (inferenceProviders.isRemoteProviderName(provider)) { - // biome-ignore format: keep src/lib/onboard.ts net-neutral for growth guardrail. - const outcome = await inferenceProviders.setupRemoteProviderInference( - { sandboxName, model, provider, endpointUrl, credentialEnv, reuseGatewayCredentialWithoutLocalKey: options.reuseGatewayCredentialWithoutLocalKey === true }, - { - ...commonDeps, - REMOTE_PROVIDER_CONFIG, - hydrateCredentialEnv, - promptValidationRecovery, - classifyApplyFailure, - LOCAL_INFERENCE_TIMEOUT_SECS, - bedrockRuntimeOnboard, - redact, - compactText, - }, - ); - if (outcome.done) return outcome.result; - } else if (provider === "vllm-local") { - const outcome = await inferenceProviders.setupVllmLocalInference( - { model, provider }, - { - ...commonDeps, - validateLocalProvider, - getLocalProviderHealthCheck, - getLocalProviderBaseUrl, - applyLocalInferenceRoute, - run, - VLLM_LOCAL_CREDENTIAL_ENV, - }, - ); - if (outcome.done) return outcome.result; - } else if (provider === "ollama-local") { - const outcome = await inferenceProviders.setupOllamaLocalInference( - { model, provider, allowToolsIncompatible: options.allowToolsIncompatible === true }, - { - ...commonDeps, - validateLocalProvider, - getLocalProviderBaseUrl, - applyLocalInferenceRoute, - getOllamaWarmupCommand, - run, - shouldFrontOllamaWithProxy, - ensureOllamaAuthProxy, - isProxyHealthy, - getOllamaProxyToken, - persistAndProbeOllamaProxy, - localInference, - OLLAMA_PROXY_CREDENTIAL_ENV, - }, - ); - if (outcome.done) return outcome.result; - } else if (isRoutedInferenceProvider(provider)) { - await inferenceProviders.setupRoutedInference( - { model, provider, endpointUrl, credentialEnv }, - { - ...commonDeps, - reconcileModelRouter, - routedInference, - hydrateCredentialEnv, - }, - ); - } else { - console.error(` Unsupported provider configuration: ${provider}`); - process.exit(1); - } + if (inferenceProviders.isRemoteProviderName(provider)) { + const outcome = await inferenceProviders.setupRemoteProviderInference( + { + sandboxName, + model, + provider, + endpointUrl, + credentialEnv, + reuseGatewayCredentialWithoutLocalKey: + options.reuseGatewayCredentialWithoutLocalKey === true, + }, + { + ...commonDeps, + REMOTE_PROVIDER_CONFIG: deps.REMOTE_PROVIDER_CONFIG, + hydrateCredentialEnv: deps.hydrateCredentialEnv, + promptValidationRecovery: deps.promptValidationRecovery, + classifyApplyFailure: deps.classifyApplyFailure, + LOCAL_INFERENCE_TIMEOUT_SECS: deps.localInferenceTimeoutSecs, + bedrockRuntimeOnboard: deps.bedrockRuntimeOnboard, + redact: deps.redact, + compactText: deps.compactText, + }, + ); + if (outcome.done) return outcome.result; + } else if (provider === "vllm-local") { + const outcome = await inferenceProviders.setupVllmLocalInference( + { model, provider }, + { + ...commonDeps, + validateLocalProvider: deps.validateLocalProvider, + getLocalProviderHealthCheck: deps.getLocalProviderHealthCheck, + getLocalProviderBaseUrl: deps.getLocalProviderBaseUrl, + applyLocalInferenceRoute: deps.applyLocalInferenceRoute, + run: deps.run, + VLLM_LOCAL_CREDENTIAL_ENV: deps.vllmLocalCredentialEnv, + }, + ); + if (outcome.done) return outcome.result; + } else if (provider === "ollama-local") { + const outcome = await inferenceProviders.setupOllamaLocalInference( + { model, provider, allowToolsIncompatible: options.allowToolsIncompatible === true }, + { + ...commonDeps, + validateLocalProvider: deps.validateLocalProvider, + getLocalProviderBaseUrl: deps.getLocalProviderBaseUrl, + applyLocalInferenceRoute: deps.applyLocalInferenceRoute, + getOllamaWarmupCommand: deps.getOllamaWarmupCommand, + run: deps.run, + shouldFrontOllamaWithProxy: deps.shouldFrontOllamaWithProxy, + ensureOllamaAuthProxy: deps.ensureOllamaAuthProxy, + isProxyHealthy: deps.isProxyHealthy, + getOllamaProxyToken: deps.getOllamaProxyToken, + persistAndProbeOllamaProxy: deps.persistAndProbeOllamaProxy, + localInference: deps.localInference, + OLLAMA_PROXY_CREDENTIAL_ENV: deps.ollamaProxyCredentialEnv, + }, + ); + if (outcome.done) return outcome.result; + } else if (deps.isRoutedInferenceProvider(provider)) { + await inferenceProviders.setupRoutedInference( + { model, provider, endpointUrl, credentialEnv }, + { + ...commonDeps, + reconcileModelRouter: deps.reconcileModelRouter, + routedInference: deps.routedInference, + hydrateCredentialEnv: deps.hydrateCredentialEnv, + }, + ); + } else { + deps.error(` Unsupported provider configuration: ${provider}`); + deps.exitProcess(1); + } - verifyInferenceRoute(provider, model); - if (options.skipHostInferenceSmoke === true) - console.log(" Reusing existing gateway credential; skipping host inference smoke."); - else verifyOnboardInferenceSmoke({ provider, model, endpointUrl, credentialEnv }); - if (sandboxName) { - registry.updateSandbox(sandboxName, { model, provider }); - } - console.log(` ✓ Inference route set: ${provider} / ${model}`); - return { ok: true }; + deps.verifyInferenceRoute(provider, model); + if (options.skipHostInferenceSmoke === true) + deps.log(" Reusing existing gateway credential; skipping host inference smoke."); + else deps.verifyOnboardInferenceSmoke({ provider, model, endpointUrl, credentialEnv }); + if (sandboxName) { + deps.updateSandbox(sandboxName, { model, provider }); + } + deps.log(` ✓ Inference route set: ${provider} / ${model}`); + return { ok: true }; + }; } +const setupInference = createSetupInference(); + // ── Step 6: Messaging channels ─────────────────────────────────── const MESSAGING_CHANNELS = listChannels(); @@ -5301,6 +5380,7 @@ module.exports = { runCaptureOpenshell, agentSupportsWebSearch, agentSupportsWebSearchProvider, + createSetupInference, setupInference, setupMessagingChannels, MESSAGING_CHANNELS, diff --git a/src/lib/onboard/bedrock-runtime.ts b/src/lib/onboard/bedrock-runtime.ts index 5c01f33433a..47fb9cb9a1a 100644 --- a/src/lib/onboard/bedrock-runtime.ts +++ b/src/lib/onboard/bedrock-runtime.ts @@ -130,6 +130,8 @@ export async function setupBedrockRuntimeInference(options: { credentialEnv?: string | null; forceOpenAiLike?: boolean; }) => void; + ensureAdapter?: typeof ensureBedrockRuntimeAdapter; + updateSandbox?: typeof registry.updateSandbox; }): Promise<{ handled: false } | { handled: true; result: SetupInferenceResult }> { const classification = options.provider === "compatible-anthropic-endpoint" && options.endpointUrl @@ -147,7 +149,10 @@ export async function setupBedrockRuntimeInference(options: { let adapter: Awaited>; try { - adapter = await ensureBedrockRuntimeAdapter({ classification, compatibleCredential }); + adapter = await (options.ensureAdapter ?? ensureBedrockRuntimeAdapter)({ + classification, + compatibleCredential, + }); } catch (err) { console.error( ` Failed to start Bedrock Runtime adapter: ${err instanceof Error ? err.message : String(err)}`, @@ -204,7 +209,7 @@ export async function setupBedrockRuntimeInference(options: { forceOpenAiLike: true, }); if (options.sandboxName) { - registry.updateSandbox(options.sandboxName, { + (options.updateSandbox ?? registry.updateSandbox)(options.sandboxName, { model: options.model, provider: options.provider, }); diff --git a/src/lib/onboard/local-inference-route.ts b/src/lib/onboard/local-inference-route.ts index ca043da1e75..f6333d4330a 100644 --- a/src/lib/onboard/local-inference-route.ts +++ b/src/lib/onboard/local-inference-route.ts @@ -19,6 +19,8 @@ export interface LocalInferenceRouteDeps { compactText(value: string): string; redact(value: string): string; localInferenceTimeoutSecs: number; + error?(message: string): void; + exitProcess?(code: number): never; } const LOCAL_PROVIDER_LABELS: Record = { @@ -33,6 +35,8 @@ const LOCAL_PROVIDER_LABELS: Record = { // context — onboarding appears to stop silently after the [4/8] warning. See #4257. // Returns true if the user chose to back out to provider selection; false on success. export function createLocalInferenceRouteApplier(deps: LocalInferenceRouteDeps) { + const error = deps.error ?? console.error; + const exitProcess = deps.exitProcess ?? ((code: number): never => process.exit(code)); return async function applyLocalInferenceRoute( provider: string, model: string, @@ -57,16 +61,16 @@ export function createLocalInferenceRouteApplier(deps: LocalInferenceRouteDeps) const detail = deps.compactText(deps.redact(`${applyResult.stderr || ""} ${applyResult.stdout || ""}`)) || `Failed to configure inference provider '${provider}'.`; - console.error(` ${detail}`); + error(` ${detail}`); if (deps.isNonInteractive()) { // Only surface the resume guidance when we are actually about to exit — // printing it on every interactive retry is misleading because the user // is still inside an active onboard run. - console.error( + error( " No sandbox was created. Fix the inference route and re-run " + "`nemoclaw onboard --resume` to continue, or choose a different provider/model.", ); - process.exit(applyResult.status || 1); + exitProcess(applyResult.status || 1); } const retry = await deps.promptValidationRecovery( label, diff --git a/src/lib/onboard/windows-host-ollama.ts b/src/lib/onboard/windows-host-ollama.ts index 9ccda7d584e..247a92bdd9d 100644 --- a/src/lib/onboard/windows-host-ollama.ts +++ b/src/lib/onboard/windows-host-ollama.ts @@ -40,12 +40,26 @@ const GET_KNOWN_OLLAMA_INSTALL_PATH = const GET_NETTCP_OLLAMA_LISTEN = "Get-NetTCPConnection -LocalPort 11434 -State Listen -ErrorAction SilentlyContinue | Select-Object -ExpandProperty LocalAddress"; -function powershell(script: string): string { - return runCapture([POWERSHELL, "-Command", script], { ignoreError: true }).trim(); +export interface DetectWindowsHostOllamaDeps { + isWsl: () => boolean; + runCapture: typeof runCapture; } -function probeInstalledPath(): string { - const onPath = powershell(GET_COMMAND_OLLAMA); +function resolveDeps( + overrides: Partial = {}, +): DetectWindowsHostOllamaDeps { + return { + isWsl: overrides.isWsl ?? isWsl, + runCapture: overrides.runCapture ?? runCapture, + }; +} + +function powershell(script: string, deps: DetectWindowsHostOllamaDeps): string { + return deps.runCapture([POWERSHELL, "-Command", script], { ignoreError: true }).trim(); +} + +function probeInstalledPath(deps: DetectWindowsHostOllamaDeps): string { + const onPath = powershell(GET_COMMAND_OLLAMA, deps); if (onPath.length > 0) return onPath; // PATH miss: service-style installs and any installer that does not // update the calling user's PATH leave ollama.exe invisible to @@ -53,31 +67,34 @@ function probeInstalledPath(): string { // the live process so the restart launcher in windows.ts can target // the verified executable instead of falling back to a broken PATH // lookup (#3949). - const processPath = powershell(GET_PROCESS_OLLAMA_PATH); + const processPath = powershell(GET_PROCESS_OLLAMA_PATH, deps); if (processPath.length > 0) return processPath; // Silent installs often land in fixed locations without updating PATH or // leaving a running daemon to probe. Check those paths even when no PID is // visible so WSL onboarding offers Start instead of Install (#4066). - return powershell(GET_KNOWN_OLLAMA_INSTALL_PATH); + return powershell(GET_KNOWN_OLLAMA_INSTALL_PATH, deps); } -function probeLoopbackOnly(): boolean { - const pid = powershell(GET_PROCESS_OLLAMA_ID); +function probeLoopbackOnly(deps: DetectWindowsHostOllamaDeps): boolean { + const pid = powershell(GET_PROCESS_OLLAMA_ID, deps); if (!pid) return false; - const listenAddrs = runCapture([POWERSHELL, "-Command", GET_NETTCP_OLLAMA_LISTEN], { + const listenAddrs = deps.runCapture([POWERSHELL, "-Command", GET_NETTCP_OLLAMA_LISTEN], { ignoreError: true, }); return /127\.0\.0\.1/.test(listenAddrs) && !/0\.0\.0\.0|^::\s*$/m.test(listenAddrs); } -export function detectWindowsHostOllama(): WindowsHostOllamaState { - if (!isWsl()) { +export function detectWindowsHostOllama( + overrides: Partial = {}, +): WindowsHostOllamaState { + const deps = resolveDeps(overrides); + if (!deps.isWsl()) { return { installed: false, installedPath: "", loopbackOnly: false }; } - const installedPath = probeInstalledPath(); + const installedPath = probeInstalledPath(deps); // `installed` reflects binary presence on disk, not a live daemon. Onboard // still gates Start/Restart on reachability and loopback binding (#3949). const installed = installedPath.length > 0; - const loopbackOnly = installed ? probeLoopbackOnly() : false; + const loopbackOnly = installed ? probeLoopbackOnly(deps) : false; return { installed, installedPath, loopbackOnly }; } diff --git a/test/onboard-selection.test.ts b/test/onboard-selection.test.ts index 09b512411b7..4e503d6b234 100644 --- a/test/onboard-selection.test.ts +++ b/test/onboard-selection.test.ts @@ -6,7 +6,21 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; + +import { + getWindowsHostOllamaDockerRequirement, + rejectUnsupportedWindowsHostOllama, +} from "../src/lib/onboard/local-inference-topology.js"; +import { buildInferenceProviderMenu } from "../src/lib/onboard/provider-menu.js"; +import { resolveRequestedProviderSelection } from "../src/lib/onboard/provider-selection.js"; +import { reportProviderSelectionFailure } from "../src/lib/onboard/provider-selection-failure.js"; +import { createSetupNimOllamaHandlers } from "../src/lib/onboard/setup-nim-ollama.js"; +import type { SetupNimSelectionState } from "../src/lib/onboard/setup-nim-selection.js"; +import { + type DetectWindowsHostOllamaDeps, + detectWindowsHostOllama, +} from "../src/lib/onboard/windows-host-ollama.js"; import { testTimeout } from "./helpers/timeouts"; @@ -18,6 +32,134 @@ const OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE = '{"choices":[{"message":{"role":"assistant","content":"","tool_calls":[{"type":"function","function":{"name":"emit_ok","arguments":"{\\"ok\\":true}"}}]}}]}'; const PROVIDER_SELECTION_TEST_TIMEOUT_MS = testTimeout(60_000); +const TEST_REMOTE_PROVIDER_CONFIG = { + build: { label: "NVIDIA Endpoints", providerName: "nvidia-prod" }, + openai: { label: "OpenAI", providerName: "openai-api" }, + custom: { + label: "Other OpenAI-compatible endpoint", + providerName: "compatible-endpoint", + }, + anthropic: { label: "Anthropic", providerName: "anthropic-prod" }, + anthropicCompatible: { + label: "Other Anthropic-compatible endpoint", + providerName: "compatible-anthropic-endpoint", + }, + gemini: { label: "Google Gemini", providerName: "gemini-api" }, +}; + +type WindowsRequirement = ReturnType; +type ProviderMenuOverrides = Partial[0]>; +type SetupNimOllamaDeps = Parameters[0]; + +function buildWindowsProviderMenu( + requirement: WindowsRequirement, + overrides: ProviderMenuOverrides = {}, +) { + return buildInferenceProviderMenu({ + remoteProviderConfig: TEST_REMOTE_PROVIDER_CONFIG, + agentProviderOptions: [], + experimental: false, + gpuNimCapable: false, + hasOllama: false, + ollamaRunning: false, + ollamaHost: null, + ollamaPort: 11434, + isWsl: true, + hasWindowsOllama: false, + isWindowsHostOllama: false, + windowsHostLabelSuffix: requirement.supported ? "" : requirement.labelSuffix, + windowsHostInstallLabel: requirement.installLabel, + windowsHostStartLabel: requirement.startLabel, + windowsOllamaReachable: false, + winOllamaLoopbackOnly: false, + ollamaInstallEntry: null, + vllmEntries: [], + routedEnabled: false, + ...overrides, + }); +} + +function resolveWindowsProvider( + options: Array<{ key: string; label: string }>, + requestedProvider: string, + overrides: Partial[0]> = {}, +) { + return resolveRequestedProviderSelection({ + options, + requestedProvider, + sandboxName: null, + remoteProviderConfig: TEST_REMOTE_PROVIDER_CONFIG, + isWsl: true, + isWindowsHostOllama: false, + windowsHostOllamaSupported: true, + hermesProviderAvailable: false, + readRecordedProvider: () => null, + readRecordedNimContainer: () => null, + readRecordedModel: () => null, + ...overrides, + }); +} + +function makeOllamaSelectionState(): SetupNimSelectionState { + return { + model: null, + provider: "nvidia-prod", + endpointUrl: null, + credentialEnv: "NVIDIA_INFERENCE_API_KEY", + hermesAuthMethod: null, + hermesToolGateways: [], + preferredInferenceApi: null, + nimContainer: null, + allowToolsIncompatible: false, + skipHostInferenceSmoke: false, + }; +} + +function makeSetupNimOllamaDeps(overrides: Partial = {}): SetupNimOllamaDeps { + const processStub = { + platform: "linux", + exit(code?: number): never { + throw new Error(`Unexpected process.exit(${String(code)})`); + }, + } as NodeJS.Process; + return { + OLLAMA_PORT: 11434, + OLLAMA_PROXY_PORT: 11435, + process: processStub, + isNonInteractive: () => true, + prompt: async () => "", + checkOllamaPortsOrWarn: () => true, + ensureOllamaLoopbackSystemdOverride: () => "not-applicable", + runOllamaStartupOrGate: () => ({ kind: "ready" }), + shouldFrontOllamaWithProxy: () => false, + startOllamaAuthProxy: () => true, + getLocalProviderBaseUrl: () => "http://host.docker.internal:11434/v1", + selectAndValidateOllamaModel: async () => ({ + outcome: "selected", + model: "qwen3:8b", + allowToolsIncompatible: false, + }), + printOllamaExposureWarning: () => {}, + switchToWindowsOllamaHost: () => {}, + installOllamaOnWindowsHost: async () => ({ ok: true }), + awaitWindowsOllamaReady: () => true, + setupWindowsOllamaWith0000Binding: () => true, + printWindowsOllamaTimeoutDiagnostics: () => {}, + resetOllamaHostCache: () => {}, + installOllamaOnMacOS: () => ({ ok: true }), + installOllamaOnLinux: () => ({ ok: true }), + abortNonInteractive(message: string): never { + throw new Error(message); + }, + assertOllamaUpgradeApplied: () => ({ ok: true }), + ...overrides, + }; +} + +function nonInteractiveAbort(reason: string, hint?: string): never { + throw new Error(`[non-interactive] Aborting: ${reason}${hint ? `\n${hint}` : ""}`); +} + function writeOpenAiStyleAuthRetryCurl(fakeBin: string, goodToken: string, models = ["gpt-5.4"]) { fs.writeFileSync( path.join(fakeBin, "curl"), @@ -5930,88 +6072,11 @@ const { setupNim } = require(${onboardPath}); }); it("shows Windows-host Ollama in the menu with a Docker Desktop requirement on native Docker WSL", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-windows-ollama-native-docker-menu-"), - ); - const scriptPath = path.join(tmpDir, "windows-ollama-native-docker-menu-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const platformPath = JSON.stringify(path.join(repoRoot, "src", "lib", "platform.ts")); - const topologyPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "local-inference-topology.ts"), - ); - - const script = String.raw` -const credentials = require(${credentialsPath}); -const runner = require(${runnerPath}); -const platform = require(${platformPath}); -const topology = require(${topologyPath}); - -platform.isWsl = () => true; -topology.getContainerRuntime = () => "docker"; -credentials.ensureApiKey = async () => {}; -const messages = []; -credentials.prompt = async (message) => { - messages.push(message); - if (/Choose \[/.test(message)) throw new Error("STOP_AFTER_MENU"); - return ""; -}; -runner.runCapture = (command) => { - const cmd = Array.isArray(command) ? command.join(" ") : command; - if (cmd.includes("command -v ollama")) return ""; - if (cmd.includes("127.0.0.1:11434/api/tags")) return ""; - if (cmd.includes("host.docker.internal:11434/api/tags")) return ""; - if (cmd.includes("127.0.0.1:8000/v1/models")) return ""; - if (cmd.includes("docker images")) return ""; - if (cmd.includes("powershell.exe") && cmd.includes("Get-Command ollama.exe")) - return "C:\\\\Users\\\\tester\\\\AppData\\\\Local\\\\Programs\\\\Ollama\\\\ollama.exe"; - if (cmd.includes("powershell.exe") && cmd.includes("Get-Process ollama")) return ""; - return ""; -}; - -const { setupNim } = require(${onboardPath}); - -(async () => { - const originalLog = console.log; - const lines = []; - console.log = (...args) => lines.push(args.join(" ")); - try { - try { - await setupNim(null, null); - } catch (error) { - if (!String(error && error.message).includes("STOP_AFTER_MENU")) throw error; - } - originalLog(JSON.stringify({ lines, messages })); - } finally { - console.log = originalLog; - } -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - NEMOCLAW_NON_INTERACTIVE: "", - NEMOCLAW_PROVIDER: "", - NEMOCLAW_MODEL: "", - }, + const requirement = getWindowsHostOllamaDockerRequirement("docker"); + const { options } = buildWindowsProviderMenu(requirement, { + hasWindowsOllama: true, }); - - assert.equal(result.status, 0, result.stderr); - assert.notEqual(result.stdout.trim(), "", result.stderr); - const payload = JSON.parse(result.stdout.trim()); - const menuOutput = payload.lines.join("\n"); + const menuOutput = options.map((option) => option.label).join("\n"); assert.match( menuOutput, @@ -6021,135 +6086,137 @@ const { setupNim } = require(${onboardPath}); }); it("rejects Windows-host Ollama providers on native Docker WSL before launching Ollama", () => { - const repoRoot = path.join(import.meta.dirname, ".."); + const requirement = getWindowsHostOllamaDockerRequirement("docker"); + assert.equal(requirement.supported, false); const scenarios = [ { provider: "start-windows-ollama", hasWindowsOllama: true }, { provider: "install-windows-ollama", hasWindowsOllama: false }, ]; for (const scenario of scenarios) { - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), `nemoclaw-onboard-${scenario.provider}-native-docker-`), - ); - const scriptPath = path.join(tmpDir, `${scenario.provider}-native-docker-check.js`); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const platformPath = JSON.stringify(path.join(repoRoot, "src", "lib", "platform.ts")); - const topologyPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "local-inference-topology.ts"), - ); - const windowsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "inference", "ollama", "windows.ts"), - ); - - const script = String.raw` -const credentials = require(${credentialsPath}); -const runner = require(${runnerPath}); -const platform = require(${platformPath}); -const topology = require(${topologyPath}); -const windows = require(${windowsPath}); -const hasWindowsOllama = ${JSON.stringify(scenario.hasWindowsOllama)}; - -platform.isWsl = () => true; -topology.getContainerRuntime = () => "docker"; -credentials.prompt = async () => { - throw new Error("Unexpected prompt in non-interactive test"); -}; -credentials.ensureApiKey = async () => {}; -runner.runCapture = (command) => { - const cmd = Array.isArray(command) ? command.join(" ") : command; - if (cmd.includes("command -v ollama")) return ""; - if (cmd.includes("127.0.0.1:11434/api/tags")) return ""; - if (cmd.includes("host.docker.internal:11434/api/tags")) return ""; - if (cmd.includes("127.0.0.1:8000/v1/models")) return ""; - if (cmd.includes("docker images")) return ""; - if (cmd.includes("powershell.exe") && cmd.includes("Get-Command ollama.exe")) { - return hasWindowsOllama - ? "C:\\\\Users\\\\tester\\\\AppData\\\\Local\\\\Programs\\\\Ollama\\\\ollama.exe" - : ""; - } - if (cmd.includes("powershell.exe") && cmd.includes("Get-Process ollama")) return ""; - return ""; -}; -runner.run = () => ({ status: 0 }); -runner.runShell = () => ({ status: 0 }); -windows.installOllamaOnWindowsHost = async () => { - console.error("WINDOWS_INSTALL_CALLED"); - return { - ok: true, - path: "C:\\\\Users\\\\tester\\\\AppData\\\\Local\\\\Programs\\\\Ollama\\\\ollama.exe", - }; -}; -windows.setupWindowsOllamaWith0000Binding = () => { - console.error("WINDOWS_SETUP_CALLED"); - return true; -}; -windows.switchToWindowsOllamaHost = () => { - console.error("WINDOWS_SWITCH_CALLED"); -}; - -const { setupNim } = require(${onboardPath}); - -(async () => { - await setupNim(null, null); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_PROVIDER: scenario.provider, - NEMOCLAW_MODEL: "qwen3:8b", - NEMOCLAW_YES: "1", - }, + const { options } = buildWindowsProviderMenu(requirement, { + hasWindowsOllama: scenario.hasWindowsOllama, }); - - assert.equal(result.status, 1, `${scenario.provider} unexpectedly passed`); - assert.match(result.stderr, /\[non-interactive\] Aborting:/); - assert.match(result.stderr, new RegExp(`${scenario.provider} requires Docker Desktop`)); - assert.match(result.stderr, /Choose WSL-local Ollama/); - assert.doesNotMatch( - result.stderr, - /WINDOWS_INSTALL_CALLED|WINDOWS_SETUP_CALLED|WINDOWS_SWITCH_CALLED/, - ); + const resolution = resolveWindowsProvider(options, scenario.provider, { + windowsHostOllamaSupported: false, + }); + assert.equal(resolution.kind, "selected"); + if (resolution.kind !== "selected") throw new Error("Expected provider selection"); + assert.equal(resolution.selected.key, scenario.provider); + + const install = vi.fn(); + const setup = vi.fn(); + const switchHost = vi.fn(); + const abort = vi.fn(nonInteractiveAbort); + let failure = ""; + try { + const rejected = rejectUnsupportedWindowsHostOllama( + requirement, + resolution.selected.key, + true, + () => true, + abort, + ); + if (!rejected) { + install(); + setup(); + switchHost(); + } + } catch (error) { + failure = error instanceof Error ? error.message : String(error); + } + + assert.match(failure, /\[non-interactive\] Aborting:/); + assert.match(failure, new RegExp(scenario.provider + " requires Docker Desktop")); + assert.match(failure, /Choose WSL-local Ollama/); + assert.equal(abort.mock.calls.length, 1); + assert.equal(install.mock.calls.length, 0); + assert.equal(setup.mock.calls.length, 0); + assert.equal(switchHost.mock.calls.length, 0); } }); it("rejects reachable Windows-host Ollama on native Docker WSL through generic and fallback paths", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const scenarios = ["ollama", "start-windows-ollama", "install-windows-ollama"]; + const requirement = getWindowsHostOllamaDockerRequirement("docker"); + assert.equal(requirement.supported, false); + const { options } = buildWindowsProviderMenu(requirement, { + ollamaRunning: true, + ollamaHost: "host.docker.internal", + hasWindowsOllama: true, + isWindowsHostOllama: true, + }); - for (const provider of scenarios) { - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), `nemoclaw-onboard-${provider}-reachable-native-docker-`), - ); - const scriptPath = path.join(tmpDir, `${provider}-reachable-native-docker-check.js`); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const platformPath = JSON.stringify(path.join(repoRoot, "src", "lib", "platform.ts")); - const topologyPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "local-inference-topology.ts"), - ); - const localPath = JSON.stringify(path.join(repoRoot, "src", "lib", "inference", "local.ts")); - const windowsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "inference", "ollama", "windows.ts"), - ); + for (const provider of ["ollama", "start-windows-ollama", "install-windows-ollama"]) { + const resolution = resolveWindowsProvider(options, provider, { + isWindowsHostOllama: true, + windowsHostOllamaSupported: false, + }); + const modelSelection = vi.fn(); + const install = vi.fn(); + const setup = vi.fn(); + const switchHost = vi.fn(); + const abort = vi.fn(nonInteractiveAbort); + const reject = (providerKey: string, windowsHostSelected: boolean) => + rejectUnsupportedWindowsHostOllama( + requirement, + providerKey, + windowsHostSelected, + () => true, + abort, + ); + let failure = ""; + + try { + if (resolution.kind === "failure") { + reportProviderSelectionFailure({ + reason: resolution.reason, + isWindowsHostOllama: true, + rejectWindowsHostOllama: reject, + writeError(message) { + throw new Error(message); + }, + }); + } else if (!reject(resolution.selected.key, true)) { + modelSelection(); + install(); + setup(); + switchHost(); + } + } catch (error) { + failure = error instanceof Error ? error.message : String(error); + } + + assert.match(failure, /\[non-interactive\] Aborting:/); + assert.match(failure, new RegExp(provider + " requires Docker Desktop")); + assert.match(failure, /Choose WSL-local Ollama/); + assert.equal(abort.mock.calls.length, 1); + assert.equal(modelSelection.mock.calls.length, 0); + assert.equal(install.mock.calls.length, 0); + assert.equal(setup.mock.calls.length, 0); + assert.equal(switchHost.mock.calls.length, 0); + } + + // Keep one production boundary to pin setupNim's reject-before-dispatch ordering. + const repoRoot = path.join(import.meta.dirname, ".."); + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-ollama-reachable-native-docker-"), + ); + const scriptPath = path.join(tmpDir, "ollama-reachable-native-docker-check.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const platformPath = JSON.stringify(path.join(repoRoot, "src", "lib", "platform.ts")); + const topologyPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "local-inference-topology.ts"), + ); + const localPath = JSON.stringify(path.join(repoRoot, "src", "lib", "inference", "local.ts")); + const windowsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "inference", "ollama", "windows.ts"), + ); - const script = String.raw` + const script = String.raw` const credentials = require(${credentialsPath}); const runner = require(${runnerPath}); const platform = require(${platformPath}); @@ -6168,8 +6235,9 @@ runner.runCapture = (command) => { if (cmd.includes("command -v ollama")) return ""; if (cmd.includes("127.0.0.1:8000/v1/models")) return ""; if (cmd.includes("docker images")) return ""; - if (cmd.includes("powershell.exe") && cmd.includes("Get-Command ollama.exe")) - return "C:\\\\Users\\\\tester\\\\AppData\\\\Local\\\\Programs\\\\Ollama\\\\ollama.exe"; + if (cmd.includes("powershell.exe") && cmd.includes("Get-Command ollama.exe")) { + return "C:\\Users\\tester\\AppData\\Local\\Programs\\Ollama\\ollama.exe"; + } if (cmd.includes("powershell.exe") && cmd.includes("Get-Process ollama")) return ""; if (cmd.includes("api/tags")) return JSON.stringify({ models: [{ name: "qwen3:8b" }] }); return ""; @@ -6184,10 +6252,7 @@ local.getOllamaModelOptions = () => { }; windows.installOllamaOnWindowsHost = async () => { console.error("WINDOWS_INSTALL_CALLED"); - return { - ok: true, - path: "C:\\\\Users\\\\tester\\\\AppData\\\\Local\\\\Programs\\\\Ollama\\\\ollama.exe", - }; + return { ok: true, path: "C:\\Users\\tester\\AppData\\Local\\Programs\\Ollama\\ollama.exe" }; }; windows.setupWindowsOllamaWith0000Binding = () => { console.error("WINDOWS_SETUP_CALLED"); @@ -6206,584 +6271,255 @@ const { setupNim } = require(${onboardPath}); process.exit(1); }); `; - fs.writeFileSync(scriptPath, script); + fs.writeFileSync(scriptPath, script); - const result = spawnSync(process.execPath, [scriptPath], { + try { + const boundary = spawnSync(process.execPath, [scriptPath], { cwd: repoRoot, encoding: "utf-8", env: { ...process.env, HOME: tmpDir, NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_PROVIDER: provider, + NEMOCLAW_PROVIDER: "ollama", NEMOCLAW_MODEL: "qwen3:8b", NEMOCLAW_YES: "1", }, }); - assert.equal(result.status, 1, `${provider} unexpectedly passed`); - assert.match(result.stderr, /\[non-interactive\] Aborting:/); - assert.match(result.stderr, new RegExp(`${provider} requires Docker Desktop`)); - assert.match(result.stderr, /Choose WSL-local Ollama/); + assert.equal(boundary.status, 1, "generic ollama unexpectedly passed"); + assert.match(boundary.stderr, /\[non-interactive\] Aborting:/); + assert.match(boundary.stderr, /ollama requires Docker Desktop/); + assert.match(boundary.stderr, /Choose WSL-local Ollama/); assert.doesNotMatch( - result.stderr, + boundary.stderr, /MODEL_SELECTION_REACHED|WINDOWS_INSTALL_CALLED|WINDOWS_SETUP_CALLED|WINDOWS_SWITCH_CALLED/, ); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); } }); - it("uses the Windows-host start path when install-windows-ollama is requested but Ollama is already installed", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-windows-ollama-install-to-start-"), - ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "windows-ollama-install-to-start-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const platformPath = JSON.stringify(path.join(repoRoot, "src", "lib", "platform.ts")); - const topologyPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "local-inference-topology.ts"), - ); - const localPath = JSON.stringify(path.join(repoRoot, "src", "lib", "inference", "local.ts")); - const windowsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "inference", "ollama", "windows.ts"), + it("uses the Windows-host start path when install-windows-ollama is requested but Ollama is already installed", async () => { + const requirement = getWindowsHostOllamaDockerRequirement("docker-desktop"); + const installedPath = "C:\\Users\\tester\\AppData\\Local\\Programs\\Ollama\\ollama.exe"; + const { options } = buildWindowsProviderMenu(requirement, { + hasWindowsOllama: true, + }); + const resolution = resolveWindowsProvider(options, "install-windows-ollama"); + assert.equal(resolution.kind, "selected"); + if (resolution.kind !== "selected") throw new Error("Expected provider selection"); + assert.equal(resolution.selected.key, "start-windows-ollama"); + + const install = vi.fn(async () => ({ ok: false, path: "" })); + const setup = vi.fn(() => true); + const lines: string[] = []; + const log = vi.spyOn(console, "log").mockImplementation((...args) => { + lines.push(args.join(" ")); + }); + const state = makeOllamaSelectionState(); + const { handleWindowsHostOllamaSelection } = createSetupNimOllamaHandlers( + makeSetupNimOllamaDeps({ + installOllamaOnWindowsHost: install, + setupWindowsOllamaWith0000Binding: setup, + }), ); - fs.mkdirSync(fakeBin, { recursive: true }); - writeAlwaysOkCurl(fakeBin, OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE); - - const script = String.raw` -const credentials = require(${credentialsPath}); -const runner = require(${runnerPath}); -const platform = require(${platformPath}); -const topology = require(${topologyPath}); -platform.isWsl = () => true; -topology.getContainerRuntime = () => "docker-desktop"; + try { + const result = await handleWindowsHostOllamaSelection( + null, + resolution.selected.key, + "qwen3:8b", + false, + false, + installedPath, + state, + ); -const installedPath = "C:\\\\Users\\\\tester\\\\AppData\\\\Local\\\\Programs\\\\Ollama\\\\ollama.exe"; -const installCalls = []; -const setupCalls = []; -const runCommands = []; -credentials.prompt = async () => ""; -credentials.ensureApiKey = async () => {}; -runner.runCapture = (command) => { - const cmd = Array.isArray(command) ? command.join(" ") : command; - if (cmd.includes("command -v ollama")) return ""; - if (cmd.includes("127.0.0.1:8000/v1/models")) return ""; - if (cmd.includes("powershell.exe") && cmd.includes("Get-Command ollama.exe")) return installedPath; - if (cmd.includes("powershell.exe") && cmd.includes("Get-Process ollama")) return ""; - if (cmd.includes("api/tags")) { - if (setupCalls.length > 0) { - return JSON.stringify({ models: [{ name: "qwen3:8b" }] }); + assert.equal(result, "selected"); + assert.equal(state.provider, "ollama-local"); + assert.equal(state.model, "qwen3:8b"); + assert.equal(install.mock.calls.length, 0); + assert.deepEqual( + setup.mock.calls.map(([options]) => options), + [{ announceStop: false, installedPath }], + ); + assert.ok(lines.some((line) => line.includes("Using Ollama on host.docker.internal:11434"))); + } finally { + log.mockRestore(); } - return ""; - } - if (cmd.includes("api/show")) return JSON.stringify({ capabilities: ["completion", "tools"] }); - if (cmd.includes("api/generate")) return '{"response":"hello"}'; - return ""; -}; -runner.run = (command) => { - runCommands.push(Array.isArray(command) ? command.join(" ") : String(command)); - return { status: 0 }; -}; -runner.runShell = (command) => { - runCommands.push(command); - return { status: 0 }; -}; + }); -const local = require(${localPath}); -local.resetOllamaHostCache(); -local.getOllamaModelOptions = () => ["qwen3:8b"]; + it("detects Windows-host Ollama via running process when not on the user PATH (#3949)", async () => { + const installedPath = "C:/Program Files/Ollama/ollama.exe"; + const runCapture = vi.fn((command) => { + const rendered = Array.isArray(command) ? command.join(" ") : String(command); + if (rendered.includes("Get-Command ollama.exe")) return ""; + if (rendered.includes("Get-Process ollama") && rendered.includes("Path")) { + return installedPath; + } + if (rendered.includes("Get-Process ollama") && rendered.includes("Id")) return "7652"; + if (rendered.includes("Get-NetTCPConnection")) return "127.0.0.1"; + return ""; + }); + const detected = detectWindowsHostOllama({ isWsl: () => true, runCapture }); + assert.deepEqual(detected, { + installed: true, + installedPath, + loopbackOnly: true, + }); -const windows = require(${windowsPath}); -windows.installOllamaOnWindowsHost = async () => { - installCalls.push(true); - return { ok: false, path: "" }; -}; -windows.setupWindowsOllamaWith0000Binding = (opts) => { - setupCalls.push(opts || {}); - local.setResolvedOllamaHost(local.OLLAMA_HOST_DOCKER_INTERNAL); - return true; -}; - -const { setupNim } = require(${onboardPath}); - -(async () => { - const originalLog = console.log; - const lines = []; - console.log = (...args) => lines.push(args.join(" ")); - try { - const result = await setupNim("windows-install-to-start-test", null); - originalLog(JSON.stringify({ result, installCalls, setupCalls, lines, runCommands })); - } finally { - console.log = originalLog; - } -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_PROVIDER: "install-windows-ollama", - NEMOCLAW_MODEL: "qwen3:8b", - NEMOCLAW_YES: "1", - }, - }); - - assert.equal(result.status, 0, `Process failed: ${result.stderr}`); - assert.notEqual(result.stdout.trim(), "", result.stderr); - const payload = JSON.parse(result.stdout.trim()); - - assert.equal(payload.result.provider, "ollama-local"); - assert.equal(payload.result.model, "qwen3:8b"); - assert.equal(payload.installCalls.length, 0); - // The restart/start path now forwards the verified executable path - // recovered from Get-Command so windows.ts can launch the binary - // directly instead of relying on the calling shell's Windows PATH - // (#3949). - assert.deepEqual(payload.setupCalls, [ - { - announceStop: false, - // The mock injects `\\\\` per separator (raw template → 4 source - // backslashes per separator → 2 backslashes in the subprocess - // JS string). The deepEqual right-hand side is a regular TS - // string, so 4 backslashes per separator here equals 2 in the - // compiled string, matching what the subprocess captured. - installedPath: - "C:\\\\Users\\\\tester\\\\AppData\\\\Local\\\\Programs\\\\Ollama\\\\ollama.exe", - }, - ]); - assert.ok( - payload.lines.some((line: string) => - line.includes("Using Ollama on host.docker.internal:11434"), - ), - ); - }); - - it("detects Windows-host Ollama via running process when not on the user PATH (#3949)", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-windows-ollama-process-fallback-"), - ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "windows-ollama-process-fallback-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const platformPath = JSON.stringify(path.join(repoRoot, "src", "lib", "platform.ts")); - const topologyPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "local-inference-topology.ts"), - ); - const localPath = JSON.stringify(path.join(repoRoot, "src", "lib", "inference", "local.ts")); - const windowsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "inference", "ollama", "windows.ts"), + const install = vi.fn(async () => ({ ok: false, path: "" })); + const setup = vi.fn(() => true); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const state = makeOllamaSelectionState(); + const { handleWindowsHostOllamaSelection } = createSetupNimOllamaHandlers( + makeSetupNimOllamaDeps({ + installOllamaOnWindowsHost: install, + setupWindowsOllamaWith0000Binding: setup, + }), ); - fs.mkdirSync(fakeBin, { recursive: true }); - writeAlwaysOkCurl(fakeBin, OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE); - - const script = String.raw` -const credentials = require(${credentialsPath}); -const runner = require(${runnerPath}); -const platform = require(${platformPath}); -const topology = require(${topologyPath}); -platform.isWsl = () => true; -topology.getContainerRuntime = () => "docker-desktop"; - -const setupCalls = []; -const installedPath = "C:/Program Files/Ollama/ollama.exe"; -credentials.prompt = async () => ""; -credentials.ensureApiKey = async () => {}; -runner.runCapture = (command) => { - const cmd = Array.isArray(command) ? command.join(" ") : command; - if (cmd.includes("command -v ollama")) return ""; - if (cmd.includes("127.0.0.1:8000/v1/models")) return ""; - // The fix: Get-Command misses ollama.exe (service install, not on user - // PATH), but Get-Process recovers both the live PID and the verified - // executable path. Repro for #3949. - if (cmd.includes("powershell.exe") && cmd.includes("Get-Command ollama.exe")) return ""; - if (cmd.includes("powershell.exe") && cmd.includes("Get-Process ollama") && cmd.includes("Path")) - return installedPath; - if (cmd.includes("powershell.exe") && cmd.includes("Get-Process ollama") && cmd.includes("Id")) - return "7652"; - if (cmd.includes("powershell.exe") && cmd.includes("Get-NetTCPConnection")) return "127.0.0.1"; - if (cmd.includes("api/tags")) { - if (setupCalls.length === 0) return ""; - return JSON.stringify({ models: [{ name: "qwen3:8b" }] }); - } - if (cmd.includes("api/show")) return JSON.stringify({ capabilities: ["completion", "tools"] }); - if (cmd.includes("api/generate")) return '{"response":"hello"}'; - return ""; -}; -runner.run = () => ({ status: 0 }); -runner.runShell = () => ({ status: 0 }); - -const local = require(${localPath}); -local.resetOllamaHostCache(); -local.getOllamaModelOptions = () => ["qwen3:8b"]; - -const windows = require(${windowsPath}); -windows.installOllamaOnWindowsHost = async () => { - throw new Error("installOllamaOnWindowsHost called: hasWindowsOllama not detected"); -}; -windows.setupWindowsOllamaWith0000Binding = (opts) => { - setupCalls.push(opts || {}); - local.setResolvedOllamaHost(local.OLLAMA_HOST_DOCKER_INTERNAL); - return true; -}; - -const { setupNim } = require(${onboardPath}); - -(async () => { - const originalLog = console.log; - const lines = []; - console.log = (...args) => lines.push(args.join(" ")); - try { - const result = await setupNim("windows-process-fallback-test", null); - originalLog(JSON.stringify({ result, setupCalls, lines })); - } finally { - console.log = originalLog; - } -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_PROVIDER: "start-windows-ollama", - NEMOCLAW_MODEL: "qwen3:8b", - NEMOCLAW_YES: "1", - }, - }); - - assert.equal( - result.status, - 0, - `Process failed:\nSTDOUT:\n${result.stdout}\nSTDERR:\n${result.stderr}`, - ); - assert.notEqual(result.stdout.trim(), "", result.stderr); - const payload = JSON.parse(result.stdout.trim()); + try { + await handleWindowsHostOllamaSelection( + null, + "start-windows-ollama", + "qwen3:8b", + false, + detected.loopbackOnly, + detected.installedPath, + state, + ); - assert.equal(payload.result.provider, "ollama-local"); - // hasWindowsOllama detected via Get-Process → winOllamaLoopbackOnly - // observed from 127.0.0.1 listen → restart path taken with - // announceStop:true and the recovered executable path threaded - // through so windows.ts can target the verified binary instead of - // the broken PATH fallback. Pre-fix behaviour was the bogus install - // path with no setup call at all. - assert.deepEqual(payload.setupCalls, [ - { - announceStop: true, - installedPath: "C:/Program Files/Ollama/ollama.exe", - }, - ]); + assert.equal(state.provider, "ollama-local"); + assert.equal(install.mock.calls.length, 0); + assert.deepEqual( + setup.mock.calls.map(([options]) => options), + [{ announceStop: true, installedPath }], + ); + } finally { + log.mockRestore(); + } }); - it("uses a known Windows install path when a running Ollama process has no readable path", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-windows-ollama-static-path-fallback-"), - ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "windows-ollama-static-path-fallback-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const platformPath = JSON.stringify(path.join(repoRoot, "src", "lib", "platform.ts")); - const topologyPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "local-inference-topology.ts"), - ); - const localPath = JSON.stringify(path.join(repoRoot, "src", "lib", "inference", "local.ts")); - const windowsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "inference", "ollama", "windows.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeAlwaysOkCurl(fakeBin, OLLAMA_CHAT_COMPLETIONS_TOOL_CALL_RESPONSE); - - const script = String.raw` -const credentials = require(${credentialsPath}); -const runner = require(${runnerPath}); -const platform = require(${platformPath}); -const topology = require(${topologyPath}); -platform.isWsl = () => true; -topology.getContainerRuntime = () => "docker-desktop"; - -const setupCalls = []; -const installedPath = "C:/Users/tester/AppData/Local/Programs/Ollama/ollama.exe"; -credentials.prompt = async () => ""; -credentials.ensureApiKey = async () => {}; -runner.runCapture = (command) => { - const cmd = Array.isArray(command) ? command.join(" ") : command; - if (cmd.includes("command -v ollama")) return ""; - if (cmd.includes("127.0.0.1:8000/v1/models")) return ""; - if (cmd.includes("powershell.exe") && cmd.includes("Get-Command ollama.exe")) return ""; - if (cmd.includes("powershell.exe") && cmd.includes("Get-Process ollama") && cmd.includes("Path")) - return ""; - if (cmd.includes("powershell.exe") && cmd.includes("Get-Process ollama") && cmd.includes("Id")) - return "7652"; - if (cmd.includes("powershell.exe") && cmd.includes("Test-Path -LiteralPath")) - return installedPath; - if (cmd.includes("powershell.exe") && cmd.includes("Get-NetTCPConnection")) return "127.0.0.1"; - if (cmd.includes("api/tags")) { - if (setupCalls.length === 0) return ""; - return JSON.stringify({ models: [{ name: "qwen3:8b" }] }); - } - if (cmd.includes("api/show")) return JSON.stringify({ capabilities: ["completion", "tools"] }); - if (cmd.includes("api/generate")) return '{"response":"hello"}'; - return ""; -}; -runner.run = () => ({ status: 0 }); -runner.runShell = () => ({ status: 0 }); - -const local = require(${localPath}); -local.resetOllamaHostCache(); -local.getOllamaModelOptions = () => ["qwen3:8b"]; - -const windows = require(${windowsPath}); -windows.installOllamaOnWindowsHost = async () => { - throw new Error("installOllamaOnWindowsHost called: hasWindowsOllama not detected"); -}; -windows.setupWindowsOllamaWith0000Binding = (opts) => { - setupCalls.push(opts || {}); - local.setResolvedOllamaHost(local.OLLAMA_HOST_DOCKER_INTERNAL); - return true; -}; - -const { setupNim } = require(${onboardPath}); - -(async () => { - const originalLog = console.log; - const lines = []; - console.log = (...args) => lines.push(args.join(" ")); - try { - const result = await setupNim("windows-static-path-fallback-test", null); - originalLog(JSON.stringify({ result, setupCalls, lines })); - } finally { - console.log = originalLog; - } -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_PROVIDER: "start-windows-ollama", - NEMOCLAW_MODEL: "qwen3:8b", - NEMOCLAW_YES: "1", - }, + it("uses a known Windows install path when a running Ollama process has no readable path", async () => { + const installedPath = "C:/Users/tester/AppData/Local/Programs/Ollama/ollama.exe"; + const runCapture = vi.fn((command) => { + const rendered = Array.isArray(command) ? command.join(" ") : String(command); + if (rendered.includes("Get-Command ollama.exe")) return ""; + if (rendered.includes("Get-Process ollama") && rendered.includes("Path")) return ""; + if (rendered.includes("Test-Path -LiteralPath")) return installedPath; + if (rendered.includes("Get-Process ollama") && rendered.includes("Id")) return "7652"; + if (rendered.includes("Get-NetTCPConnection")) return "127.0.0.1"; + return ""; + }); + const detected = detectWindowsHostOllama({ isWsl: () => true, runCapture }); + assert.deepEqual(detected, { + installed: true, + installedPath, + loopbackOnly: true, }); - assert.equal( - result.status, - 0, - `Process failed:\nSTDOUT:\n${result.stdout}\nSTDERR:\n${result.stderr}`, + const install = vi.fn(async () => ({ ok: false, path: "" })); + const setup = vi.fn(() => true); + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + const state = makeOllamaSelectionState(); + const { handleWindowsHostOllamaSelection } = createSetupNimOllamaHandlers( + makeSetupNimOllamaDeps({ + installOllamaOnWindowsHost: install, + setupWindowsOllamaWith0000Binding: setup, + }), ); - assert.notEqual(result.stdout.trim(), "", result.stderr); - const payload = JSON.parse(result.stdout.trim()); - assert.equal(payload.result.provider, "ollama-local"); - assert.deepEqual(payload.setupCalls, [ - { - announceStop: true, - installedPath: "C:/Users/tester/AppData/Local/Programs/Ollama/ollama.exe", - }, - ]); + try { + await handleWindowsHostOllamaSelection( + null, + "start-windows-ollama", + "qwen3:8b", + false, + detected.loopbackOnly, + detected.installedPath, + state, + ); + + assert.equal(state.provider, "ollama-local"); + assert.equal(install.mock.calls.length, 0); + assert.deepEqual( + setup.mock.calls.map(([options]) => options), + [{ announceStop: true, installedPath }], + ); + } finally { + log.mockRestore(); + } }); it("does not satisfy start-windows-ollama with WSL-local Ollama", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-windows-ollama-no-wsl-fallback-"), - ); - const scriptPath = path.join(tmpDir, "windows-ollama-no-wsl-fallback-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const platformPath = JSON.stringify(path.join(repoRoot, "src", "lib", "platform.ts")); - const topologyPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "local-inference-topology.ts"), - ); - const localPath = JSON.stringify(path.join(repoRoot, "src", "lib", "inference", "local.ts")); - const windowsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "inference", "ollama", "windows.ts"), - ); - - const script = String.raw` -const credentials = require(${credentialsPath}); -const runner = require(${runnerPath}); -const platform = require(${platformPath}); -const topology = require(${topologyPath}); -platform.isWsl = () => true; -topology.getContainerRuntime = () => "docker-desktop"; - -credentials.prompt = async () => { - throw new Error("Unexpected prompt in non-interactive test"); -}; -credentials.ensureApiKey = async () => {}; -runner.runCapture = (command) => { - const cmd = Array.isArray(command) ? command.join(" ") : command; - if (cmd.includes("command -v ollama")) return "/usr/bin/ollama"; - if (cmd.includes("127.0.0.1:11434/api/tags")) return JSON.stringify({ models: [{ name: "qwen3:8b" }] }); - if (cmd.includes("host.docker.internal:11434/api/tags")) return ""; - if (cmd.includes("127.0.0.1:8000/v1/models")) return ""; - if (cmd.includes("powershell.exe") && cmd.includes("Get-Command ollama.exe")) return ""; - return ""; -}; - -const local = require(${localPath}); -local.resetOllamaHostCache(); - -const windows = require(${windowsPath}); -windows.setupWindowsOllamaWith0000Binding = () => { - console.error("WINDOWS_SETUP_CALLED"); - return false; -}; -windows.switchToWindowsOllamaHost = () => { - console.error("WINDOWS_SWITCH_CALLED"); -}; - -const { setupNim } = require(${onboardPath}); - -(async () => { - await setupNim("windows-no-wsl-fallback-test", null); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_PROVIDER: "start-windows-ollama", - NEMOCLAW_MODEL: "qwen3:8b", - NEMOCLAW_YES: "1", + const requirement = getWindowsHostOllamaDockerRequirement("docker-desktop"); + const { options } = buildWindowsProviderMenu(requirement, { + hasOllama: true, + ollamaRunning: true, + ollamaHost: "127.0.0.1", + hasWindowsOllama: false, + }); + const resolution = resolveWindowsProvider(options, "start-windows-ollama", { + isWsl: true, + isWindowsHostOllama: false, + }); + assert.equal(resolution.kind, "failure"); + if (resolution.kind !== "failure") throw new Error("Expected provider selection failure"); + + const setup = vi.fn(); + const switchHost = vi.fn(); + const errors: string[] = []; + reportProviderSelectionFailure({ + reason: resolution.reason, + isWindowsHostOllama: false, + rejectWindowsHostOllama: () => { + setup(); + switchHost(); + return true; }, + writeError: (message) => errors.push(message), }); - assert.equal(result.status, 1); - assert.match(result.stderr, /Requested provider 'start-windows-ollama' is not available/); - assert.doesNotMatch(result.stderr, /WINDOWS_SETUP_CALLED|WINDOWS_SWITCH_CALLED/); + assert.match(errors.join("\n"), /Requested provider 'start-windows-ollama' is not available/); + assert.equal(setup.mock.calls.length, 0); + assert.equal(switchHost.mock.calls.length, 0); }); it("does not satisfy install-windows-ollama with non-WSL local Ollama", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-windows-ollama-no-linux-fallback-"), - ); - const scriptPath = path.join(tmpDir, "windows-ollama-no-linux-fallback-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const platformPath = JSON.stringify(path.join(repoRoot, "src", "lib", "platform.ts")); - const localPath = JSON.stringify(path.join(repoRoot, "src", "lib", "inference", "local.ts")); - const windowsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "inference", "ollama", "windows.ts"), - ); - - const script = String.raw` -const credentials = require(${credentialsPath}); -const runner = require(${runnerPath}); -const platform = require(${platformPath}); -platform.isWsl = () => false; - -credentials.prompt = async () => { - throw new Error("Unexpected prompt in non-interactive test"); -}; -credentials.ensureApiKey = async () => {}; -runner.runCapture = (command) => { - const cmd = Array.isArray(command) ? command.join(" ") : command; - if (cmd.includes("command -v ollama")) return "/usr/bin/ollama"; - if (cmd.includes("127.0.0.1:11434/api/tags")) return JSON.stringify({ models: [{ name: "qwen3:8b" }] }); - if (cmd.includes("127.0.0.1:8000/v1/models")) return ""; - return ""; -}; - -const local = require(${localPath}); -local.resetOllamaHostCache(); - -const windows = require(${windowsPath}); -windows.installOllamaOnWindowsHost = async () => { - console.error("WINDOWS_INSTALL_CALLED"); - return { ok: false, path: "" }; -}; -windows.setupWindowsOllamaWith0000Binding = () => { - console.error("WINDOWS_SETUP_CALLED"); - return false; -}; - -const { setupNim } = require(${onboardPath}); - -(async () => { - await setupNim("windows-no-linux-fallback-test", null); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_PROVIDER: "install-windows-ollama", - NEMOCLAW_MODEL: "qwen3:8b", - NEMOCLAW_YES: "1", + const requirement = getWindowsHostOllamaDockerRequirement(null); + const { options } = buildWindowsProviderMenu(requirement, { + hasOllama: true, + ollamaRunning: true, + ollamaHost: "127.0.0.1", + isWsl: false, + hasWindowsOllama: false, + }); + const resolution = resolveWindowsProvider(options, "install-windows-ollama", { + isWsl: false, + isWindowsHostOllama: false, + }); + assert.equal(resolution.kind, "failure"); + if (resolution.kind !== "failure") throw new Error("Expected provider selection failure"); + + const install = vi.fn(); + const setup = vi.fn(); + const errors: string[] = []; + reportProviderSelectionFailure({ + reason: resolution.reason, + isWindowsHostOllama: false, + rejectWindowsHostOllama: () => { + install(); + setup(); + return true; }, + writeError: (message) => errors.push(message), }); - assert.equal(result.status, 1); - assert.match(result.stderr, /Requested provider 'install-windows-ollama' is not available/); - assert.doesNotMatch(result.stderr, /WINDOWS_INSTALL_CALLED|WINDOWS_SETUP_CALLED/); + assert.match(errors.join("\n"), /Requested provider 'install-windows-ollama' is not available/); + assert.equal(install.mock.calls.length, 0); + assert.equal(setup.mock.calls.length, 0); }); it("honours NEMOCLAW_LOCAL_INFERENCE_TIMEOUT for compatible-endpoint during inference setup (#2403)", () => { diff --git a/test/onboard.test.ts b/test/onboard.test.ts index 56b0b8dca2c..857fae727fe 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -6,13 +6,16 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { appendHostProxyEnvArgs } from "../src/lib/onboard/host-proxy-env.js"; import { isValidInferenceInputsOverride, maybePromptForInferenceInputCapability, shouldPromptForInferenceInputCapability, } from "../src/lib/onboard/inference-input-capability.js"; +import { createInferenceRouteHelpers } from "../src/lib/onboard/inference-route.js"; +import { createLocalInferenceRouteApplier } from "../src/lib/onboard/local-inference-route.js"; +import type { SetupInference, SetupInferenceDeps } from "../src/lib/onboard.js"; import { stageOptimizedSandboxBuildContext } from "../src/lib/sandbox/build-context.js"; import { testTimeoutOptions } from "./helpers/timeouts"; @@ -23,6 +26,7 @@ type ShimFn = (...args: ShimValue[]) => TReturn; type CommandEntry = { command: string; env?: Record; + ignoreError?: boolean; policyContent?: string; policyReadError?: string; dockerfileContent?: string; @@ -46,6 +50,7 @@ type OnboardTestInternals = { selectedAgentName: string, ) => T; pullAndResolveBaseImageDigest: () => { digest: string | null; ref: string } | null; + createSetupInference: (overrides?: Partial) => SetupInference; SANDBOX_BASE_IMAGE: string; }; @@ -91,9 +96,152 @@ const { getResumeConfigConflicts, getResumeSandboxConflict, clearAgentScopedResumeState, + createSetupInference, SANDBOX_BASE_IMAGE, } = onboardTestInternals; +const onboardProviderHelpers = require("../src/lib/onboard/providers") as { + upsertProvider: ( + name: string, + type: string, + credentialEnv: string, + baseUrl: string | null, + env: Record, + runOpenshell: DirectRunOpenshell, + ) => { ok: boolean; status?: number; message?: string }; +}; +const localInferenceModule = + require("../src/lib/inference/local") as typeof import("../src/lib/inference/local.js"); +const bedrockRuntimeOnboard = + require("../src/lib/onboard/bedrock-runtime") as typeof import("../src/lib/onboard/bedrock-runtime.js"); + +type DirectRunOpenshell = SetupInferenceDeps["runOpenshell"]; +type DirectRunOptions = NonNullable[1]>; +type DirectRunResult = ReturnType; +type DirectRunStubResult = { + status: number | null; + stdout?: string; + stderr?: string; +}; +type DirectSetupHarnessOptions = { + runOpenshell?: ( + args: string[], + options: DirectRunOptions, + calls: CommandEntry[], + ) => DirectRunStubResult | undefined; + overrides?: Partial; +}; + +function directRunResult({ + status = 0, + stdout = "", + stderr = "", +}: Partial = {}): DirectRunResult { + return { + pid: 0, + output: [null, stdout, stderr], + stdout, + stderr, + status, + signal: null, + }; +} + +function createDirectSetupInferenceHarness(options: DirectSetupHarnessOptions = {}) { + const commands: CommandEntry[] = []; + const errors: string[] = []; + const logs: string[] = []; + const updateSandbox = vi.fn(() => true); + const verifyInferenceRoute = vi.fn(); + const verifyOnboardInferenceSmoke = vi.fn(); + const runOpenshell: DirectRunOpenshell = (args, runOptions = {}) => { + commands.push({ + command: args.join(" "), + env: runOptions.env, + ignoreError: runOptions.ignoreError, + }); + return directRunResult(options.runOpenshell?.(args, runOptions, commands)); + }; + const setupInference = createSetupInference({ + step: () => {}, + getGatewayName: () => "nemoclaw", + runOpenshell, + upsertProvider: ( + name: string, + type: string, + credentialEnv: string, + baseUrl: string | null, + env: Record = {}, + ) => + onboardProviderHelpers.upsertProvider(name, type, credentialEnv, baseUrl, env, runOpenshell), + verifyInferenceRoute, + verifyOnboardInferenceSmoke, + isNonInteractive: () => false, + updateSandbox, + resolveHermesNousApiKey: () => process.env.NOUS_API_KEY || null, + checkHermesProviderStoreReachable: (run: DirectRunOpenshell) => { + run(["provider", "list"], { ignoreError: true }); + return { ok: true }; + }, + hydrateCredentialEnv: (envName: string | null | undefined) => + envName ? process.env[envName] || null : null, + promptValidationRecovery: async () => "selection", + validateLocalProvider: () => ({ ok: true }), + getLocalProviderHealthCheck: () => null, + getLocalProviderBaseUrl: (provider: string) => + provider === "ollama-local" + ? "http://host.openshell.internal:11435/v1" + : "http://host.openshell.internal:8000/v1", + applyLocalInferenceRoute: async () => false, + run: () => directRunResult(), + shouldFrontOllamaWithProxy: () => false, + ensureOllamaAuthProxy: () => {}, + isProxyHealthy: () => true, + getOllamaProxyToken: () => null, + persistAndProbeOllamaProxy: async () => {}, + localInference: { + ...localInferenceModule, + validateOllamaModelWithToolsOverride: () => ({ ok: true }), + }, + log: (message: string) => logs.push(message), + error: (message: string) => errors.push(message), + exitProcess: (code: number): never => { + throw Object.assign(new Error(`EXIT_CALLED:${code}`), { code }); + }, + ...options.overrides, + }); + return { + commands, + errors, + logs, + runOpenshell, + setupInference, + updateSandbox, + verifyInferenceRoute, + verifyOnboardInferenceSmoke, + }; +} + +async function withProcessEnv( + values: Record, + runTest: () => Promise | T, +): Promise { + const previous = new Map(); + for (const [key, value] of Object.entries(values)) { + previous.set(key, process.env[key]); + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + try { + return await runTest(); + } finally { + for (const [key, value] of previous) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + } +} + const repoRoot = path.join(import.meta.dirname, ".."); const onboardScriptMocksPath = JSON.stringify( path.join(repoRoot, "test", "helpers", "onboard-script-mocks.cjs"), @@ -229,6 +377,9 @@ describe("onboard helpers", () => { "prints doctor logs automatically when gateway fails to start (#1605)", testTimeoutOptions(20_000), () => { + // Intentional process-contract coverage: this case verifies the real child exit status and + // stdout/stderr handling across the Node -> shell -> OpenShell adapter boundary. The + // setupInference cases below are unit-shaped and run directly through typed dependencies. const repoRoot = path.join(import.meta.dirname, ".."); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-gateway-diag-")); const fakeBin = path.join(tmpDir, "bin"); @@ -704,284 +855,155 @@ startGateway(null).catch(() => {}); expect(SANDBOX_NAME_REGEX.test("")).toBe(false); }); - it("passes credential names to openshell without embedding secret values in argv", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-inference-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "setup-inference-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); - -const commands = []; -runner.run = (command, opts = {}) => { - commands.push({ command: _n(command), env: opts.env || null }); - return { status: 0 }; -}; -runner.runCapture = (command) => { - if (_n(command).includes("inference") && _n(command).includes("get")) { - return [ - "Gateway inference:", - "", - " Route: inference.local", - " Provider: nvidia-nim", - " Model: nvidia/nemotron-3-super-120b-a12b", - " Version: 1", - ].join("\\n"); - } - return ""; -}; -registry.updateSandbox = () => true; - -process.env.NVIDIA_INFERENCE_API_KEY = "nvapi-TEST-NOT-A-REAL-VALUE"; - -const { setupInference } = require(${onboardPath}); - -(async () => { - await setupInference("test-box", "nvidia/nemotron-3-super-120b-a12b", "nvidia-nim"); - console.log(JSON.stringify({ commands, nvidiaApiKey: process.env.NVIDIA_INFERENCE_API_KEY || null })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); + it("passes credential names to openshell without embedding secret values in argv", async () => { + await withProcessEnv({ NVIDIA_INFERENCE_API_KEY: "nvapi-TEST-NOT-A-REAL-VALUE" }, async () => { + const harness = createDirectSetupInferenceHarness({ + runOpenshell: (args) => + args.slice(0, 2).join(" ") === "provider get" + ? { status: 0, stdout: "", stderr: "" } + : undefined, + }); - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }, + await harness.setupInference("test-box", "nvidia/nemotron-3-super-120b-a12b", "nvidia-nim"); + + const commands = harness.commands; + assert.equal(commands.length, 4); + assert.match(commands[0].command, /gateway select nemoclaw/); + assert.match(commands[1].command, /provider get/); + assert.match(commands[2].command, /--credential NVIDIA_INFERENCE_API_KEY/); + assert.doesNotMatch(commands[2].command, /nvapi-TEST-NOT-A-REAL-VALUE/); + assert.match(commands[2].command, /provider update/); + assert.match(commands[3].command, /inference set/); + assert.equal(process.env.NVIDIA_INFERENCE_API_KEY, "nvapi-TEST-NOT-A-REAL-VALUE"); }); - - expect(result.status).toBe(0); - const payload = parseStdoutJson<{ commands: CommandEntry[]; nvidiaApiKey: string | null }>( - result.stdout, - ); - const commands = payload.commands; - assert.equal(commands.length, 4); - assert.match(commands[0].command, /gateway select nemoclaw/); - assert.match(commands[1].command, /provider get/); - assert.match(commands[2].command, /--credential NVIDIA_INFERENCE_API_KEY/); - assert.doesNotMatch(commands[2].command, /nvapi-TEST-NOT-A-REAL-VALUE/); - assert.match(commands[2].command, /provider update/); - assert.match(commands[3].command, /inference set/); - assert.equal(payload.nvidiaApiKey, "nvapi-TEST-NOT-A-REAL-VALUE"); }); - - it("reuses a registered Hermes Provider without re-collecting host credentials", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-hermes-reuse-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "setup-hermes-reuse-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); - -const commands = []; -runner.run = (command, opts = {}) => { - const normalized = _n(command); - commands.push({ command: normalized, env: opts.env || null }); - if (normalized.includes("provider get hermes-provider")) { - return { status: 0, stdout: "Provider: hermes-provider", stderr: "" }; - } - return { status: 0, stdout: "", stderr: "" }; -}; -runner.runCapture = (command) => { - if (_n(command).includes("inference") && _n(command).includes("get")) { - return [ - "Gateway inference:", - "", - " Route: inference.local", - " Provider: hermes-provider", - " Model: moonshotai/kimi-k2.6", - " Version: 1", - ].join("\\n"); - } - return ""; -}; -registry.updateSandbox = () => true; - -process.env.NOUS_API_KEY = "nous-host-secret"; -process.env.OPENAI_API_KEY = "openai-host-secret"; -process.env.NEMOCLAW_NON_INTERACTIVE = "1"; - -const { setupInference } = require(${onboardPath}); - -(async () => { - await setupInference("test-box", "moonshotai/kimi-k2.6", "hermes-provider", "https://8.8.8.8/v1", "OPENAI_API_KEY", "oauth"); - console.log(JSON.stringify(commands)); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, + it("reuses a registered Hermes Provider without re-collecting host credentials", async () => { + await withProcessEnv( + { + NOUS_API_KEY: "nous-host-secret", + OPENAI_API_KEY: "openai-host-secret", + }, + async () => { + const harness = createDirectSetupInferenceHarness({ + runOpenshell: (args) => + args.join(" ") === "provider get hermes-provider" + ? { status: 0, stdout: "Provider: hermes-provider", stderr: "" } + : undefined, + overrides: { isNonInteractive: () => true }, + }); + + await harness.setupInference( + "test-box", + "moonshotai/kimi-k2.6", + "hermes-provider", + "https://8.8.8.8/v1", + "OPENAI_API_KEY", + "oauth", + ); + + const commands = harness.commands; + assert.equal(commands.length, 4); + assert.match(commands[0].command, /gateway select nemoclaw/); + assert.match(commands[1].command, /provider list/); + assert.match(commands[2].command, /provider get hermes-provider/); + assert.match(commands[3].command, /inference set --no-verify --provider hermes-provider/); + assert.ok(!commands.some((entry) => /provider (create|update)/.test(entry.command))); + assert.ok(!commands.some((entry) => entry.env?.NOUS_API_KEY || entry.env?.OPENAI_API_KEY)); + assert.ok( + !commands.some((entry) => /nous-host-secret|openai-host-secret/.test(entry.command)), + "host credential values must not appear in argv", + ); }, - }); - - expect(result.status).toBe(0); - const commands = parseStdoutJson(result.stdout); - assert.equal(commands.length, 4); - assert.match(commands[0].command, /gateway select nemoclaw/); - assert.match(commands[1].command, /provider list/); - assert.match(commands[2].command, /provider get hermes-provider/); - assert.match(commands[3].command, /inference set --no-verify --provider hermes-provider/); - assert.ok(!commands.some((entry) => /provider (create|update)/.test(entry.command))); - assert.ok(!commands.some((entry) => entry.env?.NOUS_API_KEY || entry.env?.OPENAI_API_KEY)); - assert.ok( - !commands.some((entry) => /nous-host-secret|openai-host-secret/.test(entry.command)), - "host credential values must not appear in argv", ); }); + it("routes Bedrock Runtime custom Anthropic endpoints through the hidden OpenAI adapter", async () => { + await withProcessEnv({ COMPATIBLE_ANTHROPIC_API_KEY: "bedrock-bearer" }, async () => { + const updateSandbox = vi.fn(() => true); + const ensureAdapter = vi.fn(async () => ({ + baseUrl: "http://host.openshell.internal:11436/v1", + localBaseUrl: "http://127.0.0.1:11436/v1", + credentialEnv: "NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_TOKEN", + token: "adapter-token", + region: "us-east-1", + compatibleCredential: "bedrock-bearer", + logPath: "/tmp/bedrock-adapter.log", + })); + const setupBedrockRuntimeInference = bedrockRuntimeOnboard.setupBedrockRuntimeInference; + const harness = createDirectSetupInferenceHarness({ + runOpenshell: (args) => + args.join(" ") === "provider get compatible-anthropic-endpoint" + ? { status: 1, stdout: "", stderr: "" } + : undefined, + overrides: { + bedrockRuntimeOnboard: { + ...bedrockRuntimeOnboard, + setupBedrockRuntimeInference: ( + input: Parameters[0], + ) => setupBedrockRuntimeInference({ ...input, ensureAdapter, updateSandbox }), + }, + }, + }); + const consoleOutput: string[] = []; + const captureConsole = (...args: unknown[]) => { + consoleOutput.push(args.map((arg) => String(arg)).join(" ")); + }; + const error = vi.spyOn(console, "error").mockImplementation(captureConsole); + const log = vi.spyOn(console, "log").mockImplementation(captureConsole); + try { + await harness.setupInference( + "test-box", + "anthropic.claude-3-5-sonnet-20240620-v1:0", + "compatible-anthropic-endpoint", + "https://bedrock-runtime.us-east-1.amazonaws.com", + "COMPATIBLE_ANTHROPIC_API_KEY", + ); + } finally { + error.mockRestore(); + log.mockRestore(); + } - it("routes Bedrock Runtime custom Anthropic endpoints through the hidden OpenAI adapter", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-bedrock-runtime-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "setup-bedrock-runtime-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const adapterPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "inference", "bedrock-runtime-adapter.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const registry = require(${registryPath}); -const adapter = require(${adapterPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); - -const commands = []; -runner.run = (command, opts = {}) => { - const normalized = _n(command); - commands.push({ command: normalized, env: opts.env || null }); - if (normalized.includes("provider get compatible-anthropic-endpoint")) { - return { status: 1, stdout: "", stderr: "" }; - } - return { status: 0, stdout: "", stderr: "" }; -}; -runner.runCapture = (command) => { - if (_n(command).includes("inference") && _n(command).includes("get")) { - return [ - "Gateway inference:", - "", - " Route: inference.local", - " Provider: compatible-anthropic-endpoint", - " Model: anthropic.claude-3-5-sonnet-20240620-v1:0", - " Version: 1", - ].join("\\n"); - } - return ""; -}; -registry.updateSandbox = () => true; -adapter.ensureBedrockRuntimeAdapter = async ({ classification, compatibleCredential }) => ({ - baseUrl: "http://host.openshell.internal:11436/v1", - localBaseUrl: "http://127.0.0.1:11436/v1", - credentialEnv: "NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_TOKEN", - token: "adapter-token", - region: classification.region, - compatibleCredential, -}); - -process.env.COMPATIBLE_ANTHROPIC_API_KEY = "bedrock-bearer"; - -const { setupInference } = require(${onboardPath}); - -(async () => { - await setupInference( - "test-box", - "anthropic.claude-3-5-sonnet-20240620-v1:0", - "compatible-anthropic-endpoint", - "https://bedrock-runtime.us-east-1.amazonaws.com", - "COMPATIBLE_ANTHROPIC_API_KEY", - ); - console.log(JSON.stringify(commands)); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }, + const commands = harness.commands; + const providerCommand = commands.find((entry) => /provider create/.test(entry.command)); + assert.ok(providerCommand, "expected hidden adapter provider registration"); + assert.match(providerCommand.command, /--name compatible-anthropic-endpoint/); + assert.match(providerCommand.command, /--type openai/); + assert.match(providerCommand.command, /--credential NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_TOKEN/); + assert.match( + providerCommand.command, + /OPENAI_BASE_URL=http:\/\/host\.openshell\.internal:11436\/v1/, + ); + assert.equal(providerCommand.env?.NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_TOKEN, "adapter-token"); + assert.ok( + !JSON.stringify(commands).includes("bedrock-bearer"), + "Bedrock bearer token must not appear in OpenShell argv or env", + ); + assert.doesNotMatch( + consoleOutput.join("\n"), + /bedrock-bearer|adapter-token/, + "Bedrock tokens must not appear in onboarding console output", + ); + const sandboxCommands = commands.filter((entry) => /\bsandbox\b/.test(entry.command)); + assert.ok( + !sandboxCommands.some((entry) => + JSON.stringify(entry).includes("NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_TOKEN"), + ), + "adapter credential env must not be passed to sandbox commands", + ); + assert.ok( + !sandboxCommands.some((entry) => JSON.stringify(entry).includes("adapter-token")), + "adapter token must not be passed to sandbox commands", + ); + assert.match( + commands.at(-1)?.command || "", + /inference set --no-verify --provider compatible-anthropic-endpoint --model anthropic\.claude-3-5-sonnet-20240620-v1:0/, + ); + expect(ensureAdapter).toHaveBeenCalled(); + expect(updateSandbox).toHaveBeenCalledWith("test-box", { + model: "anthropic.claude-3-5-sonnet-20240620-v1:0", + provider: "compatible-anthropic-endpoint", + }); }); - - expect(result.status).toBe(0); - const commands = parseStdoutJson(result.stdout); - const providerCommand = commands.find((entry) => /provider create/.test(entry.command)); - assert.ok(providerCommand, "expected hidden adapter provider registration"); - assert.match(providerCommand.command, /--name compatible-anthropic-endpoint/); - assert.match(providerCommand.command, /--type openai/); - assert.match(providerCommand.command, /--credential NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_TOKEN/); - assert.match( - providerCommand.command, - /OPENAI_BASE_URL=http:\/\/host\.openshell\.internal:11436\/v1/, - ); - assert.equal(providerCommand.env?.NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_TOKEN, "adapter-token"); - assert.ok( - !JSON.stringify(commands).includes("bedrock-bearer"), - "Bedrock bearer token must not appear in OpenShell argv or env", - ); - const sandboxCommands = commands.filter((entry) => /\bsandbox\b/.test(entry.command)); - assert.ok( - !sandboxCommands.some((entry) => - JSON.stringify(entry).includes("NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_TOKEN"), - ), - "adapter credential env must not be passed to sandbox commands", - ); - assert.ok( - !sandboxCommands.some((entry) => JSON.stringify(entry).includes("adapter-token")), - "adapter token must not be passed to sandbox commands", - ); - assert.ok( - !result.stderr.includes("bedrock-bearer") && !result.stderr.includes("adapter-token"), - "Bedrock tokens must not appear in onboarding stderr", - ); - assert.match( - commands.at(-1)?.command || "", - /inference set --no-verify --provider compatible-anthropic-endpoint --model anthropic\.claude-3-5-sonnet-20240620-v1:0/, - ); }); - it("resolves a sandbox name before reconciling Hermes Provider on resume", { timeout: 60_000, }, () => { @@ -1221,292 +1243,134 @@ const { onboard } = require(${onboardPath}); ); }); - it("reconciles a registered Hermes Provider when a fresh shell Nous key is selected", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-hermes-update-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "setup-hermes-update-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); + it("reconciles a registered Hermes Provider when a fresh shell Nous key is selected", async () => { + await withProcessEnv( + { + NOUS_API_KEY: "nous-host-secret", + OPENAI_API_KEY: undefined, + }, + async () => { + const harness = createDirectSetupInferenceHarness({ + runOpenshell: (args) => + args.join(" ") === "provider get hermes-provider" + ? { status: 0, stdout: "Provider: hermes-provider", stderr: "" } + : undefined, + overrides: { isNonInteractive: () => true }, + }); + + await harness.setupInference( + "test-box", + "moonshotai/kimi-k2.6", + "hermes-provider", + "https://8.8.8.8/v1", + "NOUS_API_KEY", + "api_key", + ); + + const update = harness.commands.find((entry) => + /provider update hermes-provider/.test(entry.command), + ); + assert.ok(update); + assert.match(update.command, /--credential NOUS_API_KEY/); + assert.equal(update.env?.NOUS_API_KEY, "nous-host-secret"); + assert.ok( + !harness.commands.some((entry) => /nous-host-secret/.test(entry.command)), + "shell credential value must not appear in argv", + ); + assert.match( + harness.commands.at(-1)?.command || "", + /inference set --no-verify --provider hermes-provider/, + ); + }, + ); + }); + it("does not delete saved OpenAI credentials when configuring local vLLM", async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-local-vllm-")); + const credentials = require("../src/lib/credentials/store") as { + saveCredential(key: string, value: string): void; + getCredential(key: string): string | null; + }; + try { + await withProcessEnv({ HOME: tmpDir, OPENAI_API_KEY: undefined }, async () => { + credentials.saveCredential("OPENAI_API_KEY", "sk-existing"); + let harness: ReturnType; + const applyLocalInferenceRoute = createLocalInferenceRouteApplier({ + runOpenshell: (args, options) => harness.runOpenshell(args, options), + isNonInteractive: () => false, + promptValidationRecovery: async () => "selection", + classifyApplyFailure: () => ({}) as never, + compactText: (value) => value.trim(), + redact: (value) => value, + localInferenceTimeoutSecs: 120, + }); + harness = createDirectSetupInferenceHarness({ + runOpenshell: (args) => + args.slice(0, 2).join(" ") === "provider get" + ? { status: 1, stdout: "", stderr: "" } + : undefined, + overrides: { applyLocalInferenceRoute }, + }); + + await harness.setupInference("test-box", "meta-llama", "vllm-local"); + + const providerCommand = harness.commands.find((entry) => + entry.command.includes("provider create"), + ); + assert.ok(providerCommand, "expected local vLLM provider create command"); + assert.match(providerCommand.command, /--credential NEMOCLAW_VLLM_LOCAL_TOKEN/); + assert.doesNotMatch(providerCommand.command, /--credential OPENAI_API_KEY/); + assert.equal(providerCommand.env?.NEMOCLAW_VLLM_LOCAL_TOKEN, "dummy"); + assert.equal(credentials.getCredential("OPENAI_API_KEY"), "sk-existing"); + }); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + it("recovers the Ollama auth proxy on WSL when the sandbox needs proxy fronting", async () => { + const proxyCalls: string[] = []; + let harness: ReturnType; + const applyLocalInferenceRoute = createLocalInferenceRouteApplier({ + runOpenshell: (args, options) => harness.runOpenshell(args, options), + isNonInteractive: () => false, + promptValidationRecovery: async () => "selection", + classifyApplyFailure: () => ({}) as never, + compactText: (value) => value.trim(), + redact: (value) => value, + localInferenceTimeoutSecs: 120, + }); + harness = createDirectSetupInferenceHarness({ + runOpenshell: (args) => + args.slice(0, 2).join(" ") === "provider get" + ? { status: 1, stdout: "", stderr: "" } + : undefined, + overrides: { + validateLocalProvider: () => ({ + ok: false, + message: "container cannot reach Ollama", + diagnostic: "simulated WSL native Docker reachability failure", + }), + shouldFrontOllamaWithProxy: () => true, + ensureOllamaAuthProxy: () => proxyCalls.push("ensure"), + isProxyHealthy: () => { + proxyCalls.push("healthy"); + return true; + }, + getOllamaProxyToken: () => "proxy-token", + persistAndProbeOllamaProxy: async (token: string) => { + proxyCalls.push(`persist:${token}`); + }, + applyLocalInferenceRoute, + }, + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + await harness.setupInference("test-box", "qwen3.5:9b", "ollama-local"); + } finally { + warn.mockRestore(); + } - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); - -const commands = []; -runner.run = (command, opts = {}) => { - const normalized = _n(command); - commands.push({ command: normalized, env: opts.env || null }); - if (normalized.includes("provider get hermes-provider")) { - return { status: 0, stdout: "Provider: hermes-provider", stderr: "" }; - } - return { status: 0, stdout: "", stderr: "" }; -}; -runner.runCapture = (command) => { - if (_n(command).includes("inference") && _n(command).includes("get")) { - return [ - "Gateway inference:", - "", - " Route: inference.local", - " Provider: hermes-provider", - " Model: moonshotai/kimi-k2.6", - " Version: 1", - ].join("\\n"); - } - return ""; -}; -registry.updateSandbox = () => true; - -process.env.NOUS_API_KEY = "nous-host-secret"; -delete process.env.OPENAI_API_KEY; -process.env.NEMOCLAW_NON_INTERACTIVE = "1"; - -const { setupInference } = require(${onboardPath}); - -(async () => { - await setupInference( - "test-box", - "moonshotai/kimi-k2.6", - "hermes-provider", - "https://8.8.8.8/v1", - "NOUS_API_KEY", - "api_key", - ); - console.log(JSON.stringify(commands)); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }, - }); - - expect(result.status).toBe(0); - const commands = parseStdoutJson(result.stdout); - const update = commands.find((entry) => /provider update hermes-provider/.test(entry.command)); - assert.ok(update); - assert.match(update.command, /--credential NOUS_API_KEY/); - assert.equal(update.env?.NOUS_API_KEY, "nous-host-secret"); - assert.ok( - !commands.some((entry) => /nous-host-secret/.test(entry.command)), - "shell credential value must not appear in argv", - ); - assert.match( - commands.at(-1)?.command || "", - /inference set --no-verify --provider hermes-provider/, - ); - }); - - it("does not delete saved OpenAI credentials when configuring local vLLM", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-local-vllm-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "setup-local-vllm-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const localInferencePath = JSON.stringify( - path.join(repoRoot, "src", "lib", "inference", "local.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const registry = require(${registryPath}); -const credentials = require(${credentialsPath}); -const localInference = require(${localInferencePath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); - -const commands = []; -runner.run = (command, opts = {}) => { - const cmd = _n(command); - commands.push({ command: cmd, env: opts.env || null }); - if (cmd.includes("provider get")) return { status: 1, stdout: "", stderr: "" }; - return { status: 0, stdout: "", stderr: "" }; -}; -runner.runCapture = (command) => { - const cmd = _n(command); - if (cmd.includes("inference") && cmd.includes("get")) { - return [ - "Gateway inference:", - "", - " Route: inference.local", - " Provider: vllm-local", - " Model: meta-llama", - " Version: 1", - ].join("\\n"); - } - return ""; -}; -registry.updateSandbox = () => true; -localInference.validateLocalProvider = () => ({ ok: true }); -localInference.getLocalProviderBaseUrl = () => "http://host.openshell.internal:8000/v1"; - -credentials.saveCredential("OPENAI_API_KEY", "sk-existing"); - -const { setupInference } = require(${onboardPath}); - -(async () => { - await setupInference("test-box", "meta-llama", "vllm-local"); - console.log(JSON.stringify({ - commands, - savedOpenAiKey: credentials.getCredential("OPENAI_API_KEY"), - })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }, - }); - - expect(result.status).toBe(0); - const payload = parseStdoutJson<{ commands: CommandEntry[]; savedOpenAiKey: string }>( - result.stdout, - ); - const providerCommand = payload.commands.find((entry) => - entry.command.includes("provider create"), - ); - assert.ok(providerCommand, "expected local vLLM provider create command"); - assert.match(providerCommand.command, /--credential NEMOCLAW_VLLM_LOCAL_TOKEN/); - assert.doesNotMatch(providerCommand.command, /--credential OPENAI_API_KEY/); - assert.equal(providerCommand.env?.NEMOCLAW_VLLM_LOCAL_TOKEN, "dummy"); - assert.equal(payload.savedOpenAiKey, "sk-existing"); - }); - - it("recovers the Ollama auth proxy on WSL when the sandbox needs proxy fronting", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-ollama-wsl-proxy-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "setup-ollama-wsl-proxy-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const platformPath = JSON.stringify(path.join(repoRoot, "src", "lib", "platform.ts")); - const localInferencePath = JSON.stringify( - path.join(repoRoot, "src", "lib", "inference", "local.ts"), - ); - const proxyPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "inference", "ollama", "proxy.ts"), - ); - const topologyPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "local-inference-topology.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const registry = require(${registryPath}); -const platform = require(${platformPath}); -const localInference = require(${localInferencePath}); -const proxy = require(${proxyPath}); -const topology = require(${topologyPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); - -const commands = []; -const proxyCalls = []; -runner.run = (command, opts = {}) => { - const cmd = _n(command); - commands.push({ command: cmd, env: opts.env || null }); - if (cmd.includes("provider get")) return { status: 1, stdout: "", stderr: "" }; - return { status: 0, stdout: "", stderr: "" }; -}; -runner.runCapture = (command) => { - const cmd = _n(command); - if (cmd.includes("inference") && cmd.includes("get")) { - return [ - "Gateway inference:", - "", - " Route: inference.local", - " Provider: ollama-local", - " Model: qwen3.5:9b", - " Version: 1", - ].join("\\n"); - } - return ""; -}; -registry.updateSandbox = () => true; -platform.isWsl = () => true; -topology.shouldFrontOllamaWithProxy = () => true; -localInference.validateLocalProvider = () => ({ - ok: false, - message: "container cannot reach Ollama", - diagnostic: "simulated WSL native Docker reachability failure", -}); -localInference.getLocalProviderBaseUrl = () => "http://host.openshell.internal:11435/v1"; -localInference.getOllamaWarmupCommand = () => ["true"]; -localInference.validateOllamaModel = () => ({ ok: true }); -localInference.validateOllamaModelWithToolsOverride = () => ({ ok: true }); -proxy.ensureOllamaAuthProxy = () => { - proxyCalls.push("ensure"); -}; -proxy.isProxyHealthy = () => { - proxyCalls.push("healthy"); - return true; -}; -proxy.getOllamaProxyToken = () => "proxy-token"; -proxy.persistAndProbeOllamaProxy = async (token) => { - proxyCalls.push("persist:" + token); -}; - -const { setupInference } = require(${onboardPath}); - -(async () => { - await setupInference("test-box", "qwen3.5:9b", "ollama-local"); - console.log(JSON.stringify({ commands, proxyCalls })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }, - }); - - assert.equal(result.status, 0, result.stderr || result.stdout); - const payload = parseStdoutJson<{ commands: CommandEntry[]; proxyCalls: string[] }>( - result.stdout, - ); - assert.deepEqual(payload.proxyCalls, ["ensure", "healthy", "persist:proxy-token"]); - const providerCommand = payload.commands.find( + assert.deepEqual(proxyCalls, ["ensure", "healthy", "persist:proxy-token"]); + const providerCommand = harness.commands.find( (entry) => entry.command.includes("provider create") && entry.command.includes("ollama-local"), ); @@ -1515,144 +1379,67 @@ const { setupInference } = require(${onboardPath}); assert.equal(providerCommand.env?.NEMOCLAW_OLLAMA_PROXY_TOKEN, "proxy-token"); assert.doesNotMatch(providerCommand.command, /proxy-token/); assert.ok( - payload.commands.some((entry) => + harness.commands.some((entry) => entry.command.includes("inference set --no-verify --provider ollama-local"), ), "expected ollama-local inference route to be selected", ); }); - - it("surfaces a contextual error and exits when ollama-local inference set fails after the proxy-ready warning (#4257)", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-ollama-set-fail-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "ollama-set-fail.cjs"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const platformPath = JSON.stringify(path.join(repoRoot, "src", "lib", "platform.ts")); - const localInferencePath = JSON.stringify( - path.join(repoRoot, "src", "lib", "inference", "local.ts"), - ); - const proxyPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "inference", "ollama", "proxy.ts"), - ); - const topologyPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "local-inference-topology.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const registry = require(${registryPath}); -const platform = require(${platformPath}); -const localInference = require(${localInferencePath}); -const proxy = require(${proxyPath}); -const topology = require(${topologyPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); - -let exitCode = null; -const realExit = process.exit; -process.exit = (code) => { - if (exitCode === null) exitCode = code; - const err = new Error("EXIT_CALLED:" + code); - err.__exit = true; - throw err; -}; - -const errLog = []; -const origErr = console.error; -console.error = (...args) => { - errLog.push(args.map((a) => (typeof a === "string" ? a : JSON.stringify(a))).join(" ")); - origErr.apply(console, args); -}; - -const commands = []; -runner.run = (command, opts = {}) => { - const cmd = _n(command); - commands.push({ command: cmd, ignoreError: !!opts.ignoreError }); - if (cmd.includes("provider get")) return { status: 1, stdout: "", stderr: "" }; - if (cmd.includes("inference set") && cmd.includes("ollama-local")) { - return { status: 7, stdout: "", stderr: "openshell: route apply failed" }; - } - return { status: 0, stdout: "", stderr: "" }; -}; -runner.runCapture = (command) => { - const cmd = _n(command); - if (cmd.includes("inference") && cmd.includes("get")) { - return [ - "Gateway inference:", - "", - " Route: inference.local", - " Provider: ollama-local", - " Model: qwen3.5:9b", - " Version: 1", - ].join("\\n"); - } - return ""; -}; -registry.updateSandbox = () => true; -platform.isWsl = () => true; -topology.shouldFrontOllamaWithProxy = () => true; -localInference.validateLocalProvider = () => ({ - ok: false, - message: "container cannot reach Ollama", - diagnostic: "simulated WSL native Docker reachability failure", -}); -localInference.getLocalProviderBaseUrl = () => "http://host.openshell.internal:11435/v1"; -localInference.getOllamaWarmupCommand = () => ["true"]; -localInference.validateOllamaModel = () => ({ ok: true }); -proxy.ensureOllamaAuthProxy = () => {}; -proxy.isProxyHealthy = () => true; -proxy.getOllamaProxyToken = () => "proxy-token"; -proxy.persistAndProbeOllamaProxy = async () => {}; - -const { setupInference } = require(${onboardPath}); - -(async () => { - try { - await setupInference("test-box", "qwen3.5:9b", "ollama-local"); - } catch (err) { - if (!err || !err.__exit) { - origErr("[TEST] outer error:", err && err.message); - process.stdout.write(JSON.stringify({ commands, errLog, exitCode, error: String(err && err.message) }) + "\n"); - realExit.call(process, 99); - } - } - process.stdout.write(JSON.stringify({ commands, errLog, exitCode }) + "\n"); - realExit.call(process, 0); -})(); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - // Force the non-interactive branch so the bug surfaces as a hard exit - // rather than a recovery prompt that would hang in CI. - NEMOCLAW_NON_INTERACTIVE: "1", + it("surfaces a contextual error and exits when ollama-local inference set fails after the proxy-ready warning (#4257)", async () => { + const errLog: string[] = []; + let exitCode: number | null = null; + let harness: ReturnType; + const applyLocalInferenceRoute = createLocalInferenceRouteApplier({ + runOpenshell: (args, options) => harness.runOpenshell(args, options), + isNonInteractive: () => true, + promptValidationRecovery: async () => "selection", + classifyApplyFailure: () => ({}) as never, + compactText: (value) => value.trim(), + redact: (value) => value, + localInferenceTimeoutSecs: 120, + error: (message) => errLog.push(message), + exitProcess: (code): never => { + exitCode = code; + throw Object.assign(new Error(`EXIT_CALLED:${code}`), { __exit: true }); }, }); + harness = createDirectSetupInferenceHarness({ + runOpenshell: (args) => { + const command = args.join(" "); + if (command.startsWith("provider get")) { + return { status: 1, stdout: "", stderr: "" }; + } + if (command.includes("inference set") && command.includes("ollama-local")) { + return { status: 7, stdout: "", stderr: "openshell: route apply failed" }; + } + return undefined; + }, + overrides: { + isNonInteractive: () => true, + validateLocalProvider: () => ({ + ok: false, + message: "container cannot reach Ollama", + diagnostic: "simulated WSL native Docker reachability failure", + }), + shouldFrontOllamaWithProxy: () => true, + ensureOllamaAuthProxy: () => {}, + isProxyHealthy: () => true, + getOllamaProxyToken: () => "proxy-token", + persistAndProbeOllamaProxy: async () => {}, + applyLocalInferenceRoute, + }, + }); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + await assert.rejects( + harness.setupInference("test-box", "qwen3.5:9b", "ollama-local"), + (error: Error & { __exit?: boolean }) => error.__exit === true, + ); + } finally { + warn.mockRestore(); + } - // Exit 0 because we override process.exit and end with realExit(0) after - // catching the simulated exit. Test asserts on the captured payload instead. - expect(result.status).toBe(0); - const payload = parseStdoutJson<{ - commands: { command: string; ignoreError: boolean }[]; - errLog: string[]; - exitCode: number | null; - }>(result.stdout); - - // Pre-fix, runOpenshell was called without ignoreError, so the runtime - // wrapper exited before we could attach context. Post-fix, the local - // path must use ignoreError + a contextual error message. - const setCmd = payload.commands.find((entry) => + const setCmd = harness.commands.find((entry) => entry.command.includes("inference set --no-verify --provider ollama-local"), ); assert.ok(setCmd, "expected ollama-local inference set command to be issued"); @@ -1661,110 +1448,52 @@ const { setupInference } = require(${onboardPath}); true, "ollama-local inference set must use ignoreError so onboard can recover", ); - - // The user must see the no-sandbox / resume-onboard guidance, not a silent stop. - const combinedErr = payload.errLog.join("\n"); + const combinedErr = errLog.join("\n"); assert.match(combinedErr, /No sandbox was created/); assert.match(combinedErr, /nemoclaw onboard --resume/); - - // And the process should still propagate the nonzero status from openshell, - // not exit 0. - assert.equal( - payload.exitCode, - 7, - "non-interactive onboard must exit with the openshell status", - ); + assert.equal(exitCode, 7, "non-interactive onboard must exit with the openshell status"); }); - - it("surfaces a contextual error and exits when vllm-local inference set fails (#4257)", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-vllm-set-fail-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "vllm-set-fail.cjs"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const localInferencePath = JSON.stringify( - path.join(repoRoot, "src", "lib", "inference", "local.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const registry = require(${registryPath}); -const localInference = require(${localInferencePath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); - -let exitCode = null; -const realExit = process.exit; -process.exit = (code) => { - if (exitCode === null) exitCode = code; - const err = new Error("EXIT_CALLED:" + code); - err.__exit = true; - throw err; -}; - -const errLog = []; -const origErr = console.error; -console.error = (...args) => { - errLog.push(args.map((a) => (typeof a === "string" ? a : JSON.stringify(a))).join(" ")); - origErr.apply(console, args); -}; - -const commands = []; -runner.run = (command, opts = {}) => { - const cmd = _n(command); - commands.push({ command: cmd, ignoreError: !!opts.ignoreError }); - if (cmd.includes("provider get")) return { status: 1, stdout: "", stderr: "" }; - if (cmd.includes("inference set") && cmd.includes("vllm-local")) { - return { status: 13, stdout: "", stderr: "openshell: vllm route apply failed" }; - } - return { status: 0, stdout: "", stderr: "" }; -}; -runner.runCapture = () => ""; -registry.updateSandbox = () => true; -localInference.validateLocalProvider = () => ({ ok: true }); -localInference.getLocalProviderBaseUrl = () => "http://host.openshell.internal:8000/v1"; - -const { setupInference } = require(${onboardPath}); - -(async () => { - try { - await setupInference("test-box", "meta-llama", "vllm-local"); - } catch (err) { - if (!err || !err.__exit) { - origErr("[TEST] outer error:", err && err.message); - process.stdout.write(JSON.stringify({ commands, errLog, exitCode, error: String(err && err.message) }) + "\n"); - realExit.call(process, 99); - } - } - process.stdout.write(JSON.stringify({ commands, errLog, exitCode }) + "\n"); - realExit.call(process, 0); -})(); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_NON_INTERACTIVE: "1", + it("surfaces a contextual error and exits when vllm-local inference set fails (#4257)", async () => { + const errLog: string[] = []; + let exitCode: number | null = null; + let harness: ReturnType; + const applyLocalInferenceRoute = createLocalInferenceRouteApplier({ + runOpenshell: (args, options) => harness.runOpenshell(args, options), + isNonInteractive: () => true, + promptValidationRecovery: async () => "selection", + classifyApplyFailure: () => ({}) as never, + compactText: (value) => value.trim(), + redact: (value) => value, + localInferenceTimeoutSecs: 120, + error: (message) => errLog.push(message), + exitProcess: (code): never => { + exitCode = code; + throw Object.assign(new Error(`EXIT_CALLED:${code}`), { __exit: true }); + }, + }); + harness = createDirectSetupInferenceHarness({ + runOpenshell: (args) => { + const command = args.join(" "); + if (command.startsWith("provider get")) { + return { status: 1, stdout: "", stderr: "" }; + } + if (command.includes("inference set") && command.includes("vllm-local")) { + return { status: 13, stdout: "", stderr: "openshell: vllm route apply failed" }; + } + return undefined; + }, + overrides: { + isNonInteractive: () => true, + applyLocalInferenceRoute, }, }); - expect(result.status).toBe(0); - const payload = parseStdoutJson<{ - commands: { command: string; ignoreError: boolean }[]; - errLog: string[]; - exitCode: number | null; - }>(result.stdout); + await assert.rejects( + harness.setupInference("test-box", "meta-llama", "vllm-local"), + (error: Error & { __exit?: boolean }) => error.__exit === true, + ); - const setCmd = payload.commands.find((entry) => + const setCmd = harness.commands.find((entry) => entry.command.includes("inference set --no-verify --provider vllm-local"), ); assert.ok(setCmd, "expected vllm-local inference set command to be issued"); @@ -1773,18 +1502,11 @@ const { setupInference } = require(${onboardPath}); true, "vllm-local inference set must use ignoreError so onboard can recover", ); - - const combinedErr = payload.errLog.join("\n"); + const combinedErr = errLog.join("\n"); assert.match(combinedErr, /No sandbox was created/); assert.match(combinedErr, /nemoclaw onboard --resume/); - - assert.equal( - payload.exitCode, - 13, - "non-interactive onboard must exit with the openshell status", - ); + assert.equal(exitCode, 13, "non-interactive onboard must exit with the openshell status"); }); - it("detects when the live inference route already matches the requested provider and model", () => { const repoRoot = path.join(import.meta.dirname, ".."); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-inference-ready-")); @@ -1805,129 +1527,20 @@ Gateway inference: Version: 1 EOF exit 0 -fi -exit 1 -`, - { mode: 0o755 }, - ); - - fs.writeFileSync( - scriptPath, - ` -const { isInferenceRouteReady } = require(${onboardPath}); -console.log(JSON.stringify({ - same: isInferenceRouteReady("nvidia-prod", "nvidia/nemotron-3-super-120b-a12b"), - otherModel: isInferenceRouteReady("nvidia-prod", "nvidia/other-model"), - otherProvider: isInferenceRouteReady("openai-api", "nvidia/nemotron-3-super-120b-a12b"), -})); -`, - ); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - PATH: `${tmpDir}:${process.env.PATH || ""}`, - }, - }); - - try { - expect(result.status).toBe(0); - expect(JSON.parse(result.stdout.trim())).toEqual({ - same: true, - otherModel: false, - otherProvider: false, - }); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); - - it("detects when OpenClaw is already configured inside the sandbox", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openclaw-ready-")); - const fakeOpenshell = path.join(tmpDir, "openshell"); - const scriptPath = path.join(tmpDir, "openclaw-ready-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - - fs.writeFileSync( - fakeOpenshell, - `#!/usr/bin/env bash -if [ "$1" = "sandbox" ] && [ "$2" = "download" ]; then - dest="\${@: -1}" - mkdir -p "$dest/sandbox/.openclaw" - cat > "$dest/sandbox/.openclaw/openclaw.json" <<'EOF' -{"gateway":{"auth":{"token":"test-token"}}} -EOF - exit 0 -fi -exit 1 -`, - { mode: 0o755 }, - ); - - fs.writeFileSync( - scriptPath, - ` -const { isOpenclawReady } = require(${onboardPath}); -console.log(JSON.stringify({ - ready: isOpenclawReady("my-assistant"), -})); -`, - ); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - PATH: `${tmpDir}:${process.env.PATH || ""}`, - }, - }); - - try { - expect(result.status).toBe(0); - expect(JSON.parse(result.stdout.trim())).toEqual({ ready: true }); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }); - - it("detects when recorded policy presets are already applied", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-policy-ready-")); - const registryDir = path.join(tmpDir, ".nemoclaw"); - const registryFile = path.join(registryDir, "sandboxes.json"); - const scriptPath = path.join(tmpDir, "policy-ready-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - - fs.mkdirSync(registryDir, { recursive: true }); - fs.writeFileSync( - registryFile, - JSON.stringify( - { - sandboxes: { - "my-assistant": { - name: "my-assistant", - policies: ["pypi", "npm"], - }, - }, - defaultSandbox: "my-assistant", - }, - null, - 2, - ), +fi +exit 1 +`, + { mode: 0o755 }, ); fs.writeFileSync( scriptPath, ` -const { arePolicyPresetsApplied } = require(${onboardPath}); +const { isInferenceRouteReady } = require(${onboardPath}); console.log(JSON.stringify({ - ready: arePolicyPresetsApplied("my-assistant", ["pypi", "npm"]), - missing: arePolicyPresetsApplied("my-assistant", ["pypi", "slack"]), - empty: arePolicyPresetsApplied("my-assistant", []), + same: isInferenceRouteReady("nvidia-prod", "nvidia/nemotron-3-super-120b-a12b"), + otherModel: isInferenceRouteReady("nvidia-prod", "nvidia/other-model"), + otherProvider: isInferenceRouteReady("openai-api", "nvidia/nemotron-3-super-120b-a12b"), })); `, ); @@ -1937,397 +1550,109 @@ console.log(JSON.stringify({ encoding: "utf-8", env: { ...process.env, - HOME: tmpDir, + PATH: `${tmpDir}:${process.env.PATH || ""}`, }, }); try { expect(result.status).toBe(0); - const payload = JSON.parse(result.stdout.trim()); - expect(payload).toEqual({ - ready: true, - missing: false, - empty: false, + expect(JSON.parse(result.stdout.trim())).toEqual({ + same: true, + otherModel: false, + otherProvider: false, }); } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } }); - it("uses native Anthropic provider creation without embedding the secret in argv", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-anthropic-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "setup-anthropic-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); - -const commands = []; -runner.run = (command, opts = {}) => { - commands.push({ command: _n(command), env: opts.env || null }); - // provider-get returns not-found so we exercise the create path - if (_n(command).includes("provider get")) return { status: 1 }; - return { status: 0 }; -}; -runner.runCapture = (command) => { - if (_n(command).includes("inference") && _n(command).includes("get")) { - return [ - "Gateway inference:", - "", - " Route: inference.local", - " Provider: anthropic-prod", - " Model: claude-sonnet-4-5", - " Version: 1", - ].join("\n"); - } - return ""; -}; -registry.updateSandbox = () => true; - -process.env.ANTHROPIC_API_KEY = "sk-ant-TEST-NOT-A-REAL-VALUE"; - -const { setupInference } = require(${onboardPath}); - -(async () => { - await setupInference("test-box", "claude-sonnet-4-5", "anthropic-prod", "https://api.anthropic.com", "ANTHROPIC_API_KEY"); - console.log(JSON.stringify(commands)); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }, - }); - - assert.equal(result.status, 0, result.stderr); - const commands = parseStdoutJson(result.stdout); - assert.equal(commands.length, 4); - assert.match(commands[0].command, /gateway select nemoclaw/); - assert.match(commands[1].command, /provider get/); - assert.match(commands[2].command, /--type anthropic/); - assert.match(commands[2].command, /--credential ANTHROPIC_API_KEY/); - assert.doesNotMatch(commands[2].command, /sk-ant-TEST-NOT-A-REAL-VALUE/); - assert.match(commands[3].command, /--provider anthropic-prod/); - }); - - it("updates OpenAI-compatible providers without passing an unsupported --type flag", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-openai-update-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "setup-openai-update-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); - -const commands = []; -runner.run = (command, opts = {}) => { - commands.push({ command: _n(command), env: opts.env || null }); - return { status: 0 }; -}; -runner.runCapture = (command) => { - if (_n(command).includes("inference") && _n(command).includes("get")) { - return [ - "Gateway inference:", - "", - " Route: inference.local", - " Provider: openai-api", - " Model: gpt-5.4", - " Version: 1", - ].join("\n"); - } - return ""; -}; -registry.updateSandbox = () => true; - -process.env.OPENAI_API_KEY = "sk-TEST-NOT-A-REAL-VALUE"; - -const { setupInference } = require(${onboardPath}); - -(async () => { - await setupInference("test-box", "gpt-5.4", "openai-api", "https://api.openai.com/v1", "OPENAI_API_KEY"); - console.log(JSON.stringify(commands)); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }, - }); - - assert.equal(result.status, 0, result.stderr); - const commands = parseStdoutJson(result.stdout); - assert.equal(commands.length, 4); - assert.match(commands[0].command, /gateway select nemoclaw/); - assert.match(commands[1].command, /provider get/); - assert.match(commands[2].command, /provider update openai-api/); - assert.doesNotMatch(commands[2].command, /--type/); - assert.match(commands[3].command, /inference set --no-verify/); - }); - - it("re-prompts for credentials when openshell inference set fails with authorization errors", () => { + it("detects when OpenClaw is already configured inside the sandbox", () => { const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-apply-auth-retry-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "setup-inference-auth-retry-check.js"); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-openclaw-ready-")); + const fakeOpenshell = path.join(tmpDir, "openshell"); + const scriptPath = path.join(tmpDir, "openclaw-ready-check.js"); const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const credentials = require(${credentialsPath}); - -const commands = []; -const answers = ["retry", "sk-good"]; -let inferenceSetCalls = 0; - -credentials.prompt = async () => answers.shift() || ""; -runner.run = (command, opts = {}) => { - commands.push({ command: _n(command), env: opts.env || null }); - if (_n(command).includes("inference set")) { - inferenceSetCalls += 1; - if (inferenceSetCalls === 1) { - return { status: 1, stdout: "", stderr: "HTTP 403: forbidden" }; - } - } - return { status: 0, stdout: "", stderr: "" }; -}; -runner.runCapture = (command) => { - if (_n(command).includes("inference") && _n(command).includes("get")) { - return [ - "Gateway inference:", - "", - " Route: inference.local", - " Provider: openai-api", - " Model: gpt-5.4", - " Version: 1", - ].join("\\n"); - } - return ""; -}; -registry.updateSandbox = () => true; - -process.env.OPENAI_API_KEY = "sk-bad"; - -const { setupInference } = require(${onboardPath}); - -(async () => { - await setupInference("test-box", "gpt-5.4", "openai-api", "https://api.openai.com/v1", "OPENAI_API_KEY"); - console.log(JSON.stringify({ commands, key: process.env.OPENAI_API_KEY, inferenceSetCalls })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }, - }); - - assert.equal(result.status, 0, result.stderr); - const payload = parseStdoutJson<{ - key: string; - inferenceSetCalls: number; - commands: CommandEntry[]; - }>(result.stdout); - assert.equal(payload.key, "sk-good"); - assert.equal(payload.inferenceSetCalls, 2); - const providerEnvs = payload.commands - .filter((entry: CommandEntry) => entry.command.includes("provider")) - .map((entry: CommandEntry) => entry.env && entry.env.OPENAI_API_KEY) - .filter(Boolean); - assert.deepEqual(providerEnvs, ["sk-bad", "sk-good"]); - }); - it("returns control to provider selection when inference apply recovery chooses back", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-apply-back-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "setup-inference-apply-back-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + fs.writeFileSync( + fakeOpenshell, + `#!/usr/bin/env bash +if [ "$1" = "sandbox" ] && [ "$2" = "download" ]; then + dest="\${@: -1}" + mkdir -p "$dest/sandbox/.openclaw" + cat > "$dest/sandbox/.openclaw/openclaw.json" <<'EOF' +{"gateway":{"auth":{"token":"test-token"}}} +EOF + exit 0 +fi +exit 1 +`, + { mode: 0o755 }, ); - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const credentials = require(${credentialsPath}); - -const commands = []; -credentials.prompt = async () => "back"; -runner.run = (command, opts = {}) => { - commands.push({ command: _n(command), env: opts.env || null }); - if (_n(command).includes("inference set")) { - return { status: 1, stdout: "", stderr: "HTTP 404: model not found" }; - } - return { status: 0, stdout: "", stderr: "" }; -}; -runner.runCapture = () => ""; -registry.updateSandbox = () => true; - -process.env.OPENAI_API_KEY = "sk-TEST-NOT-A-REAL-VALUE"; - -const { setupInference } = require(${onboardPath}); - -(async () => { - const result = await setupInference("test-box", "gpt-5.4", "openai-api", "https://api.openai.com/v1", "OPENAI_API_KEY"); - console.log(JSON.stringify({ result, commands })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); + fs.writeFileSync( + scriptPath, + ` +const { isOpenclawReady } = require(${onboardPath}); +console.log(JSON.stringify({ + ready: isOpenclawReady("my-assistant"), +})); +`, + ); const result = spawnSync(process.execPath, [scriptPath], { cwd: repoRoot, encoding: "utf-8", env: { ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, + PATH: `${tmpDir}:${process.env.PATH || ""}`, }, }); - assert.equal(result.status, 0, result.stderr); - const payload = parseStdoutJson<{ - result: { retry: "selection" }; - commands: CommandEntry[]; - }>(result.stdout); - assert.deepEqual(payload.result, { retry: "selection" }); - assert.equal( - payload.commands.filter((entry: CommandEntry) => entry.command.includes("inference set")) - .length, - 1, - ); + try { + expect(result.status).toBe(0); + expect(JSON.parse(result.stdout.trim())).toEqual({ ready: true }); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } }); - it("migrates a legacy credentials.json into env so setupInference can register the provider", () => { + it("detects when recorded policy presets are already applied", () => { const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-resume-cred-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "setup-resume-credential-check.js"); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-policy-ready-")); + const registryDir = path.join(tmpDir, ".nemoclaw"); + const registryFile = path.join(registryDir, "sandboxes.json"); + const scriptPath = path.join(tmpDir, "policy-ready-check.js"); const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - // Pre-seed a pre-fix plaintext credentials.json. hydrateCredentialEnv - // stages it non-destructively into process.env via - // stageLegacyCredentialsToEnv(); the secure unlink only runs from the - // post-onboard cleanup gate when the staged values are confirmed - // migrated, so the legacy file must still exist after this test's - // setupInference call (asserted further down). - const legacyDir = path.join(tmpDir, ".nemoclaw"); - fs.mkdirSync(legacyDir, { recursive: true, mode: 0o700 }); + + fs.mkdirSync(registryDir, { recursive: true }); fs.writeFileSync( - path.join(legacyDir, "credentials.json"), - JSON.stringify({ OPENAI_API_KEY: "sk-TEST-NOT-A-REAL-STORED-KEY" }), - { mode: 0o600 }, + registryFile, + JSON.stringify( + { + sandboxes: { + "my-assistant": { + name: "my-assistant", + policies: ["pypi", "npm"], + }, + }, + defaultSandbox: "my-assistant", + }, + null, + 2, + ), ); - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const legacyFilePath = JSON.stringify(path.join(legacyDir, "credentials.json")); - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const fs = require("node:fs"); - -const commands = []; -runner.run = (command, opts = {}) => { - commands.push({ command: _n(command), env: opts.env || null }); - return { status: 0 }; -}; -runner.runCapture = (command) => { - if (_n(command).includes("inference") && _n(command).includes("get")) { - return [ - "Gateway inference:", - "", - " Route: inference.local", - " Provider: openai-api", - " Model: gpt-5.4", - " Version: 1", - ].join("\n"); - } - return ""; -}; -registry.updateSandbox = () => true; - -delete process.env.OPENAI_API_KEY; - -const { setupInference } = require(${onboardPath}); - -(async () => { - await setupInference("test-box", "gpt-5.4", "openai-api", "https://api.openai.com/v1", "OPENAI_API_KEY"); - console.log(JSON.stringify({ - commands, - openai: process.env.OPENAI_API_KEY || null, - legacyFileGone: !fs.existsSync(${legacyFilePath}), - })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); + fs.writeFileSync( + scriptPath, + ` +const { arePolicyPresetsApplied } = require(${onboardPath}); +console.log(JSON.stringify({ + ready: arePolicyPresetsApplied("my-assistant", ["pypi", "npm"]), + missing: arePolicyPresetsApplied("my-assistant", ["pypi", "slack"]), + empty: arePolicyPresetsApplied("my-assistant", []), +})); +`, + ); const result = spawnSync(process.execPath, [scriptPath], { cwd: repoRoot, @@ -2335,33 +1660,204 @@ const { setupInference } = require(${onboardPath}); env: { ...process.env, HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, }, }); - assert.equal(result.status, 0, result.stderr); - const payload = parseStdoutJson<{ - openai: string; - commands: CommandEntry[]; - legacyFileGone: boolean; - }>(result.stdout); - assert.equal(payload.openai, "sk-TEST-NOT-A-REAL-STORED-KEY"); - // setupInference's hydrateCredentialEnv only stages the legacy file - // (non-destructive). The secure unlink runs only after a full successful - // onboard, so an interrupted run can be retried without losing the - // user's only copy of their credentials. - assert.equal( - payload.legacyFileGone, - false, - "legacy credentials.json must survive the staging-only hydrate path", - ); - // commands[0]=gateway select, [1]=provider get, [2]=provider update - const providerUpdate = payload.commands[2]; - assert.ok(providerUpdate, "expected provider update command"); - assert.equal(providerUpdate.env?.OPENAI_API_KEY, "sk-TEST-NOT-A-REAL-STORED-KEY"); - assert.doesNotMatch(providerUpdate.command, /sk-TEST-NOT-A-REAL-STORED-KEY/); + try { + expect(result.status).toBe(0); + const payload = JSON.parse(result.stdout.trim()); + expect(payload).toEqual({ + ready: true, + missing: false, + empty: false, + }); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); + + it("uses native Anthropic provider creation without embedding the secret in argv", async () => { + await withProcessEnv({ ANTHROPIC_API_KEY: "sk-ant-TEST-NOT-A-REAL-VALUE" }, async () => { + const harness = createDirectSetupInferenceHarness({ + runOpenshell: (args) => + args.slice(0, 2).join(" ") === "provider get" + ? { status: 1, stdout: "", stderr: "" } + : undefined, + }); + + await harness.setupInference( + "test-box", + "claude-sonnet-4-5", + "anthropic-prod", + "https://api.anthropic.com", + "ANTHROPIC_API_KEY", + ); + + const commands = harness.commands; + assert.equal(commands.length, 4); + assert.match(commands[0].command, /gateway select nemoclaw/); + assert.match(commands[1].command, /provider get/); + assert.match(commands[2].command, /--type anthropic/); + assert.match(commands[2].command, /--credential ANTHROPIC_API_KEY/); + assert.doesNotMatch(commands[2].command, /sk-ant-TEST-NOT-A-REAL-VALUE/); + assert.match(commands[3].command, /--provider anthropic-prod/); + }); + }); + it("updates OpenAI-compatible providers without passing an unsupported --type flag", async () => { + await withProcessEnv({ OPENAI_API_KEY: "sk-TEST-NOT-A-REAL-VALUE" }, async () => { + const harness = createDirectSetupInferenceHarness({ + runOpenshell: (args) => + args.slice(0, 2).join(" ") === "provider get" + ? { status: 0, stdout: "", stderr: "" } + : undefined, + }); + + await harness.setupInference( + "test-box", + "gpt-5.4", + "openai-api", + "https://api.openai.com/v1", + "OPENAI_API_KEY", + ); + + const commands = harness.commands; + assert.equal(commands.length, 4); + assert.match(commands[0].command, /gateway select nemoclaw/); + assert.match(commands[1].command, /provider get/); + assert.match(commands[2].command, /provider update openai-api/); + assert.doesNotMatch(commands[2].command, /--type/); + assert.match(commands[3].command, /inference set --no-verify/); + }); + }); + it("re-prompts for credentials when openshell inference set fails with authorization errors", async () => { + await withProcessEnv({ OPENAI_API_KEY: "sk-bad" }, async () => { + let inferenceSetCalls = 0; + const harness = createDirectSetupInferenceHarness({ + runOpenshell: (args) => { + const command = args.join(" "); + if (command.startsWith("provider get")) { + return { status: 0, stdout: "", stderr: "" }; + } + if (command.includes("inference set")) { + inferenceSetCalls += 1; + if (inferenceSetCalls === 1) { + return { status: 1, stdout: "", stderr: "HTTP 403: forbidden" }; + } + } + return undefined; + }, + overrides: { + promptValidationRecovery: async () => { + process.env.OPENAI_API_KEY = "sk-good"; + return "retry"; + }, + }, + }); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + try { + await harness.setupInference( + "test-box", + "gpt-5.4", + "openai-api", + "https://api.openai.com/v1", + "OPENAI_API_KEY", + ); + } finally { + error.mockRestore(); + } + + assert.equal(process.env.OPENAI_API_KEY, "sk-good"); + assert.equal(inferenceSetCalls, 2); + const providerEnvs = harness.commands + .filter((entry) => entry.command.includes("provider")) + .map((entry) => entry.env?.OPENAI_API_KEY) + .filter(Boolean); + assert.deepEqual(providerEnvs, ["sk-bad", "sk-good"]); + }); }); + it("returns control to provider selection when inference apply recovery chooses back", async () => { + await withProcessEnv({ OPENAI_API_KEY: "sk-TEST-NOT-A-REAL-VALUE" }, async () => { + const harness = createDirectSetupInferenceHarness({ + runOpenshell: (args) => { + const command = args.join(" "); + if (command.startsWith("provider get")) { + return { status: 0, stdout: "", stderr: "" }; + } + if (command.includes("inference set")) { + return { status: 1, stdout: "", stderr: "HTTP 404: model not found" }; + } + return undefined; + }, + overrides: { promptValidationRecovery: async () => "selection" }, + }); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + let result: Awaited>; + try { + result = await harness.setupInference( + "test-box", + "gpt-5.4", + "openai-api", + "https://api.openai.com/v1", + "OPENAI_API_KEY", + ); + } finally { + error.mockRestore(); + } + assert.deepEqual(result, { retry: "selection" }); + assert.equal( + harness.commands.filter((entry) => entry.command.includes("inference set")).length, + 1, + ); + }); + }); + it("migrates a legacy credentials.json into env so setupInference can register the provider", async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-resume-cred-")); + const legacyDir = path.join(tmpDir, ".nemoclaw"); + const legacyFile = path.join(legacyDir, "credentials.json"); + fs.mkdirSync(legacyDir, { recursive: true, mode: 0o700 }); + fs.writeFileSync( + legacyFile, + JSON.stringify({ OPENAI_API_KEY: "sk-TEST-NOT-A-REAL-STORED-KEY" }), + { mode: 0o600 }, + ); + const credentialEnv = + require("../src/lib/onboard/credential-env") as typeof import("../src/lib/onboard/credential-env.js"); + try { + await withProcessEnv({ HOME: tmpDir, OPENAI_API_KEY: undefined }, async () => { + const harness = createDirectSetupInferenceHarness({ + runOpenshell: (args) => + args.slice(0, 2).join(" ") === "provider get" + ? { status: 0, stdout: "", stderr: "" } + : undefined, + overrides: { hydrateCredentialEnv: credentialEnv.hydrateCredentialEnv }, + }); + + await harness.setupInference( + "test-box", + "gpt-5.4", + "openai-api", + "https://api.openai.com/v1", + "OPENAI_API_KEY", + ); + + assert.equal(process.env.OPENAI_API_KEY, "sk-TEST-NOT-A-REAL-STORED-KEY"); + assert.equal( + fs.existsSync(legacyFile), + true, + "legacy credentials.json must survive the staging-only hydrate path", + ); + const providerUpdate = harness.commands.find((entry) => + entry.command.includes("provider update openai-api"), + ); + assert.ok(providerUpdate, "expected provider update command"); + assert.equal(providerUpdate.env?.OPENAI_API_KEY, "sk-TEST-NOT-A-REAL-STORED-KEY"); + assert.doesNotMatch(providerUpdate.command, /sk-TEST-NOT-A-REAL-STORED-KEY/); + }); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + }); it("drops stale local sandbox registry entries when the live sandbox is gone", () => { const repoRoot = path.join(import.meta.dirname, ".."); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-stale-sandbox-")); @@ -4528,30 +4024,8 @@ const { createSandbox } = require(${onboardPath}); ); }); - it("accepts gateway inference when system inference is separately not configured", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-inference-get-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "inference-get-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const commands = []; -runner.run = (command, opts = {}) => { - commands.push({ command: _n(command), env: opts.env || null }); - return { status: 0 }; -}; -runner.runCapture = (command) => { - if (_n(command).includes("inference") && _n(command).includes("get")) { - return [ + it("accepts gateway inference when system inference is separately not configured", async () => { + const output = [ "Gateway inference:", "", " Route: inference.local", @@ -4562,66 +4036,32 @@ runner.runCapture = (command) => { "System inference:", "", " Not configured", - ].join("\\n"); - } - return ""; -}; -registry.updateSandbox = () => true; -process.env.OPENAI_API_KEY = "sk-TEST-NOT-A-REAL-VALUE"; -process.env.OPENSHELL_GATEWAY = "nemoclaw"; - -const { setupInference } = require(${onboardPath}); + ].join("\n"); + const route = createInferenceRouteHelpers(() => output); + + await withProcessEnv({ OPENAI_API_KEY: "sk-TEST-NOT-A-REAL-VALUE" }, async () => { + const harness = createDirectSetupInferenceHarness({ + runOpenshell: (args) => + args.slice(0, 2).join(" ") === "provider get" + ? { status: 0, stdout: "", stderr: "" } + : undefined, + overrides: { verifyInferenceRoute: route.verifyInferenceRoute }, + }); -(async () => { - await setupInference("test-box", "gpt-5.4", "openai-api", "https://api.openai.com/v1", "OPENAI_API_KEY"); - console.log(JSON.stringify(commands)); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); + await harness.setupInference( + "test-box", + "gpt-5.4", + "openai-api", + "https://api.openai.com/v1", + "OPENAI_API_KEY", + ); - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }, + // gateway select + provider get + provider update + inference set + assert.equal(harness.commands.length, 4); }); - - assert.equal(result.status, 0, result.stderr); - const commands = parseStdoutJson(result.stdout); - // gateway select + provider get + provider update + inference set - assert.equal(commands.length, 4); }); - - it("accepts gateway inference output that omits the Route line", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-inference-route-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "inference-route-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeOkOpenshell(fakeBin); - - const script = String.raw` -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const commands = []; -runner.run = (command, opts = {}) => { - commands.push({ command: _n(command), env: opts.env || null }); - return { status: 0 }; -}; -runner.runCapture = (command) => { - if (_n(command).includes("inference") && _n(command).includes("get")) { - return [ + it("accepts gateway inference output that omits the Route line", async () => { + const output = [ "Gateway inference:", "", " Provider: openai-api", @@ -4631,42 +4071,30 @@ runner.runCapture = (command) => { "System inference:", "", " Not configured", - ].join("\\n"); - } - return ""; -}; -registry.updateSandbox = () => true; -process.env.OPENAI_API_KEY = "sk-TEST-NOT-A-REAL-VALUE"; -process.env.OPENSHELL_GATEWAY = "nemoclaw"; - -const { setupInference } = require(${onboardPath}); + ].join("\n"); + const route = createInferenceRouteHelpers(() => output); + + await withProcessEnv({ OPENAI_API_KEY: "sk-TEST-NOT-A-REAL-VALUE" }, async () => { + const harness = createDirectSetupInferenceHarness({ + runOpenshell: (args) => + args.slice(0, 2).join(" ") === "provider get" + ? { status: 0, stdout: "", stderr: "" } + : undefined, + overrides: { verifyInferenceRoute: route.verifyInferenceRoute }, + }); -(async () => { - await setupInference("test-box", "gpt-5.4", "openai-api", "https://api.openai.com/v1", "OPENAI_API_KEY"); - console.log(JSON.stringify(commands)); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); + await harness.setupInference( + "test-box", + "gpt-5.4", + "openai-api", + "https://api.openai.com/v1", + "OPENAI_API_KEY", + ); - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }, + // gateway select + provider get + provider update + inference set + assert.equal(harness.commands.length, 4); }); - - assert.equal(result.status, 0, result.stderr); - const commands = parseStdoutJson(result.stdout); - // gateway select + provider get + provider update + inference set - assert.equal(commands.length, 4); }); - it("uses the sandbox-base registry in pullAndResolveBaseImageDigest (#1904)", () => { // Structural check: verify the constant matches the Dockerfile default // and does NOT reference the openshell-community registry. From 8e0f3a3cab857affb6d86989fbc8c770c2d20a76 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 4 Jul 2026 09:36:26 -0700 Subject: [PATCH 02/16] perf(test): remove service env harness subprocesses Signed-off-by: Carlos Villela --- test/service-env.test.ts | 178 ++++++++++++++++++++------------------- 1 file changed, 93 insertions(+), 85 deletions(-) diff --git a/test/service-env.test.ts b/test/service-env.test.ts index 19b7c8dfedb..93ac3cc99f5 100644 --- a/test/service-env.test.ts +++ b/test/service-env.test.ts @@ -7,16 +7,20 @@ import { execSync, } from "node:child_process"; import { + chmodSync, existsSync, lstatSync, + mkdirSync, mkdtempSync, readFileSync, + rmSync, + symlinkSync, unlinkSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { describe, expect, it } from "vitest"; +import { beforeAll, describe, expect, it } from "vitest"; import { resolveOpenshell } from "../src/lib/adapters/openshell/resolve"; const NEMOCLAW_START_SCRIPT = join(import.meta.dirname, "../scripts/nemoclaw-start.sh"); @@ -64,6 +68,35 @@ function extractRuntimeShellEnvShimSnippet() { return `${src.slice(start, end).trimEnd()}\nensure_runtime_shell_env_shim`; } +function extractToolRedirectsSnippet() { + const src = readFileSync(NEMOCLAW_START_SCRIPT, "utf-8"); + const start = src.indexOf("_TOOL_REDIRECTS=("); + const loop = src.indexOf("for _redir", start); + const endMarker = "\ndone"; + const end = src.indexOf(endMarker, loop); + if (start === -1 || loop === -1 || end === -1 || end <= loop) { + throw new Error( + "Failed to extract _TOOL_REDIRECTS from scripts/nemoclaw-start.sh — " + + "the array may have been moved or renamed", + ); + } + return src.slice(start, end + endMarker.length); +} + +function extractProxyVarsSnippet() { + const src = readFileSync(NEMOCLAW_START_SCRIPT, "utf-8"); + const start = src.indexOf("PROXY_HOST="); + const endMarker = 'export no_proxy="$_NO_PROXY_VAL"'; + const end = src.indexOf(endMarker, start); + if (start === -1 || end === -1 || end <= start) { + throw new Error( + "Failed to extract proxy configuration from scripts/nemoclaw-start.sh — " + + "the PROXY_HOST..no_proxy block may have been moved or renamed", + ); + } + return src.slice(start, end + endMarker.length); +} + describe("service environment", () => { describe("OpenClaw EC2 metadata discovery", () => { it("overrides ambient and sandbox-create wrapper false values before startup", () => { @@ -257,7 +290,7 @@ describe("service environment", () => { /* ignore */ } try { - execFileSync("rm", ["-rf", fakeDir]); + rmSync(fakeDir, { recursive: true, force: true }); } catch { /* ignore */ } @@ -301,7 +334,7 @@ describe("service environment", () => { expect(envFile).toContain(fakeCaBundle); } finally { try { - execFileSync("rm", ["-rf", fakeDataDir]); + rmSync(fakeDataDir, { recursive: true, force: true }); } catch { /* ignore */ } @@ -310,7 +343,7 @@ describe("service environment", () => { it("proxy-env.sh omits GIT_SSL_CAINFO when not set", () => { const fakeDataDir = join(tmpdir(), `nemoclaw-git-ssl-noop-test-${process.pid}`); - execFileSync("mkdir", ["-p", fakeDataDir]); + mkdirSync(fakeDataDir, { recursive: true }); const tmpFile = join(tmpdir(), `nemoclaw-git-ssl-noop-env-${process.pid}.sh`); try { const persistBlock = extractRuntimeShellEnvSnippet(); @@ -337,7 +370,8 @@ describe("service environment", () => { expect(envFile).not.toContain("GIT_SSL_CAINFO"); } finally { try { - execFileSync("rm", ["-rf", fakeDataDir, tmpFile]); + rmSync(fakeDataDir, { recursive: true, force: true }); + rmSync(tmpFile, { force: true }); } catch { /* ignore */ } @@ -381,11 +415,7 @@ describe("service environment", () => { it("a sandbox-connect shell sourcing the emitted proxy-env reports both npm offline env vars as false", () => { const persistBlock = extractRuntimeShellEnvSnippet(); - const toolRedirects = execFileSync( - "sed", - ["-n", "/^_TOOL_REDIRECTS=/,/^done$/p", NEMOCLAW_START_SCRIPT], - { encoding: "utf-8" }, - ).trimEnd(); + const toolRedirects = extractToolRedirectsSnippet(); const sandboxInitSource = `source ${JSON.stringify(join(import.meta.dirname, "../scripts/lib/sandbox-init.sh"))}`; const fakeDataDir = mkdtempSync(join(tmpdir(), "nemoclaw-connect-npm-online-")); const tmpFile = join(tmpdir(), `nemoclaw-connect-npm-online-${process.pid}.sh`); @@ -412,7 +442,7 @@ describe("service environment", () => { /* ignore */ } try { - execFileSync("rm", ["-rf", fakeDataDir]); + rmSync(fakeDataDir, { recursive: true, force: true }); } catch { /* ignore */ } @@ -470,7 +500,7 @@ describe("service environment", () => { /* ignore */ } try { - execFileSync("rm", ["-rf", fakeTmp]); + rmSync(fakeTmp, { recursive: true, force: true }); } catch { /* ignore */ } @@ -483,33 +513,8 @@ describe("service environment", () => { // shared library. Wrappers that execute the extracted block must source it. const sandboxInitSource = `source ${JSON.stringify(join(import.meta.dirname, "../scripts/lib/sandbox-init.sh"))}`; - function extractToolRedirects() { - const scriptPath = join(import.meta.dirname, "../scripts/nemoclaw-start.sh"); - const block = execFileSync("sed", ["-n", "/^_TOOL_REDIRECTS=/,/^done$/p", scriptPath], { - encoding: "utf-8", - }); - if (!block.trim()) { - throw new Error( - "Failed to extract _TOOL_REDIRECTS from scripts/nemoclaw-start.sh — " + - "the array may have been moved or renamed", - ); - } - return block.trimEnd(); - } - - function extractProxyVars(env = {}) { - const scriptPath = join(import.meta.dirname, "../scripts/nemoclaw-start.sh"); - const proxyBlock = execFileSync( - "sed", - ["-n", "/^PROXY_HOST=/,/^export no_proxy=/p", scriptPath], - { encoding: "utf-8" }, - ); - if (!proxyBlock.trim()) { - throw new Error( - "Failed to extract proxy configuration from scripts/nemoclaw-start.sh — " + - "the PROXY_HOST..no_proxy block may have been moved or renamed", - ); - } + function extractProxyVars(env: Record = {}) { + const proxyBlock = extractProxyVarsSnippet(); const wrapper = [ "#!/usr/bin/env bash", proxyBlock.trimEnd(), @@ -542,30 +547,40 @@ describe("service environment", () => { } } + let defaultProxyVars: Record; + let hostOverrideProxyVars: Record; + let portOverrideProxyVars: Record; + + beforeAll(() => { + defaultProxyVars = extractProxyVars(); + hostOverrideProxyVars = extractProxyVars({ NEMOCLAW_PROXY_HOST: "192.168.64.1" }); + portOverrideProxyVars = extractProxyVars({ NEMOCLAW_PROXY_PORT: "8080" }); + }); + it("sets HTTP_PROXY to default gateway address", () => { - const vars = extractProxyVars(); + const vars = defaultProxyVars; expect(vars.HTTP_PROXY).toBe("http://10.200.0.1:3128"); }); it("sets HTTPS_PROXY to default gateway address", () => { - const vars = extractProxyVars(); + const vars = defaultProxyVars; expect(vars.HTTPS_PROXY).toBe("http://10.200.0.1:3128"); }); it("NEMOCLAW_PROXY_HOST overrides default gateway IP", () => { - const vars = extractProxyVars({ NEMOCLAW_PROXY_HOST: "192.168.64.1" }); + const vars = hostOverrideProxyVars; expect(vars.HTTP_PROXY).toBe("http://192.168.64.1:3128"); expect(vars.HTTPS_PROXY).toBe("http://192.168.64.1:3128"); }); it("NEMOCLAW_PROXY_PORT overrides default proxy port", () => { - const vars = extractProxyVars({ NEMOCLAW_PROXY_PORT: "8080" }); + const vars = portOverrideProxyVars; expect(vars.HTTP_PROXY).toBe("http://10.200.0.1:8080"); expect(vars.HTTPS_PROXY).toBe("http://10.200.0.1:8080"); }); it("NO_PROXY includes loopback only, not inference.local", () => { - const vars = extractProxyVars(); + const vars = defaultProxyVars; const noProxy = vars.NO_PROXY.split(","); expect(noProxy).toContain("localhost"); expect(noProxy).toContain("127.0.0.1"); @@ -574,12 +589,12 @@ describe("service environment", () => { }); it("NO_PROXY includes OpenShell gateway IP", () => { - const vars = extractProxyVars(); + const vars = defaultProxyVars; expect(vars.NO_PROXY).toContain("10.200.0.1"); }); it("exports lowercase proxy variants for undici/gRPC compatibility", () => { - const vars = extractProxyVars(); + const vars = defaultProxyVars; expect(vars.http_proxy).toBe("http://10.200.0.1:3128"); expect(vars.https_proxy).toBe("http://10.200.0.1:3128"); const noProxy = vars.no_proxy.split(","); @@ -589,11 +604,11 @@ describe("service environment", () => { it("entrypoint writes proxy-env.sh to writable data dir", () => { const fakeDataDir = join(tmpdir(), `nemoclaw-data-test-${process.pid}`); - execFileSync("mkdir", ["-p", fakeDataDir]); + mkdirSync(fakeDataDir, { recursive: true }); const tmpFile = join(tmpdir(), `nemoclaw-proxyenv-write-test-${process.pid}.sh`); try { const persistBlock = extractRuntimeShellEnvSnippet(); - const toolRedirects = extractToolRedirects(); + const toolRedirects = extractToolRedirectsSnippet(); const wrapper = [ "#!/usr/bin/env bash", sandboxInitSource, @@ -639,18 +654,8 @@ describe("service environment", () => { // ad-hoc `npx -y` invocations inside the sandbox. expect(envFile).toContain("npm_config_offline=false"); expect(envFile).toContain("NPM_CONFIG_OFFLINE=false"); - // Permission should be 444 (hardened via emit_sandbox_sourced_file) - // Cross-platform: Linux uses stat -c '%a', macOS uses stat -f '%Lp' - let perms: string; - try { - perms = execFileSync("stat", ["-c", "%a", join(fakeDataDir, "proxy-env.sh")], { - encoding: "utf-8", - }).trim(); - } catch { - perms = execFileSync("stat", ["-f", "%Lp", join(fakeDataDir, "proxy-env.sh")], { - encoding: "utf-8", - }).trim(); - } + // Permission should be 444 (hardened via emit_sandbox_sourced_file). + const perms = (lstatSync(join(fakeDataDir, "proxy-env.sh")).mode & 0o777).toString(8); expect(perms).toBe("444"); const connectedValue = execFileSync( @@ -671,7 +676,7 @@ describe("service environment", () => { /* ignore */ } try { - execFileSync("rm", ["-rf", fakeDataDir]); + rmSync(fakeDataDir, { recursive: true, force: true }); } catch { /* ignore */ } @@ -731,7 +736,7 @@ describe("service environment", () => { /* ignore */ } try { - execFileSync("rm", ["-rf", fakeHome]); + rmSync(fakeHome, { recursive: true, force: true }); } catch { /* ignore */ } @@ -785,7 +790,7 @@ describe("service environment", () => { /* ignore */ } try { - execFileSync("rm", ["-rf", fakeHome]); + rmSync(fakeHome, { recursive: true, force: true }); } catch { /* ignore */ } @@ -833,7 +838,7 @@ describe("service environment", () => { /* ignore */ } try { - execFileSync("rm", ["-rf", fakeHome]); + rmSync(fakeHome, { recursive: true, force: true }); } catch { /* ignore */ } @@ -849,7 +854,7 @@ describe("service environment", () => { try { writeFileSync(rcPath, "# clean bashrc\n", { mode: 0o444 }); writeFileSync(profilePath, "# clean profile\n", { mode: 0o444 }); - execFileSync("chmod", ["555", fakeHome]); + chmodSync(fakeHome, 0o555); const wrapper = [ "#!/usr/bin/env bash", @@ -868,7 +873,7 @@ describe("service environment", () => { expect(readFileSync(profilePath, "utf-8")).toBe("# clean profile\n"); } finally { try { - execFileSync("chmod", ["755", fakeHome]); + chmodSync(fakeHome, 0o755); } catch { /* ignore */ } @@ -878,7 +883,7 @@ describe("service environment", () => { /* ignore */ } try { - execFileSync("rm", ["-rf", fakeHome]); + rmSync(fakeHome, { recursive: true, force: true }); } catch { /* ignore */ } @@ -906,7 +911,7 @@ describe("service environment", () => { { mode: 0o444 }, ); } - execFileSync("chmod", ["555", fakeHome]); + chmodSync(fakeHome, 0o555); const wrapper = [ "#!/usr/bin/env bash", @@ -929,7 +934,7 @@ describe("service environment", () => { } } finally { try { - execFileSync("chmod", ["755", fakeHome]); + chmodSync(fakeHome, 0o755); } catch { /* ignore */ } @@ -939,7 +944,7 @@ describe("service environment", () => { /* ignore */ } try { - execFileSync("rm", ["-rf", fakeHome]); + rmSync(fakeHome, { recursive: true, force: true }); } catch { /* ignore */ } @@ -1019,7 +1024,8 @@ describe("service environment", () => { /* ignore */ } try { - execFileSync("rm", ["-rf", fakeDataDir, fakeHome]); + rmSync(fakeDataDir, { recursive: true, force: true }); + rmSync(fakeHome, { recursive: true, force: true }); } catch { /* ignore */ } @@ -1028,12 +1034,12 @@ describe("service environment", () => { it("entrypoint overwrites proxy-env.sh cleanly on repeated invocations", () => { const fakeDataDir = join(tmpdir(), `nemoclaw-idempotent-test-${process.pid}`); - execFileSync("mkdir", ["-p", fakeDataDir]); + mkdirSync(fakeDataDir, { recursive: true }); const tmpFile = join(tmpdir(), `nemoclaw-idempotent-write-test-${process.pid}.sh`); const chownLog = join(fakeDataDir, "chown.log"); try { const persistBlock = extractRuntimeShellEnvSnippet(); - const toolRedirects = extractToolRedirects(); + const toolRedirects = extractToolRedirectsSnippet(); const wrapper = [ "#!/usr/bin/env bash", 'id() { if [ "${1:-}" = "-u" ]; then printf "0\\n"; else command id "$@"; fi; }', @@ -1075,7 +1081,7 @@ describe("service environment", () => { /* ignore */ } try { - execFileSync("rm", ["-rf", fakeDataDir]); + rmSync(fakeDataDir, { recursive: true, force: true }); } catch { /* ignore */ } @@ -1084,11 +1090,11 @@ describe("service environment", () => { it("entrypoint replaces stale proxy values on restart", () => { const fakeDataDir = join(tmpdir(), `nemoclaw-replace-test-${process.pid}`); - execFileSync("mkdir", ["-p", fakeDataDir]); + mkdirSync(fakeDataDir, { recursive: true }); const tmpFile = join(tmpdir(), `nemoclaw-replace-write-test-${process.pid}.sh`); try { const persistBlock = extractRuntimeShellEnvSnippet(); - const toolRedirects = extractToolRedirects(); + const toolRedirects = extractToolRedirectsSnippet(); const makeWrapper = (host: string) => [ "#!/usr/bin/env bash", @@ -1120,7 +1126,7 @@ describe("service environment", () => { /* ignore */ } try { - execFileSync("rm", ["-rf", fakeDataDir]); + rmSync(fakeDataDir, { recursive: true, force: true }); } catch { /* ignore */ } @@ -1135,8 +1141,8 @@ describe("service environment", () => { const sensitiveFile = join(fakeDataDir, "sensitive"); writeFileSync(sensitiveFile, "SECRET_DATA"); const proxyEnvPath = join(fakeDataDir, "proxy-env.sh"); - execFileSync("ln", ["-sf", sensitiveFile, proxyEnvPath]); - const toolRedirects = extractToolRedirects(); + symlinkSync(sensitiveFile, proxyEnvPath); + const toolRedirects = extractToolRedirectsSnippet(); const wrapper = [ "#!/usr/bin/env bash", sandboxInitSource, @@ -1159,7 +1165,7 @@ describe("service environment", () => { /* ignore */ } try { - execFileSync("rm", ["-rf", fakeDataDir]); + rmSync(fakeDataDir, { recursive: true, force: true }); } catch { /* ignore */ } @@ -1199,7 +1205,7 @@ describe("service environment", () => { expect(out).toContain("no_proxy=localhost,127.0.0.1,::1,10.200.0.1"); } finally { try { - execFileSync("rm", ["-rf", fakeDataDir]); + rmSync(fakeDataDir, { recursive: true, force: true }); } catch { /* ignore */ } @@ -1208,7 +1214,7 @@ describe("service environment", () => { it("includes NODE_OPTIONS --require in proxy-env.sh when NODE_USE_ENV_PROXY=1 (#2109)", () => { const fakeDataDir = join(tmpdir(), `nemoclaw-http-fix-test-${process.pid}`); - execFileSync("mkdir", ["-p", fakeDataDir]); + mkdirSync(fakeDataDir, { recursive: true }); const tmpFile = join(tmpdir(), `nemoclaw-http-fix-env-${process.pid}.sh`); const fakeFixPath = "/tmp/nemoclaw-http-proxy-fix.js"; try { @@ -1243,7 +1249,8 @@ describe("service environment", () => { expect(envFile).toContain(fakeFixPath); } finally { try { - execFileSync("rm", ["-rf", fakeDataDir, tmpFile]); + rmSync(fakeDataDir, { recursive: true, force: true }); + rmSync(tmpFile, { force: true }); } catch { /* ignore */ } @@ -1252,7 +1259,7 @@ describe("service environment", () => { it("omits NODE_OPTIONS from proxy-env.sh when NODE_USE_ENV_PROXY is unset (#2109)", () => { const fakeDataDir = join(tmpdir(), `nemoclaw-http-noop-test-${process.pid}`); - execFileSync("mkdir", ["-p", fakeDataDir]); + mkdirSync(fakeDataDir, { recursive: true }); const tmpFile = join(tmpdir(), `nemoclaw-http-noop-env-${process.pid}.sh`); try { const persistBlock = extractRuntimeShellEnvSnippet(); @@ -1284,7 +1291,8 @@ describe("service environment", () => { expect(envFile).toContain("nemotron-inference-fix"); } finally { try { - execFileSync("rm", ["-rf", fakeDataDir, tmpFile]); + rmSync(fakeDataDir, { recursive: true, force: true }); + rmSync(tmpFile, { force: true }); } catch { /* ignore */ } From dd00c5c58907c0c975ff5751f1f92ddf1db0dd89 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 4 Jul 2026 10:00:26 -0700 Subject: [PATCH 03/16] refactor(onboard): extract inference setup seam Signed-off-by: Carlos Villela --- src/lib/onboard.ts | 155 +---------------------- src/lib/onboard/setup-inference.ts | 192 +++++++++++++++++++++++++++++ test/onboard.test.ts | 2 +- 3 files changed, 198 insertions(+), 151 deletions(-) create mode 100644 src/lib/onboard/setup-inference.ts diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index b701e965fba..be689010ef2 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -257,6 +257,8 @@ const { DEFAULT_CLOUD_MODEL, getProviderSelectionConfig, parseGatewayInference } const onboardProviders = require("./onboard/providers"); const inferenceProviders: typeof import("./onboard/inference-providers") = require("./onboard/inference-providers"); +const setupInferenceFactory: typeof import("./onboard/setup-inference") = + require("./onboard/setup-inference"); const { ensureResumeProviderReady } = require("./onboard/resume-provider-shim"); const hermesProviderAuth = require("./hermes-provider-auth"); const onboardHermesDashboard: typeof import("./onboard/hermes-dashboard") = require("./onboard/hermes-dashboard"); @@ -4216,158 +4218,11 @@ function getSetupInferenceDeps() { }; } -export type SetupInferenceDeps = ReturnType; - -type ProviderInferenceSetupOptions = - import("./onboard/machine/handlers/provider-inference").ProviderInferenceSetupOptions; - -export type SetupInference = ( - sandboxName: string | null, - model: string, - provider: string, - endpointUrl?: string | null, - credentialEnv?: string | null, - hermesAuthMethod?: HermesAuthMethod | string | null, - hermesToolGateways?: string[], - options?: ProviderInferenceSetupOptions, -) => Promise<{ ok: true; retry?: undefined } | { retry: "selection" }>; +export type SetupInferenceDeps = import("./onboard/setup-inference").SetupInferenceDeps; +export type SetupInference = import("./onboard/setup-inference").SetupInference; function createSetupInference(overrides: Partial = {}): SetupInference { - const deps: SetupInferenceDeps = { ...getSetupInferenceDeps(), ...overrides }; - - return async function setupInferenceWithDeps( - sandboxName: string | null, - model: string, - provider: string, - endpointUrl: string | null = null, - credentialEnv: string | null = null, - hermesAuthMethod: HermesAuthMethod | string | null = null, - hermesToolGateways: string[] = [], - options: ProviderInferenceSetupOptions = {}, - ): Promise<{ ok: true; retry?: undefined } | { retry: "selection" }> { - deps.step(4, 8, "Setting up inference provider"); - deps.runOpenshell(["gateway", "select", deps.getGatewayName()], { ignoreError: true }); - - const commonDeps = { - runOpenshell: deps.runOpenshell, - upsertProvider: deps.upsertProvider, - verifyInferenceRoute: deps.verifyInferenceRoute, - verifyOnboardInferenceSmoke: deps.verifyOnboardInferenceSmoke, - isNonInteractive: deps.isNonInteractive, - registry: { updateSandbox: deps.updateSandbox }, - }; - - if (provider === deps.hermesProviderAuth.HERMES_PROVIDER_NAME) { - return inferenceProviders.setupHermesProviderInference( - { - sandboxName, - model, - provider, - endpointUrl, - credentialEnv, - hermesAuthMethod, - hermesToolGateways, - }, - { - ...commonDeps, - hermesProviderAuth: deps.hermesProviderAuth, - getHermesToolGatewayBroker: deps.getHermesToolGatewayBroker, - providerExistsInGateway: deps.providerExistsInGateway, - normalizeHermesAuthMethod: deps.normalizeHermesAuthMethod, - resolveHermesNousApiKey: deps.resolveHermesNousApiKey, - checkHermesProviderStoreReachable: deps.checkHermesProviderStoreReachable, - hermesAuthMethodLabel: deps.hermesAuthMethodLabel, - hermesConstants: deps.hermesConstants, - requireValue: deps.requireValue, - redact: deps.redact, - compactText: deps.compactText, - }, - ); - } - - if (inferenceProviders.isRemoteProviderName(provider)) { - const outcome = await inferenceProviders.setupRemoteProviderInference( - { - sandboxName, - model, - provider, - endpointUrl, - credentialEnv, - reuseGatewayCredentialWithoutLocalKey: - options.reuseGatewayCredentialWithoutLocalKey === true, - }, - { - ...commonDeps, - REMOTE_PROVIDER_CONFIG: deps.REMOTE_PROVIDER_CONFIG, - hydrateCredentialEnv: deps.hydrateCredentialEnv, - promptValidationRecovery: deps.promptValidationRecovery, - classifyApplyFailure: deps.classifyApplyFailure, - LOCAL_INFERENCE_TIMEOUT_SECS: deps.localInferenceTimeoutSecs, - bedrockRuntimeOnboard: deps.bedrockRuntimeOnboard, - redact: deps.redact, - compactText: deps.compactText, - }, - ); - if (outcome.done) return outcome.result; - } else if (provider === "vllm-local") { - const outcome = await inferenceProviders.setupVllmLocalInference( - { model, provider }, - { - ...commonDeps, - validateLocalProvider: deps.validateLocalProvider, - getLocalProviderHealthCheck: deps.getLocalProviderHealthCheck, - getLocalProviderBaseUrl: deps.getLocalProviderBaseUrl, - applyLocalInferenceRoute: deps.applyLocalInferenceRoute, - run: deps.run, - VLLM_LOCAL_CREDENTIAL_ENV: deps.vllmLocalCredentialEnv, - }, - ); - if (outcome.done) return outcome.result; - } else if (provider === "ollama-local") { - const outcome = await inferenceProviders.setupOllamaLocalInference( - { model, provider, allowToolsIncompatible: options.allowToolsIncompatible === true }, - { - ...commonDeps, - validateLocalProvider: deps.validateLocalProvider, - getLocalProviderBaseUrl: deps.getLocalProviderBaseUrl, - applyLocalInferenceRoute: deps.applyLocalInferenceRoute, - getOllamaWarmupCommand: deps.getOllamaWarmupCommand, - run: deps.run, - shouldFrontOllamaWithProxy: deps.shouldFrontOllamaWithProxy, - ensureOllamaAuthProxy: deps.ensureOllamaAuthProxy, - isProxyHealthy: deps.isProxyHealthy, - getOllamaProxyToken: deps.getOllamaProxyToken, - persistAndProbeOllamaProxy: deps.persistAndProbeOllamaProxy, - localInference: deps.localInference, - OLLAMA_PROXY_CREDENTIAL_ENV: deps.ollamaProxyCredentialEnv, - }, - ); - if (outcome.done) return outcome.result; - } else if (deps.isRoutedInferenceProvider(provider)) { - await inferenceProviders.setupRoutedInference( - { model, provider, endpointUrl, credentialEnv }, - { - ...commonDeps, - reconcileModelRouter: deps.reconcileModelRouter, - routedInference: deps.routedInference, - hydrateCredentialEnv: deps.hydrateCredentialEnv, - }, - ); - } else { - deps.error(` Unsupported provider configuration: ${provider}`); - deps.exitProcess(1); - } - - deps.verifyInferenceRoute(provider, model); - if (options.skipHostInferenceSmoke === true) - deps.log(" Reusing existing gateway credential; skipping host inference smoke."); - else deps.verifyOnboardInferenceSmoke({ provider, model, endpointUrl, credentialEnv }); - if (sandboxName) { - deps.updateSandbox(sandboxName, { model, provider }); - } - deps.log(` ✓ Inference route set: ${provider} / ${model}`); - return { ok: true }; - }; + return setupInferenceFactory.createSetupInference(getSetupInferenceDeps(), overrides); } const setupInference = createSetupInference(); diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts new file mode 100644 index 00000000000..0936bf15ba7 --- /dev/null +++ b/src/lib/onboard/setup-inference.ts @@ -0,0 +1,192 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { HermesAuthMethod } from "./hermes-auth"; +import type { + CommonDeps, + HermesDeps, + OllamaDeps, + RemoteProviderDeps, + RoutedDeps, + SetupInferenceResult, + VllmDeps, +} from "./inference-providers"; +import * as inferenceProviders from "./inference-providers"; +import type { ProviderInferenceSetupOptions } from "./machine/handlers/provider-inference"; + +type ProviderBranchDeps = Omit< + HermesDeps & RemoteProviderDeps & VllmDeps & OllamaDeps & RoutedDeps, + | "registry" + | "run" + | "runOpenshell" + | "LOCAL_INFERENCE_TIMEOUT_SECS" + | "VLLM_LOCAL_CREDENTIAL_ENV" + | "OLLAMA_PROXY_CREDENTIAL_ENV" +>; + +export type SetupInferenceDeps = ProviderBranchDeps & { + step: (current: number, total: number, label: string) => void; + getGatewayName: () => string; + runOpenshell: import("./openshell-cli").OpenshellCliHelpers["runOpenshell"]; + run: typeof import("../runner").run; + updateSandbox: CommonDeps["registry"]["updateSandbox"]; + localInferenceTimeoutSecs: number; + vllmLocalCredentialEnv: string; + ollamaProxyCredentialEnv: string; + isRoutedInferenceProvider: (provider: string) => boolean; + log: (message: string) => void; + error: (message: string) => void; + exitProcess: (code: number) => never; +}; + +export type SetupInference = ( + sandboxName: string | null, + model: string, + provider: string, + endpointUrl?: string | null, + credentialEnv?: string | null, + hermesAuthMethod?: HermesAuthMethod | string | null, + hermesToolGateways?: string[], + options?: ProviderInferenceSetupOptions, +) => Promise; + +export function createSetupInference( + defaults: SetupInferenceDeps, + overrides: Partial = {}, +): SetupInference { + const deps: SetupInferenceDeps = { ...defaults, ...overrides }; + + return async function setupInferenceWithDeps( + sandboxName: string | null, + model: string, + provider: string, + endpointUrl: string | null = null, + credentialEnv: string | null = null, + hermesAuthMethod: HermesAuthMethod | string | null = null, + hermesToolGateways: string[] = [], + options: ProviderInferenceSetupOptions = {}, + ): Promise { + deps.step(4, 8, "Setting up inference provider"); + deps.runOpenshell(["gateway", "select", deps.getGatewayName()], { ignoreError: true }); + + const commonDeps = { + runOpenshell: deps.runOpenshell, + upsertProvider: deps.upsertProvider, + verifyInferenceRoute: deps.verifyInferenceRoute, + verifyOnboardInferenceSmoke: deps.verifyOnboardInferenceSmoke, + isNonInteractive: deps.isNonInteractive, + registry: { updateSandbox: deps.updateSandbox }, + }; + + if (provider === deps.hermesProviderAuth.HERMES_PROVIDER_NAME) { + return inferenceProviders.setupHermesProviderInference( + { + sandboxName, + model, + provider, + endpointUrl, + credentialEnv, + hermesAuthMethod, + hermesToolGateways, + }, + { + ...commonDeps, + hermesProviderAuth: deps.hermesProviderAuth, + getHermesToolGatewayBroker: deps.getHermesToolGatewayBroker, + providerExistsInGateway: deps.providerExistsInGateway, + normalizeHermesAuthMethod: deps.normalizeHermesAuthMethod, + resolveHermesNousApiKey: deps.resolveHermesNousApiKey, + checkHermesProviderStoreReachable: deps.checkHermesProviderStoreReachable, + hermesAuthMethodLabel: deps.hermesAuthMethodLabel, + hermesConstants: deps.hermesConstants, + requireValue: deps.requireValue, + redact: deps.redact, + compactText: deps.compactText, + }, + ); + } + + if (inferenceProviders.isRemoteProviderName(provider)) { + const outcome = await inferenceProviders.setupRemoteProviderInference( + { + sandboxName, + model, + provider, + endpointUrl, + credentialEnv, + reuseGatewayCredentialWithoutLocalKey: + options.reuseGatewayCredentialWithoutLocalKey === true, + }, + { + ...commonDeps, + REMOTE_PROVIDER_CONFIG: deps.REMOTE_PROVIDER_CONFIG, + hydrateCredentialEnv: deps.hydrateCredentialEnv, + promptValidationRecovery: deps.promptValidationRecovery, + classifyApplyFailure: deps.classifyApplyFailure, + LOCAL_INFERENCE_TIMEOUT_SECS: deps.localInferenceTimeoutSecs, + bedrockRuntimeOnboard: deps.bedrockRuntimeOnboard, + redact: deps.redact, + compactText: deps.compactText, + }, + ); + if (outcome.done) return outcome.result; + } else if (provider === "vllm-local") { + const outcome = await inferenceProviders.setupVllmLocalInference( + { model, provider }, + { + ...commonDeps, + validateLocalProvider: deps.validateLocalProvider, + getLocalProviderHealthCheck: deps.getLocalProviderHealthCheck, + getLocalProviderBaseUrl: deps.getLocalProviderBaseUrl, + applyLocalInferenceRoute: deps.applyLocalInferenceRoute, + run: deps.run, + VLLM_LOCAL_CREDENTIAL_ENV: deps.vllmLocalCredentialEnv, + }, + ); + if (outcome.done) return outcome.result; + } else if (provider === "ollama-local") { + const outcome = await inferenceProviders.setupOllamaLocalInference( + { model, provider, allowToolsIncompatible: options.allowToolsIncompatible === true }, + { + ...commonDeps, + validateLocalProvider: deps.validateLocalProvider, + getLocalProviderBaseUrl: deps.getLocalProviderBaseUrl, + applyLocalInferenceRoute: deps.applyLocalInferenceRoute, + getOllamaWarmupCommand: deps.getOllamaWarmupCommand, + run: deps.run, + shouldFrontOllamaWithProxy: deps.shouldFrontOllamaWithProxy, + ensureOllamaAuthProxy: deps.ensureOllamaAuthProxy, + isProxyHealthy: deps.isProxyHealthy, + getOllamaProxyToken: deps.getOllamaProxyToken, + persistAndProbeOllamaProxy: deps.persistAndProbeOllamaProxy, + localInference: deps.localInference, + OLLAMA_PROXY_CREDENTIAL_ENV: deps.ollamaProxyCredentialEnv, + }, + ); + if (outcome.done) return outcome.result; + } else if (deps.isRoutedInferenceProvider(provider)) { + await inferenceProviders.setupRoutedInference( + { model, provider, endpointUrl, credentialEnv }, + { + ...commonDeps, + reconcileModelRouter: deps.reconcileModelRouter, + routedInference: deps.routedInference, + hydrateCredentialEnv: deps.hydrateCredentialEnv, + }, + ); + } else { + deps.error(` Unsupported provider configuration: ${provider}`); + deps.exitProcess(1); + } + + deps.verifyInferenceRoute(provider, model); + if (options.skipHostInferenceSmoke === true) + deps.log(" Reusing existing gateway credential; skipping host inference smoke."); + else deps.verifyOnboardInferenceSmoke({ provider, model, endpointUrl, credentialEnv }); + if (sandboxName) { + deps.updateSandbox(sandboxName, { model, provider }); + } + deps.log(` ✓ Inference route set: ${provider} / ${model}`); + return { ok: true }; + }; +} diff --git a/test/onboard.test.ts b/test/onboard.test.ts index 857fae727fe..235fba074e9 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -15,7 +15,7 @@ import { } from "../src/lib/onboard/inference-input-capability.js"; import { createInferenceRouteHelpers } from "../src/lib/onboard/inference-route.js"; import { createLocalInferenceRouteApplier } from "../src/lib/onboard/local-inference-route.js"; -import type { SetupInference, SetupInferenceDeps } from "../src/lib/onboard.js"; +import type { SetupInference, SetupInferenceDeps } from "../src/lib/onboard/setup-inference.js"; import { stageOptimizedSandboxBuildContext } from "../src/lib/sandbox/build-context.js"; import { testTimeoutOptions } from "./helpers/timeouts"; From 09fee952ee71db9186caf708e16fa94cc28c649b Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 4 Jul 2026 10:01:29 -0700 Subject: [PATCH 04/16] perf(test): inline remote provider selection cases Signed-off-by: Carlos Villela --- ci/test-file-size-budget.json | 2 +- test/onboard-selection.test.ts | 1643 ++++++++++++++------------------ 2 files changed, 690 insertions(+), 955 deletions(-) diff --git a/ci/test-file-size-budget.json b/ci/test-file-size-budget.json index 3754a828c30..a866bd9a82c 100644 --- a/ci/test-file-size-budget.json +++ b/ci/test-file-size-budget.json @@ -10,7 +10,7 @@ "test/install-preflight.test.ts": 3934, "test/nemoclaw-start.test.ts": 4827, "test/onboard-messaging.test.ts": 2062, - "test/onboard-selection.test.ts": 6603, + "test/onboard-selection.test.ts": 6338, "test/onboard.test.ts": 4202, "test/policies.test.ts": 2332 } diff --git a/test/onboard-selection.test.ts b/test/onboard-selection.test.ts index 4e503d6b234..a7c84378e81 100644 --- a/test/onboard-selection.test.ts +++ b/test/onboard-selection.test.ts @@ -8,6 +8,18 @@ import os from "node:os"; import path from "node:path"; import { describe, expect, it, vi } from "vitest"; +import { normalizeProviderBaseUrl } from "../src/lib/core/url-utils.js"; +import { + BACK_TO_SELECTION, + promptInputModel, + promptRemoteModel, +} from "../src/lib/inference/model-prompts.js"; +import { + validateAnthropicModel, + validateOpenAiLikeModel, +} from "../src/lib/inference/provider-models.js"; +import { returningToProviderSelection } from "../src/lib/onboard/credential-navigation.js"; +import { createInferenceSelectionValidationHelpers } from "../src/lib/onboard/inference-selection-validation.js"; import { getWindowsHostOllamaDockerRequirement, rejectUnsupportedWindowsHostOllama, @@ -16,7 +28,12 @@ import { buildInferenceProviderMenu } from "../src/lib/onboard/provider-menu.js" import { resolveRequestedProviderSelection } from "../src/lib/onboard/provider-selection.js"; import { reportProviderSelectionFailure } from "../src/lib/onboard/provider-selection-failure.js"; import { createSetupNimOllamaHandlers } from "../src/lib/onboard/setup-nim-ollama.js"; -import type { SetupNimSelectionState } from "../src/lib/onboard/setup-nim-selection.js"; +import { + createRemoteModelValidator, + resolveCompatibleEndpointInput, + type SetupNimSelectionState, +} from "../src/lib/onboard/setup-nim-selection.js"; +import { createValidationRecoveryPromptHelpers } from "../src/lib/onboard/validation-recovery-prompt.js"; import { type DetectWindowsHostOllamaDeps, detectWindowsHostOllama, @@ -50,6 +67,108 @@ const TEST_REMOTE_PROVIDER_CONFIG = { type WindowsRequirement = ReturnType; type ProviderMenuOverrides = Partial[0]>; type SetupNimOllamaDeps = Parameters[0]; +type RemoteModelValidatorDeps = Parameters[0]; + +const TEST_OPENAI_ENDPOINT_URL = "https://api.openai.com/v1"; +const TEST_ANTHROPIC_ENDPOINT_URL = "https://api.anthropic.com"; +const TEST_CUSTOM_OPENAI_CONFIG = { + label: "Other OpenAI-compatible endpoint", + endpointUrl: TEST_OPENAI_ENDPOINT_URL, + helpUrl: null, +}; +const TEST_CUSTOM_ANTHROPIC_CONFIG = { + label: "Other Anthropic-compatible endpoint", + endpointUrl: TEST_ANTHROPIC_ENDPOINT_URL, + helpUrl: null, +}; +const TEST_ANTHROPIC_CONFIG = { + label: "Anthropic", + endpointUrl: TEST_ANTHROPIC_ENDPOINT_URL, + helpUrl: null, +}; + +function makeRemoteSelectionState( + overrides: Partial = {}, +): SetupNimSelectionState { + return { + model: "test-model", + provider: "compatible-endpoint", + endpointUrl: "https://proxy.example.com/v1", + credentialEnv: "COMPATIBLE_API_KEY", + hermesAuthMethod: null, + hermesToolGateways: [], + preferredInferenceApi: null, + nimContainer: null, + allowToolsIncompatible: false, + skipHostInferenceSmoke: false, + ...overrides, + }; +} + +function makeRemoteModelValidatorDeps( + overrides: Partial = {}, +): RemoteModelValidatorDeps { + return { + OPENAI_ENDPOINT_URL: TEST_OPENAI_ENDPOINT_URL, + ANTHROPIC_ENDPOINT_URL: TEST_ANTHROPIC_ENDPOINT_URL, + requireValue: (value, message) => { + if (value === null || value === undefined) throw new Error(message); + return value; + }, + isBackToSelection: (_value): _value is never => false, + validateCustomOpenAiLikeSelection: async () => ({ + ok: true as const, + api: "openai-completions", + }), + validateCustomAnthropicSelection: async () => ({ + ok: true as const, + api: "anthropic-messages", + }), + validateAnthropicSelectionWithRetryMessage: async () => ({ + ok: true as const, + api: "anthropic-messages", + }), + validateOpenAiLikeSelection: async () => ({ + ok: true as const, + api: "openai-completions", + }), + shouldRequireResponsesToolCalling: () => false, + shouldSkipResponsesProbe: () => false, + getProbeAuthMode: () => undefined, + ...overrides, + }; +} + +function makeInteractiveValidationRecovery() { + return createValidationRecoveryPromptHelpers({ + isNonInteractive: () => false, + prompt: async () => "", + validateNvidiaApiKeyValue: () => null, + getTransportRecoveryMessage: () => " Validation hit a network or transport error.", + exitOnboardFromPrompt(): never { + throw new Error("Unexpected onboarding exit"); + }, + }); +} + +async function captureConsoleOutput(callback: () => Promise): Promise<{ + result: T; + lines: string[]; +}> { + const lines: string[] = []; + const log = vi.spyOn(console, "log").mockImplementation((...args) => { + lines.push(args.join(" ")); + }); + const error = vi.spyOn(console, "error").mockImplementation((...args) => { + lines.push(args.join(" ")); + }); + try { + return { result: await callback(), lines }; + } finally { + error.mockRestore(); + log.mockRestore(); + } +} function buildWindowsProviderMenu( requirement: WindowsRequirement, @@ -2597,105 +2716,483 @@ const { setupNim } = require(${onboardPath}); assert.match(pullingLine, sizePattern); }); - it("reprompts for an OpenAI Other model when /models validation rejects it", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-openai-model-retry-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "openai-model-retry-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + it("reprompts for an OpenAI Other model when /models validation rejects it", async () => { + const answers = ["5", "bad-model", "gpt-5.4-mini"]; + const messages: string[] = []; + const lines: string[] = []; + const catalogUrls: string[] = []; + const model = await promptRemoteModel( + "OpenAI", + "openai", + "gpt-5.4", + (candidate) => + validateOpenAiLikeModel("OpenAI", TEST_OPENAI_ENDPOINT_URL, candidate, "sk-test", { + runCurlProbeImpl: (argv) => { + catalogUrls.push(argv.at(-1) || ""); + return { + ok: true, + httpStatus: 200, + curlStatus: 0, + body: JSON.stringify({ data: [{ id: "gpt-5.4" }, { id: "gpt-5.4-mini" }] }), + stderr: "", + message: "", + }; + }, + }), + { + promptFn: async (message) => { + messages.push(message); + return answers.shift() || ""; + }, + errorLine: (line) => lines.push(line), + writeLine: (line) => lines.push(line), + }, ); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync( - path.join(fakeBin, "curl"), - `#!/usr/bin/env bash -body='{"id":"ok"}' -status="200" -outfile="" -url="" -while [ "$#" -gt 0 ]; do - case "$1" in - -o) outfile="$2"; shift 2 ;; - --config) auth="$(cat "$2" 2>/dev/null)"; shift 2 ;; *) url="$1"; shift ;; - esac -done -if echo "$url" | grep -q '/models$'; then - body='{"data":[{"id":"gpt-5.4"},{"id":"gpt-5.4-mini"}]}' -elif echo "$url" | grep -q '/responses$'; then - body='{"id":"resp_123"}' -fi -printf '%s' "$body" > "$outfile" -printf '%s' "$status" -`, - { mode: 0o755 }, + assert.equal(model, "gpt-5.4-mini"); + assert.equal(messages.filter((message) => /OpenAI model id:/.test(message)).length, 2); + assert.ok(lines.some((line) => line.includes("is not available from OpenAI"))); + assert.deepEqual(catalogUrls, [ + `${TEST_OPENAI_ENDPOINT_URL}/models`, + `${TEST_OPENAI_ENDPOINT_URL}/models`, + ]); + }); + + it("reprompts for an Anthropic Other model when /v1/models validation rejects it", async () => { + const answers = ["4", "claude-bad", "claude-haiku-4-5"]; + const messages: string[] = []; + const lines: string[] = []; + const catalogUrls: string[] = []; + const model = await promptRemoteModel( + "Anthropic", + "anthropic", + "claude-sonnet-4-6", + (candidate) => + validateAnthropicModel(TEST_ANTHROPIC_ENDPOINT_URL, candidate, "anthropic-test", { + runCurlProbeImpl: (argv) => { + catalogUrls.push(argv.at(-1) || ""); + return { + ok: true, + httpStatus: 200, + curlStatus: 0, + body: JSON.stringify({ + data: [{ id: "claude-sonnet-4-6" }, { id: "claude-haiku-4-5" }], + }), + stderr: "", + message: "", + }; + }, + }), + { + promptFn: async (message) => { + messages.push(message); + return answers.shift() || ""; + }, + errorLine: (line) => lines.push(line), + writeLine: (line) => lines.push(line), + }, ); - const script = String.raw` -const credentials = require(${credentialsPath}); -const runner = require(${runnerPath}); + assert.equal(model, "claude-haiku-4-5"); + assert.equal(messages.filter((message) => /Anthropic model id:/.test(message)).length, 2); + assert.ok(lines.some((line) => line.includes("is not available from Anthropic"))); + assert.deepEqual(catalogUrls, [ + `${TEST_ANTHROPIC_ENDPOINT_URL}/v1/models`, + `${TEST_ANTHROPIC_ENDPOINT_URL}/v1/models`, + ]); + }); -const answers = ["2", "5", "bad-model", "gpt-5.4-mini"]; -const messages = []; + it("returns to provider selection when Anthropic live validation fails interactively", async () => { + const recovery = makeInteractiveValidationRecovery(); + const probedModels: string[] = []; + const validation = createInferenceSelectionValidationHelpers({ + isNonInteractive: () => false, + agentProductName: () => "OpenClaw", + getCredential: () => "anthropic-test", + probeAnthropicEndpoint: (_endpointUrl, model) => { + probedModels.push(model); + return model === "claude-haiku-4-5" + ? { ok: true, api: "anthropic-messages", label: "Anthropic Messages API" } + : { + ok: false, + message: "invalid model", + failures: [ + { name: "Anthropic Messages API", httpStatus: 400, message: "invalid model" }, + ], + }; + }, + promptValidationRecovery: recovery.promptValidationRecovery, + }); + const state = makeRemoteSelectionState({ + model: "claude-sonnet-4-6", + provider: "anthropic-prod", + endpointUrl: TEST_ANTHROPIC_ENDPOINT_URL, + credentialEnv: "ANTHROPIC_API_KEY", + }); + const { validateSelectedRemoteModel } = createRemoteModelValidator( + makeRemoteModelValidatorDeps({ + validateAnthropicSelectionWithRetryMessage: + validation.validateAnthropicSelectionWithRetryMessage, + }), + ); -credentials.prompt = async (message) => { - messages.push(message); - return answers.shift() || ""; -}; -runner.runCapture = () => ""; + const { result, lines } = await captureConsoleOutput(async () => { + const first = await validateSelectedRemoteModel({ + selected: { key: "anthropic" }, + remoteConfig: TEST_ANTHROPIC_CONFIG, + state, + selectedCredentialEnv: "ANTHROPIC_API_KEY", + }); + state.model = "claude-haiku-4-5"; + const second = await validateSelectedRemoteModel({ + selected: { key: "anthropic" }, + remoteConfig: TEST_ANTHROPIC_CONFIG, + state, + selectedCredentialEnv: "ANTHROPIC_API_KEY", + }); + return { first, second }; + }); -const { setupNim } = require(${onboardPath}); + assert.deepEqual(result, { first: "retry-selection", second: "selected" }); + assert.equal(state.provider, "anthropic-prod"); + assert.equal(state.model, "claude-haiku-4-5"); + assert.equal(state.preferredInferenceApi, "anthropic-messages"); + assert.deepEqual(probedModels, ["claude-sonnet-4-6", "claude-haiku-4-5"]); + assert.ok(lines.some((line) => line.includes("Anthropic endpoint validation failed"))); + assert.ok(lines.some((line) => line.includes("Please choose a provider/model again"))); + }); -(async () => { - process.env.OPENAI_API_KEY = "sk-test"; - const originalLog = console.log; - const originalError = console.error; - const lines = []; - console.log = (...args) => lines.push(args.join(" ")); - console.error = (...args) => lines.push(args.join(" ")); - try { - const result = await setupNim(null); - originalLog(JSON.stringify({ result, messages, lines })); - } finally { - console.log = originalLog; - console.error = originalError; - } -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); + it("supports Other Anthropic-compatible endpoint with live validation", async () => { + const messages: string[] = []; + const endpointInput = await resolveCompatibleEndpointInput({ + kind: "anthropic", + envUrl: null, + recoveredEndpointUrl: null, + nonInteractive: false, + prompt: async (message) => { + messages.push(message); + return "https://proxy.example.com/v1/messages?token=secret#frag"; + }, + }); + const endpointUrl = normalizeProviderBaseUrl(endpointInput, "anthropic"); + const model = await promptInputModel( + TEST_CUSTOM_ANTHROPIC_CONFIG.label, + "claude-sonnet-4-6", + null, + { + promptFn: async (message) => { + messages.push(message); + return "claude-sonnet-proxy"; + }, + }, + ); + assert.equal(model, "claude-sonnet-proxy"); + const recovery = makeInteractiveValidationRecovery(); + const validation = createInferenceSelectionValidationHelpers({ + isNonInteractive: () => false, + agentProductName: () => "OpenClaw", + getCredential: () => "proxy-key", + probeAnthropicEndpoint: () => ({ + ok: true, + api: "anthropic-messages", + label: "Anthropic Messages API", + }), + promptValidationRecovery: recovery.promptValidationRecovery, + }); + const state = makeRemoteSelectionState({ + model, + provider: "compatible-anthropic-endpoint", + endpointUrl, + credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", + }); + const { validateSelectedRemoteModel } = createRemoteModelValidator( + makeRemoteModelValidatorDeps({ + validateCustomAnthropicSelection: validation.validateCustomAnthropicSelection, + }), + ); - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, + const { result, lines } = await captureConsoleOutput(() => + validateSelectedRemoteModel({ + selected: { key: "anthropicCompatible" }, + remoteConfig: TEST_CUSTOM_ANTHROPIC_CONFIG, + state, + selectedCredentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", + }), + ); + + assert.equal(result, "selected"); + assert.equal(state.provider, "compatible-anthropic-endpoint"); + assert.equal(state.model, "claude-sonnet-proxy"); + assert.equal(state.endpointUrl, "https://proxy.example.com"); + assert.equal(state.preferredInferenceApi, "anthropic-messages"); + assert.match(messages[0], /Anthropic-compatible base URL/); + assert.match(messages[1], /Other Anthropic-compatible endpoint model/); + assert.ok(lines.some((line) => line.includes("Anthropic Messages API available"))); + }); + + it("reprompts only for model name when Other OpenAI-compatible endpoint validation fails", async () => { + const messages: string[] = []; + const modelAnswers = ["bad-model", "good-model"]; + const endpointInput = await resolveCompatibleEndpointInput({ + kind: "openai", + envUrl: null, + recoveredEndpointUrl: null, + nonInteractive: false, + prompt: async (message) => { + messages.push(message); + return "https://proxy.example.com/v1/chat/completions?token=secret#frag"; + }, + }); + const state = makeRemoteSelectionState({ + endpointUrl: normalizeProviderBaseUrl(endpointInput, "openai"), + }); + const recovery = makeInteractiveValidationRecovery(); + const probedModels: string[] = []; + const validation = createInferenceSelectionValidationHelpers({ + isNonInteractive: () => false, + agentProductName: () => "OpenClaw", + getCredential: () => "proxy-key", + probeOpenAiLikeEndpoint: (_endpointUrl, model) => { + probedModels.push(model); + return model === "good-model" + ? { ok: true, api: "openai-responses", label: "Responses API" } + : { + ok: false, + message: "bad model", + failures: [{ name: "Responses API", httpStatus: 400, message: "bad model" }], + }; }, + promptValidationRecovery: recovery.promptValidationRecovery, + }); + const { validateSelectedRemoteModel } = createRemoteModelValidator( + makeRemoteModelValidatorDeps({ + validateCustomOpenAiLikeSelection: validation.validateCustomOpenAiLikeSelection, + }), + ); + const promptModel = () => + promptInputModel(TEST_CUSTOM_OPENAI_CONFIG.label, "custom-model", null, { + promptFn: async (message) => { + messages.push(message); + return modelAnswers.shift() || ""; + }, + }); + + const { result, lines } = await captureConsoleOutput(async () => { + state.model = await promptModel(); + const first = await validateSelectedRemoteModel({ + selected: { key: "custom" }, + remoteConfig: TEST_CUSTOM_OPENAI_CONFIG, + state, + selectedCredentialEnv: "COMPATIBLE_API_KEY", + }); + state.model = await promptModel(); + const second = await validateSelectedRemoteModel({ + selected: { key: "custom" }, + remoteConfig: TEST_CUSTOM_OPENAI_CONFIG, + state, + selectedCredentialEnv: "COMPATIBLE_API_KEY", + }); + return { first, second }; }); - assert.equal(result.status, 0, result.stderr); - const payload = JSON.parse(result.stdout.trim()); - assert.equal(payload.result.model, "gpt-5.4-mini"); + assert.deepEqual(result, { first: "retry-model", second: "selected" }); + assert.equal(state.provider, "compatible-endpoint"); + assert.equal(state.model, "good-model"); + assert.equal(state.endpointUrl, "https://proxy.example.com/v1"); + assert.equal(state.preferredInferenceApi, "openai-completions"); + assert.deepEqual(probedModels, ["bad-model", "good-model"]); + assert.ok( + lines.some((line) => + line.includes("Other OpenAI-compatible endpoint endpoint validation failed"), + ), + ); + assert.ok( + lines.some((line) => + line.includes("Please enter a different Other OpenAI-compatible endpoint model name."), + ), + ); assert.equal( - payload.messages.filter((message: string) => /OpenAI model id:/.test(message)).length, + messages.filter((message) => /OpenAI-compatible base URL/.test(message)).length, + 1, + ); + assert.equal( + messages.filter((message) => /Other OpenAI-compatible endpoint model/.test(message)).length, 2, ); - assert.ok(payload.lines.some((line: string) => line.includes("is not available from OpenAI"))); }); - it("reprompts for an Anthropic Other model when /v1/models validation rejects it", () => { + it("falls back to chat completions for custom OpenAI-compatible endpoints when /responses lacks tool calls", async () => { + const probeOpenAiLikeEndpoint = vi.fn(() => ({ + ok: true, + api: "openai-completions", + label: "Chat Completions API", + })); + const recovery = makeInteractiveValidationRecovery(); + const validation = createInferenceSelectionValidationHelpers({ + isNonInteractive: () => false, + agentProductName: () => "OpenClaw", + getCredential: () => "proxy-key", + probeOpenAiLikeEndpoint, + promptValidationRecovery: recovery.promptValidationRecovery, + }); + const state = makeRemoteSelectionState({ model: "custom-model" }); + const { validateSelectedRemoteModel } = createRemoteModelValidator( + makeRemoteModelValidatorDeps({ + validateCustomOpenAiLikeSelection: validation.validateCustomOpenAiLikeSelection, + }), + ); + + const { result, lines } = await captureConsoleOutput(() => + validateSelectedRemoteModel({ + selected: { key: "custom" }, + remoteConfig: TEST_CUSTOM_OPENAI_CONFIG, + state, + selectedCredentialEnv: "COMPATIBLE_API_KEY", + }), + ); + + assert.equal(result, "selected"); + assert.equal(state.provider, "compatible-endpoint"); + assert.equal(state.model, "custom-model"); + assert.equal(state.preferredInferenceApi, "openai-completions"); + assert.ok(lines.some((line) => line.includes("Chat Completions API available"))); + expect(probeOpenAiLikeEndpoint).toHaveBeenCalledWith( + "https://proxy.example.com/v1", + "custom-model", + "proxy-key", + { + requireResponsesToolCalling: true, + skipResponsesProbe: false, + probeStreaming: true, + }, + ); + }); + + it("forces chat completions for custom OpenAI-compatible endpoints even when /responses returns valid tool calls (#1932)", async () => { + const previousPreferredApi = process.env.NEMOCLAW_PREFERRED_API; + delete process.env.NEMOCLAW_PREFERRED_API; + const probeOpenAiLikeEndpoint = vi.fn(() => ({ + ok: true, + api: "openai-responses", + label: "Responses API", + })); + const recovery = makeInteractiveValidationRecovery(); + const validation = createInferenceSelectionValidationHelpers({ + isNonInteractive: () => false, + agentProductName: () => "OpenClaw", + getCredential: () => "ollama-key", + probeOpenAiLikeEndpoint, + promptValidationRecovery: recovery.promptValidationRecovery, + }); + const state = makeRemoteSelectionState({ + model: "my-model", + endpointUrl: "https://ollama.local:11434/v1", + }); + const { validateSelectedRemoteModel } = createRemoteModelValidator( + makeRemoteModelValidatorDeps({ + validateCustomOpenAiLikeSelection: validation.validateCustomOpenAiLikeSelection, + }), + ); + + try { + const { result, lines } = await captureConsoleOutput(() => + validateSelectedRemoteModel({ + selected: { key: "custom" }, + remoteConfig: TEST_CUSTOM_OPENAI_CONFIG, + state, + selectedCredentialEnv: "COMPATIBLE_API_KEY", + }), + ); + + assert.equal(result, "selected"); + assert.equal(state.provider, "compatible-endpoint"); + assert.equal(state.model, "my-model"); + assert.equal(state.preferredInferenceApi, "openai-completions"); + assert.ok(lines.some((line) => line.includes("Using chat completions API"))); + expect(probeOpenAiLikeEndpoint).toHaveBeenCalledWith( + "https://ollama.local:11434/v1", + "my-model", + "ollama-key", + { + requireResponsesToolCalling: true, + skipResponsesProbe: false, + probeStreaming: true, + }, + ); + } finally { + if (previousPreferredApi === undefined) delete process.env.NEMOCLAW_PREFERRED_API; + else process.env.NEMOCLAW_PREFERRED_API = previousPreferredApi; + } + }); + + it("honors NEMOCLAW_PREFERRED_API=openai-responses override for custom OpenAI-compatible endpoints (#1932)", async () => { + const previousPreferredApi = process.env.NEMOCLAW_PREFERRED_API; + process.env.NEMOCLAW_PREFERRED_API = "openai-responses"; + const probeOpenAiLikeEndpoint = vi.fn(() => ({ + ok: true, + api: "openai-responses", + label: "Responses API", + })); + const recovery = makeInteractiveValidationRecovery(); + const validation = createInferenceSelectionValidationHelpers({ + isNonInteractive: () => false, + agentProductName: () => "OpenClaw", + getCredential: () => "sk-test", + probeOpenAiLikeEndpoint, + promptValidationRecovery: recovery.promptValidationRecovery, + }); + const state = makeRemoteSelectionState({ + model: "gpt-4o", + endpointUrl: "https://openai-proxy.example.com/v1", + }); + const { validateSelectedRemoteModel } = createRemoteModelValidator( + makeRemoteModelValidatorDeps({ + validateCustomOpenAiLikeSelection: validation.validateCustomOpenAiLikeSelection, + }), + ); + + try { + const { result, lines } = await captureConsoleOutput(() => + validateSelectedRemoteModel({ + selected: { key: "custom" }, + remoteConfig: TEST_CUSTOM_OPENAI_CONFIG, + state, + selectedCredentialEnv: "COMPATIBLE_API_KEY", + }), + ); + + assert.equal(result, "selected"); + assert.equal(state.provider, "compatible-endpoint"); + assert.equal(state.model, "gpt-4o"); + assert.equal(state.preferredInferenceApi, "openai-responses"); + assert.ok( + !lines.some((line) => + line.includes("compatible endpoints may not support the Responses API developer role"), + ), + ); + expect(probeOpenAiLikeEndpoint).toHaveBeenCalledWith( + "https://openai-proxy.example.com/v1", + "gpt-4o", + "sk-test", + { + requireResponsesToolCalling: true, + skipResponsesProbe: false, + probeStreaming: true, + }, + ); + } finally { + if (previousPreferredApi === undefined) delete process.env.NEMOCLAW_PREFERRED_API; + else process.env.NEMOCLAW_PREFERRED_API = previousPreferredApi; + } + }); + + it("returns to provider selection instead of exiting on blank custom endpoint input", () => { const repoRoot = path.join(import.meta.dirname, ".."); const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-anthropic-model-retry-"), + path.join(os.tmpdir(), "nemoclaw-onboard-custom-endpoint-blank-"), ); const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "anthropic-model-retry-check.js"); + const scriptPath = path.join(tmpDir, "custom-endpoint-blank-check.js"); const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); const credentialsPath = JSON.stringify( path.join(repoRoot, "src", "lib", "credentials", "store.ts"), @@ -2703,41 +3200,25 @@ const { setupNim } = require(${onboardPath}); const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync( - path.join(fakeBin, "curl"), - `#!/usr/bin/env bash -body='{"data":[{"id":"claude-sonnet-4-6"},{"id":"claude-haiku-4-5"}]}' -status="200" -outfile="" -while [ "$#" -gt 0 ]; do - case "$1" in - -o) outfile="$2"; shift 2 ;; - *) shift ;; - esac -done -printf '%s' "$body" > "$outfile" -printf '%s' "$status" -`, - { mode: 0o755 }, - ); + writeAlwaysOkCurl(fakeBin, '{"id":"ok"}'); const script = String.raw` const credentials = require(${credentialsPath}); const runner = require(${runnerPath}); -const answers = ["4", "4", "claude-bad", "claude-haiku-4-5"]; +const answers = ["3", "", "", ""]; const messages = []; credentials.prompt = async (message) => { messages.push(message); return answers.shift() || ""; }; +credentials.ensureApiKey = async () => {}; runner.runCapture = () => ""; const { setupNim } = require(${onboardPath}); (async () => { - process.env.ANTHROPIC_API_KEY = "anthropic-test"; const originalLog = console.log; const originalError = console.error; const lines = []; @@ -2769,896 +3250,150 @@ const { setupNim } = require(${onboardPath}); assert.equal(result.status, 0, result.stderr); const payload = JSON.parse(result.stdout.trim()); - assert.equal(payload.result.model, "claude-haiku-4-5"); - assert.equal( - payload.messages.filter((message: string) => /Anthropic model id:/.test(message)).length, - 2, + assert.equal(payload.result.provider, "nvidia-prod"); + assert.equal(payload.result.model, "nvidia/nemotron-3-super-120b-a12b"); + assert.ok( + payload.lines.some((line: string) => + line.includes("Endpoint URL is required for Other OpenAI-compatible endpoint."), + ), + ); + assert.ok( + payload.messages.some((message: string) => /OpenAI-compatible base URL/.test(message)), ); assert.ok( - payload.lines.some((line: string) => line.includes("is not available from Anthropic")), + payload.messages.filter((message: string) => /Choose \[1\]/.test(message)).length >= 2, ); }); - it("returns to provider selection when Anthropic live validation fails interactively", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-anthropic-validation-retry-"), - ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "anthropic-validation-retry-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync( - path.join(fakeBin, "curl"), - `#!/usr/bin/env bash -body='{"error":{"message":"invalid model"}}' -status="400" -outfile="" -url="" -args="$*" -while [ "$#" -gt 0 ]; do - case "$1" in - -o) outfile="$2"; shift 2 ;; - --config) auth="$(cat "$2" 2>/dev/null)"; shift 2 ;; *) url="$1"; shift ;; - esac -done -if echo "$url" | grep -q '/v1/models$'; then - body='{"data":[{"id":"claude-sonnet-4-6"},{"id":"claude-haiku-4-5"}]}' - status="200" -elif echo "$url" | grep -q '/v1/messages$' && printf '%s' "$args" | grep -q 'claude-haiku-4-5'; then - body='{"id":"msg_123","content":[{"type":"text","text":"OK"}]}' - status="200" -fi -printf '%s' "$body" > "$outfile" -printf '%s' "$status" -`, - { mode: 0o755 }, - ); - - const script = String.raw` -const credentials = require(${credentialsPath}); -const runner = require(${runnerPath}); - -const answers = ["4", "", "4", "2"]; -const messages = []; - -credentials.prompt = async (message) => { - messages.push(message); - return answers.shift() || ""; -}; -runner.runCapture = () => ""; - -const { setupNim } = require(${onboardPath}); - -(async () => { - process.env.ANTHROPIC_API_KEY = "anthropic-test"; - const originalLog = console.log; - const originalError = console.error; - const lines = []; - console.log = (...args) => lines.push(args.join(" ")); - console.error = (...args) => lines.push(args.join(" ")); - try { - const result = await setupNim(null); - originalLog(JSON.stringify({ result, messages, lines })); - } finally { - console.log = originalLog; - console.error = originalError; - } -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }, - }); - - assert.equal(result.status, 0, result.stderr); - const payload = JSON.parse(result.stdout.trim()); - assert.equal(payload.result.provider, "anthropic-prod"); - assert.equal(payload.result.model, "claude-haiku-4-5"); - assert.ok( - payload.lines.some((line: string) => line.includes("Anthropic endpoint validation failed")), - ); - assert.ok( - payload.lines.some((line: string) => line.includes("Please choose a provider/model again")), - ); - assert.equal(payload.messages.filter((message: string) => /Choose \[/.test(message)).length, 2); - }); - - it("supports Other Anthropic-compatible endpoint with live validation", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-anthropic-compatible-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "anthropic-compatible-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync( - path.join(fakeBin, "curl"), - `#!/usr/bin/env bash -body='{"id":"msg_123","content":[{"type":"text","text":"OK"}]}' -status="200" -outfile="" -while [ "$#" -gt 0 ]; do - case "$1" in - -o) outfile="$2"; shift 2 ;; - *) shift ;; - esac -done -printf '%s' "$body" > "$outfile" -printf '%s' "$status" -`, - { mode: 0o755 }, - ); - - const script = String.raw` -const credentials = require(${credentialsPath}); -const runner = require(${runnerPath}); - -const answers = ["5", "https://proxy.example.com/v1/messages?token=secret#frag", "claude-sonnet-proxy"]; -const messages = []; - -credentials.prompt = async (message) => { - messages.push(message); - return answers.shift() || ""; -}; -runner.runCapture = () => ""; - -const { setupNim } = require(${onboardPath}); - -(async () => { - process.env.COMPATIBLE_ANTHROPIC_API_KEY = "proxy-key"; - const originalLog = console.log; - const originalError = console.error; - const lines = []; - console.log = (...args) => lines.push(args.join(" ")); - console.error = (...args) => lines.push(args.join(" ")); - try { - const result = await setupNim(null); - originalLog(JSON.stringify({ result, messages, lines })); - } finally { - console.log = originalLog; - console.error = originalError; - } -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }, - }); - - assert.equal(result.status, 0, result.stderr); - const payload = JSON.parse(result.stdout.trim()); - assert.equal(payload.result.provider, "compatible-anthropic-endpoint"); - assert.equal(payload.result.model, "claude-sonnet-proxy"); - assert.equal(payload.result.endpointUrl, "https://proxy.example.com"); - assert.equal(payload.result.preferredInferenceApi, "anthropic-messages"); - assert.match(payload.messages[1], /Anthropic-compatible base URL/); - assert.match(payload.messages[2], /Other Anthropic-compatible endpoint model/); - assert.ok( - payload.lines.some((line: string) => line.includes("Anthropic Messages API available")), - ); - }); - - it("reprompts only for model name when Other OpenAI-compatible endpoint validation fails", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-custom-openai-retry-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "custom-openai-retry-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync( - path.join(fakeBin, "curl"), - `#!/usr/bin/env bash -body='{"error":{"message":"bad model"}}' -status="400" -outfile="" -body_arg="" -url="" -while [ "$#" -gt 0 ]; do - case "$1" in - -o) outfile="$2"; shift 2 ;; - -d) body_arg="$2"; shift 2 ;; - --config) auth="$(cat "$2" 2>/dev/null)"; shift 2 ;; *) url="$1"; shift ;; - esac -done -if echo "$url" | grep -q '/responses$' && echo "$body_arg" | grep -q 'good-model'; then - body='{"id":"resp_123","output":[{"type":"message","content":[{"type":"output_text","text":"OK"}]}]}' - status="200" -elif echo "$url" | grep -q '/chat/completions$' && echo "$body_arg" | grep -q 'good-model'; then - body='{"id":"chatcmpl-123","choices":[{"message":{"content":"OK"}}]}' - status="200" -fi -printf '%s' "$body" > "$outfile" -printf '%s' "$status" -`, - { mode: 0o755 }, - ); - - const script = String.raw` -const credentials = require(${credentialsPath}); -const runner = require(${runnerPath}); - -const answers = ["3", "https://proxy.example.com/v1/chat/completions?token=secret#frag", "bad-model", "good-model"]; -const messages = []; - -credentials.prompt = async (message) => { - messages.push(message); - return answers.shift() || ""; -}; -runner.runCapture = () => ""; - -const { setupNim } = require(${onboardPath}); - -(async () => { - process.env.COMPATIBLE_API_KEY = "proxy-key"; - const originalLog = console.log; - const originalError = console.error; - const lines = []; - console.log = (...args) => lines.push(args.join(" ")); - console.error = (...args) => lines.push(args.join(" ")); - try { - const result = await setupNim(null); - originalLog(JSON.stringify({ result, messages, lines })); - } finally { - console.log = originalLog; - console.error = originalError; - } -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }, - }); - - assert.equal(result.status, 0, result.stderr); - const payload = JSON.parse(result.stdout.trim()); - assert.equal(payload.result.provider, "compatible-endpoint"); - assert.equal(payload.result.model, "good-model"); - assert.equal(payload.result.preferredInferenceApi, "openai-completions"); - assert.ok( - payload.lines.some((line: string) => - line.includes("Other OpenAI-compatible endpoint endpoint validation failed"), - ), - ); - assert.ok( - payload.lines.some((line: string) => - line.includes("Please enter a different Other OpenAI-compatible endpoint model name."), - ), - ); - assert.equal( - payload.messages.filter((message: string) => /OpenAI-compatible base URL/.test(message)) - .length, - 1, - ); - assert.equal( - payload.messages.filter((message: string) => - /Other OpenAI-compatible endpoint model/.test(message), - ).length, - 2, - ); - assert.equal(payload.messages.filter((message: string) => /Choose \[/.test(message)).length, 1); - }); - - it("falls back to chat completions for custom OpenAI-compatible endpoints when /responses lacks tool calls", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-custom-openai-responses-fallback-"), - ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "custom-openai-responses-fallback-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync( - path.join(fakeBin, "curl"), - `#!/usr/bin/env bash -body='{"error":{"message":"bad request"}}' -status="400" -outfile="" -body_arg="" -url="" -while [ "$#" -gt 0 ]; do - case "$1" in - -o) outfile="$2"; shift 2 ;; - -d) body_arg="$2"; shift 2 ;; - --config) auth="$(cat "$2" 2>/dev/null)"; shift 2 ;; *) url="$1"; shift ;; - esac -done -if echo "$url" | grep -q '/responses$'; then - body='{"id":"resp_123","output":[{"type":"message","content":[{"type":"output_text","text":"OK"}]}]}' - status="200" -elif echo "$url" | grep -q '/chat/completions$'; then - body='{"id":"chatcmpl-123","choices":[{"message":{"content":"OK"}}]}' - status="200" -fi -printf '%s' "$body" > "$outfile" -printf '%s' "$status" -`, - { mode: 0o755 }, - ); - - const script = String.raw` -const credentials = require(${credentialsPath}); -const runner = require(${runnerPath}); - -const answers = ["3", "https://proxy.example.com/v1", "custom-model"]; -const messages = []; - -credentials.prompt = async (message) => { - messages.push(message); - return answers.shift() || ""; -}; -runner.runCapture = () => ""; - -const { setupNim } = require(${onboardPath}); - -(async () => { - process.env.COMPATIBLE_API_KEY = "proxy-key"; - const originalLog = console.log; - const originalError = console.error; - const lines = []; - console.log = (...args) => lines.push(args.join(" ")); - console.error = (...args) => lines.push(args.join(" ")); - try { - const result = await setupNim(null); - originalLog(JSON.stringify({ result, messages, lines })); - } finally { - console.log = originalLog; - console.error = originalError; - } -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }, - }); - - assert.equal(result.status, 0, result.stderr); - const payload = JSON.parse(result.stdout.trim()); - assert.equal(payload.result.provider, "compatible-endpoint"); - assert.equal(payload.result.model, "custom-model"); - assert.equal(payload.result.preferredInferenceApi, "openai-completions"); - assert.ok( - payload.lines.some((line: string) => line.includes("Chat Completions API available")), - ); - }); - - it("forces chat completions for custom OpenAI-compatible endpoints even when /responses returns valid tool calls (#1932)", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-custom-openai-responses-force-completions-"), - ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "custom-openai-responses-force-completions-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - // Mock curl: /v1/responses returns a VALID response with tool calls - // (simulates Ollama 0.20+ which exposes /v1/responses successfully) - fs.writeFileSync( - path.join(fakeBin, "curl"), - `#!/usr/bin/env bash -body='{"error":{"message":"bad request"}}' -status="400" -outfile="" -body_arg="" -url="" -while [ "$#" -gt 0 ]; do - case "$1" in - -o) outfile="$2"; shift 2 ;; - -d) body_arg="$2"; shift 2 ;; - --config) auth="$(cat "$2" 2>/dev/null)"; shift 2 ;; *) url="$1"; shift ;; - esac -done -if echo "$url" | grep -q '/responses$'; then - body='{"id":"resp_123","output":[{"id":"fc_1","type":"function_call","name":"read","arguments":"{\\"path\\":\\"/tmp/test\\"}"},{"id":"msg_1","type":"message","status":"completed","role":"assistant","content":[{"type":"output_text","text":"OK"}]}]}' - status="200" -elif echo "$url" | grep -q '/chat/completions$'; then - body='{"id":"chatcmpl-123","choices":[{"message":{"content":"OK"}}]}' - status="200" -fi -printf '%s' "$body" > "$outfile" -printf '%s' "$status" -`, - { mode: 0o755 }, - ); - - const script = String.raw` -const credentials = require(${credentialsPath}); -const runner = require(${runnerPath}); - -const answers = ["3", "https://ollama.local:11434/v1", "my-model"]; -const messages = []; - -credentials.prompt = async (message) => { - messages.push(message); - return answers.shift() || ""; -}; -runner.runCapture = () => ""; - -const { setupNim } = require(${onboardPath}); - -(async () => { - process.env.COMPATIBLE_API_KEY = "ollama-key"; - const originalLog = console.log; - const originalError = console.error; - const lines = []; - console.log = (...args) => lines.push(args.join(" ")); - console.error = (...args) => lines.push(args.join(" ")); - try { - const result = await setupNim(null); - originalLog(JSON.stringify({ result, messages, lines })); - } finally { - console.log = originalLog; - console.error = originalError; - } -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }, - }); - - assert.equal(result.status, 0, result.stderr); - const payload = JSON.parse(result.stdout.trim()); - assert.equal(payload.result.provider, "compatible-endpoint"); - assert.equal(payload.result.model, "my-model"); - // Even though /v1/responses returned valid tool calls, we must force - // chat completions because many backends (Ollama, vLLM, LiteLLM) do not - // correctly handle the developer role used by the Responses API. - assert.equal(payload.result.preferredInferenceApi, "openai-completions"); - // Verify the wizard selected chat completions (either via our forced - // override or via the streaming fallback — both are correct). - assert.ok(payload.lines.some((line: string) => line.includes("openai-completions"))); - }); - - it("honors NEMOCLAW_PREFERRED_API=openai-responses override for custom OpenAI-compatible endpoints (#1932)", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-custom-openai-responses-override-"), - ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "custom-openai-responses-override-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - // Mock curl: /v1/responses returns a valid response (probe passes) - fs.writeFileSync( - path.join(fakeBin, "curl"), - `#!/usr/bin/env bash -body='{"error":{"message":"bad request"}}' -status="400" -outfile="" -url="" -while [ "$#" -gt 0 ]; do - case "$1" in - -o) outfile="$2"; shift 2 ;; - -d) shift 2 ;; - --config) auth="$(cat "$2" 2>/dev/null)"; shift 2 ;; *) url="$1"; shift ;; - esac -done -if echo "$url" | grep -q '/responses$'; then - body='{"id":"resp_123","output":[{"id":"msg_1","type":"message","status":"completed","role":"assistant","content":[{"type":"output_text","text":"OK"}]}]}' - status="200" -elif echo "$url" | grep -q '/chat/completions$'; then - body='{"id":"chatcmpl-123","choices":[{"message":{"content":"OK"}}]}' - status="200" -fi -printf '%s' "$body" > "$outfile" -printf '%s' "$status" -`, - { mode: 0o755 }, - ); - - const script = String.raw` -const credentials = require(${credentialsPath}); -const runner = require(${runnerPath}); - -const answers = ["3", "https://openai-proxy.example.com/v1", "gpt-4o"]; -const messages = []; - -credentials.prompt = async (message) => { - messages.push(message); - return answers.shift() || ""; -}; -runner.runCapture = () => ""; - -const { setupNim } = require(${onboardPath}); - -(async () => { - process.env.COMPATIBLE_API_KEY = "sk-test"; - // Explicit override: user knows their backend supports the Responses API - process.env.NEMOCLAW_PREFERRED_API = "openai-responses"; - const originalLog = console.log; - const originalError = console.error; - const lines = []; - console.log = (...args) => lines.push(args.join(" ")); - console.error = (...args) => lines.push(args.join(" ")); - try { - const result = await setupNim(null); - originalLog(JSON.stringify({ result, messages, lines })); - } finally { - console.log = originalLog; - console.error = originalError; - } -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, + it("reprompts only for model name when Other Anthropic-compatible endpoint validation fails", async () => { + const messages: string[] = []; + const modelAnswers = ["bad-claude", "good-claude"]; + const endpointInput = await resolveCompatibleEndpointInput({ + kind: "anthropic", + envUrl: null, + recoveredEndpointUrl: null, + nonInteractive: false, + prompt: async (message) => { + messages.push(message); + return "https://proxy.example.com/v1/messages?token=secret#frag"; }, }); - - assert.equal(result.status, 0, result.stderr); - const payload = JSON.parse(result.stdout.trim()); - assert.equal(payload.result.provider, "compatible-endpoint"); - assert.equal(payload.result.model, "gpt-4o"); - // With NEMOCLAW_PREFERRED_API=openai-responses, the code path that - // forces openai-completions is bypassed: our override check sees the - // env var and uses validation.api instead. In this test, the mock - // curl doesn't support SSE streaming, so the probe's streaming - // fallback returns openai-completions regardless. A real backend with - // proper streaming would yield openai-responses here. - // The important thing: the env var is read and the forced-completions - // override does NOT fire, proving the escape hatch works. - assert.equal(payload.result.preferredInferenceApi, "openai-completions"); - // Verify the forced-override message was NOT printed (env var bypassed it) - assert.ok( - !payload.lines.some((line: string) => - line.includes("compatible endpoints may not support the Responses API developer role"), - ), - ); - }); - - it("returns to provider selection instead of exiting on blank custom endpoint input", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-custom-endpoint-blank-"), - ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "custom-endpoint-blank-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeAlwaysOkCurl(fakeBin, '{"id":"ok"}'); - - const script = String.raw` -const credentials = require(${credentialsPath}); -const runner = require(${runnerPath}); - -const answers = ["3", "", "", ""]; -const messages = []; - -credentials.prompt = async (message) => { - messages.push(message); - return answers.shift() || ""; -}; -credentials.ensureApiKey = async () => {}; -runner.runCapture = () => ""; - -const { setupNim } = require(${onboardPath}); - -(async () => { - const originalLog = console.log; - const originalError = console.error; - const lines = []; - console.log = (...args) => lines.push(args.join(" ")); - console.error = (...args) => lines.push(args.join(" ")); - try { - const result = await setupNim(null); - originalLog(JSON.stringify({ result, messages, lines })); - } finally { - console.log = originalLog; - console.error = originalError; - } -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, + const state = makeRemoteSelectionState({ + provider: "compatible-anthropic-endpoint", + endpointUrl: normalizeProviderBaseUrl(endpointInput, "anthropic"), + credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", + }); + const recovery = makeInteractiveValidationRecovery(); + const probedModels: string[] = []; + const validation = createInferenceSelectionValidationHelpers({ + isNonInteractive: () => false, + agentProductName: () => "OpenClaw", + getCredential: () => "proxy-key", + probeAnthropicEndpoint: (_endpointUrl, model) => { + probedModels.push(model); + return model === "good-claude" + ? { ok: true, api: "anthropic-messages", label: "Anthropic Messages API" } + : { + ok: false, + message: "bad model", + failures: [{ name: "Anthropic Messages API", httpStatus: 400, message: "bad model" }], + }; }, + promptValidationRecovery: recovery.promptValidationRecovery, }); - - assert.equal(result.status, 0, result.stderr); - const payload = JSON.parse(result.stdout.trim()); - assert.equal(payload.result.provider, "nvidia-prod"); - assert.equal(payload.result.model, "nvidia/nemotron-3-super-120b-a12b"); - assert.ok( - payload.lines.some((line: string) => - line.includes("Endpoint URL is required for Other OpenAI-compatible endpoint."), - ), - ); - assert.ok( - payload.messages.some((message: string) => /OpenAI-compatible base URL/.test(message)), - ); - assert.ok( - payload.messages.filter((message: string) => /Choose \[1\]/.test(message)).length >= 2, - ); - }); - - it("reprompts only for model name when Other Anthropic-compatible endpoint validation fails", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-custom-anthropic-retry-"), - ); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "custom-anthropic-retry-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync( - path.join(fakeBin, "curl"), - `#!/usr/bin/env bash -body='{"error":{"message":"bad model"}}' -status="400" -outfile="" -body_arg="" -url="" -while [ "$#" -gt 0 ]; do - case "$1" in - -o) outfile="$2"; shift 2 ;; - -d) body_arg="$2"; shift 2 ;; - --config) auth="$(cat "$2" 2>/dev/null)"; shift 2 ;; *) url="$1"; shift ;; - esac -done -if echo "$url" | grep -q '/v1/messages$' && echo "$body_arg" | grep -q 'good-claude'; then - body='{"id":"msg_123","content":[{"type":"text","text":"OK"}]}' - status="200" -fi -printf '%s' "$body" > "$outfile" -printf '%s' "$status" -`, - { mode: 0o755 }, + const { validateSelectedRemoteModel } = createRemoteModelValidator( + makeRemoteModelValidatorDeps({ + validateCustomAnthropicSelection: validation.validateCustomAnthropicSelection, + }), ); + const promptModel = () => + promptInputModel(TEST_CUSTOM_ANTHROPIC_CONFIG.label, "claude-proxy", null, { + promptFn: async (message) => { + messages.push(message); + return modelAnswers.shift() || ""; + }, + }); - const script = String.raw` -const credentials = require(${credentialsPath}); -const runner = require(${runnerPath}); - -const answers = ["5", "https://proxy.example.com/v1/messages?token=secret#frag", "bad-claude", "good-claude"]; -const messages = []; - -credentials.prompt = async (message) => { - messages.push(message); - return answers.shift() || ""; -}; -runner.runCapture = () => ""; - -const { setupNim } = require(${onboardPath}); - -(async () => { - process.env.COMPATIBLE_ANTHROPIC_API_KEY = "proxy-key"; - const originalLog = console.log; - const originalError = console.error; - const lines = []; - console.log = (...args) => lines.push(args.join(" ")); - console.error = (...args) => lines.push(args.join(" ")); - try { - const result = await setupNim(null); - originalLog(JSON.stringify({ result, messages, lines })); - } finally { - console.log = originalLog; - console.error = originalError; - } -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - }, + const { result, lines } = await captureConsoleOutput(async () => { + state.model = await promptModel(); + const first = await validateSelectedRemoteModel({ + selected: { key: "anthropicCompatible" }, + remoteConfig: TEST_CUSTOM_ANTHROPIC_CONFIG, + state, + selectedCredentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", + }); + state.model = await promptModel(); + const second = await validateSelectedRemoteModel({ + selected: { key: "anthropicCompatible" }, + remoteConfig: TEST_CUSTOM_ANTHROPIC_CONFIG, + state, + selectedCredentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", + }); + return { first, second }; }); - assert.equal(result.status, 0, result.stderr); - const payload = JSON.parse(result.stdout.trim()); - assert.equal(payload.result.provider, "compatible-anthropic-endpoint"); - assert.equal(payload.result.model, "good-claude"); - assert.equal(payload.result.preferredInferenceApi, "anthropic-messages"); + assert.deepEqual(result, { first: "retry-model", second: "selected" }); + assert.equal(state.provider, "compatible-anthropic-endpoint"); + assert.equal(state.model, "good-claude"); + assert.equal(state.endpointUrl, "https://proxy.example.com"); + assert.equal(state.preferredInferenceApi, "anthropic-messages"); + assert.deepEqual(probedModels, ["bad-claude", "good-claude"]); assert.ok( - payload.lines.some((line: string) => + lines.some((line) => line.includes("Other Anthropic-compatible endpoint endpoint validation failed"), ), ); assert.ok( - payload.lines.some((line: string) => + lines.some((line) => line.includes("Please enter a different Other Anthropic-compatible endpoint model name."), ), ); assert.equal( - payload.messages.filter((message: string) => /Anthropic-compatible base URL/.test(message)) - .length, + messages.filter((message) => /Anthropic-compatible base URL/.test(message)).length, 1, ); assert.equal( - payload.messages.filter((message: string) => - /Other Anthropic-compatible endpoint model/.test(message), - ).length, + messages.filter((message) => /Other Anthropic-compatible endpoint model/.test(message)) + .length, 2, ); - assert.equal(payload.messages.filter((message: string) => /Choose \[/.test(message)).length, 1); }); - it("lets users type back at a lower-level model prompt to return to provider selection", () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-model-back-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "model-back-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - - fs.mkdirSync(fakeBin, { recursive: true }); - writeAlwaysOkCurl(fakeBin); - - const script = String.raw` -const credentials = require(${credentialsPath}); -const runner = require(${runnerPath}); - -const answers = ["3", "https://proxy.example.com/v1", "back", "1", ""]; -const messages = []; - -credentials.prompt = async (message) => { - messages.push(message); - return answers.shift() || ""; -}; -credentials.ensureApiKey = async () => { process.env.NVIDIA_INFERENCE_API_KEY = "nvapi-good"; }; -runner.runCapture = () => ""; - -const { setupNim } = require(${onboardPath}); - -(async () => { - process.env.COMPATIBLE_API_KEY = "proxy-key"; - const originalLog = console.log; - const originalError = console.error; - const lines = []; - console.log = (...args) => lines.push(args.join(" ")); - console.error = (...args) => lines.push(args.join(" ")); - try { - const result = await setupNim(null); - originalLog(JSON.stringify({ result, messages, lines })); - } finally { - console.log = originalLog; - console.error = originalError; - } -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, + it("lets users type back at a lower-level model prompt to return to provider selection", async () => { + const messages: string[] = []; + await resolveCompatibleEndpointInput({ + kind: "openai", + envUrl: null, + recoveredEndpointUrl: null, + nonInteractive: false, + prompt: async (message) => { + messages.push(message); + return "https://proxy.example.com/v1"; }, }); - assert.equal(result.status, 0, result.stderr); - const payload = JSON.parse(result.stdout.trim()); - assert.equal(payload.result.provider, "nvidia-prod"); - assert.ok( - payload.lines.some((line: string) => line.includes("Returning to provider selection.")), + const { result, lines } = await captureConsoleOutput(async () => { + const model = await promptInputModel(TEST_CUSTOM_OPENAI_CONFIG.label, "custom-model", null, { + promptFn: async (message) => { + messages.push(message); + return "back"; + }, + }); + const returned = returningToProviderSelection(model, () => { + throw new Error("Unexpected onboarding exit"); + }); + return { model, returned }; + }); + + assert.deepEqual(result.model, BACK_TO_SELECTION); + assert.equal(result.returned, true); + assert.ok(lines.some((line) => line.includes("Returning to provider selection."))); + assert.equal( + messages.filter((message) => /OpenAI-compatible base URL/.test(message)).length, + 1, ); - assert.equal(payload.messages.filter((message: string) => /Choose \[/.test(message)).length, 2); assert.equal( - payload.messages.filter((message: string) => /OpenAI-compatible base URL/.test(message)) - .length, + messages.filter((message) => /Other OpenAI-compatible endpoint model/.test(message)).length, 1, ); }); From 4a9264dfc487b23774bd1ce937194467c38c5970 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 4 Jul 2026 10:20:44 -0700 Subject: [PATCH 05/16] test(onboard): cover inference dependency failures Signed-off-by: Carlos Villela --- ci/test-file-size-budget.json | 4 +- src/lib/inference/onboard-probes.test.ts | 50 ++++ src/lib/onboard/setup-inference.ts | 1 + src/lib/onboard/windows-host-ollama.test.ts | 2 +- test/onboard-inference-failure-paths.test.ts | 297 +++++++++++++++++++ test/onboard-selection.test.ts | 168 +++++------ test/onboard.test.ts | 122 +------- test/support/setup-inference-test-harness.ts | 145 +++++++++ 8 files changed, 581 insertions(+), 208 deletions(-) create mode 100644 test/onboard-inference-failure-paths.test.ts create mode 100644 test/support/setup-inference-test-harness.ts diff --git a/ci/test-file-size-budget.json b/ci/test-file-size-budget.json index a866bd9a82c..f13b1a8d65b 100644 --- a/ci/test-file-size-budget.json +++ b/ci/test-file-size-budget.json @@ -10,8 +10,8 @@ "test/install-preflight.test.ts": 3934, "test/nemoclaw-start.test.ts": 4827, "test/onboard-messaging.test.ts": 2062, - "test/onboard-selection.test.ts": 6338, - "test/onboard.test.ts": 4202, + "test/onboard-selection.test.ts": 6334, + "test/onboard.test.ts": 4086, "test/policies.test.ts": 2332 } } diff --git a/src/lib/inference/onboard-probes.test.ts b/src/lib/inference/onboard-probes.test.ts index 1bef5cc04e0..69a04317937 100644 --- a/src/lib/inference/onboard-probes.test.ts +++ b/src/lib/inference/onboard-probes.test.ts @@ -768,6 +768,56 @@ exit 28 ); }); + it("falls back to chat completions for custom OpenAI-compatible endpoints when /responses lacks tool calls", () => { + const script = `#!/usr/bin/env bash +outfile="" +url="" +while [ "$#" -gt 0 ]; do + case "$1" in + -o) outfile="$2"; shift 2 ;; + -w|-d|--config) shift 2 ;; + http://*|https://*) url="$1"; shift ;; + *) shift ;; + esac +done +n=$(cat "${HARNESS_COUNTER}") +n=$((n + 1)) +echo "$n" > "${HARNESS_COUNTER}" +printf '%s' "$url" > "${HARNESS_TMPDIR}/request-$n-url.txt" +if echo "$url" | grep -q '/responses$'; then + printf '%s' '{"output":[{"type":"message","content":[{"type":"output_text","text":"OK"}]}]}' > "$outfile" +else + printf '%s' '{"choices":[{"message":{"content":"OK"}}]}' > "$outfile" +fi +printf '200' +`; + + withFakeCurlProbe( + { script, dirPrefix: "nemoclaw-responses-tool-fallback-" }, + ({ counter, tmpDir }) => { + const result = probeOpenAiLikeEndpoint( + "https://proxy.example.com/v1", + "custom-model", + "proxy-key", + { requireResponsesToolCalling: true }, + ); + + expect(result).toMatchObject({ + ok: true, + api: "openai-completions", + label: "Chat Completions API", + }); + expect(fs.readFileSync(counter, "utf8").trim()).toBe("2"); + expect(fs.readFileSync(path.join(tmpDir, "request-1-url.txt"), "utf8")).toBe( + "https://proxy.example.com/v1/responses", + ); + expect(fs.readFileSync(path.join(tmpDir, "request-2-url.txt"), "utf8")).toBe( + "https://proxy.example.com/v1/chat/completions", + ); + }, + ); + }); + // PR #5975 review note PRA-14 (Nemotron). Pins the silent fallback so a // future SGLang fix that removes the workaround stays observable. it("falls back to chat-completions when /responses streaming lacks required events", () => { diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index 0936bf15ba7..636a0988668 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -102,6 +102,7 @@ export function createSetupInference( requireValue: deps.requireValue, redact: deps.redact, compactText: deps.compactText, + lookup: deps.lookup, }, ); } diff --git a/src/lib/onboard/windows-host-ollama.test.ts b/src/lib/onboard/windows-host-ollama.test.ts index 79fbbb3cdc9..782e5109a26 100644 --- a/src/lib/onboard/windows-host-ollama.test.ts +++ b/src/lib/onboard/windows-host-ollama.test.ts @@ -55,7 +55,7 @@ describe("detectWindowsHostOllama", () => { it("returns uninstalled when all Windows Ollama probes miss", () => { runCapture.mockImplementation(() => ""); - expect(detectWindowsHostOllama()).toEqual({ + expect(detectWindowsHostOllama({ isWsl: () => true, runCapture })).toEqual({ installed: false, installedPath: "", loopbackOnly: false, diff --git a/test/onboard-inference-failure-paths.test.ts b/test/onboard-inference-failure-paths.test.ts new file mode 100644 index 00000000000..2b4c44dd106 --- /dev/null +++ b/test/onboard-inference-failure-paths.test.ts @@ -0,0 +1,297 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { SetupInference, SetupInferenceDeps } from "../src/lib/onboard/setup-inference.js"; +import { + createDirectSetupInferenceHarnessFactory, + directRunResult, +} from "./support/setup-inference-test-harness.js"; + +const onboard = require("../src/lib/onboard") as { + createSetupInference: (overrides?: Partial) => SetupInference; +}; +const bedrockRuntimeOnboard = + require("../src/lib/onboard/bedrock-runtime") as typeof import("../src/lib/onboard/bedrock-runtime.js"); +const createDirectSetupInferenceHarness = createDirectSetupInferenceHarnessFactory( + onboard.createSetupInference, +); + +type DirectSetupInferenceHarness = ReturnType; + +function stubProcessExit() { + return vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`EXIT_CALLED:${code ?? 0}`); + }) as typeof process.exit); +} + +function expectNoPostFailureSideEffects(harness: DirectSetupInferenceHarness): void { + expect(harness.commands.map(({ command }) => command)).toEqual(["gateway select nemoclaw"]); + expect(harness.verifyInferenceRoute).not.toHaveBeenCalled(); + expect(harness.verifyOnboardInferenceSmoke).not.toHaveBeenCalled(); + expect(harness.updateSandbox).not.toHaveBeenCalled(); +} + +describe("setupInference dependency failures", () => { + afterEach(() => { + vi.unstubAllEnvs(); + vi.restoreAllMocks(); + }); + + it("fails closed before provider registration when local vLLM validation fails", async () => { + const exit = stubProcessExit(); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + const validateLocalProvider = vi.fn(() => ({ + ok: false, + message: "vLLM is unreachable", + diagnostic: "container probe failed", + })); + const getLocalProviderHealthCheck = vi.fn(() => ["curl", "-sf", "http://127.0.0.1:8000"]); + const run = vi.fn(() => directRunResult({ status: 7 })); + const harness = createDirectSetupInferenceHarness({ + overrides: { validateLocalProvider, getLocalProviderHealthCheck, run }, + }); + + await expect(harness.setupInference("test-box", "meta-llama", "vllm-local")).rejects.toThrow( + "EXIT_CALLED:1", + ); + + expect(validateLocalProvider).toHaveBeenCalledWith("vllm-local"); + expect(getLocalProviderHealthCheck).toHaveBeenCalledWith("vllm-local"); + expect(run).toHaveBeenCalledWith(["curl", "-sf", "http://127.0.0.1:8000"], { + ignoreError: true, + suppressOutput: true, + }); + expect(exit).toHaveBeenCalledWith(1); + expect(error).toHaveBeenCalledWith(" vLLM is unreachable"); + expect(error).toHaveBeenCalledWith(" Diagnostic: container probe failed"); + expectNoPostFailureSideEffects(harness); + }); + + it("propagates local vLLM health-check errors before provider registration", async () => { + const exit = stubProcessExit(); + const run = vi.fn(() => directRunResult()); + const getLocalProviderHealthCheck = vi.fn(() => { + throw new Error("health probe exploded"); + }); + const harness = createDirectSetupInferenceHarness({ + overrides: { + validateLocalProvider: () => ({ ok: false, message: "vLLM is unreachable" }), + getLocalProviderHealthCheck, + run, + }, + }); + + await expect(harness.setupInference("test-box", "meta-llama", "vllm-local")).rejects.toThrow( + "health probe exploded", + ); + + expect(getLocalProviderHealthCheck).toHaveBeenCalledWith("vllm-local"); + expect(run).not.toHaveBeenCalled(); + expect(exit).not.toHaveBeenCalled(); + expectNoPostFailureSideEffects(harness); + }); + + it("propagates Ollama proxy startup errors before reading credentials", async () => { + const exit = stubProcessExit(); + const ensureOllamaAuthProxy = vi.fn(() => { + throw new Error("proxy startup failed"); + }); + const getOllamaProxyToken = vi.fn(() => "unused-token"); + const persistAndProbeOllamaProxy = vi.fn(async () => {}); + const harness = createDirectSetupInferenceHarness({ + overrides: { + shouldFrontOllamaWithProxy: () => true, + ensureOllamaAuthProxy, + getOllamaProxyToken, + persistAndProbeOllamaProxy, + }, + }); + + await expect(harness.setupInference("test-box", "qwen3.5:9b", "ollama-local")).rejects.toThrow( + "proxy startup failed", + ); + + expect(ensureOllamaAuthProxy).toHaveBeenCalledOnce(); + expect(getOllamaProxyToken).not.toHaveBeenCalled(); + expect(persistAndProbeOllamaProxy).not.toHaveBeenCalled(); + expect(exit).not.toHaveBeenCalled(); + expectNoPostFailureSideEffects(harness); + }); + + it("fails closed when the recovered Ollama proxy remains unhealthy", async () => { + const exit = stubProcessExit(); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + const ensureOllamaAuthProxy = vi.fn(); + const isProxyHealthy = vi.fn(() => false); + const getOllamaProxyToken = vi.fn(() => "unused-token"); + const persistAndProbeOllamaProxy = vi.fn(async () => {}); + const harness = createDirectSetupInferenceHarness({ + overrides: { + validateLocalProvider: () => ({ + ok: false, + message: "container cannot reach Ollama", + diagnostic: "proxy probe failed", + }), + shouldFrontOllamaWithProxy: () => true, + ensureOllamaAuthProxy, + isProxyHealthy, + getOllamaProxyToken, + persistAndProbeOllamaProxy, + }, + }); + + await expect(harness.setupInference("test-box", "qwen3.5:9b", "ollama-local")).rejects.toThrow( + "EXIT_CALLED:1", + ); + + expect(ensureOllamaAuthProxy).toHaveBeenCalledOnce(); + expect(isProxyHealthy).toHaveBeenCalledOnce(); + expect(getOllamaProxyToken).not.toHaveBeenCalled(); + expect(persistAndProbeOllamaProxy).not.toHaveBeenCalled(); + expect(exit).toHaveBeenCalledWith(1); + expect(error).toHaveBeenCalledWith(" container cannot reach Ollama"); + expect(error).toHaveBeenCalledWith(" Diagnostic: proxy probe failed"); + expectNoPostFailureSideEffects(harness); + }); + + it("fails closed when proxy-fronted Ollama has no credential token", async () => { + const exit = stubProcessExit(); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + const ensureOllamaAuthProxy = vi.fn(); + const getOllamaProxyToken = vi.fn(() => null); + const persistAndProbeOllamaProxy = vi.fn(async () => {}); + const harness = createDirectSetupInferenceHarness({ + overrides: { + shouldFrontOllamaWithProxy: () => true, + ensureOllamaAuthProxy, + getOllamaProxyToken, + persistAndProbeOllamaProxy, + }, + }); + + await expect(harness.setupInference("test-box", "qwen3.5:9b", "ollama-local")).rejects.toThrow( + "EXIT_CALLED:1", + ); + + expect(ensureOllamaAuthProxy).toHaveBeenCalledOnce(); + expect(getOllamaProxyToken).toHaveBeenCalledOnce(); + expect(persistAndProbeOllamaProxy).not.toHaveBeenCalled(); + expect(exit).toHaveBeenCalledWith(1); + expect(error).toHaveBeenCalledWith( + " Ollama auth proxy token is not set. Re-run onboard to initialize the proxy.", + ); + expectNoPostFailureSideEffects(harness); + }); + + it("propagates Ollama proxy persistence errors before provider registration", async () => { + const exit = stubProcessExit(); + const persistAndProbeOllamaProxy = vi.fn(async () => { + throw new Error("proxy persistence failed"); + }); + const harness = createDirectSetupInferenceHarness({ + overrides: { + shouldFrontOllamaWithProxy: () => true, + ensureOllamaAuthProxy: () => {}, + getOllamaProxyToken: () => "proxy-token", + persistAndProbeOllamaProxy, + }, + }); + + await expect(harness.setupInference("test-box", "qwen3.5:9b", "ollama-local")).rejects.toThrow( + "proxy persistence failed", + ); + + expect(persistAndProbeOllamaProxy).toHaveBeenCalledWith("proxy-token"); + expect(exit).not.toHaveBeenCalled(); + expectNoPostFailureSideEffects(harness); + }); + + it("returns to provider selection when the Bedrock adapter cannot start", async () => { + vi.stubEnv("COMPATIBLE_ANTHROPIC_API_KEY", "bedrock-bearer"); + const exit = stubProcessExit(); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + const ensureAdapter = vi.fn(async () => { + throw new Error("adapter unavailable"); + }); + const setupBedrockRuntimeInference = bedrockRuntimeOnboard.setupBedrockRuntimeInference; + const harness = createDirectSetupInferenceHarness({ + overrides: { + bedrockRuntimeOnboard: { + setupBedrockRuntimeInference: (input) => + setupBedrockRuntimeInference({ ...input, ensureAdapter }), + }, + }, + }); + + await expect( + harness.setupInference( + "test-box", + "anthropic.claude-3-5-sonnet-20240620-v1:0", + "compatible-anthropic-endpoint", + "https://bedrock-runtime.us-east-1.amazonaws.com", + "COMPATIBLE_ANTHROPIC_API_KEY", + ), + ).resolves.toEqual({ retry: "selection" }); + + expect(ensureAdapter).toHaveBeenCalledOnce(); + expect(error).toHaveBeenCalledWith( + " Failed to start Bedrock Runtime adapter: adapter unavailable", + ); + expect(exit).not.toHaveBeenCalled(); + expectNoPostFailureSideEffects(harness); + }); + + it("uses an injected Hermes DNS lookup before rejecting an unpinnable HTTPS endpoint", async () => { + const exit = stubProcessExit(); + const lookup = vi.fn>(async () => [ + { address: "8.8.8.8", family: 4 }, + ]); + const harness = createDirectSetupInferenceHarness({ overrides: { lookup } }); + + await expect( + harness.setupInference( + "test-box", + "moonshotai/kimi-k2.6", + "hermes-provider", + "https://api.public.example.test/v1", + ), + ).rejects.toThrow("DNS-backed HTTPS URLs are not supported"); + + expect(lookup).toHaveBeenCalledWith("api.public.example.test", { all: true }); + expect(exit).not.toHaveBeenCalled(); + expectNoPostFailureSideEffects(harness); + }); + + it("fails closed before routed-provider registration when model-router reconciliation fails", async () => { + const exit = stubProcessExit(); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + const reconcileModelRouter = vi.fn(async () => { + throw new Error("router unavailable"); + }); + const upsertRoutedProvider = vi.fn(() => ({ ok: true, result: {} })); + const harness = createDirectSetupInferenceHarness({ + overrides: { + isRoutedInferenceProvider: (provider) => provider === "nvidia-router", + reconcileModelRouter, + routedInference: { upsertRoutedProvider }, + }, + }); + + await expect( + harness.setupInference( + "test-box", + "router/model", + "nvidia-router", + "http://host.openshell.internal:4000/v1", + "NVIDIA_INFERENCE_API_KEY", + ), + ).rejects.toThrow("EXIT_CALLED:1"); + + expect(reconcileModelRouter).toHaveBeenCalledOnce(); + expect(upsertRoutedProvider).not.toHaveBeenCalled(); + expect(exit).toHaveBeenCalledWith(1); + expect(error).toHaveBeenCalledWith(" ✗ Failed to start model router: router unavailable"); + expectNoPostFailureSideEffects(harness); + }); +}); diff --git a/test/onboard-selection.test.ts b/test/onboard-selection.test.ts index a7c84378e81..d986f5eaffc 100644 --- a/test/onboard-selection.test.ts +++ b/test/onboard-selection.test.ts @@ -9,16 +9,11 @@ import path from "node:path"; import { describe, expect, it, vi } from "vitest"; import { normalizeProviderBaseUrl } from "../src/lib/core/url-utils.js"; -import { - BACK_TO_SELECTION, - promptInputModel, - promptRemoteModel, -} from "../src/lib/inference/model-prompts.js"; +import { promptInputModel, promptRemoteModel } from "../src/lib/inference/model-prompts.js"; import { validateAnthropicModel, validateOpenAiLikeModel, } from "../src/lib/inference/provider-models.js"; -import { returningToProviderSelection } from "../src/lib/onboard/credential-navigation.js"; import { createInferenceSelectionValidationHelpers } from "../src/lib/onboard/inference-selection-validation.js"; import { getWindowsHostOllamaDockerRequirement, @@ -3021,53 +3016,6 @@ const { setupNim } = require(${onboardPath}); ); }); - it("falls back to chat completions for custom OpenAI-compatible endpoints when /responses lacks tool calls", async () => { - const probeOpenAiLikeEndpoint = vi.fn(() => ({ - ok: true, - api: "openai-completions", - label: "Chat Completions API", - })); - const recovery = makeInteractiveValidationRecovery(); - const validation = createInferenceSelectionValidationHelpers({ - isNonInteractive: () => false, - agentProductName: () => "OpenClaw", - getCredential: () => "proxy-key", - probeOpenAiLikeEndpoint, - promptValidationRecovery: recovery.promptValidationRecovery, - }); - const state = makeRemoteSelectionState({ model: "custom-model" }); - const { validateSelectedRemoteModel } = createRemoteModelValidator( - makeRemoteModelValidatorDeps({ - validateCustomOpenAiLikeSelection: validation.validateCustomOpenAiLikeSelection, - }), - ); - - const { result, lines } = await captureConsoleOutput(() => - validateSelectedRemoteModel({ - selected: { key: "custom" }, - remoteConfig: TEST_CUSTOM_OPENAI_CONFIG, - state, - selectedCredentialEnv: "COMPATIBLE_API_KEY", - }), - ); - - assert.equal(result, "selected"); - assert.equal(state.provider, "compatible-endpoint"); - assert.equal(state.model, "custom-model"); - assert.equal(state.preferredInferenceApi, "openai-completions"); - assert.ok(lines.some((line) => line.includes("Chat Completions API available"))); - expect(probeOpenAiLikeEndpoint).toHaveBeenCalledWith( - "https://proxy.example.com/v1", - "custom-model", - "proxy-key", - { - requireResponsesToolCalling: true, - skipResponsesProbe: false, - probeStreaming: true, - }, - ); - }); - it("forces chat completions for custom OpenAI-compatible endpoints even when /responses returns valid tool calls (#1932)", async () => { const previousPreferredApi = process.env.NEMOCLAW_PREFERRED_API; delete process.env.NEMOCLAW_PREFERRED_API; @@ -3359,43 +3307,91 @@ const { setupNim } = require(${onboardPath}); ); }); - it("lets users type back at a lower-level model prompt to return to provider selection", async () => { - const messages: string[] = []; - await resolveCompatibleEndpointInput({ - kind: "openai", - envUrl: null, - recoveredEndpointUrl: null, - nonInteractive: false, - prompt: async (message) => { - messages.push(message); - return "https://proxy.example.com/v1"; - }, - }); + it("lets users type back at a lower-level model prompt to return to provider selection", () => { + const repoRoot = path.join(import.meta.dirname, ".."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-model-back-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "model-back-check.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const { result, lines } = await captureConsoleOutput(async () => { - const model = await promptInputModel(TEST_CUSTOM_OPENAI_CONFIG.label, "custom-model", null, { - promptFn: async (message) => { - messages.push(message); - return "back"; + fs.mkdirSync(fakeBin, { recursive: true }); + writeAlwaysOkCurl(fakeBin); + + const script = String.raw` +for (const key of [ + "NVIDIA_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GEMINI_API_KEY", + "COMPATIBLE_API_KEY", "COMPATIBLE_ANTHROPIC_API_KEY", "NOUS_API_KEY", + "NVIDIA_INFERENCE_API_KEY", "NGC_API_KEY", "NEMOCLAW_PROVIDER_KEY", + "NEMOCLAW_NON_INTERACTIVE", "NEMOCLAW_PROVIDER", "NEMOCLAW_MODEL", "NEMOCLAW_YES", + "NEMOCLAW_PREFERRED_API", "NEMOCLAW_EXPERIMENTAL", +]) delete process.env[key]; + +const credentials = require(${credentialsPath}); +const runner = require(${runnerPath}); + +const answers = ["3", "https://proxy.example.com/v1", "back", "1", ""]; +const messages = []; + +credentials.prompt = async (message) => { + messages.push(message); + return answers.shift() || ""; +}; +credentials.ensureApiKey = async () => { process.env.NVIDIA_INFERENCE_API_KEY = "nvapi-good"; }; +runner.runCapture = () => ""; + +const { setupNim } = require(${onboardPath}); + +(async () => { + process.env.COMPATIBLE_API_KEY = "proxy-key"; + const originalLog = console.log; + const originalError = console.error; + const lines = []; + console.log = (...args) => lines.push(args.join(" ")); + console.error = (...args) => lines.push(args.join(" ")); + try { + const result = await setupNim(null); + originalLog(JSON.stringify({ result, messages, lines })); + } finally { + console.log = originalLog; + console.error = originalError; + } +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + try { + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, }, + timeout: PROVIDER_SELECTION_TEST_TIMEOUT_MS, }); - const returned = returningToProviderSelection(model, () => { - throw new Error("Unexpected onboarding exit"); - }); - return { model, returned }; - }); - assert.deepEqual(result.model, BACK_TO_SELECTION); - assert.equal(result.returned, true); - assert.ok(lines.some((line) => line.includes("Returning to provider selection."))); - assert.equal( - messages.filter((message) => /OpenAI-compatible base URL/.test(message)).length, - 1, - ); - assert.equal( - messages.filter((message) => /Other OpenAI-compatible endpoint model/.test(message)).length, - 1, - ); + assert.equal(result.status, 0, result.stderr); + const payload = JSON.parse(result.stdout.trim()); + assert.equal(payload.result.provider, "nvidia-prod"); + assert.ok( + payload.lines.some((line: string) => line.includes("Returning to provider selection.")), + ); + const promptCount = (pattern: RegExp) => + payload.messages.filter((message: string) => pattern.test(message)).length; + assert.equal(promptCount(/Choose \[/), 2); + assert.equal(promptCount(/OpenAI-compatible base URL/), 1); + assert.equal(promptCount(/Other OpenAI-compatible endpoint model/), 1); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } }); it("lets users type back at a secret provider credential prompt to return to provider selection", () => { diff --git a/test/onboard.test.ts b/test/onboard.test.ts index 235fba074e9..c2740b10abd 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -18,6 +18,7 @@ import { createLocalInferenceRouteApplier } from "../src/lib/onboard/local-infer import type { SetupInference, SetupInferenceDeps } from "../src/lib/onboard/setup-inference.js"; import { stageOptimizedSandboxBuildContext } from "../src/lib/sandbox/build-context.js"; import { testTimeoutOptions } from "./helpers/timeouts"; +import { createDirectSetupInferenceHarnessFactory } from "./support/setup-inference-test-harness.js"; type ShimScalar = string | number | boolean | null | undefined; type ShimCallable = (...args: readonly string[]) => ShimValue; @@ -100,127 +101,10 @@ const { SANDBOX_BASE_IMAGE, } = onboardTestInternals; -const onboardProviderHelpers = require("../src/lib/onboard/providers") as { - upsertProvider: ( - name: string, - type: string, - credentialEnv: string, - baseUrl: string | null, - env: Record, - runOpenshell: DirectRunOpenshell, - ) => { ok: boolean; status?: number; message?: string }; -}; -const localInferenceModule = - require("../src/lib/inference/local") as typeof import("../src/lib/inference/local.js"); const bedrockRuntimeOnboard = require("../src/lib/onboard/bedrock-runtime") as typeof import("../src/lib/onboard/bedrock-runtime.js"); - -type DirectRunOpenshell = SetupInferenceDeps["runOpenshell"]; -type DirectRunOptions = NonNullable[1]>; -type DirectRunResult = ReturnType; -type DirectRunStubResult = { - status: number | null; - stdout?: string; - stderr?: string; -}; -type DirectSetupHarnessOptions = { - runOpenshell?: ( - args: string[], - options: DirectRunOptions, - calls: CommandEntry[], - ) => DirectRunStubResult | undefined; - overrides?: Partial; -}; - -function directRunResult({ - status = 0, - stdout = "", - stderr = "", -}: Partial = {}): DirectRunResult { - return { - pid: 0, - output: [null, stdout, stderr], - stdout, - stderr, - status, - signal: null, - }; -} - -function createDirectSetupInferenceHarness(options: DirectSetupHarnessOptions = {}) { - const commands: CommandEntry[] = []; - const errors: string[] = []; - const logs: string[] = []; - const updateSandbox = vi.fn(() => true); - const verifyInferenceRoute = vi.fn(); - const verifyOnboardInferenceSmoke = vi.fn(); - const runOpenshell: DirectRunOpenshell = (args, runOptions = {}) => { - commands.push({ - command: args.join(" "), - env: runOptions.env, - ignoreError: runOptions.ignoreError, - }); - return directRunResult(options.runOpenshell?.(args, runOptions, commands)); - }; - const setupInference = createSetupInference({ - step: () => {}, - getGatewayName: () => "nemoclaw", - runOpenshell, - upsertProvider: ( - name: string, - type: string, - credentialEnv: string, - baseUrl: string | null, - env: Record = {}, - ) => - onboardProviderHelpers.upsertProvider(name, type, credentialEnv, baseUrl, env, runOpenshell), - verifyInferenceRoute, - verifyOnboardInferenceSmoke, - isNonInteractive: () => false, - updateSandbox, - resolveHermesNousApiKey: () => process.env.NOUS_API_KEY || null, - checkHermesProviderStoreReachable: (run: DirectRunOpenshell) => { - run(["provider", "list"], { ignoreError: true }); - return { ok: true }; - }, - hydrateCredentialEnv: (envName: string | null | undefined) => - envName ? process.env[envName] || null : null, - promptValidationRecovery: async () => "selection", - validateLocalProvider: () => ({ ok: true }), - getLocalProviderHealthCheck: () => null, - getLocalProviderBaseUrl: (provider: string) => - provider === "ollama-local" - ? "http://host.openshell.internal:11435/v1" - : "http://host.openshell.internal:8000/v1", - applyLocalInferenceRoute: async () => false, - run: () => directRunResult(), - shouldFrontOllamaWithProxy: () => false, - ensureOllamaAuthProxy: () => {}, - isProxyHealthy: () => true, - getOllamaProxyToken: () => null, - persistAndProbeOllamaProxy: async () => {}, - localInference: { - ...localInferenceModule, - validateOllamaModelWithToolsOverride: () => ({ ok: true }), - }, - log: (message: string) => logs.push(message), - error: (message: string) => errors.push(message), - exitProcess: (code: number): never => { - throw Object.assign(new Error(`EXIT_CALLED:${code}`), { code }); - }, - ...options.overrides, - }); - return { - commands, - errors, - logs, - runOpenshell, - setupInference, - updateSandbox, - verifyInferenceRoute, - verifyOnboardInferenceSmoke, - }; -} +const createDirectSetupInferenceHarness = + createDirectSetupInferenceHarnessFactory(createSetupInference); async function withProcessEnv( values: Record, diff --git a/test/support/setup-inference-test-harness.ts b/test/support/setup-inference-test-harness.ts new file mode 100644 index 00000000000..3b1481c7563 --- /dev/null +++ b/test/support/setup-inference-test-harness.ts @@ -0,0 +1,145 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { vi } from "vitest"; +import type { SetupInference, SetupInferenceDeps } from "../../src/lib/onboard/setup-inference.js"; + +const onboardProviderHelpers = require("../../src/lib/onboard/providers") as { + upsertProvider: ( + name: string, + type: string, + credentialEnv: string, + baseUrl: string | null, + env: Record, + runOpenshell: DirectRunOpenshell, + ) => { ok: boolean; status?: number; message?: string }; +}; +const localInferenceModule = + require("../../src/lib/inference/local") as typeof import("../../src/lib/inference/local.js"); + +export type DirectCommandEntry = { + command: string; + env?: Record; + ignoreError?: boolean; +}; + +type CreateSetupInference = (overrides?: Partial) => SetupInference; +type DirectRunOpenshell = SetupInferenceDeps["runOpenshell"]; +type DirectRunOptions = NonNullable[1]>; +type DirectRunResult = ReturnType; + +export type DirectRunStubResult = { + status: number | null; + stdout?: string; + stderr?: string; +}; + +export type DirectSetupHarnessOptions = { + runOpenshell?: ( + args: string[], + options: DirectRunOptions, + calls: DirectCommandEntry[], + ) => DirectRunStubResult | undefined; + overrides?: Partial; +}; + +export function directRunResult({ + status = 0, + stdout = "", + stderr = "", +}: Partial = {}): DirectRunResult { + return { + pid: 0, + output: [null, stdout, stderr], + stdout, + stderr, + status, + signal: null, + }; +} + +export function createDirectSetupInferenceHarnessFactory( + createSetupInference: CreateSetupInference, +) { + return function createDirectSetupInferenceHarness(options: DirectSetupHarnessOptions = {}) { + const commands: DirectCommandEntry[] = []; + const errors: string[] = []; + const logs: string[] = []; + const updateSandbox = vi.fn(() => true); + const verifyInferenceRoute = vi.fn(); + const verifyOnboardInferenceSmoke = vi.fn(); + const runOpenshell: DirectRunOpenshell = (args, runOptions = {}) => { + commands.push({ + command: args.join(" "), + env: runOptions.env, + ignoreError: runOptions.ignoreError, + }); + return directRunResult(options.runOpenshell?.(args, runOptions, commands)); + }; + const setupInference = createSetupInference({ + step: () => {}, + getGatewayName: () => "nemoclaw", + runOpenshell, + upsertProvider: ( + name: string, + type: string, + credentialEnv: string, + baseUrl: string | null, + env: Record = {}, + ) => + onboardProviderHelpers.upsertProvider( + name, + type, + credentialEnv, + baseUrl, + env, + runOpenshell, + ), + verifyInferenceRoute, + verifyOnboardInferenceSmoke, + isNonInteractive: () => false, + updateSandbox, + resolveHermesNousApiKey: () => process.env.NOUS_API_KEY || null, + checkHermesProviderStoreReachable: (run: DirectRunOpenshell) => { + run(["provider", "list"], { ignoreError: true }); + return { ok: true }; + }, + hydrateCredentialEnv: (envName: string | null | undefined) => + envName ? process.env[envName] || null : null, + promptValidationRecovery: async () => "selection", + validateLocalProvider: () => ({ ok: true }), + getLocalProviderHealthCheck: () => null, + getLocalProviderBaseUrl: (provider: string) => + provider === "ollama-local" + ? "http://host.openshell.internal:11435/v1" + : "http://host.openshell.internal:8000/v1", + applyLocalInferenceRoute: async () => false, + run: () => directRunResult(), + shouldFrontOllamaWithProxy: () => false, + ensureOllamaAuthProxy: () => {}, + isProxyHealthy: () => true, + getOllamaProxyToken: () => null, + persistAndProbeOllamaProxy: async () => {}, + localInference: { + ...localInferenceModule, + validateOllamaModelWithToolsOverride: () => ({ ok: true }), + }, + log: (message: string) => logs.push(message), + error: (message: string) => errors.push(message), + exitProcess: (code: number): never => { + throw Object.assign(new Error(`EXIT_CALLED:${code}`), { code }); + }, + ...options.overrides, + }); + return { + commands, + errors, + logs, + runOpenshell, + setupInference, + updateSandbox, + verifyInferenceRoute, + verifyOnboardInferenceSmoke, + }; + }; +} From bf0efcef911a2ddc9ff2a125b4ff95e5a9089e5d Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 4 Jul 2026 10:39:30 -0700 Subject: [PATCH 06/16] fix(onboard): complete inference dependency wiring Signed-off-by: Carlos Villela --- ci/test-file-size-budget.json | 4 +- .../inference/onboard-probes-curl-harness.ts | 29 ++ src/lib/inference/onboard-probes.test.ts | 29 +- src/lib/onboard/inference-providers/remote.ts | 2 + src/lib/onboard/inference-providers/types.ts | 3 +- test/onboard-selection.test.ts | 296 ++++-------------- test/onboard.test.ts | 127 ++++---- .../support/onboard-selection-test-helpers.ts | 192 ++++++++++++ test/support/setup-inference-test-harness.ts | 42 +++ 9 files changed, 388 insertions(+), 336 deletions(-) create mode 100644 test/support/onboard-selection-test-helpers.ts diff --git a/ci/test-file-size-budget.json b/ci/test-file-size-budget.json index f13b1a8d65b..11a1bbadc88 100644 --- a/ci/test-file-size-budget.json +++ b/ci/test-file-size-budget.json @@ -10,8 +10,8 @@ "test/install-preflight.test.ts": 3934, "test/nemoclaw-start.test.ts": 4827, "test/onboard-messaging.test.ts": 2062, - "test/onboard-selection.test.ts": 6334, - "test/onboard.test.ts": 4086, + "test/onboard-selection.test.ts": 6146, + "test/onboard.test.ts": 4079, "test/policies.test.ts": 2332 } } diff --git a/src/lib/inference/onboard-probes-curl-harness.ts b/src/lib/inference/onboard-probes-curl-harness.ts index 9db54dbc3e1..66fb194c800 100644 --- a/src/lib/inference/onboard-probes-curl-harness.ts +++ b/src/lib/inference/onboard-probes-curl-harness.ts @@ -37,6 +37,35 @@ export function makeFakeCurlScript(bodyLogic: string): string { return `${FAKE_CURL_HEADER}${bodyLogic}`; } +// Fake curl for the strict Responses API compatibility check. It records each +// requested URL, returns a successful Responses payload without a tool call, +// then returns a successful Chat Completions payload so callers can assert the +// exact fallback order without duplicating shell parsing in a test body. +export function makeResponsesFallbackUrlRecordingFakeCurlScript(): string { + return `#!/usr/bin/env bash +outfile="" +url="" +while [ "$#" -gt 0 ]; do + case "$1" in + -o) outfile="$2"; shift 2 ;; + -w|-d|--config) shift 2 ;; + http://*|https://*) url="$1"; shift ;; + *) shift ;; + esac +done +n=$(cat "${HARNESS_COUNTER}") +n=$((n + 1)) +echo "$n" > "${HARNESS_COUNTER}" +printf '%s' "$url" > "${HARNESS_TMPDIR}/request-$n-url.txt" +if echo "$url" | grep -q '/responses$'; then + printf '%s' '{"output":[{"type":"message","content":[{"type":"output_text","text":"OK"}]}]}' > "$outfile" +else + printf '%s' '{"choices":[{"message":{"content":"OK"}}]}' > "$outfile" +fi +printf '200' +`; +} + // Restore an env var to its pre-test value without branching at the call // site (kept identical to the helper the test file uses so restore semantics // are unchanged). diff --git a/src/lib/inference/onboard-probes.test.ts b/src/lib/inference/onboard-probes.test.ts index 69a04317937..89309aa1df0 100644 --- a/src/lib/inference/onboard-probes.test.ts +++ b/src/lib/inference/onboard-probes.test.ts @@ -11,6 +11,7 @@ import { HARNESS_COUNTER, HARNESS_TMPDIR, makeFakeCurlScript, + makeResponsesFallbackUrlRecordingFakeCurlScript, withFakeCurlProbe, } from "./onboard-probes-curl-harness"; @@ -769,31 +770,11 @@ exit 28 }); it("falls back to chat completions for custom OpenAI-compatible endpoints when /responses lacks tool calls", () => { - const script = `#!/usr/bin/env bash -outfile="" -url="" -while [ "$#" -gt 0 ]; do - case "$1" in - -o) outfile="$2"; shift 2 ;; - -w|-d|--config) shift 2 ;; - http://*|https://*) url="$1"; shift ;; - *) shift ;; - esac -done -n=$(cat "${HARNESS_COUNTER}") -n=$((n + 1)) -echo "$n" > "${HARNESS_COUNTER}" -printf '%s' "$url" > "${HARNESS_TMPDIR}/request-$n-url.txt" -if echo "$url" | grep -q '/responses$'; then - printf '%s' '{"output":[{"type":"message","content":[{"type":"output_text","text":"OK"}]}]}' > "$outfile" -else - printf '%s' '{"choices":[{"message":{"content":"OK"}}]}' > "$outfile" -fi -printf '200' -`; - withFakeCurlProbe( - { script, dirPrefix: "nemoclaw-responses-tool-fallback-" }, + { + script: makeResponsesFallbackUrlRecordingFakeCurlScript(), + dirPrefix: "nemoclaw-responses-tool-fallback-", + }, ({ counter, tmpDir }) => { const result = probeOpenAiLikeEndpoint( "https://proxy.example.com/v1", diff --git a/src/lib/onboard/inference-providers/remote.ts b/src/lib/onboard/inference-providers/remote.ts index 902b3909998..8b01e3e3d4f 100644 --- a/src/lib/onboard/inference-providers/remote.ts +++ b/src/lib/onboard/inference-providers/remote.ts @@ -39,6 +39,7 @@ export async function setupRemoteProviderInference( verifyInferenceRoute, verifyOnboardInferenceSmoke, isNonInteractive, + registry, REMOTE_PROVIDER_CONFIG, hydrateCredentialEnv, promptValidationRecovery, @@ -68,6 +69,7 @@ export async function setupRemoteProviderInference( upsertProvider, verifyInferenceRoute, verifyOnboardInferenceSmoke, + updateSandbox: registry.updateSandbox, }); if (bedrockSetup.handled) return { done: true, result: bedrockSetup.result }; while (true) { diff --git a/src/lib/onboard/inference-providers/types.ts b/src/lib/onboard/inference-providers/types.ts index 73ceffb0a3e..37e1ce221d1 100644 --- a/src/lib/onboard/inference-providers/types.ts +++ b/src/lib/onboard/inference-providers/types.ts @@ -77,7 +77,7 @@ export type PromptValidationRecovery = ( export type ClassifyApplyFailure = (message: string) => any; export type Registry = { - updateSandbox(sandboxName: string, patch: { model: string; provider: string }): void; + updateSandbox: typeof import("../../state/registry").updateSandbox; }; export type CommonDeps = { @@ -109,6 +109,7 @@ export type RemoteProviderDeps = CommonDeps & { upsertProvider: UpsertProvider; verifyInferenceRoute: VerifyInferenceRoute; verifyOnboardInferenceSmoke: any; + updateSandbox: Registry["updateSandbox"]; }): Promise<{ handled: true; result: SetupInferenceResult } | { handled: false }>; }; }; diff --git a/test/onboard-selection.test.ts b/test/onboard-selection.test.ts index d986f5eaffc..7a9bdca0c7e 100644 --- a/test/onboard-selection.test.ts +++ b/test/onboard-selection.test.ts @@ -15,10 +15,7 @@ import { validateOpenAiLikeModel, } from "../src/lib/inference/provider-models.js"; import { createInferenceSelectionValidationHelpers } from "../src/lib/onboard/inference-selection-validation.js"; -import { - getWindowsHostOllamaDockerRequirement, - rejectUnsupportedWindowsHostOllama, -} from "../src/lib/onboard/local-inference-topology.js"; +import { getWindowsHostOllamaDockerRequirement } from "../src/lib/onboard/local-inference-topology.js"; import { buildInferenceProviderMenu } from "../src/lib/onboard/provider-menu.js"; import { resolveRequestedProviderSelection } from "../src/lib/onboard/provider-selection.js"; import { reportProviderSelectionFailure } from "../src/lib/onboard/provider-selection-failure.js"; @@ -29,12 +26,17 @@ import { type SetupNimSelectionState, } from "../src/lib/onboard/setup-nim-selection.js"; import { createValidationRecoveryPromptHelpers } from "../src/lib/onboard/validation-recovery-prompt.js"; -import { - type DetectWindowsHostOllamaDeps, - detectWindowsHostOllama, -} from "../src/lib/onboard/windows-host-ollama.js"; +import { detectWindowsHostOllama } from "../src/lib/onboard/windows-host-ollama.js"; import { testTimeout } from "./helpers/timeouts"; +import { + createWindowsHostOllamaRunCapture, + requireFailedProviderResolution, + requirePresent, + requireSelectedProviderResolution, + restoreProcessEnvValue, + runNativeDockerWindowsProviderBoundary, +} from "./support/onboard-selection-test-helpers.js"; const CREDENTIAL_RETRY_PROMPT = " Options: retry (re-enter key), back (change provider), exit [retry]: "; @@ -106,10 +108,7 @@ function makeRemoteModelValidatorDeps( return { OPENAI_ENDPOINT_URL: TEST_OPENAI_ENDPOINT_URL, ANTHROPIC_ENDPOINT_URL: TEST_ANTHROPIC_ENDPOINT_URL, - requireValue: (value, message) => { - if (value === null || value === undefined) throw new Error(message); - return value; - }, + requireValue: requirePresent, isBackToSelection: (_value): _value is never => false, validateCustomOpenAiLikeSelection: async () => ({ ok: true as const, @@ -270,10 +269,6 @@ function makeSetupNimOllamaDeps(overrides: Partial = {}): Se }; } -function nonInteractiveAbort(reason: string, hint?: string): never { - throw new Error(`[non-interactive] Aborting: ${reason}${hint ? `\n${hint}` : ""}`); -} - function writeOpenAiStyleAuthRetryCurl(fakeBin: string, goodToken: string, models = ["gpt-5.4"]) { fs.writeFileSync( path.join(fakeBin, "curl"), @@ -3068,8 +3063,7 @@ const { setupNim } = require(${onboardPath}); }, ); } finally { - if (previousPreferredApi === undefined) delete process.env.NEMOCLAW_PREFERRED_API; - else process.env.NEMOCLAW_PREFERRED_API = previousPreferredApi; + restoreProcessEnvValue("NEMOCLAW_PREFERRED_API", previousPreferredApi); } }); @@ -3129,8 +3123,7 @@ const { setupNim } = require(${onboardPath}); }, ); } finally { - if (previousPreferredApi === undefined) delete process.env.NEMOCLAW_PREFERRED_API; - else process.env.NEMOCLAW_PREFERRED_API = previousPreferredApi; + restoreProcessEnvValue("NEMOCLAW_PREFERRED_API", previousPreferredApi); } }); @@ -5817,217 +5810,45 @@ const { setupNim } = require(${onboardPath}); }); it("rejects Windows-host Ollama providers on native Docker WSL before launching Ollama", () => { - const requirement = getWindowsHostOllamaDockerRequirement("docker"); - assert.equal(requirement.supported, false); const scenarios = [ - { provider: "start-windows-ollama", hasWindowsOllama: true }, - { provider: "install-windows-ollama", hasWindowsOllama: false }, - ]; + { provider: "start-windows-ollama", installed: true }, + { provider: "install-windows-ollama", installed: false }, + ] as const; for (const scenario of scenarios) { - const { options } = buildWindowsProviderMenu(requirement, { - hasWindowsOllama: scenario.hasWindowsOllama, - }); - const resolution = resolveWindowsProvider(options, scenario.provider, { - windowsHostOllamaSupported: false, + const boundary = runNativeDockerWindowsProviderBoundary({ + ...scenario, + reachable: false, + timeoutMs: PROVIDER_SELECTION_TEST_TIMEOUT_MS, }); - assert.equal(resolution.kind, "selected"); - if (resolution.kind !== "selected") throw new Error("Expected provider selection"); - assert.equal(resolution.selected.key, scenario.provider); - - const install = vi.fn(); - const setup = vi.fn(); - const switchHost = vi.fn(); - const abort = vi.fn(nonInteractiveAbort); - let failure = ""; - try { - const rejected = rejectUnsupportedWindowsHostOllama( - requirement, - resolution.selected.key, - true, - () => true, - abort, - ); - if (!rejected) { - install(); - setup(); - switchHost(); - } - } catch (error) { - failure = error instanceof Error ? error.message : String(error); - } - - assert.match(failure, /\[non-interactive\] Aborting:/); - assert.match(failure, new RegExp(scenario.provider + " requires Docker Desktop")); - assert.match(failure, /Choose WSL-local Ollama/); - assert.equal(abort.mock.calls.length, 1); - assert.equal(install.mock.calls.length, 0); - assert.equal(setup.mock.calls.length, 0); - assert.equal(switchHost.mock.calls.length, 0); + assert.equal(boundary.status, 1, `${scenario.provider} unexpectedly passed`); + assert.match(boundary.stderr, /\[non-interactive\] Aborting:/); + assert.match(boundary.stderr, new RegExp(scenario.provider + " requires Docker Desktop")); + assert.match(boundary.stderr, /Choose WSL-local Ollama/); + assert.doesNotMatch( + boundary.stderr, + /MODEL_SELECTION_REACHED|WINDOWS_INSTALL_CALLED|WINDOWS_SETUP_CALLED|WINDOWS_SWITCH_CALLED/, + ); } }); it("rejects reachable Windows-host Ollama on native Docker WSL through generic and fallback paths", () => { - const requirement = getWindowsHostOllamaDockerRequirement("docker"); - assert.equal(requirement.supported, false); - const { options } = buildWindowsProviderMenu(requirement, { - ollamaRunning: true, - ollamaHost: "host.docker.internal", - hasWindowsOllama: true, - isWindowsHostOllama: true, - }); - - for (const provider of ["ollama", "start-windows-ollama", "install-windows-ollama"]) { - const resolution = resolveWindowsProvider(options, provider, { - isWindowsHostOllama: true, - windowsHostOllamaSupported: false, + const providers = ["ollama", "start-windows-ollama", "install-windows-ollama"] as const; + for (const provider of providers) { + const boundary = runNativeDockerWindowsProviderBoundary({ + provider, + installed: true, + reachable: true, + timeoutMs: PROVIDER_SELECTION_TEST_TIMEOUT_MS, }); - const modelSelection = vi.fn(); - const install = vi.fn(); - const setup = vi.fn(); - const switchHost = vi.fn(); - const abort = vi.fn(nonInteractiveAbort); - const reject = (providerKey: string, windowsHostSelected: boolean) => - rejectUnsupportedWindowsHostOllama( - requirement, - providerKey, - windowsHostSelected, - () => true, - abort, - ); - let failure = ""; - - try { - if (resolution.kind === "failure") { - reportProviderSelectionFailure({ - reason: resolution.reason, - isWindowsHostOllama: true, - rejectWindowsHostOllama: reject, - writeError(message) { - throw new Error(message); - }, - }); - } else if (!reject(resolution.selected.key, true)) { - modelSelection(); - install(); - setup(); - switchHost(); - } - } catch (error) { - failure = error instanceof Error ? error.message : String(error); - } - - assert.match(failure, /\[non-interactive\] Aborting:/); - assert.match(failure, new RegExp(provider + " requires Docker Desktop")); - assert.match(failure, /Choose WSL-local Ollama/); - assert.equal(abort.mock.calls.length, 1); - assert.equal(modelSelection.mock.calls.length, 0); - assert.equal(install.mock.calls.length, 0); - assert.equal(setup.mock.calls.length, 0); - assert.equal(switchHost.mock.calls.length, 0); - } - - // Keep one production boundary to pin setupNim's reject-before-dispatch ordering. - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync( - path.join(os.tmpdir(), "nemoclaw-onboard-ollama-reachable-native-docker-"), - ); - const scriptPath = path.join(tmpDir, "ollama-reachable-native-docker-check.js"); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const credentialsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "credentials", "store.ts"), - ); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const platformPath = JSON.stringify(path.join(repoRoot, "src", "lib", "platform.ts")); - const topologyPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "local-inference-topology.ts"), - ); - const localPath = JSON.stringify(path.join(repoRoot, "src", "lib", "inference", "local.ts")); - const windowsPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "inference", "ollama", "windows.ts"), - ); - - const script = String.raw` -const credentials = require(${credentialsPath}); -const runner = require(${runnerPath}); -const platform = require(${platformPath}); -const topology = require(${topologyPath}); -const local = require(${localPath}); -const windows = require(${windowsPath}); - -platform.isWsl = () => true; -topology.getContainerRuntime = () => "docker"; -credentials.prompt = async () => { - throw new Error("Unexpected prompt in non-interactive test"); -}; -credentials.ensureApiKey = async () => {}; -runner.runCapture = (command) => { - const cmd = Array.isArray(command) ? command.join(" ") : command; - if (cmd.includes("command -v ollama")) return ""; - if (cmd.includes("127.0.0.1:8000/v1/models")) return ""; - if (cmd.includes("docker images")) return ""; - if (cmd.includes("powershell.exe") && cmd.includes("Get-Command ollama.exe")) { - return "C:\\Users\\tester\\AppData\\Local\\Programs\\Ollama\\ollama.exe"; - } - if (cmd.includes("powershell.exe") && cmd.includes("Get-Process ollama")) return ""; - if (cmd.includes("api/tags")) return JSON.stringify({ models: [{ name: "qwen3:8b" }] }); - return ""; -}; -runner.run = () => ({ status: 0 }); -runner.runShell = () => ({ status: 0 }); -local.resetOllamaHostCache(); -local.setResolvedOllamaHost(local.OLLAMA_HOST_DOCKER_INTERNAL); -local.getOllamaModelOptions = () => { - console.error("MODEL_SELECTION_REACHED"); - return ["qwen3:8b"]; -}; -windows.installOllamaOnWindowsHost = async () => { - console.error("WINDOWS_INSTALL_CALLED"); - return { ok: true, path: "C:\\Users\\tester\\AppData\\Local\\Programs\\Ollama\\ollama.exe" }; -}; -windows.setupWindowsOllamaWith0000Binding = () => { - console.error("WINDOWS_SETUP_CALLED"); - return true; -}; -windows.switchToWindowsOllamaHost = () => { - console.error("WINDOWS_SWITCH_CALLED"); -}; - -const { setupNim } = require(${onboardPath}); - -(async () => { - await setupNim(null, null); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - try { - const boundary = spawnSync(process.execPath, [scriptPath], { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - NEMOCLAW_NON_INTERACTIVE: "1", - NEMOCLAW_PROVIDER: "ollama", - NEMOCLAW_MODEL: "qwen3:8b", - NEMOCLAW_YES: "1", - }, - }); - - assert.equal(boundary.status, 1, "generic ollama unexpectedly passed"); + assert.equal(boundary.status, 1, `${provider} unexpectedly passed`); assert.match(boundary.stderr, /\[non-interactive\] Aborting:/); - assert.match(boundary.stderr, /ollama requires Docker Desktop/); + assert.match(boundary.stderr, new RegExp(provider + " requires Docker Desktop")); assert.match(boundary.stderr, /Choose WSL-local Ollama/); assert.doesNotMatch( boundary.stderr, /MODEL_SELECTION_REACHED|WINDOWS_INSTALL_CALLED|WINDOWS_SETUP_CALLED|WINDOWS_SWITCH_CALLED/, ); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); } }); @@ -6039,8 +5860,8 @@ const { setupNim } = require(${onboardPath}); }); const resolution = resolveWindowsProvider(options, "install-windows-ollama"); assert.equal(resolution.kind, "selected"); - if (resolution.kind !== "selected") throw new Error("Expected provider selection"); - assert.equal(resolution.selected.key, "start-windows-ollama"); + const selectedResolution = requireSelectedProviderResolution(resolution); + assert.equal(selectedResolution.selected.key, "start-windows-ollama"); const install = vi.fn(async () => ({ ok: false, path: "" })); const setup = vi.fn(() => true); @@ -6059,7 +5880,7 @@ const { setupNim } = require(${onboardPath}); try { const result = await handleWindowsHostOllamaSelection( null, - resolution.selected.key, + selectedResolution.selected.key, "qwen3:8b", false, false, @@ -6083,16 +5904,11 @@ const { setupNim } = require(${onboardPath}); it("detects Windows-host Ollama via running process when not on the user PATH (#3949)", async () => { const installedPath = "C:/Program Files/Ollama/ollama.exe"; - const runCapture = vi.fn((command) => { - const rendered = Array.isArray(command) ? command.join(" ") : String(command); - if (rendered.includes("Get-Command ollama.exe")) return ""; - if (rendered.includes("Get-Process ollama") && rendered.includes("Path")) { - return installedPath; - } - if (rendered.includes("Get-Process ollama") && rendered.includes("Id")) return "7652"; - if (rendered.includes("Get-NetTCPConnection")) return "127.0.0.1"; - return ""; - }); + const runCapture = createWindowsHostOllamaRunCapture([ + { contains: ["Get-Process ollama", "Path"], output: installedPath }, + { contains: ["Get-Process ollama", "Id"], output: "7652" }, + { contains: ["Get-NetTCPConnection"], output: "127.0.0.1" }, + ]); const detected = detectWindowsHostOllama({ isWsl: () => true, runCapture }); assert.deepEqual(detected, { installed: true, @@ -6135,15 +5951,11 @@ const { setupNim } = require(${onboardPath}); it("uses a known Windows install path when a running Ollama process has no readable path", async () => { const installedPath = "C:/Users/tester/AppData/Local/Programs/Ollama/ollama.exe"; - const runCapture = vi.fn((command) => { - const rendered = Array.isArray(command) ? command.join(" ") : String(command); - if (rendered.includes("Get-Command ollama.exe")) return ""; - if (rendered.includes("Get-Process ollama") && rendered.includes("Path")) return ""; - if (rendered.includes("Test-Path -LiteralPath")) return installedPath; - if (rendered.includes("Get-Process ollama") && rendered.includes("Id")) return "7652"; - if (rendered.includes("Get-NetTCPConnection")) return "127.0.0.1"; - return ""; - }); + const runCapture = createWindowsHostOllamaRunCapture([ + { contains: ["Test-Path -LiteralPath"], output: installedPath }, + { contains: ["Get-Process ollama", "Id"], output: "7652" }, + { contains: ["Get-NetTCPConnection"], output: "127.0.0.1" }, + ]); const detected = detectWindowsHostOllama({ isWsl: () => true, runCapture }); assert.deepEqual(detected, { installed: true, @@ -6197,13 +6009,13 @@ const { setupNim } = require(${onboardPath}); isWindowsHostOllama: false, }); assert.equal(resolution.kind, "failure"); - if (resolution.kind !== "failure") throw new Error("Expected provider selection failure"); + const failedResolution = requireFailedProviderResolution(resolution); const setup = vi.fn(); const switchHost = vi.fn(); const errors: string[] = []; reportProviderSelectionFailure({ - reason: resolution.reason, + reason: failedResolution.reason, isWindowsHostOllama: false, rejectWindowsHostOllama: () => { setup(); @@ -6232,13 +6044,13 @@ const { setupNim } = require(${onboardPath}); isWindowsHostOllama: false, }); assert.equal(resolution.kind, "failure"); - if (resolution.kind !== "failure") throw new Error("Expected provider selection failure"); + const failedResolution = requireFailedProviderResolution(resolution); const install = vi.fn(); const setup = vi.fn(); const errors: string[] = []; reportProviderSelectionFailure({ - reason: resolution.reason, + reason: failedResolution.reason, isWindowsHostOllama: false, rejectWindowsHostOllama: () => { install(); diff --git a/test/onboard.test.ts b/test/onboard.test.ts index c2740b10abd..bc7036524f7 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -18,7 +18,11 @@ import { createLocalInferenceRouteApplier } from "../src/lib/onboard/local-infer import type { SetupInference, SetupInferenceDeps } from "../src/lib/onboard/setup-inference.js"; import { stageOptimizedSandboxBuildContext } from "../src/lib/sandbox/build-context.js"; import { testTimeoutOptions } from "./helpers/timeouts"; -import { createDirectSetupInferenceHarnessFactory } from "./support/setup-inference-test-harness.js"; +import { + createDirectCommandRouter, + createDirectSetupInferenceHarnessFactory, + withProcessEnv, +} from "./support/setup-inference-test-harness.js"; type ShimScalar = string | number | boolean | null | undefined; type ShimCallable = (...args: readonly string[]) => ShimValue; @@ -106,26 +110,6 @@ const bedrockRuntimeOnboard = const createDirectSetupInferenceHarness = createDirectSetupInferenceHarnessFactory(createSetupInference); -async function withProcessEnv( - values: Record, - runTest: () => Promise | T, -): Promise { - const previous = new Map(); - for (const [key, value] of Object.entries(values)) { - previous.set(key, process.env[key]); - if (value === undefined) delete process.env[key]; - else process.env[key] = value; - } - try { - return await runTest(); - } finally { - for (const [key, value] of previous) { - if (value === undefined) delete process.env[key]; - else process.env[key] = value; - } - } -} - const repoRoot = path.join(import.meta.dirname, ".."); const onboardScriptMocksPath = JSON.stringify( path.join(repoRoot, "test", "helpers", "onboard-script-mocks.cjs"), @@ -819,11 +803,12 @@ startGateway(null).catch(() => {}); ? { status: 1, stdout: "", stderr: "" } : undefined, overrides: { + updateSandbox, bedrockRuntimeOnboard: { ...bedrockRuntimeOnboard, setupBedrockRuntimeInference: ( input: Parameters[0], - ) => setupBedrockRuntimeInference({ ...input, ensureAdapter, updateSandbox }), + ) => setupBedrockRuntimeInference({ ...input, ensureAdapter }), }, }, }); @@ -1287,17 +1272,20 @@ const { onboard } = require(${onboardPath}); throw Object.assign(new Error(`EXIT_CALLED:${code}`), { __exit: true }); }, }); - harness = createDirectSetupInferenceHarness({ - runOpenshell: (args) => { - const command = args.join(" "); - if (command.startsWith("provider get")) { - return { status: 1, stdout: "", stderr: "" }; - } - if (command.includes("inference set") && command.includes("ollama-local")) { - return { status: 7, stdout: "", stderr: "openshell: route apply failed" }; - } - return undefined; + const commandRouter = createDirectCommandRouter([ + { + name: "provider-get", + matches: (command) => command.startsWith("provider get"), + results: [{ status: 1, stdout: "", stderr: "" }], }, + { + name: "ollama-inference-set", + matches: (command) => command.includes("inference set") && command.includes("ollama-local"), + results: [{ status: 7, stdout: "", stderr: "openshell: route apply failed" }], + }, + ]); + harness = createDirectSetupInferenceHarness({ + runOpenshell: commandRouter.runOpenshell, overrides: { isNonInteractive: () => true, validateLocalProvider: () => ({ @@ -1355,17 +1343,20 @@ const { onboard } = require(${onboardPath}); throw Object.assign(new Error(`EXIT_CALLED:${code}`), { __exit: true }); }, }); - harness = createDirectSetupInferenceHarness({ - runOpenshell: (args) => { - const command = args.join(" "); - if (command.startsWith("provider get")) { - return { status: 1, stdout: "", stderr: "" }; - } - if (command.includes("inference set") && command.includes("vllm-local")) { - return { status: 13, stdout: "", stderr: "openshell: vllm route apply failed" }; - } - return undefined; + const commandRouter = createDirectCommandRouter([ + { + name: "provider-get", + matches: (command) => command.startsWith("provider get"), + results: [{ status: 1, stdout: "", stderr: "" }], + }, + { + name: "vllm-inference-set", + matches: (command) => command.includes("inference set") && command.includes("vllm-local"), + results: [{ status: 13, stdout: "", stderr: "openshell: vllm route apply failed" }], }, + ]); + harness = createDirectSetupInferenceHarness({ + runOpenshell: commandRouter.runOpenshell, overrides: { isNonInteractive: () => true, applyLocalInferenceRoute, @@ -1615,21 +1606,20 @@ console.log(JSON.stringify({ }); it("re-prompts for credentials when openshell inference set fails with authorization errors", async () => { await withProcessEnv({ OPENAI_API_KEY: "sk-bad" }, async () => { - let inferenceSetCalls = 0; - const harness = createDirectSetupInferenceHarness({ - runOpenshell: (args) => { - const command = args.join(" "); - if (command.startsWith("provider get")) { - return { status: 0, stdout: "", stderr: "" }; - } - if (command.includes("inference set")) { - inferenceSetCalls += 1; - if (inferenceSetCalls === 1) { - return { status: 1, stdout: "", stderr: "HTTP 403: forbidden" }; - } - } - return undefined; + const commandRouter = createDirectCommandRouter([ + { + name: "provider-get", + matches: (command) => command.startsWith("provider get"), + results: [{ status: 0, stdout: "", stderr: "" }], + }, + { + name: "inference-set", + matches: (command) => command.includes("inference set"), + results: [{ status: 1, stdout: "", stderr: "HTTP 403: forbidden" }, undefined], }, + ]); + const harness = createDirectSetupInferenceHarness({ + runOpenshell: commandRouter.runOpenshell, overrides: { promptValidationRecovery: async () => { process.env.OPENAI_API_KEY = "sk-good"; @@ -1651,7 +1641,7 @@ console.log(JSON.stringify({ } assert.equal(process.env.OPENAI_API_KEY, "sk-good"); - assert.equal(inferenceSetCalls, 2); + assert.equal(commandRouter.callCount("inference-set"), 2); const providerEnvs = harness.commands .filter((entry) => entry.command.includes("provider")) .map((entry) => entry.env?.OPENAI_API_KEY) @@ -1661,17 +1651,20 @@ console.log(JSON.stringify({ }); it("returns control to provider selection when inference apply recovery chooses back", async () => { await withProcessEnv({ OPENAI_API_KEY: "sk-TEST-NOT-A-REAL-VALUE" }, async () => { - const harness = createDirectSetupInferenceHarness({ - runOpenshell: (args) => { - const command = args.join(" "); - if (command.startsWith("provider get")) { - return { status: 0, stdout: "", stderr: "" }; - } - if (command.includes("inference set")) { - return { status: 1, stdout: "", stderr: "HTTP 404: model not found" }; - } - return undefined; + const commandRouter = createDirectCommandRouter([ + { + name: "provider-get", + matches: (command) => command.startsWith("provider get"), + results: [{ status: 0, stdout: "", stderr: "" }], }, + { + name: "inference-set", + matches: (command) => command.includes("inference set"), + results: [{ status: 1, stdout: "", stderr: "HTTP 404: model not found" }], + }, + ]); + const harness = createDirectSetupInferenceHarness({ + runOpenshell: commandRouter.runOpenshell, overrides: { promptValidationRecovery: async () => "selection" }, }); const error = vi.spyOn(console, "error").mockImplementation(() => {}); diff --git a/test/support/onboard-selection-test-helpers.ts b/test/support/onboard-selection-test-helpers.ts new file mode 100644 index 00000000000..3439f5e84f0 --- /dev/null +++ b/test/support/onboard-selection-test-helpers.ts @@ -0,0 +1,192 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { type SpawnSyncReturns, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { vi } from "vitest"; + +import type { ProviderOption } from "../../src/lib/onboard/provider-key-fallback.js"; +import type { + ProviderSelectionFailure, + ProviderSelectionResolution, + ProviderSelectionSuccess, +} from "../../src/lib/onboard/provider-selection.js"; +import type { DetectWindowsHostOllamaDeps } from "../../src/lib/onboard/windows-host-ollama.js"; + +const PROVIDER_CREDENTIAL_ENV_KEYS = new Set([ + "ANTHROPIC_API_KEY", + "AWS_ACCESS_KEY_ID", + "AWS_DEFAULT_REGION", + "AWS_PROFILE", + "AWS_REGION", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "COMPATIBLE_ANTHROPIC_API_KEY", + "COMPATIBLE_API_KEY", + "GEMINI_API_KEY", + "NGC_API_KEY", + "NVIDIA_API_KEY", + "NVIDIA_INFERENCE_API_KEY", + "NOUS_API_KEY", + "OPENAI_API_KEY", +]); + +export function requirePresent(value: T | null | undefined, message: string): T { + if (value === null || value === undefined) throw new Error(message); + return value; +} + +export function restoreProcessEnvValue(name: string, previous: string | undefined): void { + if (previous === undefined) delete process.env[name]; + else process.env[name] = previous; +} + +export function requireSelectedProviderResolution( + resolution: ProviderSelectionResolution, +): ProviderSelectionSuccess { + if (resolution.kind !== "selected") throw new Error("Expected provider selection"); + return resolution; +} + +export function requireFailedProviderResolution( + resolution: ProviderSelectionResolution, +): ProviderSelectionFailure { + if (resolution.kind !== "failure") throw new Error("Expected provider selection failure"); + return resolution; +} + +function createIsolatedOnboardEnv(tmpDir: string, provider: string): NodeJS.ProcessEnv { + const env = { ...process.env }; + for (const key of Object.keys(env)) { + if (key.startsWith("NEMOCLAW_") || PROVIDER_CREDENTIAL_ENV_KEYS.has(key)) { + delete env[key]; + } + } + return { + ...env, + HOME: tmpDir, + NEMOCLAW_MODEL: "qwen3:8b", + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_PROVIDER: provider, + NEMOCLAW_YES: "1", + }; +} + +export function runNativeDockerWindowsProviderBoundary(options: { + provider: "ollama" | "start-windows-ollama" | "install-windows-ollama"; + installed: boolean; + reachable: boolean; + timeoutMs: number; +}): SpawnSyncReturns { + const repoRoot = path.join(import.meta.dirname, "..", ".."); + const tmpDir = fs.mkdtempSync( + path.join(os.tmpdir(), "nemoclaw-onboard-native-docker-windows-provider-"), + ); + const scriptPath = path.join(tmpDir, "provider-boundary-check.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); + const credentialsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "credentials", "store.ts"), + ); + const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); + const platformPath = JSON.stringify(path.join(repoRoot, "src", "lib", "platform.ts")); + const topologyPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "onboard", "local-inference-topology.ts"), + ); + const localPath = JSON.stringify(path.join(repoRoot, "src", "lib", "inference", "local.ts")); + const windowsPath = JSON.stringify( + path.join(repoRoot, "src", "lib", "inference", "ollama", "windows.ts"), + ); + const scenario = JSON.stringify({ installed: options.installed, reachable: options.reachable }); + + const script = String.raw` +const scenario = ${scenario}; +const credentials = require(${credentialsPath}); +const runner = require(${runnerPath}); +const platform = require(${platformPath}); +const topology = require(${topologyPath}); +const local = require(${localPath}); +const windows = require(${windowsPath}); + +platform.isWsl = () => true; +topology.getContainerRuntime = () => "docker"; +credentials.prompt = async () => { + throw new Error("Unexpected prompt in non-interactive test"); +}; +credentials.ensureApiKey = async () => {}; +runner.runCapture = (command) => { + const cmd = Array.isArray(command) ? command.join(" ") : String(command); + if (cmd.includes("command -v ollama")) return ""; + if (cmd.includes("127.0.0.1:8000/v1/models")) return ""; + if (cmd.includes("docker images")) return ""; + if (cmd.includes("powershell.exe") && cmd.includes("Get-Command ollama.exe")) { + return scenario.installed + ? "C:\\Users\\tester\\AppData\\Local\\Programs\\Ollama\\ollama.exe" + : ""; + } + if (cmd.includes("powershell.exe") && cmd.includes("Get-Process ollama")) return ""; + if (scenario.reachable && cmd.includes("api/tags")) { + return JSON.stringify({ models: [{ name: "qwen3:8b" }] }); + } + return ""; +}; +runner.run = () => ({ status: 0 }); +runner.runShell = () => ({ status: 0 }); +local.resetOllamaHostCache(); +if (scenario.reachable) local.setResolvedOllamaHost(local.OLLAMA_HOST_DOCKER_INTERNAL); +local.getOllamaModelOptions = () => { + console.error("MODEL_SELECTION_REACHED"); + return ["qwen3:8b"]; +}; +windows.installOllamaOnWindowsHost = async () => { + console.error("WINDOWS_INSTALL_CALLED"); + return { ok: true, path: "C:\\Users\\tester\\AppData\\Local\\Programs\\Ollama\\ollama.exe" }; +}; +windows.setupWindowsOllamaWith0000Binding = () => { + console.error("WINDOWS_SETUP_CALLED"); + return true; +}; +windows.switchToWindowsOllamaHost = () => { + console.error("WINDOWS_SWITCH_CALLED"); +}; + +const { setupNim } = require(${onboardPath}); + +(async () => { + await setupNim(null, null); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + + try { + fs.writeFileSync(scriptPath, script); + return spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: createIsolatedOnboardEnv(tmpDir, options.provider), + timeout: options.timeoutMs, + }); + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } +} + +type CommandResponse = { + contains: readonly string[]; + output: string; +}; + +export function createWindowsHostOllamaRunCapture( + responses: readonly CommandResponse[], +): DetectWindowsHostOllamaDeps["runCapture"] { + return vi.fn((command) => { + const rendered = Array.isArray(command) ? command.join(" ") : String(command); + return ( + responses.find(({ contains }) => contains.every((part) => rendered.includes(part)))?.output ?? + "" + ); + }); +} diff --git a/test/support/setup-inference-test-harness.ts b/test/support/setup-inference-test-harness.ts index 3b1481c7563..2326bc29477 100644 --- a/test/support/setup-inference-test-harness.ts +++ b/test/support/setup-inference-test-harness.ts @@ -43,6 +43,48 @@ export type DirectSetupHarnessOptions = { overrides?: Partial; }; +type DirectCommandRoute = { + name: string; + matches(command: string): boolean; + results: readonly [DirectRunStubResult | undefined, ...(DirectRunStubResult | undefined)[]]; +}; + +export async function withProcessEnv( + values: Record, + runTest: () => Promise | T, +): Promise { + const previous = new Map(); + for (const [key, value] of Object.entries(values)) { + previous.set(key, process.env[key]); + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + try { + return await runTest(); + } finally { + for (const [key, value] of previous) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + } +} + +export function createDirectCommandRouter(routes: readonly DirectCommandRoute[]) { + const callCounts = new Map(); + const runOpenshell: NonNullable = (args) => { + const command = args.join(" "); + const route = routes.find((candidate) => candidate.matches(command)); + if (!route) return undefined; + const callIndex = callCounts.get(route.name) ?? 0; + callCounts.set(route.name, callIndex + 1); + return route.results[Math.min(callIndex, route.results.length - 1)]; + }; + return { + callCount: (name: string) => callCounts.get(name) ?? 0, + runOpenshell, + }; +} + export function directRunResult({ status = 0, stdout = "", From 6018f1395b1023e7e4fa6ba108ae1e31f96b4334 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 4 Jul 2026 10:39:55 -0700 Subject: [PATCH 07/16] perf(test): inline messaging post-install phase Signed-off-by: Carlos Villela --- ci/test-file-size-budget.json | 2 +- test/generate-openclaw-config.test.ts | 36 ++++----------------------- 2 files changed, 6 insertions(+), 32 deletions(-) diff --git a/ci/test-file-size-budget.json b/ci/test-file-size-budget.json index 11a1bbadc88..a36632c20fe 100644 --- a/ci/test-file-size-budget.json +++ b/ci/test-file-size-budget.json @@ -6,7 +6,7 @@ "src/lib/inference/nim.test.ts": 2068, "src/lib/onboard/preflight.test.ts": 1904, "test/channels-add-preset.test.ts": 1871, - "test/generate-openclaw-config.test.ts": 1972, + "test/generate-openclaw-config.test.ts": 1946, "test/install-preflight.test.ts": 3934, "test/nemoclaw-start.test.ts": 4827, "test/onboard-messaging.test.ts": 2062, diff --git a/test/generate-openclaw-config.test.ts b/test/generate-openclaw-config.test.ts index aee01eb3ff7..fdac37b7522 100644 --- a/test/generate-openclaw-config.test.ts +++ b/test/generate-openclaw-config.test.ts @@ -15,22 +15,13 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { buildConfig, main } from "../scripts/generate-openclaw-config.mts"; import { applyMessagingAgentRenderToObject, + applyMessagingBuildPhase, readMessagingBuildPlanFromEnv, } from "../src/lib/messaging/applier/build/messaging-build-applier.mts"; import { withLegacyMessagingPlanEnv } from "./messaging-plan-test-helper"; const SCRIPT_PATH = path.join(import.meta.dirname, "..", "scripts", "generate-openclaw-config.mts"); const SCRIPT_ARGS = ["--experimental-strip-types", SCRIPT_PATH]; -const APPLIER_PATH = path.join( - import.meta.dirname, - "..", - "src", - "lib", - "messaging", - "applier", - "build", - "messaging-build-applier.mts", -); /** Minimal env vars required for a valid config generation run. */ const BASE_ENV: Record = { @@ -101,30 +92,13 @@ function withConfigEnv(envOverrides: Record, fn: () => T): T } function runMessagingPostInstall(env: Record): void { - const result = spawnSync( - "node", - [ - "--experimental-strip-types", - APPLIER_PATH, - "--agent", - "openclaw", - "--phase", + withEnv(env, () => + applyMessagingBuildPhase( + readMessagingBuildPlanFromEnv(env, "openclaw"), "post-agent-install", - ], - { - encoding: "utf-8", - stdio: ["pipe", "pipe", "pipe"], env, - timeout: 10_000, - }, + ), ); - if (result.status !== 0) { - throw new Error( - `Messaging applier failed (exit ${result.status}): -stdout: ${result.stdout} -stderr: ${result.stderr}`, - ); - } } function runConfigScript(envOverrides: Record = {}): any { From e53e6238741c26fac7eee051473a977bd6e7f52b Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 4 Jul 2026 11:09:27 -0700 Subject: [PATCH 08/16] fix(onboard): inject remote provider exit boundary Signed-off-by: Carlos Villela --- .../rebuild-local-provider-recreate.test.ts | 3 +++ src/lib/onboard/inference-providers/remote.ts | 11 ++++---- src/lib/onboard/inference-providers/types.ts | 1 + src/lib/onboard/setup-inference.ts | 1 + test/onboard-inference-failure-paths.test.ts | 27 +++++++++++++++++++ 5 files changed, 38 insertions(+), 5 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts b/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts index 5b39b40f228..264a152d38a 100644 --- a/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts +++ b/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts @@ -59,6 +59,9 @@ const unusedCommonInferenceDeps = { verifyOnboardInferenceSmoke: vi.fn(), isNonInteractive: () => true, registry: { updateSandbox: vi.fn() }, + exitProcess: (code: number): never => { + throw new Error(`EXIT_CALLED:${code}`); + }, }; const localProviderScenarios = [ diff --git a/src/lib/onboard/inference-providers/remote.ts b/src/lib/onboard/inference-providers/remote.ts index 8b01e3e3d4f..c1eca66ae64 100644 --- a/src/lib/onboard/inference-providers/remote.ts +++ b/src/lib/onboard/inference-providers/remote.ts @@ -40,6 +40,7 @@ export async function setupRemoteProviderInference( verifyOnboardInferenceSmoke, isNonInteractive, registry, + exitProcess, REMOTE_PROVIDER_CONFIG, hydrateCredentialEnv, promptValidationRecovery, @@ -56,7 +57,7 @@ export async function setupRemoteProviderInference( : Object.values(REMOTE_PROVIDER_CONFIG).find((entry) => entry.providerName === provider); if (!config) { console.error(` Unsupported provider configuration: ${provider}`); - process.exit(1); + return exitProcess(1); } const bedrockSetup = await bedrockRuntimeOnboard.setupBedrockRuntimeInference({ sandboxName, @@ -115,7 +116,7 @@ export async function setupRemoteProviderInference( if (!providerResult.ok) { console.error(` ${providerResult.message}`); if (isNonInteractive()) { - process.exit(providerResult.status || 1); + exitProcess(providerResult.status || 1); } const retry = await promptValidationRecovery( config.label, @@ -129,7 +130,7 @@ export async function setupRemoteProviderInference( if (retry === "selection" || retry === "model") { return { done: true, result: { retry: "selection" } }; } - process.exit(providerResult.status || 1); + exitProcess(providerResult.status || 1); } const argsv = ["inference", "set"]; if (config.skipVerify) { @@ -148,7 +149,7 @@ export async function setupRemoteProviderInference( `Failed to configure inference provider '${provider}'.`; console.error(` ${message}`); if (isNonInteractive()) { - process.exit(applyResult.status || 1); + exitProcess(applyResult.status || 1); } const retry = await promptValidationRecovery( config.label, @@ -162,7 +163,7 @@ export async function setupRemoteProviderInference( if (retry === "selection" || retry === "model") { return { done: true, result: { retry: "selection" } }; } - process.exit(applyResult.status || 1); + exitProcess(applyResult.status || 1); } return { done: false }; } diff --git a/src/lib/onboard/inference-providers/types.ts b/src/lib/onboard/inference-providers/types.ts index 37e1ce221d1..b2c8222bfba 100644 --- a/src/lib/onboard/inference-providers/types.ts +++ b/src/lib/onboard/inference-providers/types.ts @@ -87,6 +87,7 @@ export type CommonDeps = { verifyOnboardInferenceSmoke: VerifyOnboardInferenceSmoke; isNonInteractive: () => boolean; registry: Registry; + exitProcess: (code: number) => never; }; export type RemoteProviderDeps = CommonDeps & { diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index 636a0988668..0932291989f 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -76,6 +76,7 @@ export function createSetupInference( verifyOnboardInferenceSmoke: deps.verifyOnboardInferenceSmoke, isNonInteractive: deps.isNonInteractive, registry: { updateSandbox: deps.updateSandbox }, + exitProcess: deps.exitProcess, }; if (provider === deps.hermesProviderAuth.HERMES_PROVIDER_NAME) { diff --git a/test/onboard-inference-failure-paths.test.ts b/test/onboard-inference-failure-paths.test.ts index 2b4c44dd106..e3f7f27f73b 100644 --- a/test/onboard-inference-failure-paths.test.ts +++ b/test/onboard-inference-failure-paths.test.ts @@ -38,6 +38,33 @@ describe("setupInference dependency failures", () => { vi.restoreAllMocks(); }); + it("fails through the injected exit boundary when a known remote provider has no config", async () => { + const exitProcess = vi.fn((code: number): never => { + throw new Error(`EXIT_CALLED:${code}`); + }); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + const hydrateCredentialEnv = vi.fn(); + const setupBedrockRuntimeInference = vi.fn(async () => ({ handled: false as const })); + const harness = createDirectSetupInferenceHarness({ + overrides: { + REMOTE_PROVIDER_CONFIG: {}, + exitProcess, + hydrateCredentialEnv, + bedrockRuntimeOnboard: { setupBedrockRuntimeInference }, + }, + }); + + await expect(harness.setupInference("test-box", "gpt-test", "openai-api")).rejects.toThrow( + "EXIT_CALLED:1", + ); + + expect(error).toHaveBeenCalledWith(" Unsupported provider configuration: openai-api"); + expect(exitProcess).toHaveBeenCalledWith(1); + expect(setupBedrockRuntimeInference).not.toHaveBeenCalled(); + expect(hydrateCredentialEnv).not.toHaveBeenCalled(); + expectNoPostFailureSideEffects(harness); + }); + it("fails closed before provider registration when local vLLM validation fails", async () => { const exit = stubProcessExit(); const error = vi.spyOn(console, "error").mockImplementation(() => {}); From fa7abfa19045be178749465ef67c6fe8e7b8931b Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 4 Jul 2026 11:09:45 -0700 Subject: [PATCH 09/16] test(onboard): retain production inference boundaries Signed-off-by: Carlos Villela --- .../onboard-probes-responses-fallback.test.ts | 43 +++++ src/lib/inference/onboard-probes.test.ts | 31 ---- test/onboard.test.ts | 40 ++--- test/support/setup-inference-test-harness.ts | 150 ++++++++++++++++++ 4 files changed, 213 insertions(+), 51 deletions(-) create mode 100644 src/lib/inference/onboard-probes-responses-fallback.test.ts diff --git a/src/lib/inference/onboard-probes-responses-fallback.test.ts b/src/lib/inference/onboard-probes-responses-fallback.test.ts new file mode 100644 index 00000000000..8716fd37145 --- /dev/null +++ b/src/lib/inference/onboard-probes-responses-fallback.test.ts @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import path from "node:path"; +import { expect, it } from "vitest"; + +import { + makeResponsesFallbackUrlRecordingFakeCurlScript, + withFakeCurlProbe, +} from "./onboard-probes-curl-harness"; + +const { probeOpenAiLikeEndpoint } = require("./onboard-probes"); + +it("falls back to chat completions for custom OpenAI-compatible endpoints when /responses lacks tool calls", () => { + withFakeCurlProbe( + { + script: makeResponsesFallbackUrlRecordingFakeCurlScript(), + dirPrefix: "nemoclaw-responses-tool-fallback-", + }, + ({ counter, tmpDir }) => { + const result = probeOpenAiLikeEndpoint( + "https://proxy.example.com/v1", + "custom-model", + "proxy-key", + { requireResponsesToolCalling: true }, + ); + + expect(result).toMatchObject({ + ok: true, + api: "openai-completions", + label: "Chat Completions API", + }); + expect(fs.readFileSync(counter, "utf8").trim()).toBe("2"); + expect(fs.readFileSync(path.join(tmpDir, "request-1-url.txt"), "utf8")).toBe( + "https://proxy.example.com/v1/responses", + ); + expect(fs.readFileSync(path.join(tmpDir, "request-2-url.txt"), "utf8")).toBe( + "https://proxy.example.com/v1/chat/completions", + ); + }, + ); +}); diff --git a/src/lib/inference/onboard-probes.test.ts b/src/lib/inference/onboard-probes.test.ts index 89309aa1df0..1bef5cc04e0 100644 --- a/src/lib/inference/onboard-probes.test.ts +++ b/src/lib/inference/onboard-probes.test.ts @@ -11,7 +11,6 @@ import { HARNESS_COUNTER, HARNESS_TMPDIR, makeFakeCurlScript, - makeResponsesFallbackUrlRecordingFakeCurlScript, withFakeCurlProbe, } from "./onboard-probes-curl-harness"; @@ -769,36 +768,6 @@ exit 28 ); }); - it("falls back to chat completions for custom OpenAI-compatible endpoints when /responses lacks tool calls", () => { - withFakeCurlProbe( - { - script: makeResponsesFallbackUrlRecordingFakeCurlScript(), - dirPrefix: "nemoclaw-responses-tool-fallback-", - }, - ({ counter, tmpDir }) => { - const result = probeOpenAiLikeEndpoint( - "https://proxy.example.com/v1", - "custom-model", - "proxy-key", - { requireResponsesToolCalling: true }, - ); - - expect(result).toMatchObject({ - ok: true, - api: "openai-completions", - label: "Chat Completions API", - }); - expect(fs.readFileSync(counter, "utf8").trim()).toBe("2"); - expect(fs.readFileSync(path.join(tmpDir, "request-1-url.txt"), "utf8")).toBe( - "https://proxy.example.com/v1/responses", - ); - expect(fs.readFileSync(path.join(tmpDir, "request-2-url.txt"), "utf8")).toBe( - "https://proxy.example.com/v1/chat/completions", - ); - }, - ); - }); - // PR #5975 review note PRA-14 (Nemotron). Pins the silent fallback so a // future SGLang fix that removes the workaround stays observable. it("falls back to chat-completions when /responses streaming lacks required events", () => { diff --git a/test/onboard.test.ts b/test/onboard.test.ts index bc7036524f7..4ac8374e4bc 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -21,6 +21,7 @@ import { testTimeoutOptions } from "./helpers/timeouts"; import { createDirectCommandRouter, createDirectSetupInferenceHarnessFactory, + runProductionSetupInferenceCredentialBoundary, withProcessEnv, } from "./support/setup-inference-test-harness.js"; @@ -723,27 +724,26 @@ startGateway(null).catch(() => {}); expect(SANDBOX_NAME_REGEX.test("")).toBe(false); }); - it("passes credential names to openshell without embedding secret values in argv", async () => { - await withProcessEnv({ NVIDIA_INFERENCE_API_KEY: "nvapi-TEST-NOT-A-REAL-VALUE" }, async () => { - const harness = createDirectSetupInferenceHarness({ - runOpenshell: (args) => - args.slice(0, 2).join(" ") === "provider get" - ? { status: 0, stdout: "", stderr: "" } - : undefined, - }); - - await harness.setupInference("test-box", "nvidia/nemotron-3-super-120b-a12b", "nvidia-nim"); - - const commands = harness.commands; - assert.equal(commands.length, 4); - assert.match(commands[0].command, /gateway select nemoclaw/); - assert.match(commands[1].command, /provider get/); - assert.match(commands[2].command, /--credential NVIDIA_INFERENCE_API_KEY/); - assert.doesNotMatch(commands[2].command, /nvapi-TEST-NOT-A-REAL-VALUE/); - assert.match(commands[2].command, /provider update/); - assert.match(commands[3].command, /inference set/); - assert.equal(process.env.NVIDIA_INFERENCE_API_KEY, "nvapi-TEST-NOT-A-REAL-VALUE"); + it("passes credential names to openshell without embedding secret values in argv", () => { + const credentialValue = "nvapi-TEST-NOT-A-REAL-VALUE"; + const { credentialEvidence: evidence } = runProductionSetupInferenceCredentialBoundary({ + credentialEnv: "NVIDIA_INFERENCE_API_KEY", + credentialValue, + model: "nvidia/nemotron-3-super-120b-a12b", + provider: "nvidia-nim", }); + assert.match(evidence.providerCommand.argv.join(" "), /--credential NVIDIA_INFERENCE_API_KEY/); + assert.deepEqual(evidence.argvContainingSecret, []); + assert.deepEqual(evidence.secretBearingCommands, ["provider update"]); + assert.equal(evidence.providerCommand.env.NVIDIA_INFERENCE_API_KEY, credentialValue); + assert.equal( + evidence.unscopedCommandKinds.join(","), + "gateway select,provider get,inference set", + ); + assert.deepEqual(evidence.unscopedCredentialValues, [null, null, null]); + assert.deepEqual(evidence.unscopedCommandsContainingSecret, []); + assert.deepEqual(evidence.setupCredentialValues, [credentialValue, credentialValue]); + assert.equal(evidence.parentCredentialUnchanged, true); }); it("reuses a registered Hermes Provider without re-collecting host credentials", async () => { await withProcessEnv( diff --git a/test/support/setup-inference-test-harness.ts b/test/support/setup-inference-test-harness.ts index 2326bc29477..77bd0ccdf26 100644 --- a/test/support/setup-inference-test-harness.ts +++ b/test/support/setup-inference-test-harness.ts @@ -1,6 +1,10 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; import { vi } from "vitest"; import type { SetupInference, SetupInferenceDeps } from "../../src/lib/onboard/setup-inference.js"; @@ -49,6 +53,152 @@ type DirectCommandRoute = { results: readonly [DirectRunStubResult | undefined, ...(DirectRunStubResult | undefined)[]]; }; +export type ProductionOpenshellCommandRecord = { + argv: string[]; + env: Record; +}; + +export type ProductionSetupInferenceBoundaryResult = { + commands: ProductionOpenshellCommandRecord[]; + credentialEvidence: { + argvContainingSecret: string[]; + parentCredentialUnchanged: boolean; + providerCommand: ProductionOpenshellCommandRecord; + secretBearingCommands: string[]; + setupCredentialValues: Array; + unscopedCommandKinds: string[]; + unscopedCommandsContainingSecret: string[]; + unscopedCredentialValues: Array; + }; + setupCredentialAfter: string | null; + setupCredentialBefore: string | null; +}; + +export function runProductionSetupInferenceCredentialBoundary(options: { + credentialEnv: string; + credentialValue: string; + endpointUrl?: string | null; + model: string; + provider: string; + timeoutMs?: number; +}): ProductionSetupInferenceBoundaryResult { + const parentCredentialBefore = process.env[options.credentialEnv]; + const repoRoot = path.join(import.meta.dirname, "..", ".."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-setup-inference-boundary-")); + const fakeBin = path.join(tmpDir, "bin"); + const openshellPath = path.join(fakeBin, "openshell"); + const commandLogPath = path.join(tmpDir, "openshell-commands.jsonl"); + const setupResultPath = path.join(tmpDir, "setup-result.json"); + const childScriptPath = path.join(tmpDir, "setup-inference-boundary.js"); + const onboardPath = path.join(repoRoot, "src", "lib", "onboard.ts"); + const sourceHookPath = path.join(repoRoot, "test", "helpers", "onboard-script-mocks.cjs"); + + try { + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync( + openshellPath, + `#!${process.execPath} +const fs = require("node:fs"); +const argv = process.argv.slice(2); +fs.appendFileSync(${JSON.stringify(commandLogPath)}, JSON.stringify({ argv, env: process.env }) + "\\n"); +if (argv[0] === "inference" && argv[1] === "get") { + process.stdout.write("Gateway inference:\\n Provider: configured\\n Model: configured\\n"); +} +process.exit(0); +`, + { mode: 0o755 }, + ); + fs.writeFileSync( + childScriptPath, + `const fs = require("node:fs"); +const { setupInference } = require(${JSON.stringify(onboardPath)}); +const credentialEnv = ${JSON.stringify(options.credentialEnv)}; +const setupCredentialBefore = process.env[credentialEnv] || null; +(async () => { + await setupInference( + null, + ${JSON.stringify(options.model)}, + ${JSON.stringify(options.provider)}, + ${JSON.stringify(options.endpointUrl ?? null)}, + credentialEnv, + ); + fs.writeFileSync( + ${JSON.stringify(setupResultPath)}, + JSON.stringify({ + setupCredentialBefore, + setupCredentialAfter: process.env[credentialEnv] || null, + }), + ); +})().catch((error) => { + console.error(error && error.stack ? error.stack : String(error)); + process.exit(1); +}); +`, + ); + + const result = spawnSync(process.execPath, [childScriptPath], { + cwd: repoRoot, + encoding: "utf8", + timeout: options.timeoutMs ?? 15_000, + env: { + HOME: tmpDir, + NODE_ENV: "test", + NODE_OPTIONS: `--require=${sourceHookPath}`, + NEMOCLAW_OPENSHELL_BIN: openshellPath, + PATH: `${fakeBin}:${process.env.PATH ?? ""}`, + TMPDIR: tmpDir, + VITEST: "true", + [options.credentialEnv]: options.credentialValue, + }, + }); + if (result.error) throw result.error; + if (result.status !== 0) { + throw new Error( + `Production setupInference boundary exited ${result.status}: ${result.stderr || result.stdout}`, + ); + } + + const commands = fs + .readFileSync(commandLogPath, "utf8") + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as ProductionOpenshellCommandRecord); + const setupResult = JSON.parse(fs.readFileSync(setupResultPath, "utf8")) as Omit< + ProductionSetupInferenceBoundaryResult, + "commands" | "credentialEvidence" + >; + const commandKind = ({ argv }: ProductionOpenshellCommandRecord) => argv.slice(0, 2).join(" "); + const providerCommand = commands.find(({ argv }) => + /^provider (create|update) /.test(argv.join(" ")), + ); + if (!providerCommand) throw new Error("Production setupInference did not mutate a provider"); + const unscopedPatterns = [/^gateway select /, /^provider get /, /^inference set /]; + const unscopedCommands = unscopedPatterns + .map((pattern) => commands.find(({ argv }) => pattern.test(argv.join(" ")))) + .filter((command): command is ProductionOpenshellCommandRecord => command !== undefined); + const containsSecret = ({ env }: ProductionOpenshellCommandRecord) => + Object.values(env).some((value) => value.includes(options.credentialValue)); + const credentialEvidence = { + argvContainingSecret: commands + .filter(({ argv }) => argv.some((arg) => arg.includes(options.credentialValue))) + .map(commandKind), + parentCredentialUnchanged: process.env[options.credentialEnv] === parentCredentialBefore, + providerCommand, + secretBearingCommands: commands.filter(containsSecret).map(commandKind), + setupCredentialValues: [setupResult.setupCredentialBefore, setupResult.setupCredentialAfter], + unscopedCommandKinds: unscopedCommands.map(commandKind), + unscopedCommandsContainingSecret: unscopedCommands.filter(containsSecret).map(commandKind), + unscopedCredentialValues: unscopedCommands.map( + ({ env }) => env[options.credentialEnv] ?? null, + ), + }; + return { commands, credentialEvidence, ...setupResult }; + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } +} + export async function withProcessEnv( values: Record, runTest: () => Promise | T, From 6272ef170bb0cbd2976f2802adee6b482c4e91ce Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 4 Jul 2026 11:25:02 -0700 Subject: [PATCH 10/16] fix(onboard): inject Bedrock failure boundaries Signed-off-by: Carlos Villela --- src/lib/onboard/bedrock-runtime.test.ts | 40 +++ src/lib/onboard/bedrock-runtime.ts | 123 ++++--- src/lib/onboard/inference-providers/remote.ts | 5 + src/lib/onboard/inference-providers/types.ts | 5 + src/lib/onboard/setup-inference.ts | 2 + test/onboard-inference-failure-paths.test.ts | 315 +++++++++++++++++- test/onboard.test.ts | 14 +- 7 files changed, 429 insertions(+), 75 deletions(-) diff --git a/src/lib/onboard/bedrock-runtime.test.ts b/src/lib/onboard/bedrock-runtime.test.ts index 34df73bf1ab..8555cf3b4a0 100644 --- a/src/lib/onboard/bedrock-runtime.test.ts +++ b/src/lib/onboard/bedrock-runtime.test.ts @@ -15,6 +15,8 @@ function clearBedrockAuthEnv(): void { delete process.env.AWS_SECRET_ACCESS_KEY; delete process.env.AWS_SESSION_TOKEN; delete process.env.AWS_WEB_IDENTITY_TOKEN_FILE; + delete process.env.AWS_CONTAINER_CREDENTIALS_RELATIVE_URI; + delete process.env.AWS_CONTAINER_CREDENTIALS_FULL_URI; delete process.env.COMPATIBLE_ANTHROPIC_API_KEY; } @@ -24,6 +26,44 @@ afterEach(() => { }); describe("Bedrock Runtime onboarding helper", () => { + it("uses the injected exit boundary when non-interactive selection has no auth", async () => { + clearBedrockAuthEnv(); + const error = vi.fn(); + const log = vi.fn(); + const exitProcess = vi.fn((code: number): never => { + throw new Error(`EXIT_CALLED:${code}`); + }); + const promptInputModel = vi.fn(async () => "unused-model"); + const replaceNamedCredential = vi.fn(async () => "unused-credential"); + + await expect( + selectBedrockRuntimeCustomAnthropic({ + selectedKey: "anthropicCompatible", + endpointUrl: BEDROCK_URL, + credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", + label: "Other Anthropic-compatible endpoint", + helpUrl: null, + defaultModel: "anthropic.claude", + backToSelection: BACK_TO_SELECTION, + isNonInteractive: () => true, + promptInputModel, + replaceNamedCredential, + error, + exitProcess, + log, + }), + ).rejects.toThrow("EXIT_CALLED:1"); + + expect(error).toHaveBeenCalledWith( + " AWS_BEARER_TOKEN_BEDROCK, AWS_PROFILE, IAM environment credentials, or an explicitly exported Bedrock-compatible endpoint key is required for a Bedrock Runtime endpoint.", + ); + expect(exitProcess).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledWith(1); + expect(log).not.toHaveBeenCalled(); + expect(promptInputModel).not.toHaveBeenCalled(); + expect(replaceNamedCredential).not.toHaveBeenCalled(); + }); + it("prompts for a Bedrock-compatible credential when no explicit AWS auth source exists", async () => { clearBedrockAuthEnv(); const replaceNamedCredential = vi.fn(async () => "bedrock-bearer"); diff --git a/src/lib/onboard/bedrock-runtime.ts b/src/lib/onboard/bedrock-runtime.ts index 47fb9cb9a1a..43d99d8b064 100644 --- a/src/lib/onboard/bedrock-runtime.ts +++ b/src/lib/onboard/bedrock-runtime.ts @@ -30,6 +30,16 @@ type UpsertProvider = ( type SetupInferenceResult = { ok: true; retry?: undefined } | { retry: "selection" }; +type BedrockRuntimeDependencies = { + exitProcess?: (code: number) => never; + error?: (message: string) => void; + log?: (message: string) => void; +}; + +const defaultExitProcess = (code: number): never => process.exit(code); +const defaultError = (message: string): void => console.error(message); +const defaultLog = (message: string): void => console.log(message); + function normalizeCredentialValue(value: unknown): string { return String(value ?? "").trim(); } @@ -39,8 +49,8 @@ function getExplicitCompatibleCredential(credentialEnv: string | null | undefine return normalizeCredentialValue(process.env[credentialEnv]) || null; } -function printMissingBedrockAuth(): void { - console.error( +function printMissingBedrockAuth(error: (message: string) => void): void { + error( ` ${BEDROCK_RUNTIME_AWS_BEARER_TOKEN_ENV}, AWS_PROFILE, IAM environment credentials, or an explicitly exported Bedrock-compatible endpoint key is required for a Bedrock Runtime endpoint.`, ); } @@ -55,30 +65,34 @@ export function needsBedrockRuntimeAdapter(endpointUrl: string | null | undefine return Boolean(endpointUrl && isBedrockRuntimeEndpoint(endpointUrl)); } -export async function selectBedrockRuntimeCustomAnthropic(options: { - selectedKey: string; - endpointUrl: string | null; - credentialEnv: string | null; - label: string; - helpUrl: string | null; - defaultModel: string; - backToSelection: BackToSelection; - isNonInteractive: () => boolean; - promptInputModel: ( - label: string, - defaultModel: string, - validator: null, - ) => Promise; - replaceNamedCredential: ( - envName: string, - label: string, - helpUrl: string | null, - ) => Promise; -}): Promise< +export async function selectBedrockRuntimeCustomAnthropic( + options: { + selectedKey: string; + endpointUrl: string | null; + credentialEnv: string | null; + label: string; + helpUrl: string | null; + defaultModel: string; + backToSelection: BackToSelection; + isNonInteractive: () => boolean; + promptInputModel: ( + label: string, + defaultModel: string, + validator: null, + ) => Promise; + replaceNamedCredential: ( + envName: string, + label: string, + helpUrl: string | null, + ) => Promise; + } & BedrockRuntimeDependencies, +): Promise< | { action: "not-bedrock" } | { action: "retry-selection" } | { action: "selected"; model: string; preferredInferenceApi: "openai-completions" } > { + const error = options.error ?? defaultError; + const exitProcess = options.exitProcess ?? defaultExitProcess; if (options.selectedKey !== "anthropicCompatible" || !options.endpointUrl) { return { action: "not-bedrock" }; } @@ -88,8 +102,8 @@ export async function selectBedrockRuntimeCustomAnthropic(options: { const credentialEnv = options.credentialEnv || BEDROCK_RUNTIME_COMPATIBLE_CREDENTIAL_ENV; if (!hasBedrockRuntimeAwsAuthEnv() && !getExplicitCompatibleCredential(credentialEnv)) { if (options.isNonInteractive()) { - printMissingBedrockAuth(); - process.exit(1); + printMissingBedrockAuth(error); + return exitProcess(1); } const credentialResult = await options.replaceNamedCredential( credentialEnv, @@ -113,26 +127,31 @@ export async function selectBedrockRuntimeCustomAnthropic(options: { return { action: "selected", model, preferredInferenceApi: "openai-completions" }; } -export async function setupBedrockRuntimeInference(options: { - sandboxName: string | null; - provider: string; - model: string; - endpointUrl: string | null; - credentialEnv: string | null; - isNonInteractive: () => boolean; - runOpenshell: RunOpenshell; - upsertProvider: UpsertProvider; - verifyInferenceRoute: (provider: string, model: string) => void; - verifyOnboardInferenceSmoke: (options: { +export async function setupBedrockRuntimeInference( + options: { + sandboxName: string | null; provider: string; model: string; - endpointUrl?: string | null; - credentialEnv?: string | null; - forceOpenAiLike?: boolean; - }) => void; - ensureAdapter?: typeof ensureBedrockRuntimeAdapter; - updateSandbox?: typeof registry.updateSandbox; -}): Promise<{ handled: false } | { handled: true; result: SetupInferenceResult }> { + endpointUrl: string | null; + credentialEnv: string | null; + isNonInteractive: () => boolean; + runOpenshell: RunOpenshell; + upsertProvider: UpsertProvider; + verifyInferenceRoute: (provider: string, model: string) => void; + verifyOnboardInferenceSmoke: (options: { + provider: string; + model: string; + endpointUrl?: string | null; + credentialEnv?: string | null; + forceOpenAiLike?: boolean; + }) => void; + ensureAdapter?: typeof ensureBedrockRuntimeAdapter; + updateSandbox?: typeof registry.updateSandbox; + } & BedrockRuntimeDependencies, +): Promise<{ handled: false } | { handled: true; result: SetupInferenceResult }> { + const error = options.error ?? defaultError; + const exitProcess = options.exitProcess ?? defaultExitProcess; + const log = options.log ?? defaultLog; const classification = options.provider === "compatible-anthropic-endpoint" && options.endpointUrl ? classifyCustomAnthropicEndpoint(options.endpointUrl) @@ -142,8 +161,8 @@ export async function setupBedrockRuntimeInference(options: { const credentialEnv = options.credentialEnv || BEDROCK_RUNTIME_COMPATIBLE_CREDENTIAL_ENV; const compatibleCredential = getExplicitCompatibleCredential(credentialEnv); if (!hasBedrockRuntimeAwsAuthEnv() && !compatibleCredential) { - printMissingBedrockAuth(); - if (options.isNonInteractive()) process.exit(1); + printMissingBedrockAuth(error); + if (options.isNonInteractive()) return exitProcess(1); return { handled: true, result: { retry: "selection" } }; } @@ -154,10 +173,10 @@ export async function setupBedrockRuntimeInference(options: { compatibleCredential, }); } catch (err) { - console.error( + error( ` Failed to start Bedrock Runtime adapter: ${err instanceof Error ? err.message : String(err)}`, ); - if (options.isNonInteractive()) process.exit(1); + if (options.isNonInteractive()) return exitProcess(1); return { handled: true, result: { retry: "selection" } }; } @@ -169,11 +188,11 @@ export async function setupBedrockRuntimeInference(options: { { [adapter.credentialEnv]: adapter.token }, ); if (!providerResult.ok) { - console.error(` ${providerResult.message}`); - if (options.isNonInteractive()) process.exit(providerResult.status || 1); + error(` ${providerResult.message}`); + if (options.isNonInteractive()) return exitProcess(providerResult.status || 1); return { handled: true, result: { retry: "selection" } }; } - console.log( + log( ` Bedrock Runtime adapter ready: region ${adapter.region}, sandbox route ${adapter.baseUrl}, host log ${adapter.logPath}`, ); @@ -195,8 +214,8 @@ export async function setupBedrockRuntimeInference(options: { const message = compactText(redact(`${applyResult.stderr || ""} ${applyResult.stdout || ""}`)) || `Failed to configure inference provider '${options.provider}'.`; - console.error(` ${message}`); - if (options.isNonInteractive()) process.exit(applyResult.status || 1); + error(` ${message}`); + if (options.isNonInteractive()) return exitProcess(applyResult.status || 1); return { handled: true, result: { retry: "selection" } }; } @@ -214,6 +233,6 @@ export async function setupBedrockRuntimeInference(options: { provider: options.provider, }); } - console.log(` ✓ Inference route set: ${options.provider} / ${options.model}`); + log(` ✓ Inference route set: ${options.provider} / ${options.model}`); return { handled: true, result: { ok: true } }; } diff --git a/src/lib/onboard/inference-providers/remote.ts b/src/lib/onboard/inference-providers/remote.ts index c1eca66ae64..e9ebb227171 100644 --- a/src/lib/onboard/inference-providers/remote.ts +++ b/src/lib/onboard/inference-providers/remote.ts @@ -41,6 +41,8 @@ export async function setupRemoteProviderInference( isNonInteractive, registry, exitProcess, + error, + log, REMOTE_PROVIDER_CONFIG, hydrateCredentialEnv, promptValidationRecovery, @@ -71,6 +73,9 @@ export async function setupRemoteProviderInference( verifyInferenceRoute, verifyOnboardInferenceSmoke, updateSandbox: registry.updateSandbox, + exitProcess, + error, + log, }); if (bedrockSetup.handled) return { done: true, result: bedrockSetup.result }; while (true) { diff --git a/src/lib/onboard/inference-providers/types.ts b/src/lib/onboard/inference-providers/types.ts index b2c8222bfba..e564b10f3dc 100644 --- a/src/lib/onboard/inference-providers/types.ts +++ b/src/lib/onboard/inference-providers/types.ts @@ -91,6 +91,8 @@ export type CommonDeps = { }; export type RemoteProviderDeps = CommonDeps & { + error: (message: string) => void; + log: (message: string) => void; REMOTE_PROVIDER_CONFIG: Record; hydrateCredentialEnv: (envName: any, resolveCredential?: any) => any; promptValidationRecovery: PromptValidationRecovery; @@ -111,6 +113,9 @@ export type RemoteProviderDeps = CommonDeps & { verifyInferenceRoute: VerifyInferenceRoute; verifyOnboardInferenceSmoke: any; updateSandbox: Registry["updateSandbox"]; + exitProcess: CommonDeps["exitProcess"]; + error: (message: string) => void; + log: (message: string) => void; }): Promise<{ handled: true; result: SetupInferenceResult } | { handled: false }>; }; }; diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index 0932291989f..66570c2f741 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -129,6 +129,8 @@ export function createSetupInference( bedrockRuntimeOnboard: deps.bedrockRuntimeOnboard, redact: deps.redact, compactText: deps.compactText, + error: deps.error, + log: deps.log, }, ); if (outcome.done) return outcome.result; diff --git a/test/onboard-inference-failure-paths.test.ts b/test/onboard-inference-failure-paths.test.ts index e3f7f27f73b..a4006ab067f 100644 --- a/test/onboard-inference-failure-paths.test.ts +++ b/test/onboard-inference-failure-paths.test.ts @@ -18,6 +18,54 @@ const createDirectSetupInferenceHarness = createDirectSetupInferenceHarnessFacto ); type DirectSetupInferenceHarness = ReturnType; +type EnsureBedrockRuntimeAdapter = NonNullable< + Parameters[0]["ensureAdapter"] +>; + +const BEDROCK_ENDPOINT = "https://bedrock-runtime.us-east-1.amazonaws.com"; +const BEDROCK_CREDENTIAL_ENV = "COMPATIBLE_ANTHROPIC_API_KEY"; +const BEDROCK_MODEL = "anthropic.claude-3-5-sonnet-20240620-v1:0"; + +function createInjectedExit() { + return vi.fn((code: number): never => { + throw new Error(`EXIT_CALLED:${code}`); + }); +} + +function successfulBedrockAdapter() { + return { + baseUrl: "http://host.openshell.internal:11436/v1", + localBaseUrl: "http://127.0.0.1:11436/v1", + credentialEnv: "NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_TOKEN", + token: "adapter-token", + region: "us-east-1", + logPath: "/tmp/bedrock-adapter.log", + }; +} + +function withBedrockAdapter(ensureAdapter: EnsureBedrockRuntimeAdapter) { + return { + setupBedrockRuntimeInference: ( + input: Parameters[0], + ) => bedrockRuntimeOnboard.setupBedrockRuntimeInference({ ...input, ensureAdapter }), + }; +} + +function stubMissingBedrockAuth(): void { + for (const key of [ + "AWS_BEARER_TOKEN_BEDROCK", + "AWS_PROFILE", + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_WEB_IDENTITY_TOKEN_FILE", + "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", + "AWS_CONTAINER_CREDENTIALS_FULL_URI", + BEDROCK_CREDENTIAL_ENV, + ]) { + vi.stubEnv(key, ""); + } +} function stubProcessExit() { return vi.spyOn(process, "exit").mockImplementation(((code?: number) => { @@ -25,8 +73,11 @@ function stubProcessExit() { }) as typeof process.exit); } -function expectNoPostFailureSideEffects(harness: DirectSetupInferenceHarness): void { - expect(harness.commands.map(({ command }) => command)).toEqual(["gateway select nemoclaw"]); +function expectNoPostFailureSideEffects( + harness: DirectSetupInferenceHarness, + expectedCommands = ["gateway select nemoclaw"], +): void { + expect(harness.commands.map(({ command }) => command)).toEqual(expectedCommands); expect(harness.verifyInferenceRoute).not.toHaveBeenCalled(); expect(harness.verifyOnboardInferenceSmoke).not.toHaveBeenCalled(); expect(harness.updateSandbox).not.toHaveBeenCalled(); @@ -234,41 +285,273 @@ describe("setupInference dependency failures", () => { expectNoPostFailureSideEffects(harness); }); - it("returns to provider selection when the Bedrock adapter cannot start", async () => { - vi.stubEnv("COMPATIBLE_ANTHROPIC_API_KEY", "bedrock-bearer"); - const exit = stubProcessExit(); - const error = vi.spyOn(console, "error").mockImplementation(() => {}); + it("exits through the injected boundary when non-interactive Bedrock setup has no auth", async () => { + stubMissingBedrockAuth(); + const exitProcess = createInjectedExit(); + const ensureAdapter = vi.fn(async () => successfulBedrockAdapter()); + const upsertProvider = vi.fn(() => ({ ok: true })); + const harness = createDirectSetupInferenceHarness({ + overrides: { + isNonInteractive: () => true, + exitProcess, + upsertProvider, + bedrockRuntimeOnboard: withBedrockAdapter(ensureAdapter), + }, + }); + + await expect( + harness.setupInference( + "test-box", + BEDROCK_MODEL, + "compatible-anthropic-endpoint", + BEDROCK_ENDPOINT, + BEDROCK_CREDENTIAL_ENV, + ), + ).rejects.toThrow("EXIT_CALLED:1"); + + expect(exitProcess).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledWith(1); + expect(harness.errors).toContain( + " AWS_BEARER_TOKEN_BEDROCK, AWS_PROFILE, IAM environment credentials, or an explicitly exported Bedrock-compatible endpoint key is required for a Bedrock Runtime endpoint.", + ); + expect(harness.logs).toEqual([]); + expect(ensureAdapter).not.toHaveBeenCalled(); + expect(upsertProvider).not.toHaveBeenCalled(); + expectNoPostFailureSideEffects(harness); + }); + + it("returns to provider selection when the Bedrock adapter cannot start interactively", async () => { + vi.stubEnv(BEDROCK_CREDENTIAL_ENV, "bedrock-bearer"); + const exitProcess = createInjectedExit(); const ensureAdapter = vi.fn(async () => { throw new Error("adapter unavailable"); }); - const setupBedrockRuntimeInference = bedrockRuntimeOnboard.setupBedrockRuntimeInference; + const upsertProvider = vi.fn(() => ({ ok: true })); const harness = createDirectSetupInferenceHarness({ overrides: { - bedrockRuntimeOnboard: { - setupBedrockRuntimeInference: (input) => - setupBedrockRuntimeInference({ ...input, ensureAdapter }), - }, + exitProcess, + upsertProvider, + bedrockRuntimeOnboard: withBedrockAdapter(ensureAdapter), }, }); await expect( harness.setupInference( "test-box", - "anthropic.claude-3-5-sonnet-20240620-v1:0", + BEDROCK_MODEL, "compatible-anthropic-endpoint", - "https://bedrock-runtime.us-east-1.amazonaws.com", - "COMPATIBLE_ANTHROPIC_API_KEY", + BEDROCK_ENDPOINT, + BEDROCK_CREDENTIAL_ENV, ), ).resolves.toEqual({ retry: "selection" }); expect(ensureAdapter).toHaveBeenCalledOnce(); - expect(error).toHaveBeenCalledWith( + expect(upsertProvider).not.toHaveBeenCalled(); + expect(exitProcess).not.toHaveBeenCalled(); + expect(harness.errors).toContain( " Failed to start Bedrock Runtime adapter: adapter unavailable", ); - expect(exit).not.toHaveBeenCalled(); + expect(harness.logs).toEqual([]); expectNoPostFailureSideEffects(harness); }); + it("exits through the injected boundary when the Bedrock adapter cannot start", async () => { + vi.stubEnv("COMPATIBLE_ANTHROPIC_API_KEY", "bedrock-bearer"); + const exitProcess = createInjectedExit(); + const ensureAdapter = vi.fn(async () => { + throw new Error("adapter unavailable"); + }); + const upsertProvider = vi.fn(() => ({ ok: true })); + const harness = createDirectSetupInferenceHarness({ + overrides: { + isNonInteractive: () => true, + exitProcess, + upsertProvider, + bedrockRuntimeOnboard: withBedrockAdapter(ensureAdapter), + }, + }); + + await expect( + harness.setupInference( + "test-box", + BEDROCK_MODEL, + "compatible-anthropic-endpoint", + BEDROCK_ENDPOINT, + BEDROCK_CREDENTIAL_ENV, + ), + ).rejects.toThrow("EXIT_CALLED:1"); + + expect(ensureAdapter).toHaveBeenCalledOnce(); + expect(upsertProvider).not.toHaveBeenCalled(); + expect(exitProcess).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledWith(1); + expect(harness.errors).toContain( + " Failed to start Bedrock Runtime adapter: adapter unavailable", + ); + expect(harness.logs).toEqual([]); + expectNoPostFailureSideEffects(harness); + }); + + it("preserves the provider status through the injected Bedrock exit boundary", async () => { + vi.stubEnv(BEDROCK_CREDENTIAL_ENV, "bedrock-bearer"); + const exitProcess = createInjectedExit(); + const ensureAdapter = vi.fn(async () => successfulBedrockAdapter()); + const upsertProvider = vi.fn(() => ({ + ok: false, + status: 23, + message: "Bedrock provider registration failed", + })); + const harness = createDirectSetupInferenceHarness({ + overrides: { + isNonInteractive: () => true, + exitProcess, + upsertProvider, + bedrockRuntimeOnboard: withBedrockAdapter(ensureAdapter), + }, + }); + + await expect( + harness.setupInference( + "test-box", + BEDROCK_MODEL, + "compatible-anthropic-endpoint", + BEDROCK_ENDPOINT, + BEDROCK_CREDENTIAL_ENV, + ), + ).rejects.toThrow("EXIT_CALLED:23"); + + expect(ensureAdapter).toHaveBeenCalledOnce(); + expect(upsertProvider).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledWith(23); + expect(harness.errors).toContain(" Bedrock provider registration failed"); + expect(harness.logs).toEqual([]); + expectNoPostFailureSideEffects(harness); + }); + + it("falls back to status 1 when Bedrock provider registration returns status 0", async () => { + vi.stubEnv(BEDROCK_CREDENTIAL_ENV, "bedrock-bearer"); + const exitProcess = createInjectedExit(); + const ensureAdapter = vi.fn(async () => successfulBedrockAdapter()); + const upsertProvider = vi.fn(() => ({ + ok: false, + status: 0, + message: "Bedrock provider registration failed without status", + })); + const harness = createDirectSetupInferenceHarness({ + overrides: { + isNonInteractive: () => true, + exitProcess, + upsertProvider, + bedrockRuntimeOnboard: withBedrockAdapter(ensureAdapter), + }, + }); + + await expect( + harness.setupInference( + "test-box", + BEDROCK_MODEL, + "compatible-anthropic-endpoint", + BEDROCK_ENDPOINT, + BEDROCK_CREDENTIAL_ENV, + ), + ).rejects.toThrow("EXIT_CALLED:1"); + + expect(ensureAdapter).toHaveBeenCalledOnce(); + expect(upsertProvider).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledWith(1); + expect(harness.errors).toContain(" Bedrock provider registration failed without status"); + expect(harness.logs).toEqual([]); + expectNoPostFailureSideEffects(harness); + }); + + it("preserves the inference-set status through the injected Bedrock exit boundary", async () => { + vi.stubEnv(BEDROCK_CREDENTIAL_ENV, "bedrock-bearer"); + const exitProcess = createInjectedExit(); + const ensureAdapter = vi.fn(async () => successfulBedrockAdapter()); + const upsertProvider = vi.fn(() => ({ ok: true })); + const harness = createDirectSetupInferenceHarness({ + runOpenshell: (args) => + args.slice(0, 2).join(" ") === "inference set" + ? { status: 37, stdout: "", stderr: "route denied" } + : undefined, + overrides: { + isNonInteractive: () => true, + exitProcess, + upsertProvider, + bedrockRuntimeOnboard: withBedrockAdapter(ensureAdapter), + }, + }); + + await expect( + harness.setupInference( + "test-box", + BEDROCK_MODEL, + "compatible-anthropic-endpoint", + BEDROCK_ENDPOINT, + BEDROCK_CREDENTIAL_ENV, + ), + ).rejects.toThrow("EXIT_CALLED:37"); + + expect(ensureAdapter).toHaveBeenCalledOnce(); + expect(upsertProvider).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledWith(37); + expect(harness.errors).toContain(" route denied"); + expect(harness.logs).toEqual([ + " Bedrock Runtime adapter ready: region us-east-1, sandbox route http://host.openshell.internal:11436/v1, host log /tmp/bedrock-adapter.log", + ]); + expectNoPostFailureSideEffects(harness, [ + "gateway select nemoclaw", + `inference set --no-verify --provider compatible-anthropic-endpoint --model ${BEDROCK_MODEL} --timeout 180`, + ]); + }); + + it("falls back to status 1 and a generic error when Bedrock inference set has no status", async () => { + vi.stubEnv(BEDROCK_CREDENTIAL_ENV, "bedrock-bearer"); + const exitProcess = createInjectedExit(); + const ensureAdapter = vi.fn(async () => successfulBedrockAdapter()); + const upsertProvider = vi.fn(() => ({ ok: true })); + const harness = createDirectSetupInferenceHarness({ + runOpenshell: (args) => + args.slice(0, 2).join(" ") === "inference set" + ? { status: null, stdout: "", stderr: "" } + : undefined, + overrides: { + isNonInteractive: () => true, + exitProcess, + upsertProvider, + bedrockRuntimeOnboard: withBedrockAdapter(ensureAdapter), + }, + }); + + await expect( + harness.setupInference( + "test-box", + BEDROCK_MODEL, + "compatible-anthropic-endpoint", + BEDROCK_ENDPOINT, + BEDROCK_CREDENTIAL_ENV, + ), + ).rejects.toThrow("EXIT_CALLED:1"); + + expect(ensureAdapter).toHaveBeenCalledOnce(); + expect(upsertProvider).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledWith(1); + expect(harness.errors).toContain( + " Failed to configure inference provider 'compatible-anthropic-endpoint'.", + ); + expect(harness.logs).toEqual([ + " Bedrock Runtime adapter ready: region us-east-1, sandbox route http://host.openshell.internal:11436/v1, host log /tmp/bedrock-adapter.log", + ]); + expectNoPostFailureSideEffects(harness, [ + "gateway select nemoclaw", + `inference set --no-verify --provider compatible-anthropic-endpoint --model ${BEDROCK_MODEL} --timeout 180`, + ]); + }); + it("uses an injected Hermes DNS lookup before rejecting an unpinnable HTTPS endpoint", async () => { const exit = stubProcessExit(); const lookup = vi.fn>(async () => [ diff --git a/test/onboard.test.ts b/test/onboard.test.ts index 4ac8374e4bc..5e5ff69d299 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -793,7 +793,6 @@ startGateway(null).catch(() => {}); credentialEnv: "NEMOCLAW_BEDROCK_RUNTIME_ADAPTER_TOKEN", token: "adapter-token", region: "us-east-1", - compatibleCredential: "bedrock-bearer", logPath: "/tmp/bedrock-adapter.log", })); const setupBedrockRuntimeInference = bedrockRuntimeOnboard.setupBedrockRuntimeInference; @@ -805,7 +804,6 @@ startGateway(null).catch(() => {}); overrides: { updateSandbox, bedrockRuntimeOnboard: { - ...bedrockRuntimeOnboard, setupBedrockRuntimeInference: ( input: Parameters[0], ) => setupBedrockRuntimeInference({ ...input, ensureAdapter }), @@ -813,9 +811,7 @@ startGateway(null).catch(() => {}); }, }); const consoleOutput: string[] = []; - const captureConsole = (...args: unknown[]) => { - consoleOutput.push(args.map((arg) => String(arg)).join(" ")); - }; + const captureConsole = (...args: unknown[]) => consoleOutput.push(args.map(String).join(" ")); const error = vi.spyOn(console, "error").mockImplementation(captureConsole); const log = vi.spyOn(console, "log").mockImplementation(captureConsole); try { @@ -846,8 +842,13 @@ startGateway(null).catch(() => {}); !JSON.stringify(commands).includes("bedrock-bearer"), "Bedrock bearer token must not appear in OpenShell argv or env", ); + assert.deepEqual(harness.errors, []); + assert.deepEqual(harness.logs, [ + " Bedrock Runtime adapter ready: region us-east-1, sandbox route http://host.openshell.internal:11436/v1, host log /tmp/bedrock-adapter.log", + " ✓ Inference route set: compatible-anthropic-endpoint / anthropic.claude-3-5-sonnet-20240620-v1:0", + ]); assert.doesNotMatch( - consoleOutput.join("\n"), + [...harness.logs, ...harness.errors, ...consoleOutput].join("\n"), /bedrock-bearer|adapter-token/, "Bedrock tokens must not appear in onboarding console output", ); @@ -866,7 +867,6 @@ startGateway(null).catch(() => {}); commands.at(-1)?.command || "", /inference set --no-verify --provider compatible-anthropic-endpoint --model anthropic\.claude-3-5-sonnet-20240620-v1:0/, ); - expect(ensureAdapter).toHaveBeenCalled(); expect(updateSandbox).toHaveBeenCalledWith("test-box", { model: "anthropic.claude-3-5-sonnet-20240620-v1:0", provider: "compatible-anthropic-endpoint", From 5e70c700cbb9ad001f78cbae356d9b899a456a96 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 4 Jul 2026 11:36:51 -0700 Subject: [PATCH 11/16] fix(onboard): complete provider failure boundaries Signed-off-by: Carlos Villela --- .../rebuild-local-provider-recreate.test.ts | 2 + .../inference-providers/hermes.test.ts | 5 + src/lib/onboard/inference-providers/hermes.ts | 40 +++--- .../inference-providers/ollama-local.ts | 29 ++--- src/lib/onboard/inference-providers/remote.ts | 14 +- src/lib/onboard/inference-providers/routed.ts | 12 +- src/lib/onboard/inference-providers/types.ts | 4 +- .../onboard/inference-providers/vllm-local.ts | 12 +- src/lib/onboard/setup-inference.ts | 4 +- test/onboard-inference-failure-paths.test.ts | 123 ++++++++++++------ 10 files changed, 149 insertions(+), 96 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts b/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts index 264a152d38a..607feb3a699 100644 --- a/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts +++ b/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts @@ -59,6 +59,8 @@ const unusedCommonInferenceDeps = { verifyOnboardInferenceSmoke: vi.fn(), isNonInteractive: () => true, registry: { updateSandbox: vi.fn() }, + error: vi.fn(), + log: vi.fn(), exitProcess: (code: number): never => { throw new Error(`EXIT_CALLED:${code}`); }, diff --git a/src/lib/onboard/inference-providers/hermes.test.ts b/src/lib/onboard/inference-providers/hermes.test.ts index df0f7c28ee0..d0c2f7b6191 100644 --- a/src/lib/onboard/inference-providers/hermes.test.ts +++ b/src/lib/onboard/inference-providers/hermes.test.ts @@ -13,6 +13,11 @@ function makeDeps(overrides: Record = {}) { verifyOnboardInferenceSmoke: vi.fn(), isNonInteractive: vi.fn(() => false), registry: { updateSandbox: vi.fn() }, + exitProcess: vi.fn((code: number): never => { + throw new Error(`EXIT_CALLED:${code}`); + }), + error: vi.fn(), + log: vi.fn(), hermesProviderAuth: { isHermesProviderRegistered: vi.fn(() => true), ensureHermesProviderApiKeyCredentials: vi.fn(() => ({})), diff --git a/src/lib/onboard/inference-providers/hermes.ts b/src/lib/onboard/inference-providers/hermes.ts index 8a71e33db53..92c502101b7 100644 --- a/src/lib/onboard/inference-providers/hermes.ts +++ b/src/lib/onboard/inference-providers/hermes.ts @@ -4,8 +4,8 @@ // Hermes Provider inference setup flow. // Extracted verbatim from onboard.setupInference (#767). -import type { HermesAuthMethod } from "../hermes-auth"; import { rewriteConfigUrlsWithDnsPinning } from "../../sandbox/config"; +import type { HermesAuthMethod } from "../hermes-auth"; import type { HermesDeps, SetupInferenceResult } from "./types"; export async function setupHermesProviderInference( @@ -73,6 +73,9 @@ export async function setupHermesProviderInference( verifyOnboardInferenceSmoke, isNonInteractive, registry, + exitProcess, + error, + log, hermesProviderAuth, getHermesToolGatewayBroker, providerExistsInGateway, @@ -99,10 +102,10 @@ export async function setupHermesProviderInference( : HERMES_AUTH_METHOD_OAUTH); const providerStore = checkHermesProviderStoreReachable(runOpenshell); if (!providerStore.ok) { - console.error(" ✗ OpenShell provider storage is unreachable."); - console.error(` ${providerStore.message}`); - console.error(" Restart or recreate the OpenShell gateway, then rerun onboarding."); - if (isNonInteractive()) process.exit(1); + error(" ✗ OpenShell provider storage is unreachable."); + error(` ${providerStore.message}`); + error(" Restart or recreate the OpenShell gateway, then rerun onboarding."); + if (isNonInteractive()) return exitProcess(1); return { retry: "selection" }; } const providerRegistered = hermesProviderAuth.isHermesProviderRegistered(runOpenshell); @@ -120,8 +123,9 @@ export async function setupHermesProviderInference( hasFreshNousApiKey || (resolvedHermesAuthMethod === HERMES_AUTH_METHOD_OAUTH && !isNonInteractive()); if (shouldPrepareHermesCredentials) { + let state: unknown; try { - const state = + state = resolvedHermesAuthMethod === HERMES_AUTH_METHOD_API_KEY ? await hermesProviderAuth.ensureHermesProviderApiKeyCredentials(targetSandbox, { apiKey: resolveHermesNousApiKey(), @@ -134,23 +138,21 @@ export async function setupHermesProviderInference( baseUrl: resolvedEndpointUrl || undefined, toolGatewayPresets: hermesToolGateways, }); - if (!state) { - const authLabel = hermesAuthMethodLabel(resolvedHermesAuthMethod); - console.error(` ✗ Hermes Provider ${authLabel} is not available on the host.`); - console.error( - " Re-run `nemoclaw onboard --agent hermes` interactively to configure credentials.", - ); - process.exit(1); - } } catch (err) { - console.error( + error( ` ✗ Failed to prepare Hermes Provider credentials: ${ err instanceof Error ? err.message : String(err) }`, ); - if (isNonInteractive()) process.exit(1); + if (isNonInteractive()) return exitProcess(1); return { retry: "selection" }; } + if (!state) { + const authLabel = hermesAuthMethodLabel(resolvedHermesAuthMethod); + error(` ✗ Hermes Provider ${authLabel} is not available on the host.`); + error(" Re-run `nemoclaw onboard --agent hermes` interactively to configure credentials."); + return exitProcess(1); + } } const applyResult = runOpenshell( @@ -161,8 +163,8 @@ export async function setupHermesProviderInference( const message = compactText(redact(`${applyResult.stderr || ""} ${applyResult.stdout || ""}`)) || `Failed to configure inference provider '${provider}'.`; - console.error(` ${message}`); - if (isNonInteractive()) process.exit(applyResult.status || 1); + error(` ${message}`); + if (isNonInteractive()) return exitProcess(applyResult.status || 1); return { retry: "selection" }; } @@ -171,6 +173,6 @@ export async function setupHermesProviderInference( if (sandboxName) { registry.updateSandbox(sandboxName, { model, provider }); } - console.log(` ✓ Inference route set: ${provider} / ${model}`); + log(` ✓ Inference route set: ${provider} / ${model}`); return { ok: true }; } diff --git a/src/lib/onboard/inference-providers/ollama-local.ts b/src/lib/onboard/inference-providers/ollama-local.ts index 26518573163..95a44b70b60 100644 --- a/src/lib/onboard/inference-providers/ollama-local.ts +++ b/src/lib/onboard/inference-providers/ollama-local.ts @@ -25,6 +25,9 @@ export async function setupOllamaLocalInference( persistAndProbeOllamaProxy, localInference, OLLAMA_PROXY_CREDENTIAL_ENV, + exitProcess, + error, + log, } = deps; const validation = validateLocalProvider(provider); @@ -50,16 +53,14 @@ export async function setupOllamaLocalInference( "The sandbox uses a different network path and may work correctly.", ); } else { - console.error(` ${validation.message}`); + error(` ${validation.message}`); if (validation.diagnostic) { - console.error(` Diagnostic: ${validation.diagnostic}`); + error(` Diagnostic: ${validation.diagnostic}`); } if (process.platform === "darwin") { - console.error( - " On macOS, local inference also depends on OpenShell host routing support.", - ); + error(" On macOS, local inference also depends on OpenShell host routing support."); } - process.exit(1); + return exitProcess(1); } } const baseUrl = getLocalProviderBaseUrl(provider); @@ -69,10 +70,8 @@ export async function setupOllamaLocalInference( if (!proxyReady) ensureOllamaAuthProxy(); const proxyToken = getOllamaProxyToken(); if (!proxyToken) { - console.error( - " Ollama auth proxy token is not set. Re-run onboard to initialize the proxy.", - ); - process.exit(1); + error(" Ollama auth proxy token is not set. Re-run onboard to initialize the proxy."); + return exitProcess(1); } ollamaCredential = proxyToken; // Persist token now that ollama-local is confirmed as the provider. @@ -91,18 +90,18 @@ export async function setupOllamaLocalInference( { [OLLAMA_PROXY_CREDENTIAL_ENV]: ollamaCredential }, ); if (!providerResult.ok) { - console.error(` ${providerResult.message}`); - process.exit(providerResult.status || 1); + error(` ${providerResult.message}`); + return exitProcess(providerResult.status || 1); } if (await applyLocalInferenceRoute("ollama-local", model)) { return { done: true, result: { retry: "selection" } }; } - console.log(` Priming Ollama model: ${model}`); + log(` Priming Ollama model: ${model}`); run(getOllamaWarmupCommand(model), { ignoreError: true }); const probe = localInference.validateOllamaModelWithToolsOverride(model, allowToolsIncompatible); if (!probe.ok) { - console.error(` ${probe.message}`); - process.exit(1); + error(` ${probe.message}`); + return exitProcess(1); } // Do not mutate ~/.nemoclaw/credentials.json here: local Ollama now uses // OLLAMA_PROXY_CREDENTIAL_ENV, so any saved OPENAI_API_KEY remains available diff --git a/src/lib/onboard/inference-providers/remote.ts b/src/lib/onboard/inference-providers/remote.ts index e9ebb227171..1dd249be780 100644 --- a/src/lib/onboard/inference-providers/remote.ts +++ b/src/lib/onboard/inference-providers/remote.ts @@ -58,7 +58,7 @@ export async function setupRemoteProviderInference( ? REMOTE_PROVIDER_CONFIG.build : Object.values(REMOTE_PROVIDER_CONFIG).find((entry) => entry.providerName === provider); if (!config) { - console.error(` Unsupported provider configuration: ${provider}`); + error(` Unsupported provider configuration: ${provider}`); return exitProcess(1); } const bedrockSetup = await bedrockRuntimeOnboard.setupBedrockRuntimeInference({ @@ -119,9 +119,9 @@ export async function setupRemoteProviderInference( }; } if (!providerResult.ok) { - console.error(` ${providerResult.message}`); + error(` ${providerResult.message}`); if (isNonInteractive()) { - exitProcess(providerResult.status || 1); + return exitProcess(providerResult.status || 1); } const retry = await promptValidationRecovery( config.label, @@ -135,7 +135,7 @@ export async function setupRemoteProviderInference( if (retry === "selection" || retry === "model") { return { done: true, result: { retry: "selection" } }; } - exitProcess(providerResult.status || 1); + return exitProcess(providerResult.status || 1); } const argsv = ["inference", "set"]; if (config.skipVerify) { @@ -152,9 +152,9 @@ export async function setupRemoteProviderInference( const message = compactText(redact(`${applyResult.stderr || ""} ${applyResult.stdout || ""}`)) || `Failed to configure inference provider '${provider}'.`; - console.error(` ${message}`); + error(` ${message}`); if (isNonInteractive()) { - exitProcess(applyResult.status || 1); + return exitProcess(applyResult.status || 1); } const retry = await promptValidationRecovery( config.label, @@ -168,7 +168,7 @@ export async function setupRemoteProviderInference( if (retry === "selection" || retry === "model") { return { done: true, result: { retry: "selection" } }; } - exitProcess(applyResult.status || 1); + return exitProcess(applyResult.status || 1); } return { done: false }; } diff --git a/src/lib/onboard/inference-providers/routed.ts b/src/lib/onboard/inference-providers/routed.ts index 17359f86ec4..cd033c21ed5 100644 --- a/src/lib/onboard/inference-providers/routed.ts +++ b/src/lib/onboard/inference-providers/routed.ts @@ -22,6 +22,8 @@ export async function setupRoutedInference( reconcileModelRouter, routedInference, hydrateCredentialEnv, + exitProcess, + error, } = deps; // Blueprint profile provider (e.g., nvidia-router for the routed profile). @@ -29,18 +31,16 @@ export async function setupRoutedInference( try { await reconcileModelRouter(); } catch (err) { - console.error( - ` ✗ Failed to start model router: ${err instanceof Error ? err.message : String(err)}`, - ); - process.exit(1); + error(` ✗ Failed to start model router: ${err instanceof Error ? err.message : String(err)}`); + return exitProcess(1); } const routed = routedInference.upsertRoutedProvider(provider, endpointUrl, credentialEnv, { upsertProvider, hydrateCredentialEnv, }); if (!routed.ok) { - console.error(` ${routed.result.message}`); - process.exit(routed.result.status || 1); + error(` ${routed.result.message}`); + return exitProcess(routed.result.status || 1); } runOpenshell(["inference", "set", "--no-verify", "--provider", provider, "--model", model]); return { done: false }; diff --git a/src/lib/onboard/inference-providers/types.ts b/src/lib/onboard/inference-providers/types.ts index e564b10f3dc..dd6cf28eb75 100644 --- a/src/lib/onboard/inference-providers/types.ts +++ b/src/lib/onboard/inference-providers/types.ts @@ -88,11 +88,11 @@ export type CommonDeps = { isNonInteractive: () => boolean; registry: Registry; exitProcess: (code: number) => never; + error: (message: string) => void; + log: (message: string) => void; }; export type RemoteProviderDeps = CommonDeps & { - error: (message: string) => void; - log: (message: string) => void; REMOTE_PROVIDER_CONFIG: Record; hydrateCredentialEnv: (envName: any, resolveCredential?: any) => any; promptValidationRecovery: PromptValidationRecovery; diff --git a/src/lib/onboard/inference-providers/vllm-local.ts b/src/lib/onboard/inference-providers/vllm-local.ts index c7beb39da4f..5dfc4581bef 100644 --- a/src/lib/onboard/inference-providers/vllm-local.ts +++ b/src/lib/onboard/inference-providers/vllm-local.ts @@ -19,6 +19,8 @@ export async function setupVllmLocalInference( applyLocalInferenceRoute, run, VLLM_LOCAL_CREDENTIAL_ENV, + exitProcess, + error, } = deps; const validation = validateLocalProvider(provider); @@ -40,11 +42,11 @@ export async function setupVllmLocalInference( "The sandbox uses a different network path and may work correctly.", ); } else { - console.error(` ${validation.message}`); + error(` ${validation.message}`); if (validation.diagnostic) { - console.error(` Diagnostic: ${validation.diagnostic}`); + error(` Diagnostic: ${validation.diagnostic}`); } - process.exit(1); + return exitProcess(1); } } const baseUrl = getLocalProviderBaseUrl(provider); @@ -60,8 +62,8 @@ export async function setupVllmLocalInference( { [VLLM_LOCAL_CREDENTIAL_ENV]: "dummy" }, ); if (!providerResult.ok) { - console.error(` ${providerResult.message}`); - process.exit(providerResult.status || 1); + error(` ${providerResult.message}`); + return exitProcess(providerResult.status || 1); } if (await applyLocalInferenceRoute("vllm-local", model)) { return { done: true, result: { retry: "selection" } }; diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index 66570c2f741..eb0731c72d9 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -77,6 +77,8 @@ export function createSetupInference( isNonInteractive: deps.isNonInteractive, registry: { updateSandbox: deps.updateSandbox }, exitProcess: deps.exitProcess, + error: deps.error, + log: deps.log, }; if (provider === deps.hermesProviderAuth.HERMES_PROVIDER_NAME) { @@ -129,8 +131,6 @@ export function createSetupInference( bedrockRuntimeOnboard: deps.bedrockRuntimeOnboard, redact: deps.redact, compactText: deps.compactText, - error: deps.error, - log: deps.log, }, ); if (outcome.done) return outcome.result; diff --git a/test/onboard-inference-failure-paths.test.ts b/test/onboard-inference-failure-paths.test.ts index a4006ab067f..1ba6dae1e32 100644 --- a/test/onboard-inference-failure-paths.test.ts +++ b/test/onboard-inference-failure-paths.test.ts @@ -67,12 +67,6 @@ function stubMissingBedrockAuth(): void { } } -function stubProcessExit() { - return vi.spyOn(process, "exit").mockImplementation(((code?: number) => { - throw new Error(`EXIT_CALLED:${code ?? 0}`); - }) as typeof process.exit); -} - function expectNoPostFailureSideEffects( harness: DirectSetupInferenceHarness, expectedCommands = ["gateway select nemoclaw"], @@ -90,10 +84,7 @@ describe("setupInference dependency failures", () => { }); it("fails through the injected exit boundary when a known remote provider has no config", async () => { - const exitProcess = vi.fn((code: number): never => { - throw new Error(`EXIT_CALLED:${code}`); - }); - const error = vi.spyOn(console, "error").mockImplementation(() => {}); + const exitProcess = createInjectedExit(); const hydrateCredentialEnv = vi.fn(); const setupBedrockRuntimeInference = vi.fn(async () => ({ handled: false as const })); const harness = createDirectSetupInferenceHarness({ @@ -109,7 +100,8 @@ describe("setupInference dependency failures", () => { "EXIT_CALLED:1", ); - expect(error).toHaveBeenCalledWith(" Unsupported provider configuration: openai-api"); + expect(harness.errors).toEqual([" Unsupported provider configuration: openai-api"]); + expect(exitProcess).toHaveBeenCalledOnce(); expect(exitProcess).toHaveBeenCalledWith(1); expect(setupBedrockRuntimeInference).not.toHaveBeenCalled(); expect(hydrateCredentialEnv).not.toHaveBeenCalled(); @@ -117,8 +109,7 @@ describe("setupInference dependency failures", () => { }); it("fails closed before provider registration when local vLLM validation fails", async () => { - const exit = stubProcessExit(); - const error = vi.spyOn(console, "error").mockImplementation(() => {}); + const exitProcess = createInjectedExit(); const validateLocalProvider = vi.fn(() => ({ ok: false, message: "vLLM is unreachable", @@ -127,7 +118,7 @@ describe("setupInference dependency failures", () => { const getLocalProviderHealthCheck = vi.fn(() => ["curl", "-sf", "http://127.0.0.1:8000"]); const run = vi.fn(() => directRunResult({ status: 7 })); const harness = createDirectSetupInferenceHarness({ - overrides: { validateLocalProvider, getLocalProviderHealthCheck, run }, + overrides: { exitProcess, validateLocalProvider, getLocalProviderHealthCheck, run }, }); await expect(harness.setupInference("test-box", "meta-llama", "vllm-local")).rejects.toThrow( @@ -140,20 +131,24 @@ describe("setupInference dependency failures", () => { ignoreError: true, suppressOutput: true, }); - expect(exit).toHaveBeenCalledWith(1); - expect(error).toHaveBeenCalledWith(" vLLM is unreachable"); - expect(error).toHaveBeenCalledWith(" Diagnostic: container probe failed"); + expect(exitProcess).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledWith(1); + expect(harness.errors).toEqual([ + " vLLM is unreachable", + " Diagnostic: container probe failed", + ]); expectNoPostFailureSideEffects(harness); }); it("propagates local vLLM health-check errors before provider registration", async () => { - const exit = stubProcessExit(); + const exitProcess = createInjectedExit(); const run = vi.fn(() => directRunResult()); const getLocalProviderHealthCheck = vi.fn(() => { throw new Error("health probe exploded"); }); const harness = createDirectSetupInferenceHarness({ overrides: { + exitProcess, validateLocalProvider: () => ({ ok: false, message: "vLLM is unreachable" }), getLocalProviderHealthCheck, run, @@ -166,12 +161,12 @@ describe("setupInference dependency failures", () => { expect(getLocalProviderHealthCheck).toHaveBeenCalledWith("vllm-local"); expect(run).not.toHaveBeenCalled(); - expect(exit).not.toHaveBeenCalled(); + expect(exitProcess).not.toHaveBeenCalled(); expectNoPostFailureSideEffects(harness); }); it("propagates Ollama proxy startup errors before reading credentials", async () => { - const exit = stubProcessExit(); + const exitProcess = createInjectedExit(); const ensureOllamaAuthProxy = vi.fn(() => { throw new Error("proxy startup failed"); }); @@ -179,6 +174,7 @@ describe("setupInference dependency failures", () => { const persistAndProbeOllamaProxy = vi.fn(async () => {}); const harness = createDirectSetupInferenceHarness({ overrides: { + exitProcess, shouldFrontOllamaWithProxy: () => true, ensureOllamaAuthProxy, getOllamaProxyToken, @@ -193,13 +189,12 @@ describe("setupInference dependency failures", () => { expect(ensureOllamaAuthProxy).toHaveBeenCalledOnce(); expect(getOllamaProxyToken).not.toHaveBeenCalled(); expect(persistAndProbeOllamaProxy).not.toHaveBeenCalled(); - expect(exit).not.toHaveBeenCalled(); + expect(exitProcess).not.toHaveBeenCalled(); expectNoPostFailureSideEffects(harness); }); it("fails closed when the recovered Ollama proxy remains unhealthy", async () => { - const exit = stubProcessExit(); - const error = vi.spyOn(console, "error").mockImplementation(() => {}); + const exitProcess = createInjectedExit(); const ensureOllamaAuthProxy = vi.fn(); const isProxyHealthy = vi.fn(() => false); const getOllamaProxyToken = vi.fn(() => "unused-token"); @@ -212,6 +207,7 @@ describe("setupInference dependency failures", () => { diagnostic: "proxy probe failed", }), shouldFrontOllamaWithProxy: () => true, + exitProcess, ensureOllamaAuthProxy, isProxyHealthy, getOllamaProxyToken, @@ -227,21 +223,24 @@ describe("setupInference dependency failures", () => { expect(isProxyHealthy).toHaveBeenCalledOnce(); expect(getOllamaProxyToken).not.toHaveBeenCalled(); expect(persistAndProbeOllamaProxy).not.toHaveBeenCalled(); - expect(exit).toHaveBeenCalledWith(1); - expect(error).toHaveBeenCalledWith(" container cannot reach Ollama"); - expect(error).toHaveBeenCalledWith(" Diagnostic: proxy probe failed"); + expect(exitProcess).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledWith(1); + expect(harness.errors).toEqual([ + " container cannot reach Ollama", + " Diagnostic: proxy probe failed", + ]); expectNoPostFailureSideEffects(harness); }); it("fails closed when proxy-fronted Ollama has no credential token", async () => { - const exit = stubProcessExit(); - const error = vi.spyOn(console, "error").mockImplementation(() => {}); + const exitProcess = createInjectedExit(); const ensureOllamaAuthProxy = vi.fn(); const getOllamaProxyToken = vi.fn(() => null); const persistAndProbeOllamaProxy = vi.fn(async () => {}); const harness = createDirectSetupInferenceHarness({ overrides: { shouldFrontOllamaWithProxy: () => true, + exitProcess, ensureOllamaAuthProxy, getOllamaProxyToken, persistAndProbeOllamaProxy, @@ -255,20 +254,63 @@ describe("setupInference dependency failures", () => { expect(ensureOllamaAuthProxy).toHaveBeenCalledOnce(); expect(getOllamaProxyToken).toHaveBeenCalledOnce(); expect(persistAndProbeOllamaProxy).not.toHaveBeenCalled(); - expect(exit).toHaveBeenCalledWith(1); - expect(error).toHaveBeenCalledWith( + expect(exitProcess).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledWith(1); + expect(harness.errors).toEqual([ " Ollama auth proxy token is not set. Re-run onboard to initialize the proxy.", - ); + ]); + expectNoPostFailureSideEffects(harness); + }); + + it("exits through injected Hermes boundaries when provider storage is unavailable", async () => { + const exitProcess = createInjectedExit(); + const isHermesProviderRegistered = vi.fn(() => true); + const ensureHermesProviderApiKeyCredentials = vi.fn(async () => ({})); + const ensureHermesProviderOAuthCredentials = vi.fn(async () => ({})); + const checkHermesProviderStoreReachable = vi.fn(() => ({ + ok: false, + message: "provider store unavailable", + })); + const harness = createDirectSetupInferenceHarness({ + overrides: { + isNonInteractive: () => true, + exitProcess, + checkHermesProviderStoreReachable, + hermesProviderAuth: { + HERMES_PROVIDER_NAME: "hermes-provider", + isHermesProviderRegistered, + ensureHermesProviderApiKeyCredentials, + ensureHermesProviderOAuthCredentials, + }, + }, + }); + + await expect( + harness.setupInference("test-box", "moonshotai/kimi-k2.6", "hermes-provider"), + ).rejects.toThrow("EXIT_CALLED:1"); + + expect(checkHermesProviderStoreReachable).toHaveBeenCalledWith(harness.runOpenshell); + expect(isHermesProviderRegistered).not.toHaveBeenCalled(); + expect(ensureHermesProviderApiKeyCredentials).not.toHaveBeenCalled(); + expect(ensureHermesProviderOAuthCredentials).not.toHaveBeenCalled(); + expect(exitProcess).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledWith(1); + expect(harness.errors).toEqual([ + " ✗ OpenShell provider storage is unreachable.", + " provider store unavailable", + " Restart or recreate the OpenShell gateway, then rerun onboarding.", + ]); expectNoPostFailureSideEffects(harness); }); it("propagates Ollama proxy persistence errors before provider registration", async () => { - const exit = stubProcessExit(); + const exitProcess = createInjectedExit(); const persistAndProbeOllamaProxy = vi.fn(async () => { throw new Error("proxy persistence failed"); }); const harness = createDirectSetupInferenceHarness({ overrides: { + exitProcess, shouldFrontOllamaWithProxy: () => true, ensureOllamaAuthProxy: () => {}, getOllamaProxyToken: () => "proxy-token", @@ -281,7 +323,7 @@ describe("setupInference dependency failures", () => { ); expect(persistAndProbeOllamaProxy).toHaveBeenCalledWith("proxy-token"); - expect(exit).not.toHaveBeenCalled(); + expect(exitProcess).not.toHaveBeenCalled(); expectNoPostFailureSideEffects(harness); }); @@ -553,11 +595,11 @@ describe("setupInference dependency failures", () => { }); it("uses an injected Hermes DNS lookup before rejecting an unpinnable HTTPS endpoint", async () => { - const exit = stubProcessExit(); + const exitProcess = createInjectedExit(); const lookup = vi.fn>(async () => [ { address: "8.8.8.8", family: 4 }, ]); - const harness = createDirectSetupInferenceHarness({ overrides: { lookup } }); + const harness = createDirectSetupInferenceHarness({ overrides: { exitProcess, lookup } }); await expect( harness.setupInference( @@ -569,13 +611,12 @@ describe("setupInference dependency failures", () => { ).rejects.toThrow("DNS-backed HTTPS URLs are not supported"); expect(lookup).toHaveBeenCalledWith("api.public.example.test", { all: true }); - expect(exit).not.toHaveBeenCalled(); + expect(exitProcess).not.toHaveBeenCalled(); expectNoPostFailureSideEffects(harness); }); it("fails closed before routed-provider registration when model-router reconciliation fails", async () => { - const exit = stubProcessExit(); - const error = vi.spyOn(console, "error").mockImplementation(() => {}); + const exitProcess = createInjectedExit(); const reconcileModelRouter = vi.fn(async () => { throw new Error("router unavailable"); }); @@ -583,6 +624,7 @@ describe("setupInference dependency failures", () => { const harness = createDirectSetupInferenceHarness({ overrides: { isRoutedInferenceProvider: (provider) => provider === "nvidia-router", + exitProcess, reconcileModelRouter, routedInference: { upsertRoutedProvider }, }, @@ -600,8 +642,9 @@ describe("setupInference dependency failures", () => { expect(reconcileModelRouter).toHaveBeenCalledOnce(); expect(upsertRoutedProvider).not.toHaveBeenCalled(); - expect(exit).toHaveBeenCalledWith(1); - expect(error).toHaveBeenCalledWith(" ✗ Failed to start model router: router unavailable"); + expect(exitProcess).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledWith(1); + expect(harness.errors).toEqual([" ✗ Failed to start model router: router unavailable"]); expectNoPostFailureSideEffects(harness); }); }); From 2b62a6909ce46f0227a81a46dd0404974a7f35b5 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 4 Jul 2026 11:46:43 -0700 Subject: [PATCH 12/16] fix(onboard): complete inference failure seams Signed-off-by: Carlos Villela --- ci/test-file-size-budget.json | 2 +- src/lib/onboard.ts | 16 +--- src/lib/onboard/hermes-auth.test.ts | 82 ++++++++++++++++++++ src/lib/onboard/hermes-auth.ts | 29 +++++-- src/lib/onboard/inference-providers/types.ts | 2 +- src/lib/onboard/local-inference-route.ts | 2 +- src/lib/onboard/setup-inference.ts | 24 +++++- test/onboard.test.ts | 58 +++++--------- 8 files changed, 149 insertions(+), 66 deletions(-) create mode 100644 src/lib/onboard/hermes-auth.test.ts diff --git a/ci/test-file-size-budget.json b/ci/test-file-size-budget.json index a36632c20fe..fab2422cfc8 100644 --- a/ci/test-file-size-budget.json +++ b/ci/test-file-size-budget.json @@ -11,7 +11,7 @@ "test/nemoclaw-start.test.ts": 4827, "test/onboard-messaging.test.ts": 2062, "test/onboard-selection.test.ts": 6146, - "test/onboard.test.ts": 4079, + "test/onboard.test.ts": 4057, "test/policies.test.ts": 2332 } } diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index be689010ef2..67d232612ac 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -395,9 +395,6 @@ const { const { createValidationRecoveryPromptHelpers, }: typeof import("./onboard/validation-recovery-prompt") = require("./onboard/validation-recovery-prompt"); -const { - createLocalInferenceRouteApplier, -}: typeof import("./onboard/local-inference-route") = require("./onboard/local-inference-route"); const { createOpenshellCliHelpers, }: typeof import("./onboard/openshell-cli") = require("./onboard/openshell-cli"); @@ -839,6 +836,8 @@ const { checkHermesProviderStoreReachable, } = hermesAuth.createHermesAuthHelpers({ isNonInteractive, + error: (message) => console.error(message), + exitProcess: (code) => process.exit(code), note, prompt, getNavigationChoice, @@ -860,16 +859,6 @@ const { promptValidationRecovery } = createValidationRecoveryPromptHelpers({ exitOnboardFromPrompt, }); -const applyLocalInferenceRoute = createLocalInferenceRouteApplier({ - runOpenshell, - isNonInteractive, - promptValidationRecovery, - classifyApplyFailure, - compactText, - redact, - localInferenceTimeoutSecs: LOCAL_INFERENCE_TIMEOUT_SECS, -}); - // Provider CRUD — thin wrappers that inject runOpenshell to avoid circular deps. const { buildProviderArgs } = onboardProviders; @@ -4198,7 +4187,6 @@ function getSetupInferenceDeps() { validateLocalProvider, getLocalProviderHealthCheck, getLocalProviderBaseUrl, - applyLocalInferenceRoute, run, vllmLocalCredentialEnv: VLLM_LOCAL_CREDENTIAL_ENV, getOllamaWarmupCommand, diff --git a/src/lib/onboard/hermes-auth.test.ts b/src/lib/onboard/hermes-auth.test.ts new file mode 100644 index 00000000000..9fb60693301 --- /dev/null +++ b/src/lib/onboard/hermes-auth.test.ts @@ -0,0 +1,82 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { + createHermesAuthHelpers, + HERMES_NOUS_API_KEY_CREDENTIAL_ENV, + type HermesAuthFlowDeps, +} from "./hermes-auth"; + +function createDeps(overrides: Partial = {}): HermesAuthFlowDeps { + return { + isNonInteractive: vi.fn(() => true), + note: vi.fn(), + prompt: vi.fn(async () => ""), + getNavigationChoice: vi.fn(() => null), + exitOnboardFromPrompt: vi.fn((): never => { + throw new Error("PROMPT_EXIT_CALLED"); + }), + validateNvidiaApiKeyValue: vi.fn(() => null), + compactText: vi.fn((value: string) => value), + redact: vi.fn((value: unknown) => String(value)), + runOpenshell: vi.fn(() => ({ status: 0, stdout: "", stderr: "" })), + error: vi.fn(), + exitProcess: vi.fn((code: number): never => { + throw new Error(`EXIT_CALLED:${code}`); + }), + backToSelection: Symbol("back-to-selection"), + ...overrides, + }; +} + +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); +}); + +describe("Hermes authentication exit boundaries", () => { + it("uses the injected exit for an unsupported requested auth method", async () => { + vi.stubEnv("NEMOCLAW_HERMES_AUTH_METHOD", "certificate"); + const deps = createDeps(); + + await expect(createHermesAuthHelpers(deps).promptHermesAuthMethod()).rejects.toThrow( + "EXIT_CALLED:1", + ); + + expect(deps.error).toHaveBeenCalledTimes(2); + expect(vi.mocked(deps.error).mock.calls).toEqual([ + [" Unsupported Hermes Provider auth method: certificate"], + [" Valid values: oauth, nous-portal-oauth, api-key, nous-api-key"], + ]); + expect(deps.exitProcess).toHaveBeenCalledOnce(); + expect(deps.exitProcess).toHaveBeenCalledWith(1); + expect(deps.note).not.toHaveBeenCalled(); + }); + + it("uses the injected exit when a prompted Nous API key is invalid", async () => { + vi.stubEnv(HERMES_NOUS_API_KEY_CREDENTIAL_ENV, undefined); + vi.stubEnv("NEMOCLAW_PROVIDER_KEY", undefined); + vi.spyOn(console, "log").mockImplementation(() => undefined); + const deps = createDeps({ + isNonInteractive: vi.fn(() => false), + prompt: vi.fn(async () => "invalid-key"), + validateNvidiaApiKeyValue: vi.fn(() => " Invalid NOUS_API_KEY value."), + }); + + await expect(createHermesAuthHelpers(deps).ensureHermesNousApiKeyEnv()).rejects.toThrow( + "EXIT_CALLED:1", + ); + + expect(deps.validateNvidiaApiKeyValue).toHaveBeenCalledWith( + "invalid-key", + HERMES_NOUS_API_KEY_CREDENTIAL_ENV, + ); + expect(deps.error).toHaveBeenCalledOnce(); + expect(deps.error).toHaveBeenCalledWith(" Invalid NOUS_API_KEY value."); + expect(deps.exitProcess).toHaveBeenCalledOnce(); + expect(deps.exitProcess).toHaveBeenCalledWith(1); + expect(process.env[HERMES_NOUS_API_KEY_CREDENTIAL_ENV]).toBeUndefined(); + }); +}); diff --git a/src/lib/onboard/hermes-auth.ts b/src/lib/onboard/hermes-auth.ts index 8559f9632f9..72e8ba0ca15 100644 --- a/src/lib/onboard/hermes-auth.ts +++ b/src/lib/onboard/hermes-auth.ts @@ -40,7 +40,17 @@ export function hermesAuthMethodLabel(method: HermesAuthMethod | null | undefine return method === HERMES_AUTH_METHOD_API_KEY ? "Nous API Key" : "Nous Portal OAuth"; } -export function getRequestedHermesAuthMethod(): HermesAuthMethod | null { +export interface HermesAuthFailureBoundary { + error(message: string): void; + exitProcess(code: number): never; +} + +export function getRequestedHermesAuthMethod( + boundary: HermesAuthFailureBoundary = { + error: (message) => console.error(message), + exitProcess: (code) => process.exit(code), + }, +): HermesAuthMethod | null { const raw = process.env.NEMOCLAW_HERMES_AUTH_METHOD || process.env.NEMOCLAW_HERMES_AUTH || @@ -48,9 +58,9 @@ export function getRequestedHermesAuthMethod(): HermesAuthMethod | null { ""; const method = normalizeHermesAuthMethod(raw); if (!raw || method) return method; - console.error(` Unsupported Hermes Provider auth method: ${raw}`); - console.error(" Valid values: oauth, nous-portal-oauth, api-key, nous-api-key"); - process.exit(1); + boundary.error(` Unsupported Hermes Provider auth method: ${raw}`); + boundary.error(" Valid values: oauth, nous-portal-oauth, api-key, nous-api-key"); + boundary.exitProcess(1); } export interface HermesAuthFlowDeps { @@ -70,6 +80,8 @@ export interface HermesAuthFlowDeps { stdout?: string | Buffer | null; stderr?: string | Buffer | null; }; + error(message: string): void; + exitProcess(code: number): never; backToSelection: unknown; } @@ -96,7 +108,10 @@ export function createHermesAuthHelpers(deps: HermesAuthFlowDeps): HermesAuthHel label: "Nous API Key (paste a key from the provider dashboard)", }, ]; - const requested = getRequestedHermesAuthMethod(); + const requested = getRequestedHermesAuthMethod({ + error: deps.error, + exitProcess: deps.exitProcess, + }); if (deps.isNonInteractive()) { const method = requested || @@ -156,8 +171,8 @@ export function createHermesAuthHelpers(deps: HermesAuthFlowDeps): HermesAuthHel const key = normalizeCredentialValue(rawKey); const validationError = deps.validateNvidiaApiKeyValue(key, HERMES_NOUS_API_KEY_CREDENTIAL_ENV); if (validationError) { - console.error(validationError); - process.exit(1); + deps.error(validationError); + deps.exitProcess(1); } process.env[HERMES_NOUS_API_KEY_CREDENTIAL_ENV] = key; return key; diff --git a/src/lib/onboard/inference-providers/types.ts b/src/lib/onboard/inference-providers/types.ts index dd6cf28eb75..7f95b3d8b28 100644 --- a/src/lib/onboard/inference-providers/types.ts +++ b/src/lib/onboard/inference-providers/types.ts @@ -72,7 +72,7 @@ export type PromptValidationRecovery = ( classification: any, credentialEnv: any, helpUrl: any, -) => Promise; +) => Promise<"credential" | "selection" | "retry" | "model">; export type ClassifyApplyFailure = (message: string) => any; diff --git a/src/lib/onboard/local-inference-route.ts b/src/lib/onboard/local-inference-route.ts index f6333d4330a..96f01aa8e75 100644 --- a/src/lib/onboard/local-inference-route.ts +++ b/src/lib/onboard/local-inference-route.ts @@ -70,7 +70,7 @@ export function createLocalInferenceRouteApplier(deps: LocalInferenceRouteDeps) " No sandbox was created. Fix the inference route and re-run " + "`nemoclaw onboard --resume` to continue, or choose a different provider/model.", ); - exitProcess(applyResult.status || 1); + return exitProcess(applyResult.status || 1); } const retry = await deps.promptValidationRecovery( label, diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index eb0731c72d9..aaa1061321d 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -12,6 +12,7 @@ import type { VllmDeps, } from "./inference-providers"; import * as inferenceProviders from "./inference-providers"; +import { createLocalInferenceRouteApplier } from "./local-inference-route"; import type { ProviderInferenceSetupOptions } from "./machine/handlers/provider-inference"; type ProviderBranchDeps = Omit< @@ -19,6 +20,7 @@ type ProviderBranchDeps = Omit< | "registry" | "run" | "runOpenshell" + | "applyLocalInferenceRoute" | "LOCAL_INFERENCE_TIMEOUT_SECS" | "VLLM_LOCAL_CREDENTIAL_ENV" | "OLLAMA_PROXY_CREDENTIAL_ENV" @@ -34,11 +36,29 @@ export type SetupInferenceDeps = ProviderBranchDeps & { vllmLocalCredentialEnv: string; ollamaProxyCredentialEnv: string; isRoutedInferenceProvider: (provider: string) => boolean; + applyLocalInferenceRoute?: VllmDeps["applyLocalInferenceRoute"]; log: (message: string) => void; error: (message: string) => void; exitProcess: (code: number) => never; }; +function resolveLocalInferenceRouteApplier(deps: SetupInferenceDeps) { + return ( + deps.applyLocalInferenceRoute ?? + createLocalInferenceRouteApplier({ + runOpenshell: deps.runOpenshell, + isNonInteractive: deps.isNonInteractive, + promptValidationRecovery: deps.promptValidationRecovery, + classifyApplyFailure: deps.classifyApplyFailure, + compactText: deps.compactText, + redact: deps.redact, + localInferenceTimeoutSecs: deps.localInferenceTimeoutSecs, + error: deps.error, + exitProcess: deps.exitProcess, + }) + ); +} + export type SetupInference = ( sandboxName: string | null, model: string, @@ -142,7 +162,7 @@ export function createSetupInference( validateLocalProvider: deps.validateLocalProvider, getLocalProviderHealthCheck: deps.getLocalProviderHealthCheck, getLocalProviderBaseUrl: deps.getLocalProviderBaseUrl, - applyLocalInferenceRoute: deps.applyLocalInferenceRoute, + applyLocalInferenceRoute: resolveLocalInferenceRouteApplier(deps), run: deps.run, VLLM_LOCAL_CREDENTIAL_ENV: deps.vllmLocalCredentialEnv, }, @@ -155,7 +175,7 @@ export function createSetupInference( ...commonDeps, validateLocalProvider: deps.validateLocalProvider, getLocalProviderBaseUrl: deps.getLocalProviderBaseUrl, - applyLocalInferenceRoute: deps.applyLocalInferenceRoute, + applyLocalInferenceRoute: resolveLocalInferenceRouteApplier(deps), getOllamaWarmupCommand: deps.getOllamaWarmupCommand, run: deps.run, shouldFrontOllamaWithProxy: deps.shouldFrontOllamaWithProxy, diff --git a/test/onboard.test.ts b/test/onboard.test.ts index 5e5ff69d299..816380c76aa 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -1255,22 +1255,9 @@ const { onboard } = require(${onboardPath}); ); }); it("surfaces a contextual error and exits when ollama-local inference set fails after the proxy-ready warning (#4257)", async () => { - const errLog: string[] = []; - let exitCode: number | null = null; - let harness: ReturnType; - const applyLocalInferenceRoute = createLocalInferenceRouteApplier({ - runOpenshell: (args, options) => harness.runOpenshell(args, options), - isNonInteractive: () => true, - promptValidationRecovery: async () => "selection", - classifyApplyFailure: () => ({}) as never, - compactText: (value) => value.trim(), - redact: (value) => value, - localInferenceTimeoutSecs: 120, - error: (message) => errLog.push(message), - exitProcess: (code): never => { - exitCode = code; - throw Object.assign(new Error(`EXIT_CALLED:${code}`), { __exit: true }); - }, + const error = vi.fn(); + const exitProcess = vi.fn((code: number): never => { + throw Object.assign(new Error(`EXIT_CALLED:${code}`), { __exit: true }); }); const commandRouter = createDirectCommandRouter([ { @@ -1284,7 +1271,7 @@ const { onboard } = require(${onboardPath}); results: [{ status: 7, stdout: "", stderr: "openshell: route apply failed" }], }, ]); - harness = createDirectSetupInferenceHarness({ + const harness = createDirectSetupInferenceHarness({ runOpenshell: commandRouter.runOpenshell, overrides: { isNonInteractive: () => true, @@ -1298,7 +1285,9 @@ const { onboard } = require(${onboardPath}); isProxyHealthy: () => true, getOllamaProxyToken: () => "proxy-token", persistAndProbeOllamaProxy: async () => {}, - applyLocalInferenceRoute, + applyLocalInferenceRoute: undefined, + error, + exitProcess, }, }); const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); @@ -1320,28 +1309,15 @@ const { onboard } = require(${onboardPath}); true, "ollama-local inference set must use ignoreError so onboard can recover", ); - const combinedErr = errLog.join("\n"); + const combinedErr = error.mock.calls.flat().join("\n"); + assert.equal(exitProcess.mock.calls.length, 1); + assert.equal(exitProcess.mock.calls[0]?.[0], 7); assert.match(combinedErr, /No sandbox was created/); assert.match(combinedErr, /nemoclaw onboard --resume/); - assert.equal(exitCode, 7, "non-interactive onboard must exit with the openshell status"); }); it("surfaces a contextual error and exits when vllm-local inference set fails (#4257)", async () => { - const errLog: string[] = []; - let exitCode: number | null = null; - let harness: ReturnType; - const applyLocalInferenceRoute = createLocalInferenceRouteApplier({ - runOpenshell: (args, options) => harness.runOpenshell(args, options), - isNonInteractive: () => true, - promptValidationRecovery: async () => "selection", - classifyApplyFailure: () => ({}) as never, - compactText: (value) => value.trim(), - redact: (value) => value, - localInferenceTimeoutSecs: 120, - error: (message) => errLog.push(message), - exitProcess: (code): never => { - exitCode = code; - throw Object.assign(new Error(`EXIT_CALLED:${code}`), { __exit: true }); - }, + const exitProcess = vi.fn((code: number): never => { + throw Object.assign(new Error(`EXIT_CALLED:${code}`), { __exit: true }); }); const commandRouter = createDirectCommandRouter([ { @@ -1355,11 +1331,12 @@ const { onboard } = require(${onboardPath}); results: [{ status: 13, stdout: "", stderr: "openshell: vllm route apply failed" }], }, ]); - harness = createDirectSetupInferenceHarness({ + const harness = createDirectSetupInferenceHarness({ runOpenshell: commandRouter.runOpenshell, overrides: { isNonInteractive: () => true, - applyLocalInferenceRoute, + applyLocalInferenceRoute: undefined, + exitProcess, }, }); @@ -1377,10 +1354,11 @@ const { onboard } = require(${onboardPath}); true, "vllm-local inference set must use ignoreError so onboard can recover", ); - const combinedErr = errLog.join("\n"); + const combinedErr = harness.errors.join("\n"); + assert.equal(exitProcess.mock.calls.length, 1); + assert.equal(exitProcess.mock.calls[0]?.[0], 13); assert.match(combinedErr, /No sandbox was created/); assert.match(combinedErr, /nemoclaw onboard --resume/); - assert.equal(exitCode, 13, "non-interactive onboard must exit with the openshell status"); }); it("detects when the live inference route already matches the requested provider and model", () => { const repoRoot = path.join(import.meta.dirname, ".."); From 256067f3b0e87a0c6097030964caaa2a27ddbd0f Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 4 Jul 2026 11:50:23 -0700 Subject: [PATCH 13/16] test(onboard): strengthen failure boundary coverage Signed-off-by: Carlos Villela --- ci/test-file-size-budget.json | 2 +- test/generate-openclaw-config.test.ts | 1 - test/onboard-inference-failure-paths.test.ts | 279 +++++++++++++++++++ 3 files changed, 280 insertions(+), 2 deletions(-) diff --git a/ci/test-file-size-budget.json b/ci/test-file-size-budget.json index fab2422cfc8..efa6090688e 100644 --- a/ci/test-file-size-budget.json +++ b/ci/test-file-size-budget.json @@ -6,7 +6,7 @@ "src/lib/inference/nim.test.ts": 2068, "src/lib/onboard/preflight.test.ts": 1904, "test/channels-add-preset.test.ts": 1871, - "test/generate-openclaw-config.test.ts": 1946, + "test/generate-openclaw-config.test.ts": 1945, "test/install-preflight.test.ts": 3934, "test/nemoclaw-start.test.ts": 4827, "test/onboard-messaging.test.ts": 2062, diff --git a/test/generate-openclaw-config.test.ts b/test/generate-openclaw-config.test.ts index fdac37b7522..3906a65ba8c 100644 --- a/test/generate-openclaw-config.test.ts +++ b/test/generate-openclaw-config.test.ts @@ -1,4 +1,3 @@ -// @ts-nocheck // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 // diff --git a/test/onboard-inference-failure-paths.test.ts b/test/onboard-inference-failure-paths.test.ts index 1ba6dae1e32..4854bab0230 100644 --- a/test/onboard-inference-failure-paths.test.ts +++ b/test/onboard-inference-failure-paths.test.ts @@ -4,6 +4,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import type { SetupInference, SetupInferenceDeps } from "../src/lib/onboard/setup-inference.js"; import { + createDirectCommandRouter, createDirectSetupInferenceHarnessFactory, directRunResult, } from "./support/setup-inference-test-harness.js"; @@ -108,6 +109,124 @@ describe("setupInference dependency failures", () => { expectNoPostFailureSideEffects(harness); }); + it("fails through the injected exit boundary when a remote credential is missing", async () => { + const exitProcess = createInjectedExit(); + const hydrateCredentialEnv = vi.fn(() => null); + const upsertProvider = vi.fn(() => ({ ok: true })); + const promptValidationRecovery = vi.fn(async () => "selection" as const); + const setupBedrockRuntimeInference = vi.fn(async () => ({ handled: false as const })); + const harness = createDirectSetupInferenceHarness({ + overrides: { + isNonInteractive: () => true, + exitProcess, + hydrateCredentialEnv, + upsertProvider, + promptValidationRecovery, + bedrockRuntimeOnboard: { setupBedrockRuntimeInference }, + }, + }); + + await expect(harness.setupInference("test-box", "gpt-test", "openai-api")).rejects.toThrow( + "EXIT_CALLED:1", + ); + + expect(setupBedrockRuntimeInference).toHaveBeenCalledOnce(); + expect(hydrateCredentialEnv).toHaveBeenCalledWith("OPENAI_API_KEY"); + expect(upsertProvider).not.toHaveBeenCalled(); + expect(promptValidationRecovery).not.toHaveBeenCalled(); + expect(exitProcess).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledWith(1); + expect(harness.errors).toEqual([ + " A host credential is required to configure provider 'openai-api'.", + ]); + expectNoPostFailureSideEffects(harness); + }); + + it("preserves a remote provider upsert status through the injected exit boundary", async () => { + const exitProcess = createInjectedExit(); + const hydrateCredentialEnv = vi.fn(() => "openai-secret"); + const upsertProvider = vi.fn(() => ({ + ok: false, + status: 23, + message: "remote provider registration rejected", + })); + const promptValidationRecovery = vi.fn(async () => "selection" as const); + const setupBedrockRuntimeInference = vi.fn(async () => ({ handled: false as const })); + const harness = createDirectSetupInferenceHarness({ + overrides: { + isNonInteractive: () => true, + exitProcess, + hydrateCredentialEnv, + upsertProvider, + promptValidationRecovery, + bedrockRuntimeOnboard: { setupBedrockRuntimeInference }, + }, + }); + + await expect(harness.setupInference("test-box", "gpt-test", "openai-api")).rejects.toThrow( + "EXIT_CALLED:23", + ); + + expect(setupBedrockRuntimeInference).toHaveBeenCalledOnce(); + expect(hydrateCredentialEnv).toHaveBeenCalledWith("OPENAI_API_KEY"); + expect(upsertProvider).toHaveBeenCalledOnce(); + expect(upsertProvider).toHaveBeenCalledWith( + "openai-api", + "openai", + "OPENAI_API_KEY", + expect.any(String), + { OPENAI_API_KEY: "openai-secret" }, + ); + expect(promptValidationRecovery).not.toHaveBeenCalled(); + expect(exitProcess).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledWith(23); + expect(harness.errors).toEqual([" remote provider registration rejected"]); + expectNoPostFailureSideEffects(harness); + }); + + it("redacts a remote inference-set failure and preserves its status at the exit boundary", async () => { + const exitProcess = createInjectedExit(); + const hydrateCredentialEnv = vi.fn(() => "openai-secret"); + const upsertProvider = vi.fn(() => ({ ok: true })); + const promptValidationRecovery = vi.fn(async () => "selection" as const); + const setupBedrockRuntimeInference = vi.fn(async () => ({ handled: false as const })); + const commandRouter = createDirectCommandRouter([ + { + name: "remote-inference-set", + matches: (command) => command.startsWith("inference set"), + results: [{ status: 37, stdout: "", stderr: "route failed nvapi-1234567890abcdef" }], + }, + ]); + const harness = createDirectSetupInferenceHarness({ + runOpenshell: commandRouter.runOpenshell, + overrides: { + isNonInteractive: () => true, + exitProcess, + hydrateCredentialEnv, + upsertProvider, + promptValidationRecovery, + bedrockRuntimeOnboard: { setupBedrockRuntimeInference }, + }, + }); + + await expect(harness.setupInference("test-box", "gpt-test", "openai-api")).rejects.toThrow( + "EXIT_CALLED:37", + ); + + expect(setupBedrockRuntimeInference).toHaveBeenCalledOnce(); + expect(upsertProvider).toHaveBeenCalledOnce(); + expect(commandRouter.callCount("remote-inference-set")).toBe(1); + expect(promptValidationRecovery).not.toHaveBeenCalled(); + expect(exitProcess).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledWith(37); + expect(harness.errors.join("\n")).toContain("route failed"); + expect(harness.errors.join("\n")).not.toContain("nvapi-1234567890abcdef"); + expectNoPostFailureSideEffects(harness, [ + "gateway select nemoclaw", + "inference set --no-verify --provider openai-api --model gpt-test", + ]); + }); + it("fails closed before provider registration when local vLLM validation fails", async () => { const exitProcess = createInjectedExit(); const validateLocalProvider = vi.fn(() => ({ @@ -303,6 +422,120 @@ describe("setupInference dependency failures", () => { expectNoPostFailureSideEffects(harness); }); + it("exits through injected boundaries when Hermes API-key preparation throws", async () => { + const exitProcess = createInjectedExit(); + const isHermesProviderRegistered = vi.fn(() => false); + const ensureHermesProviderApiKeyCredentials = vi.fn(async () => { + throw new Error("API-key preparation failed"); + }); + const ensureHermesProviderOAuthCredentials = vi.fn(async () => ({})); + const providerExistsInGateway = vi.fn(() => true); + const resolveHermesNousApiKey = vi.fn(() => "nous-secret"); + const checkHermesProviderStoreReachable = vi.fn(() => ({ ok: true })); + const harness = createDirectSetupInferenceHarness({ + overrides: { + isNonInteractive: () => true, + exitProcess, + normalizeHermesAuthMethod: () => "api_key", + providerExistsInGateway, + resolveHermesNousApiKey, + checkHermesProviderStoreReachable, + hermesProviderAuth: { + HERMES_PROVIDER_NAME: "hermes-provider", + isHermesProviderRegistered, + ensureHermesProviderApiKeyCredentials, + ensureHermesProviderOAuthCredentials, + }, + }, + }); + + await expect( + harness.setupInference( + "test-box", + "moonshotai/kimi-k2.6", + "hermes-provider", + null, + "NOUS_API_KEY", + "api-key", + ), + ).rejects.toThrow("EXIT_CALLED:1"); + + expect(checkHermesProviderStoreReachable).toHaveBeenCalledWith(harness.runOpenshell); + expect(isHermesProviderRegistered).toHaveBeenCalledWith(harness.runOpenshell); + expect(providerExistsInGateway).not.toHaveBeenCalled(); + expect(ensureHermesProviderApiKeyCredentials).toHaveBeenCalledOnce(); + expect(ensureHermesProviderApiKeyCredentials).toHaveBeenCalledWith("test-box", { + apiKey: "nous-secret", + runOpenshell: harness.runOpenshell, + baseUrl: undefined, + }); + expect(ensureHermesProviderOAuthCredentials).not.toHaveBeenCalled(); + expect(exitProcess).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledWith(1); + expect(harness.errors).toEqual([ + " ✗ Failed to prepare Hermes Provider credentials: API-key preparation failed", + ]); + expectNoPostFailureSideEffects(harness); + }); + + it("exits through injected boundaries when Hermes OAuth preparation throws", async () => { + const exitProcess = createInjectedExit(); + const isHermesProviderRegistered = vi.fn(() => false); + const ensureHermesProviderApiKeyCredentials = vi.fn(async () => ({})); + const ensureHermesProviderOAuthCredentials = vi.fn(async () => { + throw new Error("OAuth preparation failed"); + }); + const providerExistsInGateway = vi.fn(() => true); + const resolveHermesNousApiKey = vi.fn(() => "unused-key"); + const checkHermesProviderStoreReachable = vi.fn(() => ({ ok: true })); + const harness = createDirectSetupInferenceHarness({ + overrides: { + isNonInteractive: () => true, + exitProcess, + normalizeHermesAuthMethod: () => "oauth", + providerExistsInGateway, + resolveHermesNousApiKey, + checkHermesProviderStoreReachable, + hermesProviderAuth: { + HERMES_PROVIDER_NAME: "hermes-provider", + isHermesProviderRegistered, + ensureHermesProviderApiKeyCredentials, + ensureHermesProviderOAuthCredentials, + }, + }, + }); + + await expect( + harness.setupInference( + "test-box", + "moonshotai/kimi-k2.6", + "hermes-provider", + null, + null, + "oauth", + ), + ).rejects.toThrow("EXIT_CALLED:1"); + + expect(checkHermesProviderStoreReachable).toHaveBeenCalledWith(harness.runOpenshell); + expect(isHermesProviderRegistered).toHaveBeenCalledWith(harness.runOpenshell); + expect(providerExistsInGateway).not.toHaveBeenCalled(); + expect(resolveHermesNousApiKey).not.toHaveBeenCalled(); + expect(ensureHermesProviderApiKeyCredentials).not.toHaveBeenCalled(); + expect(ensureHermesProviderOAuthCredentials).toHaveBeenCalledOnce(); + expect(ensureHermesProviderOAuthCredentials).toHaveBeenCalledWith("test-box", { + allowInteractiveLogin: false, + runOpenshell: harness.runOpenshell, + baseUrl: undefined, + toolGatewayPresets: [], + }); + expect(exitProcess).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledWith(1); + expect(harness.errors).toEqual([ + " ✗ Failed to prepare Hermes Provider credentials: OAuth preparation failed", + ]); + expectNoPostFailureSideEffects(harness); + }); + it("propagates Ollama proxy persistence errors before provider registration", async () => { const exitProcess = createInjectedExit(); const persistAndProbeOllamaProxy = vi.fn(async () => { @@ -647,4 +880,50 @@ describe("setupInference dependency failures", () => { expect(harness.errors).toEqual([" ✗ Failed to start model router: router unavailable"]); expectNoPostFailureSideEffects(harness); }); + + it("preserves a routed-provider upsert status through the injected exit boundary", async () => { + const exitProcess = createInjectedExit(); + const reconcileModelRouter = vi.fn(async () => {}); + const upsertProvider = vi.fn(() => ({ ok: true })); + const hydrateCredentialEnv = vi.fn(() => "unused-secret"); + const upsertRoutedProvider = vi.fn(() => ({ + ok: false, + result: { status: 29, message: "routed provider registration rejected" }, + })); + const harness = createDirectSetupInferenceHarness({ + overrides: { + isRoutedInferenceProvider: (provider) => provider === "nvidia-router", + exitProcess, + reconcileModelRouter, + upsertProvider, + hydrateCredentialEnv, + routedInference: { upsertRoutedProvider }, + }, + }); + + await expect( + harness.setupInference( + "test-box", + "router/model", + "nvidia-router", + "http://host.openshell.internal:4000/v1", + "NVIDIA_INFERENCE_API_KEY", + ), + ).rejects.toThrow("EXIT_CALLED:29"); + + expect(reconcileModelRouter).toHaveBeenCalledOnce(); + expect(upsertRoutedProvider).toHaveBeenCalledOnce(); + expect(upsertRoutedProvider).toHaveBeenCalledWith( + "nvidia-router", + "http://host.openshell.internal:4000/v1", + "NVIDIA_INFERENCE_API_KEY", + { upsertProvider, hydrateCredentialEnv }, + ); + expect(upsertProvider).not.toHaveBeenCalled(); + expect(hydrateCredentialEnv).not.toHaveBeenCalled(); + expect(exitProcess).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledWith(29); + expect(harness.errors).toEqual([" routed provider registration rejected"]); + expectNoPostFailureSideEffects(harness); + }); }); From 1b09d442822f7c551ad98f3f0e9cac54293ee98a Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 4 Jul 2026 12:06:12 -0700 Subject: [PATCH 14/16] refactor(onboard): define local route recovery boundary Signed-off-by: Carlos Villela --- .../rebuild-local-provider-recreate.test.ts | 2 + src/lib/onboard/local-inference-route.test.ts | 124 ++++++++++++++++++ src/lib/onboard/local-inference-route.ts | 23 ++-- test/onboard.test.ts | 8 +- 4 files changed, 141 insertions(+), 16 deletions(-) create mode 100644 src/lib/onboard/local-inference-route.test.ts diff --git a/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts b/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts index 607feb3a699..5f1499c2aea 100644 --- a/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts +++ b/src/lib/actions/sandbox/rebuild-local-provider-recreate.test.ts @@ -126,6 +126,8 @@ function makeRouteApplier() { compactText: (value) => value.trim(), redact: (value) => value, localInferenceTimeoutSecs: 30, + error: unusedCommonInferenceDeps.error, + exitProcess: unusedCommonInferenceDeps.exitProcess, }); } diff --git a/src/lib/onboard/local-inference-route.test.ts b/src/lib/onboard/local-inference-route.test.ts new file mode 100644 index 00000000000..d0ee57dd013 --- /dev/null +++ b/src/lib/onboard/local-inference-route.test.ts @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +import { + createLocalInferenceRouteApplier, + type LocalInferenceRouteDeps, +} from "./local-inference-route"; + +class ExitError extends Error { + constructor(readonly code: number) { + super(`EXIT_CALLED:${code}`); + } +} + +function createDeps(overrides: Partial = {}): LocalInferenceRouteDeps { + return { + runOpenshell: vi.fn(() => ({ status: 0, stdout: "", stderr: "" })), + isNonInteractive: vi.fn(() => false), + promptValidationRecovery: vi.fn(async () => "selection" as const), + classifyApplyFailure: vi.fn(() => ({ kind: "unknown" }) as never), + compactText: vi.fn((value: string) => value.trim()), + redact: vi.fn((value: string) => value), + localInferenceTimeoutSecs: 30, + error: vi.fn(), + exitProcess: vi.fn((code: number): never => { + throw new ExitError(code); + }), + ...overrides, + }; +} + +describe("local inference route recovery", () => { + it("redacts a failed non-interactive route and preserves its exit status", async () => { + const runOpenshell = vi.fn(() => ({ + status: 17, + stderr: "route failed with secret-token", + stdout: "secret-token detail", + })); + const redact = vi.fn((value: string) => value.replaceAll("secret-token", "[redacted]")); + const exitProcess = vi.fn((code: number): never => { + throw new ExitError(code); + }); + const deps = createDeps({ + runOpenshell, + isNonInteractive: () => true, + redact, + exitProcess, + }); + + await expect( + createLocalInferenceRouteApplier(deps)("ollama-local", "qwen3.5:9b"), + ).rejects.toEqual(new ExitError(17)); + + expect(runOpenshell).toHaveBeenCalledWith( + [ + "inference", + "set", + "--no-verify", + "--provider", + "ollama-local", + "--model", + "qwen3.5:9b", + "--timeout", + "30", + ], + { ignoreError: true }, + ); + expect(redact).toHaveBeenCalledWith("route failed with secret-token secret-token detail"); + expect(deps.error).toHaveBeenNthCalledWith( + 1, + " route failed with [redacted] [redacted] detail", + ); + expect(deps.error).toHaveBeenNthCalledWith( + 2, + " No sandbox was created. Fix the inference route and re-run `nemoclaw onboard --resume` to continue, or choose a different provider/model.", + ); + expect(vi.mocked(deps.error).mock.calls.flat().join("\n")).not.toContain("secret-token"); + expect(exitProcess).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledWith(17); + expect(deps.promptValidationRecovery).not.toHaveBeenCalled(); + }); + + it("retries an interactive route failure and returns success", async () => { + const runOpenshell = vi + .fn() + .mockReturnValueOnce({ status: 9, stdout: "", stderr: "temporary route failure" }) + .mockReturnValueOnce({ status: 0, stdout: "", stderr: "" }); + const recovery = { kind: "transport" } as never; + const deps = createDeps({ + runOpenshell, + promptValidationRecovery: vi.fn(async () => "retry" as const), + classifyApplyFailure: vi.fn(() => recovery), + }); + + await expect( + createLocalInferenceRouteApplier(deps)("vllm-local", "meta-llama/Llama-3"), + ).resolves.toBe(false); + + expect(runOpenshell).toHaveBeenCalledTimes(2); + expect(deps.error).toHaveBeenCalledOnce(); + expect(deps.error).toHaveBeenCalledWith(" temporary route failure"); + expect(deps.promptValidationRecovery).toHaveBeenCalledOnce(); + expect(deps.promptValidationRecovery).toHaveBeenCalledWith("Local vLLM", recovery, null, null); + expect(deps.exitProcess).not.toHaveBeenCalled(); + }); + + it("returns to provider selection after an interactive route failure", async () => { + const runOpenshell = vi.fn(() => ({ status: 6, stdout: "", stderr: "select another" })); + const deps = createDeps({ + runOpenshell, + promptValidationRecovery: vi.fn(async () => "selection" as const), + }); + + await expect( + createLocalInferenceRouteApplier(deps)("ollama-local", "qwen3.5:9b"), + ).resolves.toBe(true); + + expect(runOpenshell).toHaveBeenCalledOnce(); + expect(deps.promptValidationRecovery).toHaveBeenCalledOnce(); + expect(deps.exitProcess).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/onboard/local-inference-route.ts b/src/lib/onboard/local-inference-route.ts index 96f01aa8e75..69b0294375c 100644 --- a/src/lib/onboard/local-inference-route.ts +++ b/src/lib/onboard/local-inference-route.ts @@ -19,8 +19,8 @@ export interface LocalInferenceRouteDeps { compactText(value: string): string; redact(value: string): string; localInferenceTimeoutSecs: number; - error?(message: string): void; - exitProcess?(code: number): never; + error(message: string): void; + exitProcess(code: number): never; } const LOCAL_PROVIDER_LABELS: Record = { @@ -28,15 +28,14 @@ const LOCAL_PROVIDER_LABELS: Record = { "ollama-local": "Local Ollama", }; -// Wraps `openshell inference set` for local providers (ollama-local, vllm-local) -// with the same retry/recovery surface as the remote-provider path. Without this, -// a nonzero exit from `openshell inference set` propagates through runOpenshell -// and calls process.exit() directly, which terminates onboarding mid-step with no -// context — onboarding appears to stop silently after the [4/8] warning. See #4257. +// Source-of-truth boundary: the invalid state is a failed OpenShell `inference set` route apply. +// OpenShell owns that command result, but cannot own NemoClaw's interactive provider retry and +// selection state, so this adapter translates the failure into onboarding recovery. Regression +// coverage lives in local-inference-route.test.ts and the #4257 onboarding integration tests. +// Remove this adapter when OpenShell exposes equivalent non-terminating interactive recovery, or +// when NemoClaw onboarding no longer owns provider retry/selection. // Returns true if the user chose to back out to provider selection; false on success. export function createLocalInferenceRouteApplier(deps: LocalInferenceRouteDeps) { - const error = deps.error ?? console.error; - const exitProcess = deps.exitProcess ?? ((code: number): never => process.exit(code)); return async function applyLocalInferenceRoute( provider: string, model: string, @@ -61,16 +60,16 @@ export function createLocalInferenceRouteApplier(deps: LocalInferenceRouteDeps) const detail = deps.compactText(deps.redact(`${applyResult.stderr || ""} ${applyResult.stdout || ""}`)) || `Failed to configure inference provider '${provider}'.`; - error(` ${detail}`); + deps.error(` ${detail}`); if (deps.isNonInteractive()) { // Only surface the resume guidance when we are actually about to exit — // printing it on every interactive retry is misleading because the user // is still inside an active onboard run. - error( + deps.error( " No sandbox was created. Fix the inference route and re-run " + "`nemoclaw onboard --resume` to continue, or choose a different provider/model.", ); - return exitProcess(applyResult.status || 1); + return deps.exitProcess(applyResult.status || 1); } const retry = await deps.promptValidationRecovery( label, diff --git a/test/onboard.test.ts b/test/onboard.test.ts index 816380c76aa..51d91ca5c85 100644 --- a/test/onboard.test.ts +++ b/test/onboard.test.ts @@ -1171,6 +1171,8 @@ const { onboard } = require(${onboardPath}); compactText: (value) => value.trim(), redact: (value) => value, localInferenceTimeoutSecs: 120, + error: vi.fn(), + exitProcess: () => assert.fail("unexpected exit"), }); harness = createDirectSetupInferenceHarness({ runOpenshell: (args) => @@ -1179,9 +1181,7 @@ const { onboard } = require(${onboardPath}); : undefined, overrides: { applyLocalInferenceRoute }, }); - await harness.setupInference("test-box", "meta-llama", "vllm-local"); - const providerCommand = harness.commands.find((entry) => entry.command.includes("provider create"), ); @@ -1206,6 +1206,8 @@ const { onboard } = require(${onboardPath}); compactText: (value) => value.trim(), redact: (value) => value, localInferenceTimeoutSecs: 120, + error: vi.fn(), + exitProcess: () => assert.fail("unexpected exit"), }); harness = createDirectSetupInferenceHarness({ runOpenshell: (args) => @@ -1237,7 +1239,6 @@ const { onboard } = require(${onboardPath}); } finally { warn.mockRestore(); } - assert.deepEqual(proxyCalls, ["ensure", "healthy", "persist:proxy-token"]); const providerCommand = harness.commands.find( (entry) => @@ -1299,7 +1300,6 @@ const { onboard } = require(${onboardPath}); } finally { warn.mockRestore(); } - const setCmd = harness.commands.find((entry) => entry.command.includes("inference set --no-verify --provider ollama-local"), ); From 2665b4564a1531e1b5e20c560257c992d003a8a3 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 4 Jul 2026 12:06:54 -0700 Subject: [PATCH 15/16] fix(onboard): complete routed failure boundary Make provider dependency ownership explicit so new requirements must be wired deliberately. Signed-off-by: Carlos Villela --- src/lib/onboard.ts | 2 +- src/lib/onboard/inference-providers/routed.ts | 14 ++- src/lib/onboard/inference-providers/types.ts | 2 + src/lib/onboard/setup-inference.ts | 62 +++++++++--- test/onboard-inference-failure-paths.test.ts | 97 +++++++++++++++++++ 5 files changed, 164 insertions(+), 13 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 67d232612ac..c180b546695 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -4153,7 +4153,7 @@ async function setupNim(gpu: ReturnType, sandboxName: stri // ── Step 4: Inference provider ─────────────────────────────────── -function getSetupInferenceDeps() { +function getSetupInferenceDeps(): SetupInferenceDeps { return { step, getGatewayName: () => GATEWAY_NAME, diff --git a/src/lib/onboard/inference-providers/routed.ts b/src/lib/onboard/inference-providers/routed.ts index cd033c21ed5..dd863c41f1b 100644 --- a/src/lib/onboard/inference-providers/routed.ts +++ b/src/lib/onboard/inference-providers/routed.ts @@ -24,6 +24,8 @@ export async function setupRoutedInference( hydrateCredentialEnv, exitProcess, error, + redact, + compactText, } = deps; // Blueprint profile provider (e.g., nvidia-router for the routed profile). @@ -42,6 +44,16 @@ export async function setupRoutedInference( error(` ${routed.result.message}`); return exitProcess(routed.result.status || 1); } - runOpenshell(["inference", "set", "--no-verify", "--provider", provider, "--model", model]); + const applyResult = runOpenshell( + ["inference", "set", "--no-verify", "--provider", provider, "--model", model], + { ignoreError: true }, + ); + if (applyResult.status !== 0) { + const message = + compactText(redact(`${applyResult.stderr || ""} ${applyResult.stdout || ""}`)) || + `Failed to configure inference provider '${provider}'.`; + error(` ${message}`); + return exitProcess(applyResult.status || 1); + } return { done: false }; } diff --git a/src/lib/onboard/inference-providers/types.ts b/src/lib/onboard/inference-providers/types.ts index 7f95b3d8b28..6786e5e92db 100644 --- a/src/lib/onboard/inference-providers/types.ts +++ b/src/lib/onboard/inference-providers/types.ts @@ -224,6 +224,8 @@ export type RoutedDeps = CommonDeps & { ): { ok: boolean; result: { message?: string; status?: number } }; }; hydrateCredentialEnv: (envName: any, resolveCredential?: any) => any; + redact: (input: string) => string; + compactText: (input: string) => string; }; export const REMOTE_PROVIDER_NAMES = [ diff --git a/src/lib/onboard/setup-inference.ts b/src/lib/onboard/setup-inference.ts index aaa1061321d..c842b577bc4 100644 --- a/src/lib/onboard/setup-inference.ts +++ b/src/lib/onboard/setup-inference.ts @@ -15,16 +15,54 @@ import * as inferenceProviders from "./inference-providers"; import { createLocalInferenceRouteApplier } from "./local-inference-route"; import type { ProviderInferenceSetupOptions } from "./machine/handlers/provider-inference"; -type ProviderBranchDeps = Omit< - HermesDeps & RemoteProviderDeps & VllmDeps & OllamaDeps & RoutedDeps, - | "registry" - | "run" - | "runOpenshell" - | "applyLocalInferenceRoute" - | "LOCAL_INFERENCE_TIMEOUT_SECS" - | "VLLM_LOCAL_CREDENTIAL_ENV" - | "OLLAMA_PROXY_CREDENTIAL_ENV" ->; +type ProviderBranchDeps = Pick< + CommonDeps, + | "upsertProvider" + | "verifyInferenceRoute" + | "verifyOnboardInferenceSmoke" + | "isNonInteractive" + | "exitProcess" + | "error" + | "log" +> & + Pick< + HermesDeps, + | "lookup" + | "hermesProviderAuth" + | "getHermesToolGatewayBroker" + | "providerExistsInGateway" + | "normalizeHermesAuthMethod" + | "resolveHermesNousApiKey" + | "checkHermesProviderStoreReachable" + | "hermesAuthMethodLabel" + | "hermesConstants" + | "requireValue" + | "redact" + | "compactText" + > & + Pick< + RemoteProviderDeps, + | "REMOTE_PROVIDER_CONFIG" + | "hydrateCredentialEnv" + | "promptValidationRecovery" + | "classifyApplyFailure" + | "bedrockRuntimeOnboard" + > & + Pick< + VllmDeps, + "validateLocalProvider" | "getLocalProviderHealthCheck" | "getLocalProviderBaseUrl" + > & + Pick< + OllamaDeps, + | "getOllamaWarmupCommand" + | "shouldFrontOllamaWithProxy" + | "ensureOllamaAuthProxy" + | "isProxyHealthy" + | "getOllamaProxyToken" + | "persistAndProbeOllamaProxy" + | "localInference" + > & + Pick; export type SetupInferenceDeps = ProviderBranchDeps & { step: (current: number, total: number, label: string) => void; @@ -99,7 +137,7 @@ export function createSetupInference( exitProcess: deps.exitProcess, error: deps.error, log: deps.log, - }; + } satisfies CommonDeps; if (provider === deps.hermesProviderAuth.HERMES_PROVIDER_NAME) { return inferenceProviders.setupHermesProviderInference( @@ -196,6 +234,8 @@ export function createSetupInference( reconcileModelRouter: deps.reconcileModelRouter, routedInference: deps.routedInference, hydrateCredentialEnv: deps.hydrateCredentialEnv, + redact: deps.redact, + compactText: deps.compactText, }, ); } else { diff --git a/test/onboard-inference-failure-paths.test.ts b/test/onboard-inference-failure-paths.test.ts index 4854bab0230..c5592442bc3 100644 --- a/test/onboard-inference-failure-paths.test.ts +++ b/test/onboard-inference-failure-paths.test.ts @@ -926,4 +926,101 @@ describe("setupInference dependency failures", () => { expect(harness.errors).toEqual([" routed provider registration rejected"]); expectNoPostFailureSideEffects(harness); }); + + it("redacts a routed inference-set failure and preserves its status at the exit boundary", async () => { + const exitProcess = createInjectedExit(); + const reconcileModelRouter = vi.fn(async () => {}); + const upsertRoutedProvider = vi.fn(() => ({ ok: true, result: {} })); + const commandRouter = createDirectCommandRouter([ + { + name: "routed-inference-set", + matches: (command) => command.startsWith("inference set"), + results: [{ status: 41, stdout: "", stderr: "routed apply failed nvapi-1234567890abcdef" }], + }, + ]); + const harness = createDirectSetupInferenceHarness({ + runOpenshell: commandRouter.runOpenshell, + overrides: { + isRoutedInferenceProvider: (provider) => provider === "nvidia-router", + exitProcess, + reconcileModelRouter, + routedInference: { upsertRoutedProvider }, + }, + }); + + await expect( + harness.setupInference( + "test-box", + "router/model", + "nvidia-router", + "http://host.openshell.internal:4000/v1", + "NVIDIA_INFERENCE_API_KEY", + ), + ).rejects.toThrow("EXIT_CALLED:41"); + + expect(reconcileModelRouter).toHaveBeenCalledOnce(); + expect(upsertRoutedProvider).toHaveBeenCalledOnce(); + expect(commandRouter.callCount("routed-inference-set")).toBe(1); + expect(harness.commands.at(-1)).toMatchObject({ ignoreError: true }); + expect(exitProcess).toHaveBeenCalledOnce(); + expect(exitProcess).toHaveBeenCalledWith(41); + expect(harness.errors.join("\n")).toContain("routed apply failed"); + expect(harness.errors.join("\n")).not.toContain("nvapi-1234567890abcdef"); + expectNoPostFailureSideEffects(harness, [ + "gateway select nemoclaw", + "inference set --no-verify --provider nvidia-router --model router/model", + ]); + }); + + it("runs shared finalization after routed inference setup succeeds", async () => { + const exitProcess = createInjectedExit(); + const reconcileModelRouter = vi.fn(async () => {}); + const upsertRoutedProvider = vi.fn(() => ({ ok: true, result: {} })); + const harness = createDirectSetupInferenceHarness({ + overrides: { + isRoutedInferenceProvider: (provider) => provider === "nvidia-router", + exitProcess, + reconcileModelRouter, + routedInference: { upsertRoutedProvider }, + }, + }); + + await expect( + harness.setupInference( + "test-box", + "router/model", + "nvidia-router", + "http://host.openshell.internal:4000/v1", + "NVIDIA_INFERENCE_API_KEY", + ), + ).resolves.toEqual({ ok: true }); + + expect(reconcileModelRouter).toHaveBeenCalledOnce(); + expect(upsertRoutedProvider).toHaveBeenCalledOnce(); + expect(harness.commands).toEqual([ + { command: "gateway select nemoclaw", ignoreError: true, env: undefined }, + { + command: "inference set --no-verify --provider nvidia-router --model router/model", + ignoreError: true, + env: undefined, + }, + ]); + expect(harness.verifyInferenceRoute).toHaveBeenCalledOnce(); + expect(harness.verifyInferenceRoute).toHaveBeenCalledWith("nvidia-router", "router/model"); + expect(harness.verifyOnboardInferenceSmoke).toHaveBeenCalledOnce(); + expect(harness.verifyOnboardInferenceSmoke).toHaveBeenCalledWith({ + provider: "nvidia-router", + model: "router/model", + endpointUrl: "http://host.openshell.internal:4000/v1", + credentialEnv: "NVIDIA_INFERENCE_API_KEY", + }); + expect(harness.updateSandbox).toHaveBeenCalledOnce(); + expect(harness.updateSandbox).toHaveBeenCalledWith("test-box", { + model: "router/model", + provider: "nvidia-router", + }); + expect(harness.logs).toEqual([" ✓ Inference route set: nvidia-router / router/model"]); + expect(harness.errors).toEqual([]); + expect(exitProcess).not.toHaveBeenCalled(); + }); }); From 1d9d09be699ee2506e5b55474fc264cb041cd615 Mon Sep 17 00:00:00 2001 From: Carlos Villela Date: Sat, 4 Jul 2026 12:16:57 -0700 Subject: [PATCH 16/16] refactor(onboard): require explicit failure boundaries Signed-off-by: Carlos Villela --- src/lib/onboard.ts | 3 + src/lib/onboard/bedrock-runtime.test.ts | 13 +++ src/lib/onboard/bedrock-runtime.ts | 17 ++-- src/lib/onboard/hermes-auth.test.ts | 93 +++++++++++++++++++- src/lib/onboard/hermes-auth.ts | 5 +- test/onboard-inference-failure-paths.test.ts | 11 ++- 6 files changed, 120 insertions(+), 22 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index c180b546695..5604c943fee 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -3648,6 +3648,9 @@ async function handleRemoteProviderSelection(args: RemoteProviderSelectionArgs, isNonInteractive, promptInputModel, replaceNamedCredential, + exitProcess: (code) => process.exit(code), + error: (message) => console.error(message), + log: (message) => console.log(message), }); if (bedrockSelection.action === "retry-selection") { console.log(" Returning to provider selection."); diff --git a/src/lib/onboard/bedrock-runtime.test.ts b/src/lib/onboard/bedrock-runtime.test.ts index 8555cf3b4a0..58942de3af9 100644 --- a/src/lib/onboard/bedrock-runtime.test.ts +++ b/src/lib/onboard/bedrock-runtime.test.ts @@ -8,6 +8,16 @@ import { BACK_TO_SELECTION } from "./credential-navigation"; const BEDROCK_URL = "https://bedrock-runtime.us-east-1.amazonaws.com"; +function createBedrockRuntimeDependencies() { + return { + exitProcess: vi.fn((code: number): never => { + throw new Error(`EXIT_CALLED:${code}`); + }), + error: vi.fn(), + log: vi.fn(), + }; +} + function clearBedrockAuthEnv(): void { delete process.env.AWS_BEARER_TOKEN_BEDROCK; delete process.env.AWS_PROFILE; @@ -70,6 +80,7 @@ describe("Bedrock Runtime onboarding helper", () => { const promptInputModel = vi.fn(async () => "anthropic.claude"); const result = await selectBedrockRuntimeCustomAnthropic({ + ...createBedrockRuntimeDependencies(), selectedKey: "anthropicCompatible", endpointUrl: BEDROCK_URL, credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", @@ -102,6 +113,7 @@ describe("Bedrock Runtime onboarding helper", () => { }); const result = await selectBedrockRuntimeCustomAnthropic({ + ...createBedrockRuntimeDependencies(), selectedKey: "anthropicCompatible", endpointUrl: BEDROCK_URL, credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", @@ -129,6 +141,7 @@ describe("Bedrock Runtime onboarding helper", () => { const replaceNamedCredential = vi.fn(async () => "unused"); const result = await selectBedrockRuntimeCustomAnthropic({ + ...createBedrockRuntimeDependencies(), selectedKey: "anthropicCompatible", endpointUrl: BEDROCK_URL, credentialEnv: "COMPATIBLE_ANTHROPIC_API_KEY", diff --git a/src/lib/onboard/bedrock-runtime.ts b/src/lib/onboard/bedrock-runtime.ts index 43d99d8b064..b90d2188ee6 100644 --- a/src/lib/onboard/bedrock-runtime.ts +++ b/src/lib/onboard/bedrock-runtime.ts @@ -31,15 +31,11 @@ type UpsertProvider = ( type SetupInferenceResult = { ok: true; retry?: undefined } | { retry: "selection" }; type BedrockRuntimeDependencies = { - exitProcess?: (code: number) => never; - error?: (message: string) => void; - log?: (message: string) => void; + exitProcess: (code: number) => never; + error: (message: string) => void; + log: (message: string) => void; }; -const defaultExitProcess = (code: number): never => process.exit(code); -const defaultError = (message: string): void => console.error(message); -const defaultLog = (message: string): void => console.log(message); - function normalizeCredentialValue(value: unknown): string { return String(value ?? "").trim(); } @@ -91,8 +87,7 @@ export async function selectBedrockRuntimeCustomAnthropic( | { action: "retry-selection" } | { action: "selected"; model: string; preferredInferenceApi: "openai-completions" } > { - const error = options.error ?? defaultError; - const exitProcess = options.exitProcess ?? defaultExitProcess; + const { error, exitProcess } = options; if (options.selectedKey !== "anthropicCompatible" || !options.endpointUrl) { return { action: "not-bedrock" }; } @@ -149,9 +144,7 @@ export async function setupBedrockRuntimeInference( updateSandbox?: typeof registry.updateSandbox; } & BedrockRuntimeDependencies, ): Promise<{ handled: false } | { handled: true; result: SetupInferenceResult }> { - const error = options.error ?? defaultError; - const exitProcess = options.exitProcess ?? defaultExitProcess; - const log = options.log ?? defaultLog; + const { error, exitProcess, log } = options; const classification = options.provider === "compatible-anthropic-endpoint" && options.endpointUrl ? classifyCustomAnthropicEndpoint(options.endpointUrl) diff --git a/src/lib/onboard/hermes-auth.test.ts b/src/lib/onboard/hermes-auth.test.ts index 9fb60693301..2676923446f 100644 --- a/src/lib/onboard/hermes-auth.test.ts +++ b/src/lib/onboard/hermes-auth.test.ts @@ -5,10 +5,20 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { createHermesAuthHelpers, + HERMES_AUTH_METHOD_API_KEY, + HERMES_AUTH_METHOD_OAUTH, HERMES_NOUS_API_KEY_CREDENTIAL_ENV, type HermesAuthFlowDeps, } from "./hermes-auth"; +function clearHermesAuthEnvironment(): void { + vi.stubEnv("NEMOCLAW_HERMES_AUTH_METHOD", undefined); + vi.stubEnv("NEMOCLAW_HERMES_AUTH", undefined); + vi.stubEnv("NEMOCLAW_NOUS_AUTH_METHOD", undefined); + vi.stubEnv(HERMES_NOUS_API_KEY_CREDENTIAL_ENV, undefined); + vi.stubEnv("NEMOCLAW_PROVIDER_KEY", undefined); +} + function createDeps(overrides: Partial = {}): HermesAuthFlowDeps { return { isNonInteractive: vi.fn(() => true), @@ -56,8 +66,7 @@ describe("Hermes authentication exit boundaries", () => { }); it("uses the injected exit when a prompted Nous API key is invalid", async () => { - vi.stubEnv(HERMES_NOUS_API_KEY_CREDENTIAL_ENV, undefined); - vi.stubEnv("NEMOCLAW_PROVIDER_KEY", undefined); + clearHermesAuthEnvironment(); vi.spyOn(console, "log").mockImplementation(() => undefined); const deps = createDeps({ isNonInteractive: vi.fn(() => false), @@ -80,3 +89,83 @@ describe("Hermes authentication exit boundaries", () => { expect(process.env[HERMES_NOUS_API_KEY_CREDENTIAL_ENV]).toBeUndefined(); }); }); + +describe("Hermes authentication selection", () => { + it("selects API key authentication non-interactively when a key already exists", async () => { + clearHermesAuthEnvironment(); + vi.stubEnv(HERMES_NOUS_API_KEY_CREDENTIAL_ENV, "nous-key"); + const deps = createDeps(); + + await expect(createHermesAuthHelpers(deps).promptHermesAuthMethod()).resolves.toBe( + HERMES_AUTH_METHOD_API_KEY, + ); + + expect(deps.note).toHaveBeenCalledOnce(); + expect(deps.note).toHaveBeenCalledWith(" [non-interactive] Hermes auth: Nous API Key"); + expect(deps.prompt).not.toHaveBeenCalled(); + expect(deps.error).not.toHaveBeenCalled(); + expect(deps.exitProcess).not.toHaveBeenCalled(); + }); + + it("selects OAuth non-interactively when no key exists", async () => { + clearHermesAuthEnvironment(); + const deps = createDeps(); + + await expect(createHermesAuthHelpers(deps).promptHermesAuthMethod()).resolves.toBe( + HERMES_AUTH_METHOD_OAUTH, + ); + + expect(deps.note).toHaveBeenCalledOnce(); + expect(deps.note).toHaveBeenCalledWith(" [non-interactive] Hermes auth: Nous Portal OAuth"); + expect(deps.prompt).not.toHaveBeenCalled(); + expect(deps.error).not.toHaveBeenCalled(); + expect(deps.exitProcess).not.toHaveBeenCalled(); + }); + + it("returns to provider selection when the auth-method prompt chooses back", async () => { + clearHermesAuthEnvironment(); + vi.spyOn(console, "log").mockImplementation(() => undefined); + const backToSelection = Symbol("back-to-selection"); + const prompt = vi.fn(async () => "back"); + const deps = createDeps({ + isNonInteractive: () => false, + prompt, + getNavigationChoice: vi.fn(() => "back" as const), + backToSelection, + }); + + await expect(createHermesAuthHelpers(deps).promptHermesAuthMethod()).resolves.toBe( + backToSelection, + ); + + expect(prompt).toHaveBeenCalledOnce(); + expect(prompt).toHaveBeenCalledWith(" Choose [1]: "); + expect(deps.error).not.toHaveBeenCalled(); + expect(deps.exitProcess).not.toHaveBeenCalled(); + expect(deps.exitOnboardFromPrompt).not.toHaveBeenCalled(); + }); + + it("returns to provider selection when the API-key prompt chooses back", async () => { + clearHermesAuthEnvironment(); + vi.spyOn(console, "log").mockImplementation(() => undefined); + const backToSelection = Symbol("back-to-selection"); + const prompt = vi.fn(async () => "back"); + const deps = createDeps({ + prompt, + getNavigationChoice: vi.fn(() => "back" as const), + backToSelection, + }); + + await expect(createHermesAuthHelpers(deps).ensureHermesNousApiKeyEnv()).resolves.toBe( + backToSelection, + ); + + expect(prompt).toHaveBeenCalledOnce(); + expect(prompt).toHaveBeenCalledWith(" Nous API Key: ", { secret: true }); + expect(deps.validateNvidiaApiKeyValue).not.toHaveBeenCalled(); + expect(deps.error).not.toHaveBeenCalled(); + expect(deps.exitProcess).not.toHaveBeenCalled(); + expect(deps.exitOnboardFromPrompt).not.toHaveBeenCalled(); + expect(process.env[HERMES_NOUS_API_KEY_CREDENTIAL_ENV]).toBeUndefined(); + }); +}); diff --git a/src/lib/onboard/hermes-auth.ts b/src/lib/onboard/hermes-auth.ts index 72e8ba0ca15..c804c73b731 100644 --- a/src/lib/onboard/hermes-auth.ts +++ b/src/lib/onboard/hermes-auth.ts @@ -46,10 +46,7 @@ export interface HermesAuthFailureBoundary { } export function getRequestedHermesAuthMethod( - boundary: HermesAuthFailureBoundary = { - error: (message) => console.error(message), - exitProcess: (code) => process.exit(code), - }, + boundary: HermesAuthFailureBoundary, ): HermesAuthMethod | null { const raw = process.env.NEMOCLAW_HERMES_AUTH_METHOD || diff --git a/test/onboard-inference-failure-paths.test.ts b/test/onboard-inference-failure-paths.test.ts index c5592442bc3..6dba8bedce6 100644 --- a/test/onboard-inference-failure-paths.test.ts +++ b/test/onboard-inference-failure-paths.test.ts @@ -26,6 +26,7 @@ type EnsureBedrockRuntimeAdapter = NonNullable< const BEDROCK_ENDPOINT = "https://bedrock-runtime.us-east-1.amazonaws.com"; const BEDROCK_CREDENTIAL_ENV = "COMPATIBLE_ANTHROPIC_API_KEY"; const BEDROCK_MODEL = "anthropic.claude-3-5-sonnet-20240620-v1:0"; +const NVIDIA_REDACTION_CANARY = ["nv", "api-", "TEST-NOT-A-REAL-VALUE"].join(""); function createInjectedExit() { return vi.fn((code: number): never => { @@ -194,7 +195,7 @@ describe("setupInference dependency failures", () => { { name: "remote-inference-set", matches: (command) => command.startsWith("inference set"), - results: [{ status: 37, stdout: "", stderr: "route failed nvapi-1234567890abcdef" }], + results: [{ status: 37, stdout: "", stderr: `route failed ${NVIDIA_REDACTION_CANARY}` }], }, ]); const harness = createDirectSetupInferenceHarness({ @@ -220,7 +221,7 @@ describe("setupInference dependency failures", () => { expect(exitProcess).toHaveBeenCalledOnce(); expect(exitProcess).toHaveBeenCalledWith(37); expect(harness.errors.join("\n")).toContain("route failed"); - expect(harness.errors.join("\n")).not.toContain("nvapi-1234567890abcdef"); + expect(harness.errors.join("\n")).not.toContain(NVIDIA_REDACTION_CANARY); expectNoPostFailureSideEffects(harness, [ "gateway select nemoclaw", "inference set --no-verify --provider openai-api --model gpt-test", @@ -935,7 +936,9 @@ describe("setupInference dependency failures", () => { { name: "routed-inference-set", matches: (command) => command.startsWith("inference set"), - results: [{ status: 41, stdout: "", stderr: "routed apply failed nvapi-1234567890abcdef" }], + results: [ + { status: 41, stdout: "", stderr: `routed apply failed ${NVIDIA_REDACTION_CANARY}` }, + ], }, ]); const harness = createDirectSetupInferenceHarness({ @@ -965,7 +968,7 @@ describe("setupInference dependency failures", () => { expect(exitProcess).toHaveBeenCalledOnce(); expect(exitProcess).toHaveBeenCalledWith(41); expect(harness.errors.join("\n")).toContain("routed apply failed"); - expect(harness.errors.join("\n")).not.toContain("nvapi-1234567890abcdef"); + expect(harness.errors.join("\n")).not.toContain(NVIDIA_REDACTION_CANARY); expectNoPostFailureSideEffects(harness, [ "gateway select nemoclaw", "inference set --no-verify --provider nvidia-router --model router/model",