From 24e90667f5439f331e59e599b77c257cdbd9a338 Mon Sep 17 00:00:00 2001 From: Benedikt Schackenberg <6381261+BenediktSchackenberg@users.noreply.github.com> Date: Thu, 4 Jun 2026 15:59:51 +0000 Subject: [PATCH 1/2] refactor(onboard): extract inference provider flows into modules (#767) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit src/lib/onboard.ts had grown to ~7.2k lines with the inference provider selection logic (hermes / ollama-local / vllm-local / remote / routed) inlined across the main onboarding flow. This pulls each provider out behind a small shared interface so the orchestrator can focus on flow control. - New src/lib/onboard/inference-providers/ module: - types.ts — shared provider interface + helper types - hermes.ts - ollama-local.ts - vllm-local.ts - remote.ts - routed.ts - index.ts — registry / dispatcher - onboard.ts is now a thin dispatcher into the provider registry (7204 → 6983 lines; -314 / +93 from the file diff) Pure refactor — no behaviour changes, no UX changes, identical persisted state. Existing tests under src/lib/onboard/ pass unchanged (1072/1072 vitest, tsc clean). Refs #767 #924 Signed-off-by: Benedikt Schackenberg <6381261+BenediktSchackenberg@users.noreply.github.com> --- src/lib/onboard.ts | 407 ++++-------------- src/lib/onboard/inference-providers/hermes.ts | 138 ++++++ src/lib/onboard/inference-providers/index.ts | 30 ++ .../inference-providers/ollama-local.ts | 114 +++++ src/lib/onboard/inference-providers/remote.ts | 136 ++++++ src/lib/onboard/inference-providers/routed.ts | 47 ++ src/lib/onboard/inference-providers/types.ts | 217 ++++++++++ .../onboard/inference-providers/vllm-local.ts | 76 ++++ 8 files changed, 851 insertions(+), 314 deletions(-) create mode 100644 src/lib/onboard/inference-providers/hermes.ts create mode 100644 src/lib/onboard/inference-providers/index.ts create mode 100644 src/lib/onboard/inference-providers/ollama-local.ts create mode 100644 src/lib/onboard/inference-providers/remote.ts create mode 100644 src/lib/onboard/inference-providers/routed.ts create mode 100644 src/lib/onboard/inference-providers/types.ts create mode 100644 src/lib/onboard/inference-providers/vllm-local.ts diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index b287a2b5567..156a27981d3 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -220,6 +220,7 @@ const { } = inferenceConfig; const onboardProviders = require("./onboard/providers"); +const inferenceProviders: typeof import("./onboard/inference-providers") = require("./onboard/inference-providers"); const { ensureResumeProviderReady } = require("./onboard/resume-provider-shim"); const hermesProviderAuth = require("./hermes-provider-auth"); const onboardHermesDashboard: typeof import("./onboard/hermes-dashboard") = require("./onboard/hermes-dashboard"); @@ -5199,329 +5200,107 @@ async function setupInference( step(4, 8, "Setting up inference provider"); runOpenshell(["gateway", "select", GATEWAY_NAME], { ignoreError: true }); - if (provider === hermesProviderAuth.HERMES_PROVIDER_NAME) { - const targetSandbox = requireValue(sandboxName, "Hermes Provider requires a sandbox name"); - const resolvedHermesAuthMethod = - normalizeHermesAuthMethod(hermesAuthMethod) || - (credentialEnv === HERMES_NOUS_API_KEY_CREDENTIAL_ENV - ? HERMES_AUTH_METHOD_API_KEY - : HERMES_AUTH_METHOD_OAUTH); - const providerStore = checkHermesProviderStoreReachable(runOpenshell); - if (!providerStore.ok) { - console.error(" ✗ OpenShell provider storage is unreachable."); - console.error(` ${providerStore.message}`); - console.error(" Restart or recreate the OpenShell gateway, then rerun onboarding."); - if (isNonInteractive()) process.exit(1); - return { retry: "selection" }; - } - const providerRegistered = hermesProviderAuth.isHermesProviderRegistered(runOpenshell); - const toolGatewayProviderRegistered = - hermesToolGateways.length === 0 - ? true - : providerExistsInGateway( - getHermesToolGatewayBroker().getHermesToolGatewayProviderName(targetSandbox), - ); - const hasFreshNousApiKey = - resolvedHermesAuthMethod === HERMES_AUTH_METHOD_API_KEY && !!resolveHermesNousApiKey(); - const shouldPrepareHermesCredentials = - !providerRegistered || - !toolGatewayProviderRegistered || - hasFreshNousApiKey || - (resolvedHermesAuthMethod === HERMES_AUTH_METHOD_OAUTH && !isNonInteractive()); - if (shouldPrepareHermesCredentials) { - try { - const state = - resolvedHermesAuthMethod === HERMES_AUTH_METHOD_API_KEY - ? await hermesProviderAuth.ensureHermesProviderApiKeyCredentials(targetSandbox, { - apiKey: resolveHermesNousApiKey(), - runOpenshell, - baseUrl: endpointUrl || undefined, - }) - : await hermesProviderAuth.ensureHermesProviderOAuthCredentials(targetSandbox, { - allowInteractiveLogin: !isNonInteractive(), - runOpenshell, - baseUrl: endpointUrl || undefined, - toolGatewayPresets: hermesToolGateways, - }); - if (!state) { - const authLabel = hermesAuthMethodLabel(resolvedHermesAuthMethod); - console.error(` ✗ Hermes Provider ${authLabel} is not available on the host.`); - console.error( - " Re-run `nemoclaw onboard --agent hermes` interactively to configure credentials.", - ); - process.exit(1); - } - } catch (err) { - console.error( - ` ✗ Failed to prepare Hermes Provider credentials: ${ - err instanceof Error ? err.message : String(err) - }`, - ); - if (isNonInteractive()) process.exit(1); - return { retry: "selection" }; - } - } + const commonDeps = { + runOpenshell, + upsertProvider, + verifyInferenceRoute, + verifyOnboardInferenceSmoke, + isNonInteractive, + registry, + }; - const applyResult = runOpenshell( - ["inference", "set", "--no-verify", "--provider", provider, "--model", model], - { ignoreError: true }, + if (provider === hermesProviderAuth.HERMES_PROVIDER_NAME) { + return inferenceProviders.setupHermesProviderInference( + { + sandboxName, + model, + provider, + endpointUrl, + credentialEnv, + hermesAuthMethod, + hermesToolGateways, + }, + { + ...commonDeps, + hermesProviderAuth, + getHermesToolGatewayBroker, + providerExistsInGateway, + normalizeHermesAuthMethod, + resolveHermesNousApiKey, + checkHermesProviderStoreReachable, + hermesAuthMethodLabel, + hermesConstants: { + HERMES_NOUS_API_KEY_CREDENTIAL_ENV, + HERMES_AUTH_METHOD_API_KEY, + HERMES_AUTH_METHOD_OAUTH, + }, + requireValue, + redact, + compactText, + }, ); - if (applyResult.status !== 0) { - const message = - compactText(redact(`${applyResult.stderr || ""} ${applyResult.stdout || ""}`)) || - `Failed to configure inference provider '${provider}'.`; - console.error(` ${message}`); - if (isNonInteractive()) process.exit(applyResult.status || 1); - return { retry: "selection" }; - } - - verifyInferenceRoute(provider, model); - verifyOnboardInferenceSmoke({ provider, model, endpointUrl, credentialEnv }); - if (sandboxName) { - registry.updateSandbox(sandboxName, { model, provider }); - } - console.log(` ✓ Inference route set: ${provider} / ${model}`); - return { ok: true }; } - if ( - provider === "nvidia-prod" || - provider === "nvidia-nim" || - provider === "openai-api" || - provider === "anthropic-prod" || - provider === "compatible-anthropic-endpoint" || - provider === "gemini-api" || - provider === "compatible-endpoint" - ) { - const config = - provider === "nvidia-nim" - ? REMOTE_PROVIDER_CONFIG.build - : Object.values(REMOTE_PROVIDER_CONFIG).find((entry) => entry.providerName === provider); - if (!config) { - console.error(` Unsupported provider configuration: ${provider}`); - process.exit(1); - } - const bedrockSetup = await bedrockRuntimeOnboard.setupBedrockRuntimeInference({ - sandboxName, - provider, - model, - endpointUrl, - credentialEnv, - isNonInteractive, - runOpenshell, - upsertProvider, - verifyInferenceRoute, - verifyOnboardInferenceSmoke, - }); - if (bedrockSetup.handled) return bedrockSetup.result; - while (true) { - const resolvedCredentialEnv = credentialEnv || (config && config.credentialEnv); - const resolvedEndpointUrl = endpointUrl || (config && config.endpointUrl); - const credentialValue = hydrateCredentialEnv(resolvedCredentialEnv); - const env = - resolvedCredentialEnv && credentialValue - ? { [resolvedCredentialEnv]: credentialValue } - : {}; - const providerResult = upsertProvider( - provider, - config.providerType, - resolvedCredentialEnv, - resolvedEndpointUrl, - env, - ); - if (!providerResult.ok) { - console.error(` ${providerResult.message}`); - if (isNonInteractive()) { - process.exit(providerResult.status || 1); - } - const retry = await promptValidationRecovery( - config.label, - classifyApplyFailure(providerResult.message), - resolvedCredentialEnv, - config.helpUrl, - ); - if (retry === "credential" || retry === "retry") { - continue; - } - if (retry === "selection" || retry === "model") { - return { retry: "selection" }; - } - process.exit(providerResult.status || 1); - } - const args = ["inference", "set"]; - if (config.skipVerify) { - args.push("--no-verify"); - } - args.push("--provider", provider, "--model", model); - if (provider === "compatible-endpoint") { - args.push("--timeout", String(LOCAL_INFERENCE_TIMEOUT_SECS)); - } - const applyResult = runOpenshell(args, { ignoreError: true }); - if (applyResult.status === 0) { - break; - } - const message = - compactText(redact(`${applyResult.stderr || ""} ${applyResult.stdout || ""}`)) || - `Failed to configure inference provider '${provider}'.`; - console.error(` ${message}`); - if (isNonInteractive()) { - process.exit(applyResult.status || 1); - } - const retry = await promptValidationRecovery( - config.label, - classifyApplyFailure(message), - resolvedCredentialEnv, - config.helpUrl, - ); - if (retry === "credential" || retry === "retry") { - continue; - } - if (retry === "selection" || retry === "model") { - return { retry: "selection" }; - } - process.exit(applyResult.status || 1); - } + if (inferenceProviders.isRemoteProviderName(provider)) { + const outcome = await inferenceProviders.setupRemoteProviderInference( + { sandboxName, model, provider, endpointUrl, credentialEnv }, + { + ...commonDeps, + REMOTE_PROVIDER_CONFIG, + hydrateCredentialEnv, + promptValidationRecovery, + classifyApplyFailure, + LOCAL_INFERENCE_TIMEOUT_SECS, + bedrockRuntimeOnboard, + redact, + compactText, + }, + ); + if (outcome.done) return outcome.result; } else if (provider === "vllm-local") { - const validation = validateLocalProvider(provider); - if (!validation.ok) { - const hostCheck = getLocalProviderHealthCheck(provider); - // Use run() and check exit status rather than coercing runCapture() output - // to boolean — curl -sf can leave output even on failure in edge cases. - const hostResponding = hostCheck - ? run(hostCheck, { ignoreError: true, suppressOutput: true }).status === 0 - : false; - - if (hostResponding) { - console.warn(` ⚠ ${validation.message}`); - if (validation.diagnostic) { - console.warn(` Diagnostic: ${validation.diagnostic}`); - } - console.warn( - " The server is healthy on the host — continuing. " + - "The sandbox uses a different network path and may work correctly.", - ); - } else { - console.error(` ${validation.message}`); - if (validation.diagnostic) { - console.error(` Diagnostic: ${validation.diagnostic}`); - } - process.exit(1); - } - } - const baseUrl = getLocalProviderBaseUrl(provider); - // Use a dedicated internal credential env so the gateway does not pick - // up the user's host OPENAI_API_KEY for local vLLM. vLLM does not enforce - // the bearer at runtime, but a dedicated env name prevents accidental - // hijacking. See GH #2519. - const providerResult = upsertProvider( - "vllm-local", - "openai", - VLLM_LOCAL_CREDENTIAL_ENV, - baseUrl, - { [VLLM_LOCAL_CREDENTIAL_ENV]: "dummy" }, + const outcome = await inferenceProviders.setupVllmLocalInference( + { model, provider }, + { + ...commonDeps, + validateLocalProvider, + getLocalProviderHealthCheck, + getLocalProviderBaseUrl, + applyLocalInferenceRoute, + run, + VLLM_LOCAL_CREDENTIAL_ENV, + }, ); - if (!providerResult.ok) { - console.error(` ${providerResult.message}`); - process.exit(providerResult.status || 1); - } - if (await applyLocalInferenceRoute("vllm-local", model)) return { retry: "selection" }; - // Do not mutate ~/.nemoclaw/credentials.json here: local vLLM now uses - // VLLM_LOCAL_CREDENTIAL_ENV, so any saved OPENAI_API_KEY remains available - // to unrelated OpenAI-backed sandboxes. + if (outcome.done) return outcome.result; } else if (provider === "ollama-local") { - const validation = validateLocalProvider(provider); - let proxyReady = false; - const frontOllamaWithProxy = shouldFrontOllamaWithProxy(); - if (!validation.ok) { - // The container reachability check uses Docker's --add-host host-gateway, - // which may not work on all Docker configurations (e.g., Brev, rootless). - // The real sandbox uses k3s CoreDNS + NodeHosts — a different path. - // Try to start/restart the auth proxy before probing — this recovers - // from stale or missing proxy processes before we decide to abort. - if (frontOllamaWithProxy) { - ensureOllamaAuthProxy(); - proxyReady = isProxyHealthy(); - } - if (proxyReady) { - console.warn(` ⚠ ${validation.message}`); - if (validation.diagnostic) { - console.warn(` Diagnostic: ${validation.diagnostic}`); - } - console.warn( - " The auth proxy is healthy on the host — continuing. " + - "The sandbox uses a different network path and may work correctly.", - ); - } else { - console.error(` ${validation.message}`); - if (validation.diagnostic) { - console.error(` Diagnostic: ${validation.diagnostic}`); - } - if (process.platform === "darwin") { - console.error( - " On macOS, local inference also depends on OpenShell host routing support.", - ); - } - process.exit(1); - } - } - const baseUrl = getLocalProviderBaseUrl(provider); - let ollamaCredential = "ollama"; - if (frontOllamaWithProxy) { - // Skip if already started during the fallback recovery above. - if (!proxyReady) ensureOllamaAuthProxy(); - const proxyToken = getOllamaProxyToken(); - if (!proxyToken) { - console.error( - " Ollama auth proxy token is not set. Re-run onboard to initialize the proxy.", - ); - process.exit(1); - } - ollamaCredential = proxyToken; - // Persist token now that ollama-local is confirmed as the provider. - // Not persisted earlier in case the user backs out to a different provider. - await persistAndProbeOllamaProxy(proxyToken); - } - // Use a dedicated internal credential env (NEMOCLAW_OLLAMA_PROXY_TOKEN) - // so the gateway never reads the user's host OPENAI_API_KEY for local - // Ollama. GH #2519: a stale host OPENAI_API_KEY was leaking into the - // inference path and producing 401s. - const providerResult = upsertProvider( - "ollama-local", - "openai", - OLLAMA_PROXY_CREDENTIAL_ENV, - baseUrl, - { [OLLAMA_PROXY_CREDENTIAL_ENV]: ollamaCredential }, + const outcome = await inferenceProviders.setupOllamaLocalInference( + { model, provider, allowToolsIncompatible: options.allowToolsIncompatible === true }, + { + ...commonDeps, + validateLocalProvider, + getLocalProviderBaseUrl, + applyLocalInferenceRoute, + getOllamaWarmupCommand, + run, + shouldFrontOllamaWithProxy, + ensureOllamaAuthProxy, + isProxyHealthy, + getOllamaProxyToken, + persistAndProbeOllamaProxy, + localInference, + OLLAMA_PROXY_CREDENTIAL_ENV, + }, ); - if (!providerResult.ok) { - console.error(` ${providerResult.message}`); - process.exit(providerResult.status || 1); - } - if (await applyLocalInferenceRoute("ollama-local", model)) return { retry: "selection" }; - console.log(` Priming Ollama model: ${model}`); - run(getOllamaWarmupCommand(model), { ignoreError: true }); - const probe = localInference.validateOllamaModelWithToolsOverride(model, options.allowToolsIncompatible === true); - if (!probe.ok) { - console.error(` ${probe.message}`); - process.exit(1); - } - // Do not mutate ~/.nemoclaw/credentials.json here: local Ollama now uses - // OLLAMA_PROXY_CREDENTIAL_ENV, so any saved OPENAI_API_KEY remains available - // to unrelated OpenAI-backed sandboxes. + if (outcome.done) return outcome.result; } else if (isRoutedInferenceProvider(provider)) { - // Blueprint profile provider (e.g., nvidia-router for the routed profile). - // reconcileModelRouter also probes sandbox→router reachability (#4564). - try { - await reconcileModelRouter(); - } catch (err) { - console.error(` ✗ Failed to start model router: ${err instanceof Error ? err.message : String(err)}`); - process.exit(1); - } - const routed = routedInference.upsertRoutedProvider(provider, endpointUrl, credentialEnv, { upsertProvider, hydrateCredentialEnv }); - if (!routed.ok) { - console.error(` ${routed.result.message}`); - process.exit(routed.result.status || 1); - } - runOpenshell(["inference", "set", "--no-verify", "--provider", provider, "--model", model]); + await inferenceProviders.setupRoutedInference( + { model, provider, endpointUrl, credentialEnv }, + { + ...commonDeps, + reconcileModelRouter, + routedInference, + hydrateCredentialEnv, + }, + ); } else { console.error(` Unsupported provider configuration: ${provider}`); process.exit(1); diff --git a/src/lib/onboard/inference-providers/hermes.ts b/src/lib/onboard/inference-providers/hermes.ts new file mode 100644 index 00000000000..c0ab3e82bc0 --- /dev/null +++ b/src/lib/onboard/inference-providers/hermes.ts @@ -0,0 +1,138 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Hermes Provider inference setup flow. +// Extracted verbatim from onboard.setupInference (#767). + +import type { HermesAuthMethod } from "../hermes-auth"; +import type { HermesDeps, SetupInferenceResult } from "./types"; + +export async function setupHermesProviderInference( + args: { + sandboxName: string | null; + model: string; + provider: string; + endpointUrl: string | null; + credentialEnv: string | null; + hermesAuthMethod: HermesAuthMethod | string | null; + hermesToolGateways: string[]; + }, + deps: HermesDeps, +): Promise { + const { + sandboxName, + model, + provider, + endpointUrl, + credentialEnv, + hermesAuthMethod, + hermesToolGateways, + } = args; + const { + runOpenshell, + upsertProvider: _upsertProvider, // intentionally unused; matches inline branch + verifyInferenceRoute, + verifyOnboardInferenceSmoke, + isNonInteractive, + registry, + hermesProviderAuth, + getHermesToolGatewayBroker, + providerExistsInGateway, + normalizeHermesAuthMethod, + resolveHermesNousApiKey, + checkHermesProviderStoreReachable, + hermesAuthMethodLabel, + hermesConstants: { + HERMES_NOUS_API_KEY_CREDENTIAL_ENV, + HERMES_AUTH_METHOD_API_KEY, + HERMES_AUTH_METHOD_OAUTH, + }, + requireValue, + redact, + compactText, + } = deps; + void _upsertProvider; + + const targetSandbox = requireValue(sandboxName, "Hermes Provider requires a sandbox name"); + const resolvedHermesAuthMethod = + normalizeHermesAuthMethod(hermesAuthMethod) || + (credentialEnv === HERMES_NOUS_API_KEY_CREDENTIAL_ENV + ? HERMES_AUTH_METHOD_API_KEY + : HERMES_AUTH_METHOD_OAUTH); + const providerStore = checkHermesProviderStoreReachable(runOpenshell); + if (!providerStore.ok) { + console.error(" ✗ OpenShell provider storage is unreachable."); + console.error(` ${providerStore.message}`); + console.error(" Restart or recreate the OpenShell gateway, then rerun onboarding."); + if (isNonInteractive()) process.exit(1); + return { retry: "selection" }; + } + const providerRegistered = hermesProviderAuth.isHermesProviderRegistered(runOpenshell); + const toolGatewayProviderRegistered = + hermesToolGateways.length === 0 + ? true + : providerExistsInGateway( + getHermesToolGatewayBroker().getHermesToolGatewayProviderName(targetSandbox), + ); + const hasFreshNousApiKey = + resolvedHermesAuthMethod === HERMES_AUTH_METHOD_API_KEY && !!resolveHermesNousApiKey(); + const shouldPrepareHermesCredentials = + !providerRegistered || + !toolGatewayProviderRegistered || + hasFreshNousApiKey || + (resolvedHermesAuthMethod === HERMES_AUTH_METHOD_OAUTH && !isNonInteractive()); + if (shouldPrepareHermesCredentials) { + try { + const state = + resolvedHermesAuthMethod === HERMES_AUTH_METHOD_API_KEY + ? await hermesProviderAuth.ensureHermesProviderApiKeyCredentials(targetSandbox, { + apiKey: resolveHermesNousApiKey(), + runOpenshell, + baseUrl: endpointUrl || undefined, + }) + : await hermesProviderAuth.ensureHermesProviderOAuthCredentials(targetSandbox, { + allowInteractiveLogin: !isNonInteractive(), + runOpenshell, + baseUrl: endpointUrl || undefined, + toolGatewayPresets: hermesToolGateways, + }); + if (!state) { + const authLabel = hermesAuthMethodLabel(resolvedHermesAuthMethod); + console.error(` ✗ Hermes Provider ${authLabel} is not available on the host.`); + console.error( + " Re-run `nemoclaw onboard --agent hermes` interactively to configure credentials.", + ); + process.exit(1); + } + } catch (err) { + console.error( + ` ✗ Failed to prepare Hermes Provider credentials: ${ + err instanceof Error ? err.message : String(err) + }`, + ); + if (isNonInteractive()) process.exit(1); + return { retry: "selection" }; + } + } + + const applyResult = runOpenshell( + ["inference", "set", "--no-verify", "--provider", provider, "--model", model], + { ignoreError: true }, + ); + if (applyResult.status !== 0) { + const message = + compactText(redact(`${applyResult.stderr || ""} ${applyResult.stdout || ""}`)) || + `Failed to configure inference provider '${provider}'.`; + console.error(` ${message}`); + if (isNonInteractive()) process.exit(applyResult.status || 1); + return { retry: "selection" }; + } + + verifyInferenceRoute(provider, model); + verifyOnboardInferenceSmoke({ provider, model, endpointUrl, credentialEnv }); + if (sandboxName) { + registry.updateSandbox(sandboxName, { model, provider }); + } + console.log(` ✓ Inference route set: ${provider} / ${model}`); + return { ok: true }; +} diff --git a/src/lib/onboard/inference-providers/index.ts b/src/lib/onboard/inference-providers/index.ts new file mode 100644 index 00000000000..60122e5242d --- /dev/null +++ b/src/lib/onboard/inference-providers/index.ts @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Inference provider setup modules. +// +// `setupInference` in `src/lib/onboard.ts` is the orchestrator: it owns the +// step banner, the shared verify + registry-update finalization, and the +// "unsupported provider" error path. Each provider-specific branch lives in +// its own module here so the flows can be read, reviewed, and tested in +// isolation. See issue #767 for the broader provider extraction plan. + +export { setupHermesProviderInference } from "./hermes"; +export { setupOllamaLocalInference } from "./ollama-local"; +export { setupRemoteProviderInference } from "./remote"; +export { setupRoutedInference } from "./routed"; +export { setupVllmLocalInference } from "./vllm-local"; +export { + isRemoteProviderName, + REMOTE_PROVIDER_NAMES, +} from "./types"; +export type { + CommonDeps, + HermesDeps, + OllamaDeps, + RemoteProviderDeps, + RemoteProviderName, + RoutedDeps, + SetupInferenceResult, + VllmDeps, +} from "./types"; diff --git a/src/lib/onboard/inference-providers/ollama-local.ts b/src/lib/onboard/inference-providers/ollama-local.ts new file mode 100644 index 00000000000..0ec1e32dea7 --- /dev/null +++ b/src/lib/onboard/inference-providers/ollama-local.ts @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Ollama local inference provider setup flow. +// Extracted verbatim from onboard.setupInference (#767). + +import type { + OllamaDeps, + SetupInferenceResult, +} from "./types"; + +export async function setupOllamaLocalInference( + args: { model: string; provider: string; allowToolsIncompatible: boolean }, + deps: OllamaDeps, +): Promise<{ done: true; result: SetupInferenceResult } | { done: false }> { + const { model, provider, allowToolsIncompatible } = args; + const { + upsertProvider, + validateLocalProvider, + getLocalProviderBaseUrl, + applyLocalInferenceRoute, + getOllamaWarmupCommand, + run, + shouldFrontOllamaWithProxy, + ensureOllamaAuthProxy, + isProxyHealthy, + getOllamaProxyToken, + persistAndProbeOllamaProxy, + localInference, + OLLAMA_PROXY_CREDENTIAL_ENV, + } = deps; + + const validation = validateLocalProvider(provider); + let proxyReady = false; + const frontOllamaWithProxy = shouldFrontOllamaWithProxy(); + if (!validation.ok) { + // The container reachability check uses Docker's --add-host host-gateway, + // which may not work on all Docker configurations (e.g., Brev, rootless). + // The real sandbox uses k3s CoreDNS + NodeHosts — a different path. + // Try to start/restart the auth proxy before probing — this recovers + // from stale or missing proxy processes before we decide to abort. + if (frontOllamaWithProxy) { + ensureOllamaAuthProxy(); + proxyReady = isProxyHealthy(); + } + if (proxyReady) { + console.warn(` ⚠ ${validation.message}`); + if (validation.diagnostic) { + console.warn(` Diagnostic: ${validation.diagnostic}`); + } + console.warn( + " The auth proxy is healthy on the host — continuing. " + + "The sandbox uses a different network path and may work correctly.", + ); + } else { + console.error(` ${validation.message}`); + if (validation.diagnostic) { + console.error(` Diagnostic: ${validation.diagnostic}`); + } + if (process.platform === "darwin") { + console.error( + " On macOS, local inference also depends on OpenShell host routing support.", + ); + } + process.exit(1); + } + } + const baseUrl = getLocalProviderBaseUrl(provider); + let ollamaCredential = "ollama"; + if (frontOllamaWithProxy) { + // Skip if already started during the fallback recovery above. + if (!proxyReady) ensureOllamaAuthProxy(); + const proxyToken = getOllamaProxyToken(); + if (!proxyToken) { + console.error( + " Ollama auth proxy token is not set. Re-run onboard to initialize the proxy.", + ); + process.exit(1); + } + ollamaCredential = proxyToken; + // Persist token now that ollama-local is confirmed as the provider. + // Not persisted earlier in case the user backs out to a different provider. + await persistAndProbeOllamaProxy(proxyToken); + } + // Use a dedicated internal credential env (NEMOCLAW_OLLAMA_PROXY_TOKEN) + // so the gateway never reads the user's host OPENAI_API_KEY for local + // Ollama. GH #2519: a stale host OPENAI_API_KEY was leaking into the + // inference path and producing 401s. + const providerResult = upsertProvider( + "ollama-local", + "openai", + OLLAMA_PROXY_CREDENTIAL_ENV, + baseUrl, + { [OLLAMA_PROXY_CREDENTIAL_ENV]: ollamaCredential }, + ); + if (!providerResult.ok) { + console.error(` ${providerResult.message}`); + process.exit(providerResult.status || 1); + } + if (await applyLocalInferenceRoute("ollama-local", model)) { + return { done: true, result: { retry: "selection" } }; + } + console.log(` Priming Ollama model: ${model}`); + run(getOllamaWarmupCommand(model), { ignoreError: true }); + const probe = localInference.validateOllamaModelWithToolsOverride(model, allowToolsIncompatible); + if (!probe.ok) { + console.error(` ${probe.message}`); + process.exit(1); + } + // Do not mutate ~/.nemoclaw/credentials.json here: local Ollama now uses + // OLLAMA_PROXY_CREDENTIAL_ENV, so any saved OPENAI_API_KEY remains available + // to unrelated OpenAI-backed sandboxes. + return { done: false }; +} diff --git a/src/lib/onboard/inference-providers/remote.ts b/src/lib/onboard/inference-providers/remote.ts new file mode 100644 index 00000000000..3095efea1b4 --- /dev/null +++ b/src/lib/onboard/inference-providers/remote.ts @@ -0,0 +1,136 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Remote provider inference setup flow (NVIDIA, OpenAI, Anthropic, Gemini, +// compatible endpoints, Bedrock Runtime). Extracted verbatim from +// onboard.setupInference (#767). Bedrock Runtime is delegated to +// `onboard/bedrock-runtime.ts` exactly as the inline branch did. + +import type { + RemoteProviderDeps, + SetupInferenceResult, +} from "./types"; + +/** + * Returns `{ done: true, result }` when the flow handled the request + * (e.g. Bedrock short-circuit or a retry-to-selection); returns + * `{ done: false }` so the dispatcher can run the shared verify + registry + * finalization that used to live after the provider branches. + */ +export async function setupRemoteProviderInference( + args: { + sandboxName: string | null; + model: string; + provider: string; + endpointUrl: string | null; + credentialEnv: string | null; + }, + deps: RemoteProviderDeps, +): Promise<{ done: true; result: SetupInferenceResult } | { done: false }> { + const { sandboxName, model, provider, endpointUrl, credentialEnv } = args; + const { + runOpenshell, + upsertProvider, + verifyInferenceRoute, + verifyOnboardInferenceSmoke, + isNonInteractive, + REMOTE_PROVIDER_CONFIG, + hydrateCredentialEnv, + promptValidationRecovery, + classifyApplyFailure, + LOCAL_INFERENCE_TIMEOUT_SECS, + bedrockRuntimeOnboard, + redact, + compactText, + } = deps; + + const config = + provider === "nvidia-nim" + ? REMOTE_PROVIDER_CONFIG.build + : Object.values(REMOTE_PROVIDER_CONFIG).find((entry) => entry.providerName === provider); + if (!config) { + console.error(` Unsupported provider configuration: ${provider}`); + process.exit(1); + } + const bedrockSetup = await bedrockRuntimeOnboard.setupBedrockRuntimeInference({ + sandboxName, + provider, + model, + endpointUrl, + credentialEnv, + isNonInteractive, + runOpenshell, + upsertProvider, + verifyInferenceRoute, + verifyOnboardInferenceSmoke, + }); + if (bedrockSetup.handled) return { done: true, result: bedrockSetup.result }; + while (true) { + const resolvedCredentialEnv = credentialEnv || (config && config.credentialEnv); + const resolvedEndpointUrl = endpointUrl || (config && config.endpointUrl); + const credentialValue = hydrateCredentialEnv(resolvedCredentialEnv); + const env = + resolvedCredentialEnv && credentialValue + ? { [resolvedCredentialEnv]: credentialValue } + : {}; + const providerResult = upsertProvider( + provider, + config.providerType, + resolvedCredentialEnv, + resolvedEndpointUrl, + env, + ); + if (!providerResult.ok) { + console.error(` ${providerResult.message}`); + if (isNonInteractive()) { + process.exit(providerResult.status || 1); + } + const retry = await promptValidationRecovery( + config.label, + classifyApplyFailure(providerResult.message || ""), + resolvedCredentialEnv, + config.helpUrl, + ); + if (retry === "credential" || retry === "retry") { + continue; + } + if (retry === "selection" || retry === "model") { + return { done: true, result: { retry: "selection" } }; + } + process.exit(providerResult.status || 1); + } + const argsv = ["inference", "set"]; + if (config.skipVerify) { + argsv.push("--no-verify"); + } + argsv.push("--provider", provider, "--model", model); + if (provider === "compatible-endpoint") { + argsv.push("--timeout", String(LOCAL_INFERENCE_TIMEOUT_SECS)); + } + const applyResult = runOpenshell(argsv, { ignoreError: true }); + if (applyResult.status === 0) { + break; + } + const message = + compactText(redact(`${applyResult.stderr || ""} ${applyResult.stdout || ""}`)) || + `Failed to configure inference provider '${provider}'.`; + console.error(` ${message}`); + if (isNonInteractive()) { + process.exit(applyResult.status || 1); + } + const retry = await promptValidationRecovery( + config.label, + classifyApplyFailure(message), + resolvedCredentialEnv, + config.helpUrl, + ); + if (retry === "credential" || retry === "retry") { + continue; + } + if (retry === "selection" || retry === "model") { + return { done: true, result: { retry: "selection" } }; + } + process.exit(applyResult.status || 1); + } + return { done: false }; +} diff --git a/src/lib/onboard/inference-providers/routed.ts b/src/lib/onboard/inference-providers/routed.ts new file mode 100644 index 00000000000..17359f86ec4 --- /dev/null +++ b/src/lib/onboard/inference-providers/routed.ts @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Routed (blueprint profile, e.g. nvidia-router) inference setup flow. +// Extracted verbatim from onboard.setupInference (#767). + +import type { RoutedDeps } from "./types"; + +export async function setupRoutedInference( + args: { + model: string; + provider: string; + endpointUrl: string | null; + credentialEnv: string | null; + }, + deps: RoutedDeps, +): Promise<{ done: false }> { + const { model, provider, endpointUrl, credentialEnv } = args; + const { + runOpenshell, + upsertProvider, + reconcileModelRouter, + routedInference, + hydrateCredentialEnv, + } = deps; + + // Blueprint profile provider (e.g., nvidia-router for the routed profile). + // reconcileModelRouter also probes sandbox→router reachability (#4564). + try { + await reconcileModelRouter(); + } catch (err) { + console.error( + ` ✗ Failed to start model router: ${err instanceof Error ? err.message : String(err)}`, + ); + process.exit(1); + } + const routed = routedInference.upsertRoutedProvider(provider, endpointUrl, credentialEnv, { + upsertProvider, + hydrateCredentialEnv, + }); + if (!routed.ok) { + console.error(` ${routed.result.message}`); + process.exit(routed.result.status || 1); + } + runOpenshell(["inference", "set", "--no-verify", "--provider", provider, "--model", model]); + return { done: false }; +} diff --git a/src/lib/onboard/inference-providers/types.ts b/src/lib/onboard/inference-providers/types.ts new file mode 100644 index 00000000000..cf88629666a --- /dev/null +++ b/src/lib/onboard/inference-providers/types.ts @@ -0,0 +1,217 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Shared types for per-provider inference setup modules. +// +// Each module under `src/lib/onboard/inference-providers/` exports a function +// that owns the setup flow for one provider (or one closely related group of +// providers) used by `onboard.setupInference`. The orchestrator stays in +// `src/lib/onboard.ts`; this directory holds the provider-specific branches it +// used to inline. Behavior is preserved exactly — these are pure extractions. +// +// Many dependency signatures here are intentionally loose (`unknown` / +// permissive unions). The dispatcher in `onboard.ts` already has stricter +// types for the underlying helpers; these modules just plumb values through +// so they accept whatever the orchestrator hands in without needing to +// duplicate every helper's exact signature. + +import type { HermesAuthMethod } from "../hermes-auth"; + +export type SetupInferenceResult = + | { ok: true; retry?: undefined } + | { retry: "selection" }; + +export type RunOpenshell = ( + args: string[], + options?: { ignoreError?: boolean; suppressOutput?: boolean; timeout?: number }, +) => { status: number | null; stdout?: unknown; stderr?: unknown }; + +export type RunResult = { + status: number | null; + stdout?: unknown; + stderr?: unknown; +}; + +export type UpsertProviderResult = { + ok: boolean; + message?: string; + status?: number; +}; + +// Loose to match the various concrete signatures across onboard.ts. +export type UpsertProvider = ( + name: string, + type: string, + credentialEnv: any, + baseUrl: any, + env?: NodeJS.ProcessEnv, +) => UpsertProviderResult; + +export type RemoteProviderConfigEntry = { + label: string; + providerName: string; + providerType: string; + credentialEnv: string; + endpointUrl: string; + helpUrl: string | null; + modelMode: string; + defaultModel: string; + skipVerify?: boolean; +}; + +export type VerifyInferenceRoute = (provider: string, model: string) => void; + +export type VerifyOnboardInferenceSmoke = (input: { + provider: string; + model: string; + endpointUrl?: string | null; + credentialEnv?: string | null; + forceOpenAiLike?: boolean; +}) => void; + +export type PromptValidationRecovery = ( + label: string, + classification: any, + credentialEnv: any, + helpUrl: any, +) => Promise; + +export type ClassifyApplyFailure = (message: string) => any; + +export type Registry = { + updateSandbox(sandboxName: string, patch: { model: string; provider: string }): void; +}; + +export type CommonDeps = { + runOpenshell: RunOpenshell; + upsertProvider: UpsertProvider; + verifyInferenceRoute: VerifyInferenceRoute; + verifyOnboardInferenceSmoke: VerifyOnboardInferenceSmoke; + isNonInteractive: () => boolean; + registry: Registry; +}; + +export type RemoteProviderDeps = CommonDeps & { + REMOTE_PROVIDER_CONFIG: Record; + hydrateCredentialEnv: (envName: any, resolveCredential?: any) => any; + promptValidationRecovery: PromptValidationRecovery; + classifyApplyFailure: ClassifyApplyFailure; + LOCAL_INFERENCE_TIMEOUT_SECS: number; + redact: (input: string) => string; + compactText: (input: string) => string; + bedrockRuntimeOnboard: { + setupBedrockRuntimeInference(input: { + sandboxName: string | null; + provider: string; + model: string; + endpointUrl: string | null; + credentialEnv: string | null; + isNonInteractive: () => boolean; + runOpenshell: RunOpenshell; + upsertProvider: UpsertProvider; + verifyInferenceRoute: VerifyInferenceRoute; + verifyOnboardInferenceSmoke: any; + }): Promise<{ handled: true; result: SetupInferenceResult } | { handled: false }>; + }; +}; + +export type HermesDeps = CommonDeps & { + hermesProviderAuth: { + HERMES_PROVIDER_NAME: string; + isHermesProviderRegistered(runOpenshell: any): boolean; + ensureHermesProviderApiKeyCredentials( + sandboxName: string, + opts: { apiKey: unknown; runOpenshell: any; baseUrl?: string | undefined }, + ): Promise; + ensureHermesProviderOAuthCredentials( + sandboxName: string, + opts: { + allowInteractiveLogin: boolean; + runOpenshell: any; + baseUrl?: string | undefined; + toolGatewayPresets: string[]; + }, + ): Promise; + }; + getHermesToolGatewayBroker: () => { + getHermesToolGatewayProviderName(sandboxName: string): string; + }; + providerExistsInGateway: (name: string) => boolean; + normalizeHermesAuthMethod: (m: HermesAuthMethod | string | null) => HermesAuthMethod | null; + resolveHermesNousApiKey: () => any; + checkHermesProviderStoreReachable: (runOpenshell: any) => { ok: boolean; message?: string }; + hermesAuthMethodLabel: (m: HermesAuthMethod) => string; + hermesConstants: { + HERMES_NOUS_API_KEY_CREDENTIAL_ENV: string; + HERMES_AUTH_METHOD_API_KEY: HermesAuthMethod; + HERMES_AUTH_METHOD_OAUTH: HermesAuthMethod; + }; + requireValue: (value: T | null | undefined, message: string) => T; + redact: (input: string) => string; + compactText: (input: string) => string; +}; + +// `run` accepts an array form (execa-style) in the real onboard.ts; we type it +// loosely so callers can pass either shape without casting. +export type RunFn = (cmd: any, opts?: { ignoreError?: boolean; suppressOutput?: boolean }) => RunResult; + +export type VllmDeps = CommonDeps & { + validateLocalProvider: (provider: string) => { ok: boolean; message?: string; diagnostic?: string }; + getLocalProviderHealthCheck: (provider: string) => any; + getLocalProviderBaseUrl: (provider: string) => any; + applyLocalInferenceRoute: (provider: string, model: string) => Promise; + run: RunFn; + VLLM_LOCAL_CREDENTIAL_ENV: string; +}; + +export type OllamaDeps = CommonDeps & { + validateLocalProvider: (provider: string) => { ok: boolean; message?: string; diagnostic?: string }; + getLocalProviderBaseUrl: (provider: string) => any; + applyLocalInferenceRoute: (provider: string, model: string) => Promise; + getOllamaWarmupCommand: (model: string) => any; + run: RunFn; + shouldFrontOllamaWithProxy: () => boolean; + ensureOllamaAuthProxy: () => void; + isProxyHealthy: () => boolean; + getOllamaProxyToken: () => string | null | undefined; + persistAndProbeOllamaProxy: (token: string) => Promise; + localInference: { + validateOllamaModelWithToolsOverride( + model: string, + allowToolsIncompatible: boolean, + ): { ok: boolean; message?: string }; + }; + OLLAMA_PROXY_CREDENTIAL_ENV: string; +}; + +export type RoutedDeps = CommonDeps & { + reconcileModelRouter: () => Promise; + routedInference: { + upsertRoutedProvider( + provider: string, + endpointUrl: string | null, + credentialEnv: string | null, + helpers: { + upsertProvider: UpsertProvider; + hydrateCredentialEnv: (envName: any, resolveCredential?: any) => any; + }, + ): { ok: boolean; result: { message?: string; status?: number } }; + }; + hydrateCredentialEnv: (envName: any, resolveCredential?: any) => any; +}; + +export const REMOTE_PROVIDER_NAMES = [ + "nvidia-prod", + "nvidia-nim", + "openai-api", + "anthropic-prod", + "compatible-anthropic-endpoint", + "gemini-api", + "compatible-endpoint", +] as const; + +export type RemoteProviderName = (typeof REMOTE_PROVIDER_NAMES)[number]; + +export function isRemoteProviderName(value: string): value is RemoteProviderName { + return (REMOTE_PROVIDER_NAMES as readonly string[]).includes(value); +} diff --git a/src/lib/onboard/inference-providers/vllm-local.ts b/src/lib/onboard/inference-providers/vllm-local.ts new file mode 100644 index 00000000000..406fb1108a3 --- /dev/null +++ b/src/lib/onboard/inference-providers/vllm-local.ts @@ -0,0 +1,76 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// vLLM local inference provider setup flow. +// Extracted verbatim from onboard.setupInference (#767). + +import type { + SetupInferenceResult, + VllmDeps, +} from "./types"; + +export async function setupVllmLocalInference( + args: { model: string; provider: string }, + deps: VllmDeps, +): Promise<{ done: true; result: SetupInferenceResult } | { done: false }> { + const { model, provider } = args; + const { + upsertProvider, + validateLocalProvider, + getLocalProviderHealthCheck, + getLocalProviderBaseUrl, + applyLocalInferenceRoute, + run, + VLLM_LOCAL_CREDENTIAL_ENV, + } = deps; + + const validation = validateLocalProvider(provider); + if (!validation.ok) { + const hostCheck = getLocalProviderHealthCheck(provider); + // Use run() and check exit status rather than coercing runCapture() output + // to boolean — curl -sf can leave output even on failure in edge cases. + const hostResponding = hostCheck + ? run(hostCheck, { ignoreError: true, suppressOutput: true }).status === 0 + : false; + + if (hostResponding) { + console.warn(` ⚠ ${validation.message}`); + if (validation.diagnostic) { + console.warn(` Diagnostic: ${validation.diagnostic}`); + } + console.warn( + " The server is healthy on the host — continuing. " + + "The sandbox uses a different network path and may work correctly.", + ); + } else { + console.error(` ${validation.message}`); + if (validation.diagnostic) { + console.error(` Diagnostic: ${validation.diagnostic}`); + } + process.exit(1); + } + } + const baseUrl = getLocalProviderBaseUrl(provider); + // Use a dedicated internal credential env so the gateway does not pick + // up the user's host OPENAI_API_KEY for local vLLM. vLLM does not enforce + // the bearer at runtime, but a dedicated env name prevents accidental + // hijacking. See GH #2519. + const providerResult = upsertProvider( + "vllm-local", + "openai", + VLLM_LOCAL_CREDENTIAL_ENV, + baseUrl, + { [VLLM_LOCAL_CREDENTIAL_ENV]: "dummy" }, + ); + if (!providerResult.ok) { + console.error(` ${providerResult.message}`); + process.exit(providerResult.status || 1); + } + if (await applyLocalInferenceRoute("vllm-local", model)) { + return { done: true, result: { retry: "selection" } }; + } + // Do not mutate ~/.nemoclaw/credentials.json here: local vLLM now uses + // VLLM_LOCAL_CREDENTIAL_ENV, so any saved OPENAI_API_KEY remains available + // to unrelated OpenAI-backed sandboxes. + return { done: false }; +} From a645dafa2f00ca12b61ed40bdd25df0f9a084736 Mon Sep 17 00:00:00 2001 From: Benedikt Schackenberg <6381261+BenediktSchackenberg@users.noreply.github.com> Date: Thu, 4 Jun 2026 18:54:09 +0000 Subject: [PATCH 2/2] chore: retrigger DCO check Signed-off-by: Benedikt Schackenberg <6381261+BenediktSchackenberg@users.noreply.github.com>