diff --git a/src/lib/onboard/model-router-command.ts b/src/lib/onboard/model-router-command.ts new file mode 100644 index 00000000000..fc226e972c2 --- /dev/null +++ b/src/lib/onboard/model-router-command.ts @@ -0,0 +1,283 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +const MODEL_ROUTER_FINGERPRINT_FILE = ".nemoclaw-source-fingerprint"; +const MODEL_ROUTER_FINGERPRINT_IGNORED_NAMES = new Set([ + ".git", + ".hg", + ".mypy_cache", + ".pytest_cache", + ".ruff_cache", + ".svn", + ".venv", + "__pycache__", + "build", + "dist", + "node_modules", + "venv", +]); + +type RunOptions = { + ignoreError?: boolean; + timeout?: number; +}; + +type PrepareModelRouterVenvOptions = { + venvDir: string; + allowReplaceExisting?: boolean; +}; + +export type ModelRouterCommandPaths = { + rootDir: string; + routerDir: string; + venvDir: string; + defaultVenvDir: string; +}; + +export type ModelRouterCommandDeps = { + run: (command: string[], options?: RunOptions) => { status: number | null }; + runCapture: (command: string[], options?: RunOptions) => string; + prepareModelRouterVenv: (options: PrepareModelRouterVenvOptions) => string; + packageVersion: () => string; + log?: (message: string) => void; + sourceFingerprint?: (routerDir: string) => string | null; +}; + +export type ModelRouterCommandProvisioner = { + ensureModelRouterCommand(): string; + isManagedModelRouterCurrent(): boolean; +}; + +function modelRouterCommandPath(venvDir: string): string { + return path.join(venvDir, "bin", "model-router"); +} + +function modelRouterFingerprintPath(venvDir: string): string { + return path.join(venvDir, MODEL_ROUTER_FINGERPRINT_FILE); +} + +function isExecutableFile(filePath: string): boolean { + try { + fs.accessSync(filePath, fs.constants.X_OK); + return true; + } catch { + return false; + } +} + +function isModelRouterPackageReady(routerDir: string): boolean { + return ( + fs.existsSync(path.join(routerDir, "pyproject.toml")) || + fs.existsSync(path.join(routerDir, "setup.py")) + ); +} + +function shouldSkipModelRouterFingerprintEntry(name: string): boolean { + return MODEL_ROUTER_FINGERPRINT_IGNORED_NAMES.has(name) || name.endsWith(".egg-info"); +} + +function hashModelRouterSourceTree(routerDir: string): string | null { + const sourceHash = crypto.createHash("sha256"); + + const hashDirectory = (currentDir: string): boolean => { + let entries: fs.Dirent[]; + try { + entries = fs + .readdirSync(currentDir, { withFileTypes: true }) + .sort((left: fs.Dirent, right: fs.Dirent) => left.name.localeCompare(right.name)); + } catch { + return false; + } + + let hashedSourceFile = false; + for (const entry of entries) { + if (shouldSkipModelRouterFingerprintEntry(entry.name)) continue; + if (entry.name.endsWith(".pyc") || entry.name.endsWith(".pyo")) continue; + + const entryPath = path.join(currentDir, entry.name); + const relativePath = path.relative(routerDir, entryPath).split(path.sep).join("/"); + if (entry.isDirectory()) { + hashedSourceFile = hashDirectory(entryPath) || hashedSourceFile; + continue; + } + if (entry.isSymbolicLink()) { + try { + sourceHash.update(`link:${relativePath}\0`); + sourceHash.update(fs.readlinkSync(entryPath)); + sourceHash.update("\0"); + hashedSourceFile = true; + } catch { + // Ignore unreadable links; the install step will fail if they are required. + } + continue; + } + if (!entry.isFile()) continue; + sourceHash.update(`file:${relativePath}\0`); + sourceHash.update(fs.readFileSync(entryPath)); + sourceHash.update("\0"); + hashedSourceFile = true; + } + return hashedSourceFile; + }; + + return hashDirectory(routerDir) ? `files:${sourceHash.digest("hex")}` : null; +} + +function readModelRouterInstalledFingerprint(venvDir: string): string | null { + try { + const fingerprint = fs.readFileSync(modelRouterFingerprintPath(venvDir), "utf8").trim(); + return fingerprint || null; + } catch { + return null; + } +} + +function writeModelRouterInstalledFingerprint(fingerprint: string | null, venvDir: string): void { + if (!fingerprint) return; + fs.writeFileSync(modelRouterFingerprintPath(venvDir), `${fingerprint}\n`, { mode: 0o600 }); +} + +/** + * Build the managed Model Router command boundary around explicit process + * dependencies. Keeping command discovery and source fingerprinting here lets + * focused tests exercise provisioning directly without loading all of onboard + * or replacing CommonJS module-cache entries. + */ +export function createModelRouterCommandProvisioner( + paths: ModelRouterCommandPaths, + deps: ModelRouterCommandDeps, +): ModelRouterCommandProvisioner { + const relativeRouterDir = path.relative(paths.rootDir, paths.routerDir).split(path.sep).join("/"); + + const getSourceFingerprint = (): string | null => { + if (deps.sourceFingerprint) return deps.sourceFingerprint(paths.routerDir); + + const gitHead = deps + .runCapture(["git", "-C", paths.routerDir, "rev-parse", "HEAD"], { + ignoreError: true, + }) + .trim(); + if (/^[0-9a-f]{40}$/i.test(gitHead)) return `git:${gitHead}`; + + const gitLink = deps + .runCapture(["git", "-C", paths.rootDir, "rev-parse", `HEAD:${relativeRouterDir}`], { + ignoreError: true, + }) + .trim(); + if (/^[0-9a-f]{40}$/i.test(gitLink)) return `gitlink:${gitLink}`; + + return hashModelRouterSourceTree(paths.routerDir); + }; + + const isManagedModelRouterCurrent = (): boolean => { + if (!isExecutableFile(modelRouterCommandPath(paths.venvDir))) return false; + const sourceFingerprint = getSourceFingerprint(); + if (sourceFingerprint) { + return readModelRouterInstalledFingerprint(paths.venvDir) === sourceFingerprint; + } + // When source fingerprint is unavailable (no git), accept an existing + // install-prefixed fingerprint to avoid reinstalling on every onboard. + const installed = readModelRouterInstalledFingerprint(paths.venvDir); + return installed !== null && installed.startsWith("install:"); + }; + + const initializeModelRouterSubmodule = (): void => { + if (isModelRouterPackageReady(paths.routerDir)) return; + if ( + !fs.existsSync(path.join(paths.rootDir, ".gitmodules")) || + !fs.existsSync(path.join(paths.rootDir, ".git")) + ) { + return; + } + (deps.log ?? console.log)(" Initializing Model Router source..."); + deps.run( + [ + "git", + "-C", + paths.rootDir, + "submodule", + "update", + "--init", + "--depth", + "1", + relativeRouterDir, + ], + { ignoreError: true }, + ); + }; + + const installModelRouterCommand = (): string => { + initializeModelRouterSubmodule(); + if (!isModelRouterPackageReady(paths.routerDir)) { + throw new Error( + `Model Router source is not initialized at ${paths.routerDir}. ` + + `Run: git -C ${paths.rootDir} submodule update --init --depth 1 ${relativeRouterDir}`, + ); + } + + const routerCommand = modelRouterCommandPath(paths.venvDir); + const sourceFingerprint = getSourceFingerprint(); + const allowReplaceExistingVenv = + path.resolve(paths.venvDir) === path.resolve(paths.defaultVenvDir) || + readModelRouterInstalledFingerprint(paths.venvDir) !== null; + const venvPython = deps.prepareModelRouterVenv({ + venvDir: paths.venvDir, + allowReplaceExisting: allowReplaceExistingVenv, + }); + + const installResult = deps.run( + [ + venvPython, + "-m", + "pip", + "install", + "--quiet", + "--upgrade", + `${paths.routerDir}[prefill,proxy]`, + ], + { ignoreError: true, timeout: 600_000 }, + ); + if (installResult.status !== 0) { + throw new Error("Failed to install Model Router dependencies."); + } + if (!isExecutableFile(routerCommand)) { + throw new Error("Model Router install did not produce the model-router command."); + } + const effectiveFingerprint = sourceFingerprint ?? `install:${deps.packageVersion()}`; + writeModelRouterInstalledFingerprint(effectiveFingerprint, paths.venvDir); + return routerCommand; + }; + + const resolveHostCommandPath = (): string | null => { + const result = deps.runCapture(["sh", "-c", 'command -v "$1"', "--", "model-router"], { + ignoreError: true, + }); + return result.trim() || null; + }; + + const ensureModelRouterCommand = (): string => { + const managedCommand = modelRouterCommandPath(paths.venvDir); + + if (isModelRouterPackageReady(paths.routerDir) && isManagedModelRouterCurrent()) { + return managedCommand; + } + + if (!isModelRouterPackageReady(paths.routerDir)) { + initializeModelRouterSubmodule(); + } + + if (isModelRouterPackageReady(paths.routerDir)) { + if (isManagedModelRouterCurrent()) return managedCommand; + return installModelRouterCommand(); + } + + if (isExecutableFile(managedCommand)) return managedCommand; + return resolveHostCommandPath() || installModelRouterCommand(); + }; + + return { ensureModelRouterCommand, isManagedModelRouterCurrent }; +} diff --git a/src/lib/onboard/model-router.ts b/src/lib/onboard/model-router.ts index 5ff7fccf29c..a40afe0cc65 100644 --- a/src/lib/onboard/model-router.ts +++ b/src/lib/onboard/model-router.ts @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache-2.0 import { spawn, spawnSync } from "node:child_process"; -import crypto from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -22,6 +21,7 @@ import { formatHostServiceUnreachableMessage, probeHostServiceSandboxReachability, } from "./host-service-reachability"; +import { createModelRouterCommandProvisioner } from "./model-router-command"; import { doesModelRouterProcessOwnPort, findModelRouterPidForPort, @@ -30,25 +30,17 @@ import { } from "./model-router-process"; import { prepareModelRouterVenv } from "./model-router-python"; +export { + createModelRouterCommandProvisioner, + type ModelRouterCommandDeps, + type ModelRouterCommandPaths, + type ModelRouterCommandProvisioner, +} from "./model-router-command"; + const ROUTER_HEALTH_RETRIES = 15; const ROUTER_HEALTH_INTERVAL_MS = 2000; const MODEL_ROUTER_RELATIVE_DIR = path.join("nemoclaw-blueprint", "router", "llm-router"); const MODEL_ROUTER_VENV_DIR = path.join(os.homedir(), ".nemoclaw", "model-router-venv"); -const MODEL_ROUTER_FINGERPRINT_FILE = ".nemoclaw-source-fingerprint"; -const MODEL_ROUTER_FINGERPRINT_IGNORED_NAMES = new Set([ - ".git", - ".hg", - ".mypy_cache", - ".pytest_cache", - ".ruff_cache", - ".svn", - ".venv", - "__pycache__", - "build", - "dist", - "node_modules", - "venv", -]); export const DEFAULT_MODEL_ROUTER_CREDENTIAL_ENV = "NVIDIA_INFERENCE_API_KEY"; export type BlueprintRouterConfig = { @@ -67,6 +59,48 @@ export type BlueprintInferenceProfile = { router: BlueprintRouterConfig; }; +type ModelRouterProxyConfigResult = { + status: number | null; + stderr?: string | Buffer; + error?: Error; +}; + +type ModelRouterSpawnedProcess = { + pid: number | undefined; + onError(listener: (error: Error) => void): void; + onExit(listener: (code: number | null, signal: string | null) => void): void; + unref(): void; +}; + +export type StartModelRouterDeps = { + rootDir: string; + homeDir: string; + ensureModelRouterCommand: () => string; + mkdirSync: (directory: string) => void; + runProxyConfig: ( + command: string, + args: string[], + options: { encoding: "utf8"; timeout: number; cwd: string }, + ) => ModelRouterProxyConfigResult; + spawnProxy: ( + command: string, + args: string[], + options: { + detached: true; + stdio: "ignore"; + cwd: string; + env: Record; + }, + ) => ModelRouterSpawnedProcess; + resolveProviderCredential: (name: string) => string | null; + buildSubprocessEnv: (extra: Record) => Record; + isRouterHealthy: (port: number) => Promise; + sleep: (milliseconds: number) => Promise; + isProcessAlive: (pid: number) => boolean; + terminateProcess: (pid: number) => void; + getProviderKey: () => string; +}; + /** * Load a named inference profile and router config from blueprint.yaml. * Returns null if the blueprint or profile is missing. @@ -93,13 +127,6 @@ export function loadBlueprintProfile( } } -function resolveHostCommandPath(commandName: string): string | null { - const result = runCapture(["sh", "-c", 'command -v "$1"', "--", commandName], { - ignoreError: true, - }).trim(); - return result || null; -} - function modelRouterPackageDir(): string { return path.join(ROOT, MODEL_ROUTER_RELATIVE_DIR); } @@ -108,204 +135,39 @@ function modelRouterVenvDir(): string { return process.env.NEMOCLAW_MODEL_ROUTER_VENV || MODEL_ROUTER_VENV_DIR; } -function modelRouterCommandPath(venvDir = modelRouterVenvDir()): string { - return path.join(venvDir, "bin", "model-router"); -} - -function modelRouterFingerprintPath(venvDir = modelRouterVenvDir()): string { - return path.join(venvDir, MODEL_ROUTER_FINGERPRINT_FILE); -} - -function isExecutableFile(filePath: string): boolean { - try { - fs.accessSync(filePath, fs.constants.X_OK); - return true; - } catch { - return false; - } -} - -function isModelRouterPackageReady(routerDir = modelRouterPackageDir()): boolean { - return ( - fs.existsSync(path.join(routerDir, "pyproject.toml")) || - fs.existsSync(path.join(routerDir, "setup.py")) - ); -} - -function shouldSkipModelRouterFingerprintEntry(name: string): boolean { - return MODEL_ROUTER_FINGERPRINT_IGNORED_NAMES.has(name) || name.endsWith(".egg-info"); -} - -function hashModelRouterSourceTree(routerDir = modelRouterPackageDir()): string | null { - const sourceHash = crypto.createHash("sha256"); - - const hashDirectory = (currentDir: string): boolean => { - let entries: fs.Dirent[]; - try { - entries = fs - .readdirSync(currentDir, { withFileTypes: true }) - .sort((left: fs.Dirent, right: fs.Dirent) => left.name.localeCompare(right.name)); - } catch { - return false; - } - - let hashedSourceFile = false; - for (const entry of entries) { - if (shouldSkipModelRouterFingerprintEntry(entry.name)) continue; - if (entry.name.endsWith(".pyc") || entry.name.endsWith(".pyo")) continue; - - const entryPath = path.join(currentDir, entry.name); - const relativePath = path.relative(routerDir, entryPath).split(path.sep).join("/"); - if (entry.isDirectory()) { - hashedSourceFile = hashDirectory(entryPath) || hashedSourceFile; - continue; - } - if (entry.isSymbolicLink()) { - try { - sourceHash.update(`link:${relativePath}\0`); - sourceHash.update(fs.readlinkSync(entryPath)); - sourceHash.update("\0"); - hashedSourceFile = true; - } catch { - // Ignore unreadable links; the install step will fail if they are required. - } - continue; - } - if (!entry.isFile()) continue; - sourceHash.update(`file:${relativePath}\0`); - sourceHash.update(fs.readFileSync(entryPath)); - sourceHash.update("\0"); - hashedSourceFile = true; - } - return hashedSourceFile; - }; - - return hashDirectory(routerDir) ? `files:${sourceHash.digest("hex")}` : null; -} - -function getModelRouterSourceFingerprint(routerDir = modelRouterPackageDir()): string | null { - const gitHead = runCapture(["git", "-C", routerDir, "rev-parse", "HEAD"], { - ignoreError: true, - }).trim(); - if (/^[0-9a-f]{40}$/i.test(gitHead)) return `git:${gitHead}`; - - const gitLink = runCapture( - ["git", "-C", ROOT, "rev-parse", `HEAD:${MODEL_ROUTER_RELATIVE_DIR}`], +export function createProductionModelRouterCommandProvisioner( + routerDir = modelRouterPackageDir(), + venvDir = modelRouterVenvDir(), +) { + return createModelRouterCommandProvisioner( { - ignoreError: true, + rootDir: ROOT, + routerDir, + venvDir, + defaultVenvDir: MODEL_ROUTER_VENV_DIR, }, - ).trim(); - if (/^[0-9a-f]{40}$/i.test(gitLink)) return `gitlink:${gitLink}`; - - return hashModelRouterSourceTree(routerDir); -} - -function readModelRouterInstalledFingerprint(venvDir = modelRouterVenvDir()): string | null { - try { - const fingerprint = fs.readFileSync(modelRouterFingerprintPath(venvDir), "utf8").trim(); - return fingerprint || null; - } catch { - return null; - } -} - -function writeModelRouterInstalledFingerprint( - fingerprint: string | null, - venvDir = modelRouterVenvDir(), -): void { - if (!fingerprint) return; - fs.writeFileSync(modelRouterFingerprintPath(venvDir), `${fingerprint}\n`, { mode: 0o600 }); + { + run, + runCapture, + prepareModelRouterVenv, + packageVersion: () => + JSON.parse(fs.readFileSync(path.join(ROOT, "package.json"), "utf8")).version ?? "unknown", + }, + ); } export function isManagedModelRouterCurrent( routerDir = modelRouterPackageDir(), venvDir = modelRouterVenvDir(), ): boolean { - if (!isExecutableFile(modelRouterCommandPath(venvDir))) return false; - const sourceFingerprint = getModelRouterSourceFingerprint(routerDir); - if (sourceFingerprint) { - return readModelRouterInstalledFingerprint(venvDir) === sourceFingerprint; - } - // When source fingerprint is unavailable (no git), accept an existing - // install-prefixed fingerprint to avoid reinstalling on every onboard. - const installed = readModelRouterInstalledFingerprint(venvDir); - return installed !== null && installed.startsWith("install:"); -} - -function initializeModelRouterSubmodule(routerDir = modelRouterPackageDir()): void { - if (isModelRouterPackageReady(routerDir)) return; - if (!fs.existsSync(path.join(ROOT, ".gitmodules")) || !fs.existsSync(path.join(ROOT, ".git"))) { - return; - } - console.log(" Initializing Model Router source..."); - run( - ["git", "-C", ROOT, "submodule", "update", "--init", "--depth", "1", MODEL_ROUTER_RELATIVE_DIR], - { - ignoreError: true, - }, - ); -} - -function installModelRouterCommand(routerDir = modelRouterPackageDir()): string { - initializeModelRouterSubmodule(routerDir); - if (!isModelRouterPackageReady(routerDir)) { - throw new Error( - `Model Router source is not initialized at ${routerDir}. ` + - `Run: git -C ${ROOT} submodule update --init --depth 1 ${MODEL_ROUTER_RELATIVE_DIR}`, - ); - } - - const venvDir = modelRouterVenvDir(); - const routerCommand = modelRouterCommandPath(venvDir); - const sourceFingerprint = getModelRouterSourceFingerprint(routerDir); - const allowReplaceExistingVenv = - path.resolve(venvDir) === path.resolve(MODEL_ROUTER_VENV_DIR) || - readModelRouterInstalledFingerprint(venvDir) !== null; - const venvPython = prepareModelRouterVenv({ + return createProductionModelRouterCommandProvisioner( + routerDir, venvDir, - allowReplaceExisting: allowReplaceExistingVenv, - }); - - const installResult = run( - [venvPython, "-m", "pip", "install", "--quiet", "--upgrade", `${routerDir}[prefill,proxy]`], - { - ignoreError: true, - timeout: 600_000, - }, - ); - if (installResult.status !== 0) { - throw new Error("Failed to install Model Router dependencies."); - } - if (!isExecutableFile(routerCommand)) { - throw new Error("Model Router install did not produce the model-router command."); - } - const version = - JSON.parse(fs.readFileSync(path.join(ROOT, "package.json"), "utf8")).version ?? "unknown"; - const effectiveFingerprint = sourceFingerprint ?? `install:${version}`; - writeModelRouterInstalledFingerprint(effectiveFingerprint, venvDir); - return routerCommand; + ).isManagedModelRouterCurrent(); } function ensureModelRouterCommand(): string { - const routerDir = modelRouterPackageDir(); - const venvDir = modelRouterVenvDir(); - const managedCommand = modelRouterCommandPath(venvDir); - - if (isModelRouterPackageReady(routerDir) && isManagedModelRouterCurrent(routerDir, venvDir)) { - return managedCommand; - } - - if (!isModelRouterPackageReady(routerDir)) { - initializeModelRouterSubmodule(routerDir); - } - - if (isModelRouterPackageReady(routerDir)) { - if (isManagedModelRouterCurrent(routerDir, venvDir)) return managedCommand; - return installModelRouterCommand(routerDir); - } - - if (isExecutableFile(managedCommand)) return managedCommand; - return resolveHostCommandPath("model-router") || installModelRouterCommand(); + return createProductionModelRouterCommandProvisioner().ensureModelRouterCommand(); } /** @@ -313,20 +175,61 @@ function ensureModelRouterCommand(): string { * Follows the same pattern as Ollama startup (spawn detached, poll health). * Returns the PID of the child process. */ -async function startModelRouter(routerCfg: BlueprintRouterConfig): Promise { - const routerCommand = ensureModelRouterCommand(); +function createStartModelRouterDeps(): StartModelRouterDeps { + return { + rootDir: ROOT, + homeDir: os.homedir(), + ensureModelRouterCommand, + mkdirSync: (directory) => fs.mkdirSync(directory, { recursive: true }), + runProxyConfig: (command, args, options) => spawnSync(command, args, options), + spawnProxy: (command, args, options) => { + const child = spawn(command, args, options); + return { + pid: child.pid, + onError: (listener) => { + child.once("error", listener); + }, + onExit: (listener) => { + child.once("exit", listener); + }, + unref: () => child.unref(), + }; + }, + resolveProviderCredential, + buildSubprocessEnv, + isRouterHealthy, + sleep: (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)), + isProcessAlive: (pid) => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } + }, + terminateProcess: (pid) => process.kill(pid, "SIGTERM"), + getProviderKey: () => (process.env.NEMOCLAW_PROVIDER_KEY || "").trim(), + }; +} + +export async function startModelRouter( + routerCfg: BlueprintRouterConfig, + overrides: Partial = {}, +): Promise { + const deps: StartModelRouterDeps = { ...createStartModelRouterDeps(), ...overrides }; + const routerCommand = deps.ensureModelRouterCommand(); const port = routerCfg.port || 4000; - const blueprintDir = path.join(ROOT, "nemoclaw-blueprint"); + const blueprintDir = path.join(deps.rootDir, "nemoclaw-blueprint"); const poolConfigPath = path.join( blueprintDir, routerCfg.pool_config_path || "router/pool-config.yaml", ); - const stateDir = path.join(os.homedir(), ".nemoclaw", "state"); + const stateDir = path.join(deps.homeDir, ".nemoclaw", "state"); const litellmConfigPath = path.join(stateDir, "litellm-proxy.yaml"); - fs.mkdirSync(stateDir, { recursive: true }); + deps.mkdirSync(stateDir); - const proxyConfigResult = spawnSync( + const proxyConfigResult = deps.runProxyConfig( routerCommand, ["proxy-config", "--config", poolConfigPath, "--output", litellmConfigPath], { encoding: "utf8", timeout: 30_000, cwd: blueprintDir }, @@ -339,26 +242,26 @@ async function startModelRouter(routerCfg: BlueprintRouterConfig): Promise = {}; const credName = routerCfg.credential_env || DEFAULT_MODEL_ROUTER_CREDENTIAL_ENV; - const routedCredential = resolveProviderCredential(credName); - const openAiCredential = resolveProviderCredential("OPENAI_API_KEY"); + const routedCredential = deps.resolveProviderCredential(credName); + const openAiCredential = deps.resolveProviderCredential("OPENAI_API_KEY"); if (routedCredential) { credEnvVars[credName] = routedCredential; if (!openAiCredential) credEnvVars.OPENAI_API_KEY = routedCredential; } if (openAiCredential) credEnvVars.OPENAI_API_KEY = openAiCredential; - const _providerKey = (process.env.NEMOCLAW_PROVIDER_KEY || "").trim(); + const _providerKey = deps.getProviderKey(); if (_providerKey) { if (!credEnvVars[credName]) credEnvVars[credName] = _providerKey; if (!credEnvVars.OPENAI_API_KEY) credEnvVars.OPENAI_API_KEY = _providerKey; } - if (await isRouterHealthy(port)) { + if (await deps.isRouterHealthy(port)) { throw new Error( `Port ${port} already has a healthy router endpoint; refusing to start a second router.`, ); } - const child = spawn( + const child = deps.spawnProxy( routerCommand, [ "proxy", @@ -375,16 +278,16 @@ async function startModelRouter(routerCfg: BlueprintRouterConfig): Promise { + child.onError((err: Error) => { childExited = true; childExitDetail = `child failed to start: ${err.message}`; }); - child.once("exit", (code: number | null, signal: string | null) => { + child.onExit((code: number | null, signal: string | null) => { childExited = true; if (!childExitDetail) { childExitDetail = `child exited with code ${code ?? "null"}${signal ? ` signal ${signal}` : ""}`; @@ -401,15 +304,10 @@ async function startModelRouter(routerCfg: BlueprintRouterConfig): Promise setTimeout(resolve, ROUTER_HEALTH_INTERVAL_MS)); + await deps.sleep(ROUTER_HEALTH_INTERVAL_MS); if (childExited) break; - const healthy = await isRouterHealthy(port); - let processAlive = true; - try { - process.kill(pid, 0); - } catch { - processAlive = false; - } + const healthy = await deps.isRouterHealthy(port); + const processAlive = deps.isProcessAlive(pid); if (healthy && processAlive) return pid; if (!processAlive) { childExited = true; @@ -418,7 +316,7 @@ async function startModelRouter(routerCfg: BlueprintRouterConfig): Promise; -}; - -type SandboxInferenceConfig = { - providerKey: string; - primaryModelRef: string; - inferenceBaseUrl: string; - inferenceApi: string; - inferenceCompat: unknown; +import { afterEach, describe, it, vi } from "vitest"; + +import { getSandboxInferenceConfig } from "../src/lib/inference/config"; +import { + createProductionModelRouterCommandProvisioner, + isManagedModelRouterCurrent, + startModelRouter, +} from "../src/lib/onboard/model-router"; +import { + createModelRouterCommandProvisioner, + type ModelRouterCommandDeps, +} from "../src/lib/onboard/model-router-command"; +import type { SetupInference, SetupInferenceDeps } from "../src/lib/onboard/setup-inference.js"; +import { run, runCapture } from "../src/lib/runner"; +import { + createProductionModelRouterInstallFixture, + readRouterLaunchLog, + stopTestProcess, +} from "./support/model-router-process-test-helpers.js"; +import { + createDirectSetupInferenceHarnessFactory, + type DirectCommandEntry, + withProcessEnv, +} from "./support/setup-inference-test-harness.js"; + +const onboard = require("../src/lib/onboard") as { + createSetupInference: (overrides?: Partial) => SetupInference; }; - -function parseStdoutJson(stdout: string): T { - const line = stdout.trim().split("\n").pop(); - assert.ok(line, `expected JSON payload in stdout:\n${stdout}`); - return JSON.parse(line); -} +const createDirectSetupInferenceHarness = createDirectSetupInferenceHarnessFactory( + onboard.createSetupInference, +); const MODEL_ROUTER_FINGERPRINT_FILE = ".nemoclaw-source-fingerprint"; const MODEL_ROUTER_TEST_SOURCE_SHA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; +const MODEL_ROUTER_TEST_VERSION = "0.1.0"; +const NVIDIA_TEST_CREDENTIAL = "nvapi-TEST-NOT-A-REAL-ROUTER-KEY"; -describe("onboard Model Router setup", () => { - it( - "configures Model Router as a host provider while sandboxes keep inference.local", - testTimeoutOptions(60_000), - () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-router-inference-")); - const fakeBin = path.join(tmpDir, "bin"); - const venvDir = path.join(tmpDir, "model-router-venv"); - const venvBin = path.join(venvDir, "bin"); - const scriptPath = path.join(tmpDir, "setup-router-check.js"); - const routerPort = 44000 + (process.pid % 10000); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "registry.ts"), - ); - - fs.mkdirSync(fakeBin, { recursive: true }); - fs.mkdirSync(venvBin, { recursive: true }); - fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { - mode: 0o755, - }); - fs.writeFileSync( - path.join(venvBin, "model-router"), - [ - "#!/usr/bin/env node", - 'const fs = require("fs");', - 'const http = require("http");', - 'const path = require("path");', - "const args = process.argv.slice(2);", - 'if (args[0] === "proxy-config") {', - ' const output = args[args.indexOf("--output") + 1];', - " fs.mkdirSync(path.dirname(output), { recursive: true });", - ' fs.writeFileSync(output, "model_list: []\\n");', - " process.exit(0);", - "}", - 'if (args[0] === "proxy") {', - ' const port = Number(args[args.indexOf("--port") + 1] || "4000");', - " const server = http.createServer((req, res) => {", - ' if (req.url === "/health") { res.statusCode = 200; res.end("ok"); return; }', - " res.statusCode = 404;", - " res.end();", - " });", - ' server.listen(port, "127.0.0.1");', - " setTimeout(() => process.exit(0), 10000);", - "} else {", - " process.exit(1);", - "}", - "", - ].join("\n"), - { mode: 0o755 }, - ); - fs.writeFileSync( - path.join(venvDir, MODEL_ROUTER_FINGERPRINT_FILE), - `git:${MODEL_ROUTER_TEST_SOURCE_SHA}\n`, - { mode: 0o600 }, - ); - - const script = String.raw` -const fs = require("fs"); -const path = require("path"); -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const routerPort = ${routerPort}; -const blueprintPath = path.join(${JSON.stringify(repoRoot)}, "nemoclaw-blueprint", "blueprint.yaml"); -const routerPyproject = path.join(${JSON.stringify(repoRoot)}, "nemoclaw-blueprint", "router", "llm-router", "pyproject.toml"); -const originalReadFileSync = fs.readFileSync; -const originalExistsSync = fs.existsSync; -fs.existsSync = (filePath) => { - if (filePath === routerPyproject || String(filePath) === routerPyproject) return true; - return originalExistsSync(filePath); -}; -fs.readFileSync = (filePath, ...args) => { - const raw = originalReadFileSync(filePath, ...args); - if (filePath === blueprintPath || String(filePath) === blueprintPath) { - return String(raw) - .replace('endpoint: "http://localhost:4000/v1"', 'endpoint: "http://localhost:' + routerPort + '/v1"') - .replace("port: 4000", "port: " + routerPort); - } - return raw; +type PrepareCall = { + venvDir: string; + allowReplaceExisting?: boolean; }; -const commands = []; -runner.run = (command, opts = {}) => { - const cmd = _n(command); - if (/\bpython3(?:\.\d+)? -m venv\b/.test(cmd) || cmd.includes("/bin/python -m pip")) { - throw new Error("unexpected managed-router reinstall in reuse test: " + cmd); - } - if (/(^|[\/\s])pip3(?:\s|$)/.test(cmd)) { - throw new Error("unexpected pip3 invocation in test harness: " + cmd); - } - if (cmd.includes("git -C") || /^git(?:\s|$)/.test(cmd)) { - throw new Error("unexpected git invocation in test harness: " + cmd); - } - commands.push({ command: cmd, env: opts.env || null }); - if (cmd.includes("provider get")) return { status: 1, stdout: "", stderr: "" }; - if ( - cmd.includes("gateway select") || - cmd.includes("provider create") || - cmd.includes("provider update") || - cmd.includes("inference set") - ) { - return { status: 0, stdout: "", stderr: "" }; - } - throw new Error("unexpected command in managed-router reuse test: " + cmd); +type CommandHarnessOptions = { + installedFingerprint?: string; + managedCommand?: boolean; + pathCommand?: string; + sourceFingerprint?: ModelRouterCommandDeps["sourceFingerprint"]; }; -runner.runCapture = (command) => { - const cmd = _n(command); - if (cmd.includes("git -C") && cmd.includes("rev-parse HEAD")) { - return ${JSON.stringify(MODEL_ROUTER_TEST_SOURCE_SHA)}; - } - if (cmd.includes("command -v") && /model-router$/.test(cmd)) { - return ${JSON.stringify(path.join(fakeBin, "model-router"))}; - } - if (cmd.includes("inference") && cmd.includes("get")) { - return [ - "Gateway inference:", - "", - " Route: inference.local", - " Provider: nvidia-router", - " Model: nvidia-routed", - " Version: 1", - ].join(String.fromCharCode(10)); - } - return ""; -}; -registry.updateSandbox = () => true; -process.env.NVIDIA_INFERENCE_API_KEY = "nvapi-TEST-NOT-A-REAL-ROUTER-KEY"; +const tempDirs = new Set(); -const { setupInference, getSandboxInferenceConfig } = require(${onboardPath}); +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllEnvs(); + for (const tmpDir of tempDirs) { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } + tempDirs.clear(); +}); -(async () => { - await setupInference( - "router-box", - "nvidia-routed", - "nvidia-router", - "http://host.openshell.internal:" + routerPort + "/v1", - "NVIDIA_INFERENCE_API_KEY", +function createCommandHarness(options: CommandHarnessOptions = {}) { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-router-command-")); + tempDirs.add(tmpDir); + const rootDir = path.join(tmpDir, "repo"); + const routerDir = path.join(rootDir, "nemoclaw-blueprint", "router", "llm-router"); + const venvDir = path.join(tmpDir, "model-router-venv"); + const defaultVenvDir = path.join(tmpDir, "default-model-router-venv"); + const managedCommand = path.join(venvDir, "bin", "model-router"); + const fingerprintPath = path.join(venvDir, MODEL_ROUTER_FINGERPRINT_FILE); + const runCalls: string[][] = []; + const runCaptureCalls: string[][] = []; + const prepareCalls: PrepareCall[] = []; + + fs.mkdirSync(routerDir, { recursive: true }); + fs.writeFileSync(path.join(routerDir, "pyproject.toml"), "[project]\nname = 'model-router'\n"); + const writeManagedCommand = () => { + fs.mkdirSync(path.dirname(managedCommand), { recursive: true }); + fs.writeFileSync(managedCommand, "#!/usr/bin/env sh\nexit 0\n", { mode: 0o755 }); + }; + options.managedCommand ? writeManagedCommand() : undefined; + const writeInstalledFingerprint = (fingerprint: string) => { + fs.mkdirSync(venvDir, { recursive: true }); + fs.writeFileSync(fingerprintPath, `${fingerprint}\n`, { mode: 0o600 }); + }; + options.installedFingerprint === undefined + ? undefined + : writeInstalledFingerprint(options.installedFingerprint); + + const deps: ModelRouterCommandDeps = { + run(command) { + runCalls.push(command); + command.includes("pip") && command.includes("install") && writeManagedCommand(); + return { status: 0 }; + }, + runCapture(command) { + runCaptureCalls.push(command); + return command[0] === "git" && command.includes("HEAD") + ? MODEL_ROUTER_TEST_SOURCE_SHA + : command[0] === "sh" + ? (options.pathCommand ?? "") + : ""; + }, + prepareModelRouterVenv(prepareOptions) { + prepareCalls.push(prepareOptions); + const venvPython = path.join(prepareOptions.venvDir, "bin", "python"); + fs.mkdirSync(path.dirname(venvPython), { recursive: true }); + fs.writeFileSync(venvPython, "#!/usr/bin/env sh\nexit 0\n", { mode: 0o755 }); + return venvPython; + }, + packageVersion: () => MODEL_ROUTER_TEST_VERSION, + ...(options.sourceFingerprint ? { sourceFingerprint: options.sourceFingerprint } : {}), + }; + const provisioner = createModelRouterCommandProvisioner( + { rootDir, routerDir, venvDir, defaultVenvDir }, + deps, ); - console.log(JSON.stringify({ - commands, - sandboxConfig: getSandboxInferenceConfig("nvidia-routed", "nvidia-router", "openai-completions"), - })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - const result = spawnSync( - process.execPath, - [ - "--require", - path.join(repoRoot, "test", "helpers", "onboard-script-mocks.cjs"), - scriptPath, - ], - { - cwd: repoRoot, - encoding: "utf-8", - env: { - ...process.env, - HOME: tmpDir, - PATH: `${fakeBin}:${process.env.PATH || ""}`, - NEMOCLAW_MODEL_ROUTER_VENV: venvDir, - }, + return { + fingerprintPath, + managedCommand, + prepareCalls, + provisioner, + routerDir, + runCalls, + runCaptureCalls, + venvDir, + }; +} + +function findCommand(commands: DirectCommandEntry[], pattern: RegExp): DirectCommandEntry { + const command = commands.find((entry) => pattern.test(entry.command)); + assert.ok(command, JSON.stringify(commands)); + return command; +} + +describe("onboard Model Router setup", () => { + it("configures Model Router as a host provider while sandboxes keep inference.local", async () => { + await withProcessEnv({ NVIDIA_INFERENCE_API_KEY: NVIDIA_TEST_CREDENTIAL }, async () => { + const reconcileModelRouter = vi.fn(async () => undefined); + const harness = createDirectSetupInferenceHarness({ + runOpenshell: (args) => + args[0] === "provider" && args[1] === "get" ? { status: 1 } : undefined, + overrides: { + isRoutedInferenceProvider: (provider: string) => provider === "nvidia-router", + reconcileModelRouter, }, - ); + }); + const routerPort = 44000 + (process.pid % 10000); - assert.equal(result.status, 0, result.stderr); - const payload = parseStdoutJson<{ - commands: CommandEntry[]; - sandboxConfig: SandboxInferenceConfig; - }>(result.stdout); - const providerCommand = payload.commands.find((entry) => - /provider create/.test(entry.command), + await harness.setupInference( + "router-box", + "nvidia-routed", + "nvidia-router", + `http://host.openshell.internal:${routerPort}/v1`, + "NVIDIA_INFERENCE_API_KEY", ); - assert.ok(providerCommand, JSON.stringify(payload.commands)); + + assert.equal(reconcileModelRouter.mock.calls.length, 1); + const providerCommand = findCommand(harness.commands, /provider create/); assert.match(providerCommand.command, /--name nvidia-router/); assert.match(providerCommand.command, /--credential NVIDIA_INFERENCE_API_KEY/); assert.match( providerCommand.command, new RegExp(`OPENAI_BASE_URL=http:\\/\\/host\\.openshell\\.internal:${routerPort}\\/v1`), ); - assert.doesNotMatch(providerCommand.command, /nvapi-TEST-NOT-A-REAL-ROUTER-KEY/); - assert.equal( - providerCommand.env?.NVIDIA_INFERENCE_API_KEY, - "nvapi-TEST-NOT-A-REAL-ROUTER-KEY", - ); + assert.doesNotMatch(providerCommand.command, new RegExp(NVIDIA_TEST_CREDENTIAL)); + assert.equal(providerCommand.env?.NVIDIA_INFERENCE_API_KEY, NVIDIA_TEST_CREDENTIAL); - const inferenceCommand = payload.commands.find((entry) => - /inference set/.test(entry.command), - ); - assert.ok(inferenceCommand, JSON.stringify(payload.commands)); + const inferenceCommand = findCommand(harness.commands, /inference set/); assert.match(inferenceCommand.command, /--provider nvidia-router/); assert.match(inferenceCommand.command, /--model nvidia-routed/); - - assert.deepEqual(payload.sandboxConfig, { + assert.deepEqual(getSandboxInferenceConfig("nvidia-routed", "nvidia-router"), { providerKey: "inference", primaryModelRef: "inference/nvidia-routed", inferenceBaseUrl: "https://inference.local/v1", inferenceApi: "openai-completions", inferenceCompat: null, }); - }, - ); - - it( - "prepares managed Model Router dependencies instead of using PATH when managed command is absent", - testTimeoutOptions(30_000), - () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-router-venv-")); - const fakeBin = path.join(tmpDir, "bin"); - const scriptPath = path.join(tmpDir, "setup-router-venv-check.js"); - const fakeRouterSource = path.join(tmpDir, "model-router-source.js"); - const setupLog = path.join(tmpDir, "router-setup.log"); - const venvDir = path.join(tmpDir, "model-router-venv"); - const routerPort = 45000 + (process.pid % 10000); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "registry.ts"), - ); - - try { - fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { - mode: 0o755, - }); - fs.writeFileSync( - path.join(fakeBin, "model-router"), - [ - "#!/usr/bin/env bash", - `printf "path-router %s\\n" "$*" >> ${JSON.stringify(setupLog)}`, - "exit 89", - "", - ].join("\n"), - { mode: 0o755 }, - ); - fs.writeFileSync( - path.join(fakeBin, "python3"), - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - `printf "python3 %s\\n" "$*" >> ${JSON.stringify(setupLog)}`, - // pickHostPython probe (#3781) — emit a healthy probe response so - // the helper proceeds to the venv step instead of falling back. - 'if [ "$1" = "-c" ]; then', - ' printf \'{"version": [3, 12, 7], "error": null}\\n\'', - " exit 0", - "fi", - 'if [ "$1" = "-m" ] && [ "$2" = "venv" ]; then', - ' venv_dir="$3"', - ' mkdir -p "$venv_dir/bin"', - " cat > \"$venv_dir/bin/python\" <<'PY'", - "#!/usr/bin/env bash", - "set -euo pipefail", - `printf "venv-python %s\\n" "$*" >> ${JSON.stringify(setupLog)}`, - 'if [ "$1" = "-m" ] && [ "$2" = "pip" ] && [ "$3" = "install" ]; then', - ' venv_bin="$(cd "$(dirname "$0")" && pwd)"', - ` cp ${JSON.stringify(fakeRouterSource)} "$venv_bin/model-router"`, - ' chmod +x "$venv_bin/model-router"', - " exit 0", - "fi", - "exit 97", - "PY", - ' chmod +x "$venv_dir/bin/python"', - " exit 0", - "fi", - "exit 96", - "", - ].join("\n"), - { mode: 0o755 }, - ); - fs.writeFileSync( - path.join(fakeBin, "pip3"), - [ - "#!/usr/bin/env bash", - 'printf "pip3 %s\\n" "$*" >> "$ROUTER_SETUP_LOG"', - "exit 88", - "", - ].join("\n"), - { mode: 0o755 }, - ); - fs.writeFileSync( - fakeRouterSource, - [ - `#!${process.execPath}`, - 'const fs = require("fs");', - 'const http = require("http");', - 'const path = require("path");', - "const args = process.argv.slice(2);", - 'if (args[0] === "proxy-config") {', - ' const output = args[args.indexOf("--output") + 1];', - " fs.mkdirSync(path.dirname(output), { recursive: true });", - ' fs.writeFileSync(output, "model_list: []\\n");', - " process.exit(0);", - "}", - 'if (args[0] === "proxy") {', - ' const port = Number(args[args.indexOf("--port") + 1] || "4000");', - " const server = http.createServer((req, res) => {", - ' if (req.url === "/health") { res.statusCode = 200; res.end("ok"); return; }', - " res.statusCode = 404;", - " res.end();", - " });", - ' server.listen(port, "127.0.0.1");', - " setTimeout(() => process.exit(0), 10000);", - "} else {", - " process.exit(1);", - "}", - "", - ].join("\n"), - { mode: 0o755 }, - ); - - const script = String.raw` -const fs = require("fs"); -const path = require("path"); -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const routerPort = ${routerPort}; -const repoRoot = ${JSON.stringify(repoRoot)}; -const blueprintPath = path.join(repoRoot, "nemoclaw-blueprint", "blueprint.yaml"); -const routerPyproject = path.join(repoRoot, "nemoclaw-blueprint", "router", "llm-router", "pyproject.toml"); -const originalReadFileSync = fs.readFileSync; -const originalExistsSync = fs.existsSync; -const originalRun = runner.run; -fs.existsSync = (filePath) => { - if (filePath === routerPyproject || String(filePath) === routerPyproject) return true; - return originalExistsSync(filePath); -}; -fs.readFileSync = (filePath, ...args) => { - const raw = originalReadFileSync(filePath, ...args); - if (filePath === blueprintPath || String(filePath) === blueprintPath) { - return String(raw) - .replace('endpoint: "http://localhost:4000/v1"', 'endpoint: "http://localhost:' + routerPort + '/v1"') - .replace("port: 4000", "port: " + routerPort); - } - return raw; -}; - -const commands = []; -runner.run = (command, opts = {}) => { - const cmd = _n(command); - if (/\bpython3(?:\.\d+)? -m venv\b/.test(cmd) || cmd.includes("/bin/python -m pip")) { - return originalRun(command, opts); - } - if (/(^|[\/\s])pip3(?:\s|$)/.test(cmd)) { - throw new Error("unexpected pip3 invocation in test harness: " + cmd); - } - if (cmd.includes("git -C") || /^git(?:\s|$)/.test(cmd)) { - throw new Error("unexpected git invocation in test harness: " + cmd); - } - commands.push({ command: cmd, env: opts.env || null }); - if (cmd.includes("provider get")) return { status: 1, stdout: "", stderr: "" }; - return { status: 0, stdout: "", stderr: "" }; -}; -runner.runCapture = (command) => { - const cmd = _n(command); - if (cmd.includes("git -C") && cmd.includes("rev-parse HEAD")) { - return ${JSON.stringify(MODEL_ROUTER_TEST_SOURCE_SHA)}; - } - if (cmd.includes("command -v") && /model-router$/.test(cmd)) { - return ${JSON.stringify(path.join(fakeBin, "model-router"))}; - } - if (cmd.includes("command -v") && /python3$/.test(cmd)) { - return ${JSON.stringify(path.join(fakeBin, "python3"))}; - } - if (cmd.includes("inference") && cmd.includes("get")) { - return [ - "Gateway inference:", - "", - " Route: inference.local", - " Provider: nvidia-router", - " Model: nvidia-routed", - " Version: 1", - ].join(String.fromCharCode(10)); - } - return ""; -}; -registry.updateSandbox = () => true; - -process.env.NVIDIA_INFERENCE_API_KEY = "nvapi-TEST-NOT-A-REAL-ROUTER-KEY"; - -const { setupInference } = require(${onboardPath}); - -(async () => { - await setupInference( - "router-box", - "nvidia-routed", - "nvidia-router", - "http://host.openshell.internal:" + routerPort + "/v1", - "NVIDIA_INFERENCE_API_KEY", - ); - console.log(JSON.stringify({ commands })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync( - process.execPath, - [ - "--require", - path.join(repoRoot, "test", "helpers", "onboard-script-mocks.cjs"), - scriptPath, - ], - { - cwd: repoRoot, - encoding: "utf-8", - env: { - HOME: tmpDir, - PATH: `${fakeBin}:/usr/bin:/bin`, - FAKE_ROUTER_SOURCE: fakeRouterSource, - ROUTER_SETUP_LOG: setupLog, - NEMOCLAW_MODEL_ROUTER_VENV: venvDir, - }, - }, - ); - - assert.equal(result.status, 0, result.stderr); - const log = fs.readFileSync(setupLog, "utf-8"); - assert.ok(log.includes(`python3 -m venv ${venvDir}`), log); - assert.ok( - log.includes( - `venv-python -m pip install --quiet --upgrade ${path.join(repoRoot, "nemoclaw-blueprint", "router", "llm-router")}[prefill,proxy]`, - ), - log, - ); - assert.doesNotMatch(log, /path-router/); - assert.doesNotMatch(log, /pip3 /); - const payload = parseStdoutJson<{ commands: CommandEntry[] }>(result.stdout); - assert.ok(payload.commands.some((entry) => /provider create/.test(entry.command))); - assert.ok(payload.commands.some((entry) => /inference set/.test(entry.command))); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }, - ); + }); + }); - it("prefers the managed Model Router command over PATH", testTimeoutOptions(60_000), () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-router-managed-")); - const fakeBin = path.join(tmpDir, "bin"); + it("recognizes the current managed command through the production command adapter", () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-router-current-")); + tempDirs.add(tmpDir); + const routerDir = path.join(tmpDir, "model-router-source"); const venvDir = path.join(tmpDir, "model-router-venv"); - const venvBin = path.join(venvDir, "bin"); - const setupLog = path.join(tmpDir, "router-managed.log"); - const scriptPath = path.join(tmpDir, "setup-router-managed-check.js"); - const routerPort = 46000 + (process.pid % 10000); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify(path.join(repoRoot, "src", "lib", "state", "registry.ts")); - - try { - fs.mkdirSync(fakeBin, { recursive: true }); - fs.mkdirSync(venvBin, { recursive: true }); - fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { - mode: 0o755, - }); - fs.writeFileSync( - path.join(fakeBin, "model-router"), - [ - "#!/usr/bin/env bash", - `printf "path-router %s\\n" "$*" >> ${JSON.stringify(setupLog)}`, - "exit 89", - "", - ].join("\n"), - { mode: 0o755 }, - ); - fs.writeFileSync( - path.join(venvBin, "model-router"), - [ - `#!${process.execPath}`, - 'const fs = require("fs");', - 'const http = require("http");', - 'const path = require("path");', - "const args = process.argv.slice(2);", - `fs.appendFileSync(${JSON.stringify(setupLog)}, \`managed \${args[0]}\\n\`);`, - 'if (args[0] === "proxy-config") {', - ' const output = args[args.indexOf("--output") + 1];', - " fs.mkdirSync(path.dirname(output), { recursive: true });", - ' fs.writeFileSync(output, "model_list: []\\n");', - " process.exit(0);", - "}", - 'if (args[0] === "proxy") {', - ' const port = Number(args[args.indexOf("--port") + 1] || "4000");', - " const server = http.createServer((req, res) => {", - ' if (req.url === "/health") { res.statusCode = 200; res.end("ok"); return; }', - " res.statusCode = 404;", - " res.end();", - " });", - ' server.listen(port, "127.0.0.1");', - " setTimeout(() => process.exit(0), 10000);", - "} else {", - " process.exit(1);", - "}", - "", - ].join("\n"), - { mode: 0o755 }, - ); - fs.writeFileSync( - path.join(venvDir, MODEL_ROUTER_FINGERPRINT_FILE), - `git:${MODEL_ROUTER_TEST_SOURCE_SHA}\n`, - { mode: 0o600 }, - ); - - const script = String.raw` -const fs = require("fs"); -const path = require("path"); -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const routerPort = ${routerPort}; -const repoRoot = ${JSON.stringify(repoRoot)}; -const blueprintPath = path.join(repoRoot, "nemoclaw-blueprint", "blueprint.yaml"); -const routerPyproject = path.join(repoRoot, "nemoclaw-blueprint", "router", "llm-router", "pyproject.toml"); -const originalReadFileSync = fs.readFileSync; -const originalExistsSync = fs.existsSync; -fs.existsSync = (filePath) => { - if (filePath === routerPyproject || String(filePath) === routerPyproject) return true; - return originalExistsSync(filePath); -}; -fs.readFileSync = (filePath, ...args) => { - const raw = originalReadFileSync(filePath, ...args); - if (filePath === blueprintPath || String(filePath) === blueprintPath) { - return String(raw) - .replace('endpoint: "http://localhost:4000/v1"', 'endpoint: "http://localhost:' + routerPort + '/v1"') - .replace("port: 4000", "port: " + routerPort); - } - return raw; -}; - -const commands = []; -runner.run = (command, opts = {}) => { - const cmd = _n(command); - if (/\bpython3(?:\.\d+)? -m venv\b/.test(cmd) || cmd.includes("/bin/python -m pip")) { - throw new Error("unexpected managed-router reinstall in reuse test: " + cmd); - } - if (/(^|[\/\s])pip3(?:\s|$)/.test(cmd)) { - throw new Error("unexpected pip3 invocation in test harness: " + cmd); - } - if (cmd.includes("git -C") || /^git(?:\s|$)/.test(cmd)) { - throw new Error("unexpected git invocation in test harness: " + cmd); - } - commands.push({ command: cmd, env: opts.env || null }); - if (cmd.includes("provider get")) return { status: 1, stdout: "", stderr: "" }; - if ( - cmd.includes("gateway select") || - cmd.includes("provider create") || - cmd.includes("provider update") || - cmd.includes("inference set") - ) { - return { status: 0, stdout: "", stderr: "" }; - } - throw new Error("unexpected command in managed-router reuse test: " + cmd); -}; -runner.runCapture = (command) => { - const cmd = _n(command); - if (cmd.includes("git -C") && cmd.includes("rev-parse HEAD")) { - return ${JSON.stringify(MODEL_ROUTER_TEST_SOURCE_SHA)}; - } - if (cmd.includes("command -v") && /model-router$/.test(cmd)) { - return ${JSON.stringify(path.join(fakeBin, "model-router"))}; - } - if (cmd.includes("inference") && cmd.includes("get")) { - return [ - "Gateway inference:", - "", - " Route: inference.local", - " Provider: nvidia-router", - " Model: nvidia-routed", - " Version: 1", - ].join(String.fromCharCode(10)); - } - return ""; -}; -registry.updateSandbox = () => true; - -process.env.NVIDIA_INFERENCE_API_KEY = "nvapi-TEST-NOT-A-REAL-ROUTER-KEY"; - -const { setupInference } = require(${onboardPath}); - -(async () => { - await setupInference( - "router-box", - "nvidia-routed", - "nvidia-router", - "http://host.openshell.internal:" + routerPort + "/v1", - "NVIDIA_INFERENCE_API_KEY", - ); - console.log(JSON.stringify({ commands })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); - - const result = spawnSync( - process.execPath, - [ - "--require", - path.join(repoRoot, "test", "helpers", "onboard-script-mocks.cjs"), - scriptPath, - ], - { - cwd: repoRoot, - encoding: "utf-8", - env: { - HOME: tmpDir, - PATH: `${fakeBin}:/usr/bin:/bin`, - ROUTER_SETUP_LOG: setupLog, - NEMOCLAW_MODEL_ROUTER_VENV: venvDir, - }, - }, - ); - - assert.equal(result.status, 0, result.stderr); - const log = fs.readFileSync(setupLog, "utf-8"); - assert.match(log, /managed proxy-config/); - assert.doesNotMatch(log, /path-router/); - const payload = parseStdoutJson<{ commands: CommandEntry[] }>(result.stdout); - assert.ok(payload.commands.some((entry) => /provider create/.test(entry.command))); - assert.ok(payload.commands.some((entry) => /inference set/.test(entry.command))); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } + const managedCommand = path.join(venvDir, "bin", "model-router"); + const runGit = (args: string[]) => { + const result = run(["git", ...args], { ignoreError: true, suppressOutput: true }); + assert.equal(result.status, 0, String(result.stderr || result.error || "git failed")); + }; + runGit(["init", "--quiet", routerDir]); + fs.writeFileSync(path.join(routerDir, "router.py"), "ROUTER_VERSION = 1\n"); + runGit(["-C", routerDir, "add", "router.py"]); + runGit([ + "-C", + routerDir, + "-c", + "user.name=NemoClaw Test", + "-c", + "user.email=nemoclaw-test@example.invalid", + "-c", + "commit.gpgsign=false", + "-c", + "core.hooksPath=/dev/null", + "commit", + "--quiet", + "-m", + "test: create model router source fixture", + ]); + const sourceHead = runCapture(["git", "-C", routerDir, "rev-parse", "HEAD"], { + ignoreError: true, + }).trim(); + assert.match(sourceHead, /^[0-9a-f]{40}$/i); + assert.equal( + runCapture(["git", "-C", routerDir, "rev-parse", "--show-toplevel"]).trim(), + routerDir, + ); + fs.mkdirSync(path.dirname(managedCommand), { recursive: true }); + fs.writeFileSync(managedCommand, "#!/usr/bin/env sh\nexit 0\n", { mode: 0o755 }); + fs.writeFileSync(path.join(venvDir, MODEL_ROUTER_FINGERPRINT_FILE), `git:${sourceHead}\n`, { + mode: 0o600, + }); + + assert.equal(isManagedModelRouterCurrent(routerDir, venvDir), true); }); - it( - "refreshes stale managed Model Router command when source fingerprint changes", - testTimeoutOptions(60_000), - () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-router-refresh-")); - const fakeBin = path.join(tmpDir, "bin"); - const venvDir = path.join(tmpDir, "model-router-venv"); - const venvBin = path.join(venvDir, "bin"); - const fakeRouterSource = path.join(tmpDir, "model-router-source.js"); - const setupLog = path.join(tmpDir, "router-refresh.log"); - const scriptPath = path.join(tmpDir, "setup-router-refresh-check.js"); - const routerPort = 47000 + (process.pid % 10000); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "registry.ts"), - ); - - try { - fs.mkdirSync(fakeBin, { recursive: true }); - fs.mkdirSync(venvBin, { recursive: true }); - fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { - mode: 0o755, - }); - fs.writeFileSync( - path.join(fakeBin, "model-router"), - [ - "#!/usr/bin/env bash", - `printf "path-router %s\\n" "$*" >> ${JSON.stringify(setupLog)}`, - "exit 89", - "", - ].join("\n"), - { mode: 0o755 }, + it("installs the managed command through the production provisioning adapters", async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-router-install-")); + tempDirs.add(tmpDir); + const fixture = createProductionModelRouterInstallFixture(tmpDir); + + await withProcessEnv( + { + NEMOCLAW_MODEL_ROUTER_PYTHON: undefined, + PATH: `${fixture.fakeBin}:/usr/bin:/bin`, + }, + async () => { + const provisioner = createProductionModelRouterCommandProvisioner( + fixture.routerDir, + fixture.venvDir, ); - fs.writeFileSync( - path.join(fakeBin, "python3"), - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - `printf "python3 %s\\n" "$*" >> ${JSON.stringify(setupLog)}`, - // pickHostPython probe (#3781) — emit a healthy probe response so - // the helper proceeds to the venv step instead of falling back. - 'if [ "$1" = "-c" ]; then', - ' printf \'{"version": [3, 12, 7], "error": null}\\n\'', - " exit 0", - "fi", - 'if [ "$1" = "-m" ] && [ "$2" = "venv" ]; then', - ' venv_dir="$3"', - ' mkdir -p "$venv_dir/bin"', - " cat > \"$venv_dir/bin/python\" <<'PY'", - "#!/usr/bin/env bash", - "set -euo pipefail", - `printf "venv-python %s\\n" "$*" >> ${JSON.stringify(setupLog)}`, - 'if [ "$1" = "-m" ] && [ "$2" = "pip" ] && [ "$3" = "install" ]; then', - ' venv_bin="$(cd "$(dirname "$0")" && pwd)"', - ` cp ${JSON.stringify(fakeRouterSource)} "$venv_bin/model-router"`, - ' chmod +x "$venv_bin/model-router"', - " exit 0", - "fi", - "exit 97", - "PY", - ' chmod +x "$venv_dir/bin/python"', - " exit 0", - "fi", - "exit 96", - "", - ].join("\n"), - { mode: 0o755 }, - ); - fs.writeFileSync( - path.join(venvBin, "model-router"), - [ - "#!/usr/bin/env bash", - `printf "stale-managed %s\\n" "$*" >> ${JSON.stringify(setupLog)}`, - "exit 89", - "", - ].join("\n"), - { mode: 0o755 }, - ); - fs.writeFileSync(path.join(venvDir, MODEL_ROUTER_FINGERPRINT_FILE), "git:stale\n", { - mode: 0o600, - }); - fs.writeFileSync( - fakeRouterSource, - [ - `#!${process.execPath}`, - 'const fs = require("fs");', - 'const http = require("http");', - 'const path = require("path");', - "const args = process.argv.slice(2);", - `fs.appendFileSync(${JSON.stringify(setupLog)}, \`fresh \${args[0]}\\n\`);`, - 'if (args[0] === "proxy-config") {', - ' const output = args[args.indexOf("--output") + 1];', - " fs.mkdirSync(path.dirname(output), { recursive: true });", - ' fs.writeFileSync(output, "model_list: []\\n");', - " process.exit(0);", - "}", - 'if (args[0] === "proxy") {', - ' const port = Number(args[args.indexOf("--port") + 1] || "4000");', - " const server = http.createServer((req, res) => {", - ' if (req.url === "/health") { res.statusCode = 200; res.end("ok"); return; }', - " res.statusCode = 404;", - " res.end();", - " });", - ' server.listen(port, "127.0.0.1");', - " setTimeout(() => process.exit(0), 10000);", - "} else {", - " process.exit(1);", - "}", - "", - ].join("\n"), - { mode: 0o755 }, - ); - - const script = String.raw` -const fs = require("fs"); -const path = require("path"); -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const routerPort = ${routerPort}; -const repoRoot = ${JSON.stringify(repoRoot)}; -const blueprintPath = path.join(repoRoot, "nemoclaw-blueprint", "blueprint.yaml"); -const routerPyproject = path.join(repoRoot, "nemoclaw-blueprint", "router", "llm-router", "pyproject.toml"); -const originalReadFileSync = fs.readFileSync; -const originalRun = runner.run; -const originalExistsSync = fs.existsSync; -fs.existsSync = (filePath) => { - if (filePath === routerPyproject || String(filePath) === routerPyproject) return true; - return originalExistsSync(filePath); -}; -fs.readFileSync = (filePath, ...args) => { - const raw = originalReadFileSync(filePath, ...args); - if (filePath === blueprintPath || String(filePath) === blueprintPath) { - return String(raw) - .replace('endpoint: "http://localhost:4000/v1"', 'endpoint: "http://localhost:' + routerPort + '/v1"') - .replace("port: 4000", "port: " + routerPort); - } - return raw; -}; - -const commands = []; -runner.run = (command, opts = {}) => { - const cmd = _n(command); - if (/\bpython3(?:\.\d+)? -m venv\b/.test(cmd) || cmd.includes("/bin/python -m pip")) { - return originalRun(command, opts); - } - if (/(^|[\/\s])pip3(?:\s|$)/.test(cmd)) { - throw new Error("unexpected pip3 invocation in test harness: " + cmd); - } - if (cmd.includes("git -C") || /^git(?:\s|$)/.test(cmd)) { - throw new Error("unexpected git invocation in test harness: " + cmd); - } - commands.push({ command: cmd, env: opts.env || null }); - if (cmd.includes("provider get")) return { status: 1, stdout: "", stderr: "" }; - return { status: 0, stdout: "", stderr: "" }; -}; -runner.runCapture = (command) => { - const cmd = _n(command); - if (cmd.includes("git -C") && cmd.includes("rev-parse HEAD")) { - return ${JSON.stringify(MODEL_ROUTER_TEST_SOURCE_SHA)}; - } - if (cmd.includes("command -v") && /model-router$/.test(cmd)) { - return ${JSON.stringify(path.join(fakeBin, "model-router"))}; - } - if (cmd.includes("command -v") && /python3$/.test(cmd)) { - return ${JSON.stringify(path.join(fakeBin, "python3"))}; - } - if (cmd.includes("inference") && cmd.includes("get")) { - return [ - "Gateway inference:", - "", - " Route: inference.local", - " Provider: nvidia-router", - " Model: nvidia-routed", - " Version: 1", - ].join(String.fromCharCode(10)); - } - return ""; -}; -registry.updateSandbox = () => true; - -process.env.NVIDIA_INFERENCE_API_KEY = "nvapi-TEST-NOT-A-REAL-ROUTER-KEY"; - -const { setupInference } = require(${onboardPath}); - -(async () => { - await setupInference( - "router-box", - "nvidia-routed", - "nvidia-router", - "http://host.openshell.internal:" + routerPort + "/v1", - "NVIDIA_INFERENCE_API_KEY", - ); - console.log(JSON.stringify({ commands })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); + assert.equal(provisioner.ensureModelRouterCommand(), fixture.managedCommand); + assert.equal(provisioner.isManagedModelRouterCurrent(), true); + }, + ); + + const setupLog = fs.readFileSync(fixture.setupLog, "utf8"); + assert.match(setupLog, new RegExp(`python3 -m venv ${fixture.venvDir}`)); + assert.match( + setupLog, + new RegExp( + `venv-python -m pip install --quiet --upgrade ${fixture.routerDir}\\[prefill,proxy\\]`, + ), + ); + assert.doesNotMatch(setupLog, /path-router/); + assert.equal( + fs.readFileSync(fixture.fingerprintPath, "utf8").trim(), + `git:${fixture.sourceHead}`, + ); + }); - const result = spawnSync( - process.execPath, - [ - "--require", - path.join(repoRoot, "test", "helpers", "onboard-script-mocks.cjs"), - scriptPath, - ], - { - cwd: repoRoot, - encoding: "utf-8", - env: { - HOME: tmpDir, - PATH: `${fakeBin}:/usr/bin:/bin`, - FAKE_ROUTER_SOURCE: fakeRouterSource, - ROUTER_SETUP_LOG: setupLog, - NEMOCLAW_MODEL_ROUTER_VENV: venvDir, + it("starts the managed command through the production process adapters", async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-router-start-")); + tempDirs.add(tmpDir); + const rootDir = path.join(tmpDir, "repo"); + const homeDir = path.join(tmpDir, "home"); + const routerCommand = path.join(tmpDir, "managed", "model-router"); + const launchLogPath = path.join(tmpDir, "router-launch.jsonl"); + const port = 45_678; + const healthChecks: number[] = []; + const sleepCalls: number[] = []; + let healthProbe = 0; + let pid: number | null = null; + + const blueprintDir = path.join(rootDir, "nemoclaw-blueprint"); + const poolConfigPath = path.join(blueprintDir, "router", "test-pool.yaml"); + const stateDir = path.join(homeDir, ".nemoclaw", "state"); + const litellmConfigPath = path.join(stateDir, "litellm-proxy.yaml"); + fs.mkdirSync(path.dirname(poolConfigPath), { recursive: true }); + fs.mkdirSync(path.dirname(routerCommand), { recursive: true }); + fs.writeFileSync( + routerCommand, + [ + `#!${process.execPath}`, + 'const fs = require("node:fs");', + "const args = process.argv.slice(2);", + "const env = {};", + 'for (const key of ["ROUTER_API_KEY", "OPENAI_API_KEY", "NEMOCLAW_PROVIDER_KEY"]) {', + " env[key] = process.env[key] || null;", + "}", + `fs.appendFileSync(${JSON.stringify(launchLogPath)}, JSON.stringify({ args, cwd: process.cwd(), env, pid: process.pid }) + "\\n");`, + 'if (args[0] === "proxy-config") process.exit(0);', + 'if (args[0] !== "proxy") process.exit(2);', + "setTimeout(() => process.exit(0), 5000);", + "setInterval(() => {}, 1000);", + "", + ].join("\n"), + { mode: 0o755 }, + ); + + await withProcessEnv( + { + ROUTER_API_KEY: undefined, + OPENAI_API_KEY: undefined, + NEMOCLAW_PROVIDER_KEY: undefined, + }, + async () => { + try { + pid = await startModelRouter( + { + port, + pool_config_path: "router/test-pool.yaml", + credential_env: "ROUTER_API_KEY", }, - }, - ); - - assert.equal(result.status, 0, result.stderr); - const log = fs.readFileSync(setupLog, "utf-8"); - assert.ok(log.includes(`python3 -m venv ${venvDir}`), log); - assert.ok( - log.includes( - `venv-python -m pip install --quiet --upgrade ${path.join(repoRoot, "nemoclaw-blueprint", "router", "llm-router")}[prefill,proxy]`, - ), - log, - ); - assert.match(log, /fresh proxy-config/); - assert.doesNotMatch(log, /stale-managed/); - assert.doesNotMatch(log, /path-router/); - const payload = parseStdoutJson<{ commands: CommandEntry[] }>(result.stdout); - assert.ok(payload.commands.some((entry) => /provider create/.test(entry.command))); - assert.ok(payload.commands.some((entry) => /inference set/.test(entry.command))); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }, - ); - - it( - "writes fallback fingerprint file when git source fingerprint is unavailable", - testTimeoutOptions(30_000), - () => { - const repoRoot = path.join(import.meta.dirname, ".."); - const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-router-fallback-fp-")); - const fakeBin = path.join(tmpDir, "bin"); - const venvDir = path.join(tmpDir, "model-router-venv"); - const fakeRouterSource = path.join(tmpDir, "model-router-source.js"); - const setupLog = path.join(tmpDir, "router-setup.log"); - const scriptPath = path.join(tmpDir, "setup-router-fallback-fp-check.js"); - const routerPort = 48000 + (process.pid % 10000); - const onboardPath = JSON.stringify(path.join(repoRoot, "src", "lib", "onboard.ts")); - const runnerPath = JSON.stringify(path.join(repoRoot, "src", "lib", "runner.ts")); - const registryPath = JSON.stringify( - path.join(repoRoot, "src", "lib", "state", "registry.ts"), - ); - - try { - fs.mkdirSync(fakeBin, { recursive: true }); - fs.writeFileSync(path.join(fakeBin, "openshell"), "#!/usr/bin/env bash\nexit 0\n", { - mode: 0o755, - }); - fs.writeFileSync( - path.join(fakeBin, "python3"), - [ - "#!/usr/bin/env bash", - "set -euo pipefail", - `printf "python3 %s\\n" "$*" >> ${JSON.stringify(setupLog)}`, - 'if [ "$1" = "-c" ]; then', - ' printf \'{"version": [3, 12, 7], "error": null}\\n\'', - " exit 0", - "fi", - 'if [ "$1" = "-m" ] && [ "$2" = "venv" ]; then', - ' venv_dir="$3"', - ' mkdir -p "$venv_dir/bin"', - " cat > \"$venv_dir/bin/python\" <<'PY'", - "#!/usr/bin/env bash", - "set -euo pipefail", - `printf "venv-python %s\\n" "$*" >> ${JSON.stringify(setupLog)}`, - 'if [ "$1" = "-m" ] && [ "$2" = "pip" ] && [ "$3" = "install" ]; then', - ' venv_bin="$(cd "$(dirname "$0")" && pwd)"', - ` cp ${JSON.stringify(fakeRouterSource)} "$venv_bin/model-router"`, - ' chmod +x "$venv_bin/model-router"', - " exit 0", - "fi", - "exit 97", - "PY", - ' chmod +x "$venv_dir/bin/python"', - " exit 0", - "fi", - "exit 96", - "", - ].join("\n"), - { mode: 0o755 }, - ); - fs.writeFileSync( - fakeRouterSource, - [ - `#!${process.execPath}`, - 'const fs = require("fs");', - 'const http = require("http");', - 'const path = require("path");', - "const args = process.argv.slice(2);", - 'if (args[0] === "proxy-config") {', - ' const output = args[args.indexOf("--output") + 1];', - " fs.mkdirSync(path.dirname(output), { recursive: true });", - ' fs.writeFileSync(output, "model_list: []\\n");', - " process.exit(0);", - "}", - 'if (args[0] === "proxy") {', - ' const port = Number(args[args.indexOf("--port") + 1] || "4000");', - " const server = http.createServer((req, res) => {", - ' if (req.url === "/health") { res.statusCode = 200; res.end("ok"); return; }', - " res.statusCode = 404;", - " res.end();", - " });", - ' server.listen(port, "127.0.0.1");', - " setTimeout(() => process.exit(0), 10000);", - "} else {", - " process.exit(1);", - "}", - "", - ].join("\n"), - { mode: 0o755 }, - ); - - const script = String.raw` -const fs = require("fs"); -const path = require("path"); -const runner = require(${runnerPath}); -const _n = (c) => (Array.isArray(c) ? c.join(" ") : String(c)).replace(/'/g, ""); -const registry = require(${registryPath}); -const routerPort = ${routerPort}; -const repoRoot = ${JSON.stringify(repoRoot)}; -const blueprintPath = path.join(repoRoot, "nemoclaw-blueprint", "blueprint.yaml"); -const routerPyproject = path.join(repoRoot, "nemoclaw-blueprint", "router", "llm-router", "pyproject.toml"); -const originalReadFileSync = fs.readFileSync; -const originalRun = runner.run; -const originalExistsSync = fs.existsSync; -fs.existsSync = (filePath) => { - if (filePath === routerPyproject || String(filePath) === routerPyproject) return true; - return originalExistsSync(filePath); -}; -fs.readFileSync = (filePath, ...args) => { - const raw = originalReadFileSync(filePath, ...args); - if (filePath === blueprintPath || String(filePath) === blueprintPath) { - return String(raw) - .replace('endpoint: "http://localhost:4000/v1"', 'endpoint: "http://localhost:' + routerPort + '/v1"') - .replace("port: 4000", "port: " + routerPort); - } - return raw; -}; - -const commands = []; -runner.run = (command, opts = {}) => { - const cmd = _n(command); - if (/\bpython3(?:\.\d+)? -m venv\b/.test(cmd) || cmd.includes("/bin/python -m pip")) { - return originalRun(command, opts); - } - if (/(^|[\/\s])pip3(?:\s|$)/.test(cmd)) { - throw new Error("unexpected pip3 invocation in test harness: " + cmd); - } - if (cmd.includes("git -C") || /^git(?:\s|$)/.test(cmd)) { - throw new Error("unexpected git invocation in test harness: " + cmd); - } - commands.push({ command: cmd, env: opts.env || null }); - if (cmd.includes("provider get")) return { status: 1, stdout: "", stderr: "" }; - return { status: 0, stdout: "", stderr: "" }; -}; -runner.runCapture = (command) => { - const cmd = _n(command); - // Return empty for ALL git commands so source fingerprint is null - if (cmd.includes("git ")) return ""; - if (cmd.includes("command -v") && /model-router$/.test(cmd)) return ""; - if (cmd.includes("command -v") && /python3$/.test(cmd)) { - return ${JSON.stringify(path.join(fakeBin, "python3"))}; - } - if (cmd.includes("inference") && cmd.includes("get")) { - return [ - "Gateway inference:", - "", - " Route: inference.local", - " Provider: nvidia-router", - " Model: nvidia-routed", - " Version: 1", - ].join(String.fromCharCode(10)); - } - return ""; -}; -registry.updateSandbox = () => true; - -process.env.NVIDIA_INFERENCE_API_KEY = "nvapi-TEST-NOT-A-REAL-ROUTER-KEY"; - -const { setupInference } = require(${onboardPath}); + { + rootDir, + homeDir, + ensureModelRouterCommand: () => routerCommand, + resolveProviderCredential: (name) => + name === "ROUTER_API_KEY" ? "router-secret" : null, + isRouterHealthy: async (routerPort) => { + healthChecks.push(routerPort); + healthProbe += 1; + return healthProbe > 1; + }, + sleep: async (milliseconds) => { + sleepCalls.push(milliseconds); + }, + }, + ); + const entries = await readRouterLaunchLog(launchLogPath, 2); + const proxyConfig = entries.find(({ args }) => args[0] === "proxy-config"); + const proxy = entries.find(({ args }) => args[0] === "proxy"); + assert.ok(proxyConfig); + assert.ok(proxy); + assert.deepEqual(proxyConfig.args, [ + "proxy-config", + "--config", + poolConfigPath, + "--output", + litellmConfigPath, + ]); + assert.equal(proxyConfig.cwd, blueprintDir); + assert.deepEqual(proxy.args, [ + "proxy", + "--litellm-config", + litellmConfigPath, + "--router-config", + poolConfigPath, + "--host", + "0.0.0.0", + "--port", + String(port), + ]); + assert.equal(proxy.cwd, blueprintDir); + assert.deepEqual(proxy.env, { + ROUTER_API_KEY: "router-secret", + OPENAI_API_KEY: "router-secret", + NEMOCLAW_PROVIDER_KEY: null, + }); + assert.equal(proxy.pid, pid); + assert.equal(fs.existsSync(stateDir), true); + assert.deepEqual(healthChecks, [port, port]); + assert.deepEqual(sleepCalls, [2000]); + } finally { + await stopTestProcess(pid); + } + }, + ); + }); -(async () => { - await setupInference( - "router-box", - "nvidia-routed", - "nvidia-router", - "http://host.openshell.internal:" + routerPort + "/v1", - "NVIDIA_INFERENCE_API_KEY", - ); - const fpPath = path.join(${JSON.stringify(venvDir)}, ${JSON.stringify(MODEL_ROUTER_FINGERPRINT_FILE)}); - const fpExists = fs.existsSync(fpPath); - const fpContent = fpExists ? fs.readFileSync(fpPath, "utf8").trim() : null; + it("prepares managed Model Router dependencies instead of using PATH when managed command is absent", () => { + const pathCommand = "/tmp/path-model-router"; + const harness = createCommandHarness({ pathCommand }); + + assert.equal(harness.provisioner.ensureModelRouterCommand(), harness.managedCommand); + assert.deepEqual(harness.prepareCalls, [ + { venvDir: harness.venvDir, allowReplaceExisting: false }, + ]); + assert.deepEqual(harness.runCalls, [ + [ + path.join(harness.venvDir, "bin", "python"), + "-m", + "pip", + "install", + "--quiet", + "--upgrade", + `${harness.routerDir}[prefill,proxy]`, + ], + ]); + assert.equal( + harness.runCaptureCalls.some((command) => command[0] === "sh"), + false, + "PATH command discovery must not run when managed source is available", + ); + assert.equal( + fs.readFileSync(harness.fingerprintPath, "utf8").trim(), + `git:${MODEL_ROUTER_TEST_SOURCE_SHA}`, + ); + }); - // Verify isManagedModelRouterCurrent returns true on a subsequent check - // when sourceFingerprint is null but the install: fingerprint file exists. - // Import the module and call it directly. - const modelRouter = require(${JSON.stringify( - path.join(repoRoot, "src", "lib", "onboard", "model-router.ts"), - )}); - const isCurrent = modelRouter.isManagedModelRouterCurrent( - ${JSON.stringify(path.join(tmpDir, "nonexistent-router-dir"))}, - ${JSON.stringify(venvDir)}, - ); + it("prefers the managed Model Router command over PATH", () => { + const harness = createCommandHarness({ + managedCommand: true, + installedFingerprint: `git:${MODEL_ROUTER_TEST_SOURCE_SHA}`, + pathCommand: "/tmp/path-model-router", + }); + + assert.equal(harness.provisioner.ensureModelRouterCommand(), harness.managedCommand); + assert.deepEqual(harness.prepareCalls, []); + assert.deepEqual(harness.runCalls, []); + assert.equal( + harness.runCaptureCalls.some((command) => command[0] === "sh"), + false, + ); + }); - console.log(JSON.stringify({ fpExists, fpContent, isCurrent })); -})().catch((error) => { - console.error(error); - process.exit(1); -}); -`; - fs.writeFileSync(scriptPath, script); + it("refreshes stale managed Model Router command when source fingerprint changes", () => { + const harness = createCommandHarness({ + managedCommand: true, + installedFingerprint: "git:stale", + pathCommand: "/tmp/path-model-router", + }); + + assert.equal(harness.provisioner.ensureModelRouterCommand(), harness.managedCommand); + assert.deepEqual(harness.prepareCalls, [ + { venvDir: harness.venvDir, allowReplaceExisting: true }, + ]); + assert.equal(harness.runCalls.length, 1); + assert.equal( + harness.runCaptureCalls.some((command) => command[0] === "sh"), + false, + ); + assert.equal( + fs.readFileSync(harness.fingerprintPath, "utf8").trim(), + `git:${MODEL_ROUTER_TEST_SOURCE_SHA}`, + ); + }); - const result = spawnSync( - process.execPath, - [ - "--require", - path.join(repoRoot, "test", "helpers", "onboard-script-mocks.cjs"), - scriptPath, - ], - { - cwd: repoRoot, - encoding: "utf-8", - env: { - HOME: tmpDir, - PATH: `${fakeBin}:/usr/bin:/bin`, - ROUTER_SETUP_LOG: setupLog, - NEMOCLAW_MODEL_ROUTER_VENV: venvDir, - }, - }, - ); + it("writes fallback fingerprint file when git source fingerprint is unavailable", () => { + const harness = createCommandHarness({ sourceFingerprint: () => null }); - assert.equal(result.status, 0, result.stderr); - const payload = parseStdoutJson<{ - fpExists: boolean; - fpContent: string | null; - isCurrent: boolean; - }>(result.stdout); - assert.ok(payload.fpExists, "fingerprint file must exist after install even without git"); - assert.ok(payload.fpContent, "fingerprint content must not be empty"); - assert.match( - payload.fpContent!, - /^install:.+$/, - "fallback fingerprint must use install: format", - ); - assert.doesNotMatch( - payload.fpContent!, - /^install:\d{13,}$/, - "fallback fingerprint must not use a timestamp", - ); - assert.ok( - payload.isCurrent, - "isManagedModelRouterCurrent must return true when install: fingerprint exists and source is unavailable", - ); - } finally { - fs.rmSync(tmpDir, { recursive: true, force: true }); - } - }, - ); + assert.equal(harness.provisioner.ensureModelRouterCommand(), harness.managedCommand); + const fingerprint = fs.readFileSync(harness.fingerprintPath, "utf8").trim(); + assert.equal(fingerprint, `install:${MODEL_ROUTER_TEST_VERSION}`); + assert.doesNotMatch(fingerprint, /^install:\d{13,}$/); + assert.equal(harness.provisioner.isManagedModelRouterCurrent(), true); + }); }); diff --git a/test/support/model-router-process-test-helpers.ts b/test/support/model-router-process-test-helpers.ts new file mode 100644 index 00000000000..9a7a858a3c1 --- /dev/null +++ b/test/support/model-router-process-test-helpers.ts @@ -0,0 +1,171 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; + +export type RouterLaunchLog = { + args: string[]; + cwd: string; + env: Record; + pid: number; +}; + +export type ProductionModelRouterInstallFixture = { + fakeBin: string; + fingerprintPath: string; + managedCommand: string; + routerDir: string; + setupLog: string; + sourceHead: string; + venvDir: string; +}; + +function runGit(args: string[]): string { + const result = spawnSync("git", args, { encoding: "utf8" }); + if (result.status !== 0) { + throw new Error(result.stderr || result.error?.message || `git ${args.join(" ")} failed`); + } + return result.stdout.trim(); +} + +export function createProductionModelRouterInstallFixture( + tmpDir: string, +): ProductionModelRouterInstallFixture { + const routerDir = path.join(tmpDir, "model-router-source"); + const fakeBin = path.join(tmpDir, "bin"); + const venvDir = path.join(tmpDir, "model-router-venv"); + const managedCommand = path.join(venvDir, "bin", "model-router"); + const fingerprintPath = path.join(venvDir, ".nemoclaw-source-fingerprint"); + const setupLog = path.join(tmpDir, "model-router-install.log"); + const fakeRouterSource = path.join(tmpDir, "installed-model-router"); + + fs.mkdirSync(routerDir, { recursive: true }); + runGit(["init", "--quiet", routerDir]); + fs.writeFileSync(path.join(routerDir, "pyproject.toml"), "[project]\nname = 'model-router'\n"); + runGit(["-C", routerDir, "add", "pyproject.toml"]); + runGit([ + "-C", + routerDir, + "-c", + "user.name=NemoClaw Test", + "-c", + "user.email=nemoclaw-test@example.invalid", + "-c", + "commit.gpgsign=false", + "-c", + "core.hooksPath=/dev/null", + "commit", + "--quiet", + "-m", + "test: create model router install fixture", + ]); + const sourceHead = runGit(["-C", routerDir, "rev-parse", "HEAD"]); + + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync( + path.join(fakeBin, "model-router"), + [ + "#!/usr/bin/env bash", + `printf "path-router %s\\n" "$*" >> ${JSON.stringify(setupLog)}`, + "exit 89", + "", + ].join("\n"), + { mode: 0o755 }, + ); + fs.writeFileSync(fakeRouterSource, "#!/usr/bin/env bash\nexit 0\n", { mode: 0o755 }); + const fakePython = path.join(fakeBin, "python3.13"); + fs.writeFileSync( + fakePython, + [ + "#!/usr/bin/env bash", + "set -euo pipefail", + `printf "python3 %s\\n" "$*" >> ${JSON.stringify(setupLog)}`, + 'if [ "$1" = "-c" ]; then', + ' printf \'{"version": [3, 13, 7], "error": null}\\n\'', + " exit 0", + "fi", + 'if [ "$1" = "-m" ] && [ "$2" = "venv" ]; then', + ' venv_dir="$3"', + ' mkdir -p "$venv_dir/bin"', + " cat > \"$venv_dir/bin/python\" <<'PY'", + "#!/usr/bin/env bash", + "set -euo pipefail", + `printf "venv-python %s\\n" "$*" >> ${JSON.stringify(setupLog)}`, + 'if [ "$1" = "-m" ] && [ "$2" = "pip" ] && [ "$3" = "install" ]; then', + ' venv_bin="$(cd "$(dirname "$0")" && pwd)"', + ` cp ${JSON.stringify(fakeRouterSource)} "$venv_bin/model-router"`, + ' chmod +x "$venv_bin/model-router"', + " exit 0", + "fi", + "exit 97", + "PY", + ' chmod +x "$venv_dir/bin/python"', + " exit 0", + "fi", + "exit 96", + "", + ].join("\n"), + { mode: 0o755 }, + ); + for (const candidate of ["python3.12", "python3.11", "python3.10", "python3"]) { + const candidatePath = path.join(fakeBin, candidate); + fs.copyFileSync(fakePython, candidatePath); + fs.chmodSync(candidatePath, 0o755); + } + + return { + fakeBin, + fingerprintPath, + managedCommand, + routerDir, + setupLog, + sourceHead, + venvDir, + }; +} + +function isProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +export async function stopTestProcess(pid: number | null): Promise { + if (pid === null) return; + try { + process.kill(pid, "SIGTERM"); + } catch { + return; + } + for (let attempt = 0; attempt < 100; attempt++) { + if (!isProcessAlive(pid)) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + try { + process.kill(pid, "SIGKILL"); + } catch { + // Process exited between the final probe and cleanup. + } +} + +export async function readRouterLaunchLog( + logPath: string, + expectedEntries: number, +): Promise { + for (let attempt = 0; attempt < 100; attempt++) { + if (fs.existsSync(logPath)) { + const contents = fs.readFileSync(logPath, "utf8"); + const lines = contents.split("\n"); + if (!contents.endsWith("\n")) lines.pop(); + const entries = lines.filter(Boolean).map((line) => JSON.parse(line) as RouterLaunchLog); + if (entries.length >= expectedEntries) return entries; + } + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error(`Timed out waiting for ${expectedEntries} Model Router launch log entries`); +}