From cc47fb89dbf190319014cff9331101778f11dcb3 Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 12 May 2026 10:22:42 -0700 Subject: [PATCH 1/4] fix(onboard): offer running vLLM in provider menu Signed-off-by: Aaron Erickson --- docs/inference/inference-options.md | 10 ++- docs/inference/use-local-inference.md | 10 +-- src/lib/onboard.ts | 24 +++--- test/onboard-selection.test.ts | 101 ++++++++++++++++++++++++++ 4 files changed, 120 insertions(+), 25 deletions(-) diff --git a/docs/inference/inference-options.md b/docs/inference/inference-options.md index c939db0a352..05aa07d9096 100644 --- a/docs/inference/inference-options.md +++ b/docs/inference/inference-options.md @@ -51,7 +51,7 @@ NemoClaw uses provider-specific local tokens for those routes, and rebuilds of l | Hermes Provider | Hermes only | OpenAI-compatible route | Available when onboarding Hermes Agent through `nemohermes` | | Local Ollama | Caveated | Local Ollama API | Available when Ollama is installed or running on the host | | Local NVIDIA NIM | Experimental | Local OpenAI-compatible | Requires `NEMOCLAW_EXPERIMENTAL=1` and a NIM-capable GPU | -| Local vLLM | Experimental | Local OpenAI-compatible | Requires `NEMOCLAW_EXPERIMENTAL=1` and a server already running on `localhost:8000` | +| Local vLLM | Experimental | Local OpenAI-compatible | Appears when a vLLM server is already running on `localhost:8000`; managed install/start requires `NEMOCLAW_EXPERIMENTAL=1` | ## Provider Options @@ -59,7 +59,8 @@ NemoClaw uses provider-specific local tokens for those routes, and rebuilds of l The onboard wizard presents the following provider options by default. The first six are always available. Ollama appears when it is installed or running on the host. -Experimental local vLLM appears when you opt in and NemoClaw detects either a running vLLM server or a supported NVIDIA GPU host profile. +Experimental local vLLM appears when NemoClaw detects a running vLLM server. +The managed install/start vLLM entry appears when you opt in and NemoClaw detects a supported NVIDIA GPU host profile. | Option | Description | Curated models | |--------|-------------|----------------| @@ -103,12 +104,13 @@ $ NEMOCLAW_PROVIDER=routed NVIDIA_API_KEY= nemoclaw onboard --non-inte ## Experimental Options -The following local inference options require `NEMOCLAW_EXPERIMENTAL=1` and, when prerequisites are met, appear in the onboarding selection list. +The following local inference options are experimental. +Local NIM and managed vLLM install/start require `NEMOCLAW_EXPERIMENTAL=1`; an already-running vLLM server appears directly in the onboarding selection list. | Option | Condition | Notes | |--------|-----------|-------| | Local NVIDIA NIM | NIM-capable GPU detected | Pulls and manages a NIM container. | -| Local vLLM | vLLM running on `localhost:8000`, or a supported DGX Spark, DGX Station, or Linux NVIDIA GPU profile | Auto-detects the loaded model. Can install or start a managed vLLM container for supported profiles. | +| Local vLLM | vLLM running on `localhost:8000`, or a supported DGX Spark, DGX Station, or Linux NVIDIA GPU profile | Auto-detects the loaded model when vLLM is already running. Can install or start a managed vLLM container for supported profiles after experimental opt-in. | For setup instructions, refer to [Use a Local Inference Server](use-local-inference.md). diff --git a/docs/inference/use-local-inference.md b/docs/inference/use-local-inference.md index 31ebb5bb62f..63563e6c8ec 100644 --- a/docs/inference/use-local-inference.md +++ b/docs/inference/use-local-inference.md @@ -235,15 +235,14 @@ $ NEMOCLAW_PROVIDER=anthropicCompatible \ When vLLM is already running on `localhost:8000`, NemoClaw can detect it automatically and query the `/v1/models` endpoint to determine the loaded model. On supported Linux hosts with NVIDIA GPUs, the onboard wizard can also install or start a managed vLLM container for you. -Set the experimental flag and run onboard. +For an already-running vLLM server, run onboard and select **Local vLLM [experimental]** from the provider list. ```console -$ NEMOCLAW_EXPERIMENTAL=1 nemoclaw onboard +$ nemoclaw onboard ``` -Select **Local vLLM [experimental]** from the provider list. If vLLM is already running, NemoClaw detects the running model and validates the endpoint. -If vLLM is not running and your host matches a managed profile, select the **Install vLLM** or **Start vLLM** entry. +If vLLM is not running and your host matches a managed profile, set `NEMOCLAW_EXPERIMENTAL=1`, rerun onboard, and select the **Install vLLM** or **Start vLLM** entry. NemoClaw pulls the vLLM image, downloads model weights into `~/.cache/huggingface`, starts the `nemoclaw-vllm` container on `localhost:8000`, and prints progress markers while the model loads. The first run can take 10 to 30 minutes. Later runs reuse the cached image and model weights. @@ -266,8 +265,7 @@ The vLLM `/v1/responses` endpoint does not run the `--tool-call-parser`, so tool Use an already-running vLLM server: ```console -$ NEMOCLAW_EXPERIMENTAL=1 \ - NEMOCLAW_PROVIDER=vllm \ +$ NEMOCLAW_PROVIDER=vllm \ nemoclaw onboard --non-interactive ``` diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index fafa2b9a9dc..feeedcd7e4f 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -7441,29 +7441,23 @@ async function setupNim( if (EXPERIMENTAL && gpu && gpu.nimCapable) { options.push({ key: "nim-local", label: "Local NVIDIA NIM [experimental]" }); } - // vLLM: profiles in inference/vllm.ts surface as menu entries only when - // the user explicitly opts in via NEMOCLAW_PROVIDER, or when - // NEMOCLAW_EXPERIMENTAL=1 is set. + // vLLM: an already-running local server is safe to offer in-place because + // selecting it is an explicit user action. Managed install/start remains + // gated by NEMOCLAW_PROVIDER or NEMOCLAW_EXPERIMENTAL because it pulls images + // and starts containers. // Read NEMOCLAW_PROVIDER directly so interactive runs with an explicit // env-var opt-in surface the menu entry too — requestedProvider is null // outside non-interactive mode. const explicitProvider = (process.env.NEMOCLAW_PROVIDER || "").trim().toLowerCase(); const userChoseVllm = explicitProvider === "vllm" || explicitProvider === "install-vllm"; - if (vllmProfile && (userChoseVllm || EXPERIMENTAL)) { - if (vllmRunning) { - options.push({ - key: "vllm", - label: `Local vLLM (localhost:${VLLM_PORT}) — running (suggested)`, - }); - } else { - const verb = hasVllmImage ? "Start" : "Install"; - options.push({ key: "install-vllm", label: `${verb} vLLM (${vllmProfile.name})` }); - } - } else if (EXPERIMENTAL && vllmRunning) { + if (vllmRunning) { options.push({ key: "vllm", - label: "Local vLLM [experimental] — running", + label: `Local vLLM [experimental] (localhost:${VLLM_PORT}) — running (suggested)`, }); + } else if (vllmProfile && (userChoseVllm || EXPERIMENTAL)) { + const verb = hasVllmImage ? "Start" : "Install"; + options.push({ key: "install-vllm", label: `${verb} vLLM (${vllmProfile.name})` }); } // Skipped when Windows-host already won the cache: the running entry // above already covers that case. diff --git a/test/onboard-selection.test.ts b/test/onboard-selection.test.ts index 88ec7af18c7..c49a0f8e66e 100644 --- a/test/onboard-selection.test.ts +++ b/test/onboard-selection.test.ts @@ -202,6 +202,107 @@ const { setupNim } = require(${onboardPath}); ); }); + it("offers detected running vLLM without requiring a rerun", () => { + const repoRoot = path.join(import.meta.dirname, ".."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-vllm-running-")); + const fakeBin = path.join(tmpDir, "bin"); + const scriptPath = path.join(tmpDir, "vllm-running-check.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "onboard.js")); + const credentialsPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "credentials", "store.js")); + const runnerPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "runner.js")); + + fs.mkdirSync(fakeBin, { recursive: true }); + fs.writeFileSync( + path.join(fakeBin, "curl"), + `#!/usr/bin/env bash +body='{"id":"ok"}' +status="200" +outfile="" +while [ "$#" -gt 0 ]; do + case "$1" in + -o) outfile="$2"; shift 2 ;; + *) shift ;; + esac +done +printf '%s' "$body" > "$outfile" +printf '%s' "$status" +`, + { mode: 0o755 }, + ); + const script = String.raw` +const credentials = require(${credentialsPath}); +const runner = require(${runnerPath}); + +const answers = ["7"]; +const messages = []; + +credentials.prompt = async (message) => { + messages.push(message); + return answers.shift() || ""; +}; +credentials.ensureApiKey = async () => {}; +runner.runCapture = (command) => { + const cmd = Array.isArray(command) ? command.join(" ") : command; + if (cmd.includes("command -v ollama")) return ""; + if (cmd.includes("127.0.0.1:11434/api/tags")) return ""; + if (cmd.includes("127.0.0.1:8000/v1/models")) return JSON.stringify({ data: [{ id: "meta-llama/Llama-3.3-70B-Instruct" }] }); + if (cmd.includes("docker images")) return ""; + return ""; +}; + +const { setupNim } = require(${onboardPath}); + +(async () => { + const originalLog = console.log; + const lines = []; + console.log = (...args) => lines.push(args.join(" ")); + try { + const result = await setupNim({ type: "nvidia" }, null); + originalLog(JSON.stringify({ result, messages, lines })); + } finally { + console.log = originalLog; + } +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + PATH: `${fakeBin}:${process.env.PATH || ""}`, + NEMOCLAW_EXPERIMENTAL: "", + NEMOCLAW_PROVIDER: "", + }, + }); + + expect(result.status).toBe(0); + expect(result.stdout.trim()).not.toBe(""); + const payload = JSON.parse(result.stdout.trim()); + assert.equal(payload.result.provider, "vllm-local"); + assert.equal(payload.result.model, "meta-llama/Llama-3.3-70B-Instruct"); + assert.equal(payload.result.preferredInferenceApi, "openai-completions"); + assert.equal(payload.messages.filter((message: string) => /Choose \[/.test(message)).length, 1); + assert.ok( + payload.lines.some((line: string) => + line.includes("Detected local inference option: vLLM"), + ), + ); + assert.ok( + payload.lines.some((line: string) => + /^\s*\d+\) Local vLLM \[experimental\] \(localhost:8000\) — running \(suggested\)/.test( + line, + ), + ), + ); + assert.ok(!payload.lines.some((line: string) => line.includes("rerun the same command"))); + }); + it("does not label NVIDIA Endpoints as recommended in the provider list", () => { const repoRoot = path.join(import.meta.dirname, ".."); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-no-recommended-label-")); From 7dcebda24eeeb518e1e6ccfec330f76b0374f69d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 12 May 2026 10:50:35 -0700 Subject: [PATCH 2/4] docs(inference): format vLLM onboard command references Signed-off-by: Aaron Erickson --- docs/inference/use-local-inference.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/inference/use-local-inference.md b/docs/inference/use-local-inference.md index 63563e6c8ec..1c5af8b7f67 100644 --- a/docs/inference/use-local-inference.md +++ b/docs/inference/use-local-inference.md @@ -235,14 +235,14 @@ $ NEMOCLAW_PROVIDER=anthropicCompatible \ When vLLM is already running on `localhost:8000`, NemoClaw can detect it automatically and query the `/v1/models` endpoint to determine the loaded model. On supported Linux hosts with NVIDIA GPUs, the onboard wizard can also install or start a managed vLLM container for you. -For an already-running vLLM server, run onboard and select **Local vLLM [experimental]** from the provider list. +For an already-running vLLM server, run `nemoclaw onboard` and select **Local vLLM [experimental]** from the provider list. ```console $ nemoclaw onboard ``` If vLLM is already running, NemoClaw detects the running model and validates the endpoint. -If vLLM is not running and your host matches a managed profile, set `NEMOCLAW_EXPERIMENTAL=1`, rerun onboard, and select the **Install vLLM** or **Start vLLM** entry. +If vLLM is not running and your host matches a managed profile, set `NEMOCLAW_EXPERIMENTAL=1`, rerun `nemoclaw onboard`, and select the **Install vLLM** or **Start vLLM** entry. NemoClaw pulls the vLLM image, downloads model weights into `~/.cache/huggingface`, starts the `nemoclaw-vllm` container on `localhost:8000`, and prints progress markers while the model loads. The first run can take 10 to 30 minutes. Later runs reuse the cached image and model weights. From bb2e1362b5da18d449a8c79eb990a0e0ab0ad78d Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 12 May 2026 11:02:31 -0700 Subject: [PATCH 3/4] fix(onboard): keep vLLM install opt-in explicit Signed-off-by: Aaron Erickson --- src/lib/onboard.ts | 14 ++++----- test/onboard-selection.test.ts | 53 ++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index feeedcd7e4f..07212fa9a47 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -7443,19 +7443,19 @@ async function setupNim( } // vLLM: an already-running local server is safe to offer in-place because // selecting it is an explicit user action. Managed install/start remains - // gated by NEMOCLAW_PROVIDER or NEMOCLAW_EXPERIMENTAL because it pulls images - // and starts containers. + // gated by NEMOCLAW_PROVIDER=install-vllm or NEMOCLAW_EXPERIMENTAL because it + // pulls images and starts containers. // Read NEMOCLAW_PROVIDER directly so interactive runs with an explicit // env-var opt-in surface the menu entry too — requestedProvider is null // outside non-interactive mode. const explicitProvider = (process.env.NEMOCLAW_PROVIDER || "").trim().toLowerCase(); - const userChoseVllm = explicitProvider === "vllm" || explicitProvider === "install-vllm"; + const userChoseManagedVllm = explicitProvider === "install-vllm"; if (vllmRunning) { options.push({ key: "vllm", label: `Local vLLM [experimental] (localhost:${VLLM_PORT}) — running (suggested)`, }); - } else if (vllmProfile && (userChoseVllm || EXPERIMENTAL)) { + } else if (vllmProfile && (userChoseManagedVllm || EXPERIMENTAL)) { const verb = hasVllmImage ? "Start" : "Install"; options.push({ key: "install-vllm", label: `${verb} vLLM (${vllmProfile.name})` }); } @@ -7594,15 +7594,13 @@ async function setupNim( } selected = options.find((o) => o.key === providerKey); if (!selected) { - // Action keys fall back to the equivalent running-provider key - // when the menu only emits the running entry (the install would + // Install action keys fall back to the equivalent running-provider + // key when the menu only emits the running entry (the install would // have been a no-op anyway). if (providerKey === "install-ollama") { selected = options.find((o) => o.key === "ollama"); } else if (providerKey === "install-vllm") { selected = options.find((o) => o.key === "vllm"); - } else if (providerKey === "vllm") { - selected = options.find((o) => o.key === "install-vllm"); } else if (providerKey === "ollama") { selected = options.find((o) => o.key === "install-ollama"); } diff --git a/test/onboard-selection.test.ts b/test/onboard-selection.test.ts index c49a0f8e66e..acb13496527 100644 --- a/test/onboard-selection.test.ts +++ b/test/onboard-selection.test.ts @@ -303,6 +303,59 @@ const { setupNim } = require(${onboardPath}); assert.ok(!payload.lines.some((line: string) => line.includes("rerun the same command"))); }); + it("does not turn non-interactive NEMOCLAW_PROVIDER=vllm into managed install-vllm", () => { + const repoRoot = path.join(import.meta.dirname, ".."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-vllm-no-install-")); + const scriptPath = path.join(tmpDir, "vllm-no-install-check.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "onboard.js")); + const runnerPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "runner.js")); + const vllmPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "inference", "vllm.js")); + + const script = String.raw` +const runner = require(${runnerPath}); +const vllm = require(${vllmPath}); + +vllm.installVllm = async () => { + console.error("INSTALL_VLLM_CALLED"); + return { ok: false }; +}; +runner.runCapture = (command) => { + const cmd = Array.isArray(command) ? command.join(" ") : command; + if (cmd.includes("command -v ollama")) return ""; + if (cmd.includes("127.0.0.1:11434/api/tags")) return ""; + if (cmd.includes("127.0.0.1:8000/v1/models")) return ""; + if (cmd.includes("docker images")) return ""; + return ""; +}; + +const { setupNim } = require(${onboardPath}); + +(async () => { + await setupNim({ type: "nvidia" }, null); +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + NEMOCLAW_NON_INTERACTIVE: "1", + NEMOCLAW_PROVIDER: "vllm", + NEMOCLAW_EXPERIMENTAL: "", + }, + }); + + assert.equal(result.status, 1); + assert.match(result.stderr, /Requested provider 'vllm' is not available/); + assert.doesNotMatch(result.stderr, /INSTALL_VLLM_CALLED/); + }); + it("does not label NVIDIA Endpoints as recommended in the provider list", () => { const repoRoot = path.join(import.meta.dirname, ".."); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-no-recommended-label-")); From 9c4a2b49dd388ec608abbdbeb13d8fd863fc388e Mon Sep 17 00:00:00 2001 From: Aaron Erickson Date: Tue, 12 May 2026 12:48:01 -0700 Subject: [PATCH 4/4] test(onboard): harden vLLM selection checks Signed-off-by: Aaron Erickson --- test/onboard-selection.test.ts | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/test/onboard-selection.test.ts b/test/onboard-selection.test.ts index acb13496527..b506f48084e 100644 --- a/test/onboard-selection.test.ts +++ b/test/onboard-selection.test.ts @@ -233,12 +233,25 @@ printf '%s' "$status" const credentials = require(${credentialsPath}); const runner = require(${runnerPath}); -const answers = ["7"]; const messages = []; +const lines = []; +const originalLog = console.log; + +function findRunningVllmChoice() { + const option = lines.find((line) => + /^\s*\d+\) Local vLLM \[experimental\] \(localhost:8000\) — running \(suggested\)/.test(line) + ); + const match = option && option.match(/^\s*(\d+)\)/); + if (!match) { + throw new Error("Could not find running vLLM option in menu:\\n" + lines.join("\\n")); + } + return match[1]; +} credentials.prompt = async (message) => { messages.push(message); - return answers.shift() || ""; + if (/Choose \[/.test(message)) return findRunningVllmChoice(); + return ""; }; credentials.ensureApiKey = async () => {}; runner.runCapture = (command) => { @@ -253,8 +266,6 @@ runner.runCapture = (command) => { const { setupNim } = require(${onboardPath}); (async () => { - const originalLog = console.log; - const lines = []; console.log = (...args) => lines.push(args.join(" ")); try { const result = await setupNim({ type: "nvidia" }, null); @@ -308,13 +319,21 @@ const { setupNim } = require(${onboardPath}); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-vllm-no-install-")); const scriptPath = path.join(tmpDir, "vllm-no-install-check.js"); const onboardPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "onboard.js")); + const credentialsPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "credentials", "store.js")); const runnerPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "runner.js")); const vllmPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "inference", "vllm.js")); const script = String.raw` +const credentials = require(${credentialsPath}); const runner = require(${runnerPath}); const vllm = require(${vllmPath}); +credentials.prompt = async () => { + throw new Error("Unexpected prompt in non-interactive test"); +}; +credentials.ensureApiKey = async () => { + throw new Error("Unexpected ensureApiKey call in non-interactive test"); +}; vllm.installVllm = async () => { console.error("INSTALL_VLLM_CALLED"); return { ok: false };