From efcbe4705dbad026d70eeeaed0e512378081ce63 Mon Sep 17 00:00:00 2001 From: zyang-dev <267119621+zyang-dev@users.noreply.github.com> Date: Tue, 2 Jun 2026 16:51:07 -0700 Subject: [PATCH 1/2] fix(inference): use vLLM runtime max_model_len for context window Signed-off-by: zyang-dev <267119621+zyang-dev@users.noreply.github.com> --- src/lib/inference/local.ts | 10 +++ .../inference/vllm-runtime-context.test.ts | 90 +++++++++++++++++++ src/lib/inference/vllm-runtime-context.ts | 63 +++++++++++++ src/lib/onboard.ts | 4 +- test/onboard-selection.test.ts | 20 ++++- 5 files changed, 183 insertions(+), 4 deletions(-) create mode 100644 src/lib/inference/vllm-runtime-context.test.ts create mode 100644 src/lib/inference/vllm-runtime-context.ts diff --git a/src/lib/inference/local.ts b/src/lib/inference/local.ts index ccd418e66da..5fc5b023cc0 100644 --- a/src/lib/inference/local.ts +++ b/src/lib/inference/local.ts @@ -22,6 +22,9 @@ import { resolveOllamaRuntimeContextWindow as resolveOllamaRuntimeContextWindowWithHost, } from "./ollama-runtime-context"; import type { OllamaRuntimeModelStatus } from "./ollama-runtime-context"; +import { + applyVllmRuntimeContextWindow as applyVllmRuntimeContextWindowFromModels, +} from "./vllm-runtime-context"; export type { OllamaRuntimeModelStatus } from "./ollama-runtime-context"; const { shellQuote, runCapture, runCaptureEx } = require("../runner"); @@ -771,6 +774,13 @@ export function applyOllamaRuntimeContextWindow(selectedModel: string): void { applyOllamaRuntimeContextWindowWithHost(selectedModel, getResolvedOllamaHost); } +export function applyVllmRuntimeContextWindow( + modelsResponse: unknown, + modelId: string | null | undefined, +): void { + applyVllmRuntimeContextWindowFromModels(modelsResponse, modelId); +} + function formatOllamaCpuOnlyDiagnostic(model: string, status: OllamaRuntimeModelStatus): string { const observed: string[] = []; if (status.processor) observed.push(`processor=${status.processor}`); diff --git a/src/lib/inference/vllm-runtime-context.test.ts b/src/lib/inference/vllm-runtime-context.test.ts new file mode 100644 index 00000000000..8cbb2370073 --- /dev/null +++ b/src/lib/inference/vllm-runtime-context.test.ts @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it } from "vitest"; + +import { applyVllmRuntimeContextWindow } from "../../../dist/lib/inference/vllm-runtime-context"; + +function applyContextWindow( + modelsResponse: unknown, + modelId = "model-a", + env: NodeJS.ProcessEnv = {}, +): { env: NodeJS.ProcessEnv; messages: string[] } { + const messages: string[] = []; + applyVllmRuntimeContextWindow(modelsResponse, modelId, { + env, + logger: { + log: (message: string) => messages.push(message), + warn: (message: string) => messages.push(message), + }, + }); + return { env, messages }; +} + +describe("vLLM runtime context helpers", () => { + it("applies valid vLLM /v1/models max_model_len values", () => { + expect( + applyContextWindow({ data: [{ id: "model-a", max_model_len: 65_536 }] }).env + .NEMOCLAW_CONTEXT_WINDOW, + ).toBe("65536"); + expect( + applyContextWindow({ data: [{ id: "model-a", max_model_len: "262144" }] }).env + .NEMOCLAW_CONTEXT_WINDOW, + ).toBe("262144"); + }); + + it("treats omitted max_model_len values as compatibility no-ops", () => { + for (const value of [undefined, null, " "]) { + const { env, messages } = applyContextWindow({ + data: [{ id: "model-a", max_model_len: value }], + }); + expect(env.NEMOCLAW_CONTEXT_WINDOW).toBeUndefined(); + expect(messages).toEqual([]); + } + }); + + it("warns and ignores malformed or non-positive max_model_len values", () => { + for (const value of ["bogus", "1.5", 1.5, 0, -1]) { + const { env, messages } = applyContextWindow({ + data: [{ id: "model-a", max_model_len: value }], + }); + expect(env.NEMOCLAW_CONTEXT_WINDOW).toBeUndefined(); + expect(messages.at(-1)).toContain("non-positive or malformed max_model_len"); + } + }); + + it("warns and ignores implausibly large max_model_len values", () => { + const { env, messages } = applyContextWindow({ + data: [{ id: "model-a", max_model_len: 10_000_000 }], + }); + expect(env.NEMOCLAW_CONTEXT_WINDOW).toBeUndefined(); + expect(messages.at(-1)).toContain("above NemoClaw's auto-detect ceiling"); + }); + + it("matches max_model_len by model id, then falls back to the first entry", () => { + const response = { + data: [ + { id: "model-a", max_model_len: 32_768 }, + { id: "model-b", max_model_len: 65_536 }, + ], + }; + + expect(applyContextWindow(response, "model-b").env.NEMOCLAW_CONTEXT_WINDOW).toBe("65536"); + expect(applyContextWindow(response, "missing").env.NEMOCLAW_CONTEXT_WINDOW).toBe("32768"); + expect(applyContextWindow(response, "").env.NEMOCLAW_CONTEXT_WINDOW).toBe("32768"); + }); + + it("applies detected max_model_len only when no explicit override is set", () => { + const response = { data: [{ id: "model-a", max_model_len: 65_536 }] }; + + const { env, messages } = applyContextWindow(response); + expect(env.NEMOCLAW_CONTEXT_WINDOW).toBe("65536"); + expect(messages.at(-1)).toContain("Using vLLM max_model_len"); + + const explicit = applyContextWindow(response, "model-a", { + NEMOCLAW_CONTEXT_WINDOW: "131072", + }); + expect(explicit.env.NEMOCLAW_CONTEXT_WINDOW).toBe("131072"); + expect(explicit.messages.at(-1)).toContain("Keeping configured context window"); + }); +}); diff --git a/src/lib/inference/vllm-runtime-context.ts b/src/lib/inference/vllm-runtime-context.ts new file mode 100644 index 00000000000..2fa8f611541 --- /dev/null +++ b/src/lib/inference/vllm-runtime-context.ts @@ -0,0 +1,63 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { + hasExplicitContextWindow, + parsePositiveInteger, +} from "./ollama-runtime-context"; + +const MAX_AUTODETECTED_VLLM_CONTEXT_WINDOW = 4_194_304; + +type ModelEntry = { id?: unknown; max_model_len?: unknown }; +type ApplyOptions = { env?: NodeJS.ProcessEnv; logger?: Pick }; + +export function applyVllmRuntimeContextWindow( + modelsResponse: unknown, + modelId: string | null | undefined, + options: ApplyOptions = {}, +): void { + const env = options.env ?? process.env; + const logger = options.logger ?? console; + + if (hasExplicitContextWindow(env.NEMOCLAW_CONTEXT_WINDOW)) { + logger.log(` ℹ Keeping configured context window: ${env.NEMOCLAW_CONTEXT_WINDOW} tokens`); + return; + } + + const data = (modelsResponse as { data?: unknown } | null | undefined)?.data; + const entries = Array.isArray(data) ? (data as ModelEntry[]) : []; + if (entries.length === 0) return; + + const target = String(modelId ?? "").trim(); + const entry = + (target && entries.find((candidate) => String(candidate.id ?? "").trim() === target)) || + entries[0]; + const rawMaxModelLen = entry?.max_model_len; + if ( + rawMaxModelLen === undefined || + rawMaxModelLen === null || + String(rawMaxModelLen).trim() === "" + ) { + return; + } + + const contextLength = parsePositiveInteger(rawMaxModelLen); + if (!contextLength) { + logger.warn( + ` ⚠ vLLM /v1/models returned a non-positive or malformed max_model_len ` + + `(${String(rawMaxModelLen)}); ignoring it.`, + ); + return; + } + if (contextLength > MAX_AUTODETECTED_VLLM_CONTEXT_WINDOW) { + logger.warn( + ` ⚠ vLLM /v1/models returned max_model_len=${contextLength}, above NemoClaw's ` + + `auto-detect ceiling (${MAX_AUTODETECTED_VLLM_CONTEXT_WINDOW}); ignoring it.`, + ); + return; + } + + const value = String(contextLength); + env.NEMOCLAW_CONTEXT_WINDOW = value; + logger.log(` ✓ Using vLLM max_model_len: ${value} tokens`); +} diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 8fc4c51c97c..2c0f79a4cba 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -5079,6 +5079,7 @@ async function setupNim( process.exit(1); } console.log(` Detected model: ${model}`); + localInference.applyVllmRuntimeContextWindow(vllmModels, detectedModel); } else { console.error(" Could not detect model from vLLM. Please specify manually."); process.exit(1); @@ -5108,8 +5109,7 @@ async function setupNim( } preferredInferenceApi = validation.api; // Force chat completions — vLLM's /v1/responses endpoint does not - // run the --tool-call-parser, so tool calls arrive as raw text. - // See: https://github.com/NVIDIA/NemoClaw/issues/976 + // run the --tool-call-parser, so tool calls arrive as raw text (#976). if (preferredInferenceApi !== "openai-completions") { console.log( " ℹ Using chat completions API (tool-call-parser requires /v1/chat/completions)", diff --git a/test/onboard-selection.test.ts b/test/onboard-selection.test.ts index 662c3d170af..6d81763c038 100644 --- a/test/onboard-selection.test.ts +++ b/test/onboard-selection.test.ts @@ -531,7 +531,11 @@ runner.runCapture = (command) => { const cmd = Array.isArray(command) ? command.join(" ") : command; if (cmd.includes("command -v ollama")) return ""; if (cmd.includes("127.0.0.1:11434/api/tags")) return ""; - if (cmd.includes("127.0.0.1:8000/v1/models")) return JSON.stringify({ data: [{ id: "meta-llama/Llama-3.3-70B-Instruct" }] }); + if (cmd.includes("127.0.0.1:8000/v1/models")) { + return JSON.stringify({ + data: [{ id: "meta-llama/Llama-3.3-70B-Instruct", max_model_len: 65536 }], + }); + } if (cmd.includes("docker images")) return ""; return ""; }; @@ -542,7 +546,14 @@ const { setupNim } = require(${onboardPath}); console.log = (...args) => lines.push(args.join(" ")); try { const result = await setupNim({ type: "nvidia" }, null); - originalLog(JSON.stringify({ result, messages, lines })); + originalLog( + JSON.stringify({ + result, + messages, + lines, + contextWindow: process.env.NEMOCLAW_CONTEXT_WINDOW, + }), + ); } finally { console.log = originalLog; } @@ -562,6 +573,7 @@ const { setupNim } = require(${onboardPath}); PATH: `${fakeBin}:${process.env.PATH || ""}`, NEMOCLAW_EXPERIMENTAL: "", NEMOCLAW_PROVIDER: "", + NEMOCLAW_CONTEXT_WINDOW: "", }, }); @@ -571,12 +583,16 @@ const { setupNim } = require(${onboardPath}); assert.equal(payload.result.provider, "vllm-local"); assert.equal(payload.result.model, "meta-llama/Llama-3.3-70B-Instruct"); assert.equal(payload.result.preferredInferenceApi, "openai-completions"); + assert.equal(payload.contextWindow, "65536"); assert.equal(payload.messages.filter((message: string) => /Choose \[/.test(message)).length, 1); assert.ok( payload.lines.some((line: string) => line.includes("Detected local inference option: vLLM"), ), ); + assert.ok( + payload.lines.some((line: string) => line.includes("Using vLLM max_model_len: 65536")), + ); assert.ok( payload.lines.some((line: string) => /^\s*\d+\) Local vLLM \[experimental\] \(localhost:8000\) — running \(suggested\)/.test( From 2ce7ecfd0e89019bd0db5edeedb3534eacaa4205 Mon Sep 17 00:00:00 2001 From: zyang-dev <267119621+zyang-dev@users.noreply.github.com> Date: Tue, 2 Jun 2026 17:16:08 -0700 Subject: [PATCH 2/2] fix(onboard): apply vLLM context window after validation Signed-off-by: zyang-dev <267119621+zyang-dev@users.noreply.github.com> --- src/lib/onboard.ts | 9 ++- test/onboard-selection.test.ts | 114 +++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+), 5 deletions(-) diff --git a/src/lib/onboard.ts b/src/lib/onboard.ts index 2c0f79a4cba..e61222b2f32 100644 --- a/src/lib/onboard.ts +++ b/src/lib/onboard.ts @@ -5068,8 +5068,9 @@ async function setupNim( ignoreError: true, }, ); + let vllmModels: { data?: Array<{ id?: unknown }> } = {}; try { - const vllmModels = JSON.parse(vllmModelsRaw); + vllmModels = JSON.parse(vllmModelsRaw); if (vllmModels.data && vllmModels.data.length > 0) { const detectedModel = typeof vllmModels.data[0]?.id === "string" ? vllmModels.data[0].id : null; @@ -5079,7 +5080,6 @@ async function setupNim( process.exit(1); } console.log(` Detected model: ${model}`); - localInference.applyVllmRuntimeContextWindow(vllmModels, detectedModel); } else { console.error(" Could not detect model from vLLM. Please specify manually."); process.exit(1); @@ -5104,9 +5104,8 @@ async function setupNim( if (validation.retry === "selection" || validation.retry === "model") { continue selectionLoop; } - if (!validation.ok) { - continue selectionLoop; - } + if (!validation.ok) continue selectionLoop; + localInference.applyVllmRuntimeContextWindow(vllmModels, model as string); preferredInferenceApi = validation.api; // Force chat completions — vLLM's /v1/responses endpoint does not // run the --tool-call-parser, so tool calls arrive as raw text (#976). diff --git a/test/onboard-selection.test.ts b/test/onboard-selection.test.ts index 6d81763c038..a420a83e011 100644 --- a/test/onboard-selection.test.ts +++ b/test/onboard-selection.test.ts @@ -603,6 +603,120 @@ const { setupNim } = require(${onboardPath}); assert.ok(!payload.lines.some((line: string) => line.includes("rerun the same command"))); }); + it("does not apply detected vLLM max_model_len when validation returns to provider selection", () => { + const repoRoot = path.join(import.meta.dirname, ".."); + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-vllm-validation-")); + const scriptPath = path.join(tmpDir, "vllm-validation-context-check.js"); + const onboardPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "onboard.js")); + const credentialsPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "credentials", "store.js")); + const runnerPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "runner.js")); + const validationPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "onboard", "inference-selection-validation.js")); + + const script = String.raw` +const credentials = require(${credentialsPath}); +const runner = require(${runnerPath}); +const validationHelpers = require(${validationPath}); + +class StopAfterValidationBackout extends Error {} + +const messages = []; +const lines = []; +const originalLog = console.log; +let chooseCount = 0; + +function findRunningVllmChoice() { + const option = lines.find((line) => + /^\s*\d+\) Local vLLM \[experimental\] \(localhost:8000\) — running \(suggested\)/.test(line) + ); + const match = option && option.match(/^\s*(\d+)\)/); + if (!match) { + throw new Error("Could not find running vLLM option in menu:\\n" + lines.join("\\n")); + } + return match[1]; +} + +credentials.prompt = async (message) => { + messages.push(message); + if (/Choose \[/.test(message)) { + chooseCount += 1; + if (chooseCount === 1) return findRunningVllmChoice(); + throw new StopAfterValidationBackout("validation returned to provider selection"); + } + return ""; +}; +credentials.ensureApiKey = async () => {}; +runner.runCapture = (command) => { + const cmd = Array.isArray(command) ? command.join(" ") : command; + if (cmd.includes("command -v ollama")) return ""; + if (cmd.includes("127.0.0.1:11434/api/tags")) return ""; + if (cmd.includes("127.0.0.1:8000/v1/models")) { + return JSON.stringify({ + data: [{ id: "meta-llama/Llama-3.3-70B-Instruct", max_model_len: 65536 }], + }); + } + if (cmd.includes("docker images")) return ""; + return ""; +}; +validationHelpers.createInferenceSelectionValidationHelpers = () => ({ + validateOpenAiLikeSelection: async () => ({ ok: false, retry: "selection" }), + validateAnthropicSelectionWithRetryMessage: async () => ({ ok: false, retry: "selection" }), + validateCustomOpenAiLikeSelection: async () => ({ ok: false, retry: "selection" }), + validateCustomAnthropicSelection: async () => ({ ok: false, retry: "selection" }), +}); + +const { setupNim } = require(${onboardPath}); + +(async () => { + console.log = (...args) => lines.push(args.join(" ")); + try { + await setupNim({ type: "nvidia" }, null); + throw new Error("setupNim unexpectedly completed"); + } catch (error) { + if (!(error instanceof StopAfterValidationBackout)) throw error; + originalLog( + JSON.stringify({ + messages, + lines, + contextWindow: process.env.NEMOCLAW_CONTEXT_WINDOW || null, + }), + ); + } finally { + console.log = originalLog; + } +})().catch((error) => { + console.error(error); + process.exit(1); +}); +`; + fs.writeFileSync(scriptPath, script); + + const result = spawnSync(process.execPath, [scriptPath], { + cwd: repoRoot, + encoding: "utf-8", + env: { + ...process.env, + HOME: tmpDir, + NEMOCLAW_EXPERIMENTAL: "", + NEMOCLAW_PROVIDER: "", + NEMOCLAW_CONTEXT_WINDOW: "", + }, + }); + + expect(result.status).toBe(0); + expect(result.stdout.trim()).not.toBe(""); + const payload = JSON.parse(result.stdout.trim()); + assert.equal(payload.contextWindow, null); + assert.equal(payload.messages.filter((message: string) => /Choose \[/.test(message)).length, 2); + assert.ok( + payload.lines.some((line: string) => + line.includes("Detected model: meta-llama/Llama-3.3-70B-Instruct"), + ), + ); + assert.ok( + !payload.lines.some((line: string) => line.includes("Using vLLM max_model_len: 65536")), + ); + }); + it("does not turn non-interactive NEMOCLAW_PROVIDER=vllm into managed install-vllm", () => { const repoRoot = path.join(import.meta.dirname, ".."); const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-vllm-no-install-"));