fix(inference): use vLLM runtime max_model_len for context window

src/lib/inference/local.ts

@@ -22,6 +22,9 @@ import {
   resolveOllamaRuntimeContextWindow as resolveOllamaRuntimeContextWindowWithHost,
 } from "./ollama-runtime-context";
 import type { OllamaRuntimeModelStatus } from "./ollama-runtime-context";
+import {
+  applyVllmRuntimeContextWindow as applyVllmRuntimeContextWindowFromModels,
+} from "./vllm-runtime-context";
 export type { OllamaRuntimeModelStatus } from "./ollama-runtime-context";
 
 const { shellQuote, runCapture, runCaptureEx } = require("../runner");
@@ -771,6 +774,13 @@ export function applyOllamaRuntimeContextWindow(selectedModel: string): void {
   applyOllamaRuntimeContextWindowWithHost(selectedModel, getResolvedOllamaHost);
 }
 
+export function applyVllmRuntimeContextWindow(
+  modelsResponse: unknown,
+  modelId: string | null | undefined,
+): void {
+  applyVllmRuntimeContextWindowFromModels(modelsResponse, modelId);
+}
+
 function formatOllamaCpuOnlyDiagnostic(model: string, status: OllamaRuntimeModelStatus): string {
   const observed: string[] = [];
   if (status.processor) observed.push(`processor=${status.processor}`);

src/lib/inference/vllm-runtime-context.test.ts

@@ -0,0 +1,90 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it } from "vitest";
+
+import { applyVllmRuntimeContextWindow } from "../../../dist/lib/inference/vllm-runtime-context";
+
+function applyContextWindow(
+  modelsResponse: unknown,
+  modelId = "model-a",
+  env: NodeJS.ProcessEnv = {},
+): { env: NodeJS.ProcessEnv; messages: string[] } {
+  const messages: string[] = [];
+  applyVllmRuntimeContextWindow(modelsResponse, modelId, {
+    env,
+    logger: {
+      log: (message: string) => messages.push(message),
+      warn: (message: string) => messages.push(message),
+    },
+  });
+  return { env, messages };
+}
+
+describe("vLLM runtime context helpers", () => {
+  it("applies valid vLLM /v1/models max_model_len values", () => {
+    expect(
+      applyContextWindow({ data: [{ id: "model-a", max_model_len: 65_536 }] }).env
+        .NEMOCLAW_CONTEXT_WINDOW,
+    ).toBe("65536");
+    expect(
+      applyContextWindow({ data: [{ id: "model-a", max_model_len: "262144" }] }).env
+        .NEMOCLAW_CONTEXT_WINDOW,
+    ).toBe("262144");
+  });
+
+  it("treats omitted max_model_len values as compatibility no-ops", () => {
+    for (const value of [undefined, null, "   "]) {
+      const { env, messages } = applyContextWindow({
+        data: [{ id: "model-a", max_model_len: value }],
+      });
+      expect(env.NEMOCLAW_CONTEXT_WINDOW).toBeUndefined();
+      expect(messages).toEqual([]);
+    }
+  });
+
+  it("warns and ignores malformed or non-positive max_model_len values", () => {
+    for (const value of ["bogus", "1.5", 1.5, 0, -1]) {
+      const { env, messages } = applyContextWindow({
+        data: [{ id: "model-a", max_model_len: value }],
+      });
+      expect(env.NEMOCLAW_CONTEXT_WINDOW).toBeUndefined();
+      expect(messages.at(-1)).toContain("non-positive or malformed max_model_len");
+    }
+  });
+
+  it("warns and ignores implausibly large max_model_len values", () => {
+    const { env, messages } = applyContextWindow({
+      data: [{ id: "model-a", max_model_len: 10_000_000 }],
+    });
+    expect(env.NEMOCLAW_CONTEXT_WINDOW).toBeUndefined();
+    expect(messages.at(-1)).toContain("above NemoClaw's auto-detect ceiling");
+  });
+
+  it("matches max_model_len by model id, then falls back to the first entry", () => {
+    const response = {
+      data: [
+        { id: "model-a", max_model_len: 32_768 },
+        { id: "model-b", max_model_len: 65_536 },
+      ],
+    };
+
+    expect(applyContextWindow(response, "model-b").env.NEMOCLAW_CONTEXT_WINDOW).toBe("65536");
+    expect(applyContextWindow(response, "missing").env.NEMOCLAW_CONTEXT_WINDOW).toBe("32768");
+    expect(applyContextWindow(response, "").env.NEMOCLAW_CONTEXT_WINDOW).toBe("32768");
+  });
+
+  it("applies detected max_model_len only when no explicit override is set", () => {
+    const response = { data: [{ id: "model-a", max_model_len: 65_536 }] };
+
+    const { env, messages } = applyContextWindow(response);
+    expect(env.NEMOCLAW_CONTEXT_WINDOW).toBe("65536");
+    expect(messages.at(-1)).toContain("Using vLLM max_model_len");
+
+    const explicit = applyContextWindow(response, "model-a", {
+      NEMOCLAW_CONTEXT_WINDOW: "131072",
+    });
+    expect(explicit.env.NEMOCLAW_CONTEXT_WINDOW).toBe("131072");
+    expect(explicit.messages.at(-1)).toContain("Keeping configured context window");
+  });
+});

src/lib/inference/vllm-runtime-context.ts

@@ -0,0 +1,63 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import {
+  hasExplicitContextWindow,
+  parsePositiveInteger,
+} from "./ollama-runtime-context";
+
+const MAX_AUTODETECTED_VLLM_CONTEXT_WINDOW = 4_194_304;
+
+type ModelEntry = { id?: unknown; max_model_len?: unknown };
+type ApplyOptions = { env?: NodeJS.ProcessEnv; logger?: Pick<Console, "log" | "warn"> };
+
+export function applyVllmRuntimeContextWindow(
+  modelsResponse: unknown,
+  modelId: string | null | undefined,
+  options: ApplyOptions = {},
+): void {
+  const env = options.env ?? process.env;
+  const logger = options.logger ?? console;
+
+  if (hasExplicitContextWindow(env.NEMOCLAW_CONTEXT_WINDOW)) {
+    logger.log(`  ℹ Keeping configured context window: ${env.NEMOCLAW_CONTEXT_WINDOW} tokens`);
+    return;
+  }
+
+  const data = (modelsResponse as { data?: unknown } | null | undefined)?.data;
+  const entries = Array.isArray(data) ? (data as ModelEntry[]) : [];
+  if (entries.length === 0) return;
+
+  const target = String(modelId ?? "").trim();
+  const entry =
+    (target && entries.find((candidate) => String(candidate.id ?? "").trim() === target)) ||
+    entries[0];
+  const rawMaxModelLen = entry?.max_model_len;
+  if (
+    rawMaxModelLen === undefined ||
+    rawMaxModelLen === null ||
+    String(rawMaxModelLen).trim() === ""
+  ) {
+    return;
+  }
+
+  const contextLength = parsePositiveInteger(rawMaxModelLen);
+  if (!contextLength) {
+    logger.warn(
+      `  ⚠ vLLM /v1/models returned a non-positive or malformed max_model_len ` +
+        `(${String(rawMaxModelLen)}); ignoring it.`,
+    );
+    return;
+  }
+  if (contextLength > MAX_AUTODETECTED_VLLM_CONTEXT_WINDOW) {
+    logger.warn(
+      `  ⚠ vLLM /v1/models returned max_model_len=${contextLength}, above NemoClaw's ` +
+        `auto-detect ceiling (${MAX_AUTODETECTED_VLLM_CONTEXT_WINDOW}); ignoring it.`,
+    );
+    return;
+  }
+
+  const value = String(contextLength);
+  env.NEMOCLAW_CONTEXT_WINDOW = value;
+  logger.log(`  ✓ Using vLLM max_model_len: ${value} tokens`);
+}

src/lib/onboard.ts

@@ -5068,8 +5068,9 @@ async function setupNim(
             ignoreError: true,
           },
         );
+        let vllmModels: { data?: Array<{ id?: unknown }> } = {};
         try {
-          const vllmModels = JSON.parse(vllmModelsRaw);
+          vllmModels = JSON.parse(vllmModelsRaw);
           if (vllmModels.data && vllmModels.data.length > 0) {
             const detectedModel =
               typeof vllmModels.data[0]?.id === "string" ? vllmModels.data[0].id : null;
@@ -5103,13 +5104,11 @@ async function setupNim(
         if (validation.retry === "selection" || validation.retry === "model") {
           continue selectionLoop;
         }
-        if (!validation.ok) {
-          continue selectionLoop;
-        }
+        if (!validation.ok) continue selectionLoop;
+        localInference.applyVllmRuntimeContextWindow(vllmModels, model as string);
         preferredInferenceApi = validation.api;
         // Force chat completions — vLLM's /v1/responses endpoint does not
-        // run the --tool-call-parser, so tool calls arrive as raw text.
-        // See: https://github.com/NVIDIA/NemoClaw/issues/976
+        // run the --tool-call-parser, so tool calls arrive as raw text (#976).
         if (preferredInferenceApi !== "openai-completions") {
           console.log(
             "  ℹ Using chat completions API (tool-call-parser requires /v1/chat/completions)",

test/onboard-selection.test.ts

@@ -531,7 +531,11 @@ runner.runCapture = (command) => {
   const cmd = Array.isArray(command) ? command.join(" ") : command;
   if (cmd.includes("command -v ollama")) return "";
   if (cmd.includes("127.0.0.1:11434/api/tags")) return "";
-  if (cmd.includes("127.0.0.1:8000/v1/models")) return JSON.stringify({ data: [{ id: "meta-llama/Llama-3.3-70B-Instruct" }] });
+  if (cmd.includes("127.0.0.1:8000/v1/models")) {
+    return JSON.stringify({
+      data: [{ id: "meta-llama/Llama-3.3-70B-Instruct", max_model_len: 65536 }],
+    });
+  }
   if (cmd.includes("docker images")) return "";
   return "";
 };
@@ -542,7 +546,14 @@ const { setupNim } = require(${onboardPath});
   console.log = (...args) => lines.push(args.join(" "));
   try {
     const result = await setupNim({ type: "nvidia" }, null);
-    originalLog(JSON.stringify({ result, messages, lines }));
+    originalLog(
+      JSON.stringify({
+        result,
+        messages,
+        lines,
+        contextWindow: process.env.NEMOCLAW_CONTEXT_WINDOW,
+      }),
+    );
   } finally {
     console.log = originalLog;
   }
@@ -562,6 +573,7 @@ const { setupNim } = require(${onboardPath});
         PATH: `${fakeBin}:${process.env.PATH || ""}`,
         NEMOCLAW_EXPERIMENTAL: "",
         NEMOCLAW_PROVIDER: "",
+        NEMOCLAW_CONTEXT_WINDOW: "",
       },
     });
 
@@ -571,12 +583,16 @@ const { setupNim } = require(${onboardPath});
     assert.equal(payload.result.provider, "vllm-local");
     assert.equal(payload.result.model, "meta-llama/Llama-3.3-70B-Instruct");
     assert.equal(payload.result.preferredInferenceApi, "openai-completions");
+    assert.equal(payload.contextWindow, "65536");
     assert.equal(payload.messages.filter((message: string) => /Choose \[/.test(message)).length, 1);
     assert.ok(
       payload.lines.some((line: string) =>
         line.includes("Detected local inference option: vLLM"),
       ),
     );
+    assert.ok(
+      payload.lines.some((line: string) => line.includes("Using vLLM max_model_len: 65536")),
+    );
     assert.ok(
       payload.lines.some((line: string) =>
         /^\s*\d+\) Local vLLM \[experimental\] \(localhost:8000\) — running \(suggested\)/.test(
@@ -587,6 +603,120 @@ const { setupNim } = require(${onboardPath});
     assert.ok(!payload.lines.some((line: string) => line.includes("rerun the same command")));
   });
 
+  it("does not apply detected vLLM max_model_len when validation returns to provider selection", () => {
+    const repoRoot = path.join(import.meta.dirname, "..");
+    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-vllm-validation-"));
+    const scriptPath = path.join(tmpDir, "vllm-validation-context-check.js");
+    const onboardPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "onboard.js"));
+    const credentialsPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "credentials", "store.js"));
+    const runnerPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "runner.js"));
+    const validationPath = JSON.stringify(path.join(repoRoot, "dist", "lib", "onboard", "inference-selection-validation.js"));
+
+    const script = String.raw`
+const credentials = require(${credentialsPath});
+const runner = require(${runnerPath});
+const validationHelpers = require(${validationPath});
+
+class StopAfterValidationBackout extends Error {}
+
+const messages = [];
+const lines = [];
+const originalLog = console.log;
+let chooseCount = 0;
+
+function findRunningVllmChoice() {
+  const option = lines.find((line) =>
+    /^\s*\d+\) Local vLLM \[experimental\] \(localhost:8000\) — running \(suggested\)/.test(line)
+  );
+  const match = option && option.match(/^\s*(\d+)\)/);
+  if (!match) {
+    throw new Error("Could not find running vLLM option in menu:\\n" + lines.join("\\n"));
+  }
+  return match[1];
+}
+
+credentials.prompt = async (message) => {
+  messages.push(message);
+  if (/Choose \[/.test(message)) {
+    chooseCount += 1;
+    if (chooseCount === 1) return findRunningVllmChoice();
+    throw new StopAfterValidationBackout("validation returned to provider selection");
+  }
+  return "";
+};
+credentials.ensureApiKey = async () => {};
+runner.runCapture = (command) => {
+  const cmd = Array.isArray(command) ? command.join(" ") : command;
+  if (cmd.includes("command -v ollama")) return "";
+  if (cmd.includes("127.0.0.1:11434/api/tags")) return "";
+  if (cmd.includes("127.0.0.1:8000/v1/models")) {
+    return JSON.stringify({
+      data: [{ id: "meta-llama/Llama-3.3-70B-Instruct", max_model_len: 65536 }],
+    });
+  }
+  if (cmd.includes("docker images")) return "";
+  return "";
+};
+validationHelpers.createInferenceSelectionValidationHelpers = () => ({
+  validateOpenAiLikeSelection: async () => ({ ok: false, retry: "selection" }),
+  validateAnthropicSelectionWithRetryMessage: async () => ({ ok: false, retry: "selection" }),
+  validateCustomOpenAiLikeSelection: async () => ({ ok: false, retry: "selection" }),
+  validateCustomAnthropicSelection: async () => ({ ok: false, retry: "selection" }),
+});
+
+const { setupNim } = require(${onboardPath});
+
+(async () => {
+  console.log = (...args) => lines.push(args.join(" "));
+  try {
+    await setupNim({ type: "nvidia" }, null);
+    throw new Error("setupNim unexpectedly completed");
+  } catch (error) {
+    if (!(error instanceof StopAfterValidationBackout)) throw error;
+    originalLog(
+      JSON.stringify({
+        messages,
+        lines,
+        contextWindow: process.env.NEMOCLAW_CONTEXT_WINDOW || null,
+      }),
+    );
+  } finally {
+    console.log = originalLog;
+  }
+})().catch((error) => {
+  console.error(error);
+  process.exit(1);
+});
+`;
+    fs.writeFileSync(scriptPath, script);
+
+    const result = spawnSync(process.execPath, [scriptPath], {
+      cwd: repoRoot,
+      encoding: "utf-8",
+      env: {
+        ...process.env,
+        HOME: tmpDir,
+        NEMOCLAW_EXPERIMENTAL: "",
+        NEMOCLAW_PROVIDER: "",
+        NEMOCLAW_CONTEXT_WINDOW: "",
+      },
+    });
+
+    expect(result.status).toBe(0);
+    expect(result.stdout.trim()).not.toBe("");
+    const payload = JSON.parse(result.stdout.trim());
+    assert.equal(payload.contextWindow, null);
+    assert.equal(payload.messages.filter((message: string) => /Choose \[/.test(message)).length, 2);
+    assert.ok(
+      payload.lines.some((line: string) =>
+        line.includes("Detected model: meta-llama/Llama-3.3-70B-Instruct"),
+      ),
+    );
+    assert.ok(
+      !payload.lines.some((line: string) => line.includes("Using vLLM max_model_len: 65536")),
+    );
+  });
+
   it("does not turn non-interactive NEMOCLAW_PROVIDER=vllm into managed install-vllm", () => {
     const repoRoot = path.join(import.meta.dirname, "..");
     const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-onboard-vllm-no-install-"));