diff --git a/src/lib/actions/inference-set.test.ts b/src/lib/actions/inference-set.test.ts index e8fb8ce4c4d..3b5fe8f084d 100644 --- a/src/lib/actions/inference-set.test.ts +++ b/src/lib/actions/inference-set.test.ts @@ -16,6 +16,10 @@ vi.mock("../inference/local", () => ({ DEFAULT_OLLAMA_MODEL: "llama3.1", })); +vi.mock("../inference/context-window", () => ({ + resolveContextWindowForModel: vi.fn(() => null), +})); + vi.mock("../sandbox/config", () => ({ readSandboxConfig: vi.fn(), recomputeSandboxConfigHash: vi.fn(), @@ -104,6 +108,7 @@ function createDeps(options: { target?: AgentConfigTarget; session?: Session | null; openshellStatus?: number; + contextWindow?: number | null; }): InferenceSetDeps & { calls: { runOpenshell: ReturnType; @@ -113,6 +118,7 @@ function createDeps(options: { updateSession: ReturnType; appendAuditEntry: ReturnType; log: ReturnType; + resolveContextWindowForModel: ReturnType; }; getSession: () => Session | null; } { @@ -136,6 +142,9 @@ function createDeps(options: { }), appendAuditEntry: vi.fn(), log: vi.fn(), + resolveContextWindowForModel: vi.fn((_provider: string, _model: string) => + options.contextWindow === undefined ? null : options.contextWindow, + ), }; return { getDefaultSandbox: () => defaultSandbox, @@ -152,6 +161,7 @@ function createDeps(options: { runOpenshell: calls.runOpenshell, appendAuditEntry: calls.appendAuditEntry, log: calls.log, + resolveContextWindowForModel: calls.resolveContextWindowForModel, calls, getSession: () => session, }; @@ -927,3 +937,49 @@ describe("runInferenceSet", () => { expect(logged).not.toMatch(/Inference route synced/); }); }); + +describe("runInferenceSet context window", () => { + const ollamaConfig = (): ConfigObject => ({ + agents: { defaults: { model: { primary: "inference/llama3.2:3b" } } }, + models: { + providers: { + inference: { + api: "openai-completions", + models: [{ id: "llama3.2:3b", name: "inference/llama3.2:3b", contextWindow: 131072 }], + }, + }, + }, + }); + + function inferenceModels(config: ConfigObject): Array> { + const models = config.models as { providers: { inference: { models: unknown } } }; + return models.providers.inference.models as Array>; + } + + it("writes the recomputed context window into the in-sandbox config", async () => { + const config = ollamaConfig(); + const deps = createDeps({ config, session: baseSession(), contextWindow: 16384 }); + + await runInferenceSet({ provider: "ollama-local", model: "qwen2.5:7b", noVerify: true }, deps); + + expect(deps.calls.resolveContextWindowForModel).toHaveBeenCalledWith( + "ollama-local", + "qwen2.5:7b", + ); + expect(inferenceModels(config)[0].contextWindow).toBe(16384); + const logged = deps.calls.log.mock.calls.map((a) => String(a[0])).join("\n"); + expect(logged).toMatch(/Context window for 'qwen2\.5:7b': 16384 tokens/); + }); + + it("keeps the existing window and warns when it cannot be determined", async () => { + const config = ollamaConfig(); + const deps = createDeps({ config, session: baseSession(), contextWindow: null }); + + await runInferenceSet({ provider: "ollama-local", model: "qwen2.5:7b", noVerify: true }, deps); + + expect(inferenceModels(config)[0].contextWindow).toBe(131072); + const logged = deps.calls.log.mock.calls.map((a) => String(a[0])).join("\n"); + expect(logged).toMatch(/could not determine the context window/i); + expect(logged).toMatch(/rebuild/); + }); +}); diff --git a/src/lib/actions/inference-set.ts b/src/lib/actions/inference-set.ts index 2f376134dcf..0f69b434761 100644 --- a/src/lib/actions/inference-set.ts +++ b/src/lib/actions/inference-set.ts @@ -11,6 +11,7 @@ import { getSandboxInferenceConfig, type SandboxInferenceConfig, } from "../inference/config"; +import { resolveContextWindowForModel } from "../inference/context-window"; import { type AgentConfigTarget, readSandboxConfig, @@ -68,6 +69,7 @@ export interface InferenceSetDeps { runOpenshell: (args: string[], opts?: { ignoreError?: boolean }) => OpenshellRunResult; appendAuditEntry: typeof appendAuditEntry; log: (message: string) => void; + resolveContextWindowForModel: (provider: string, model: string) => number | null; } export class InferenceSetError extends Error { @@ -110,6 +112,7 @@ function defaultDeps(): InferenceSetDeps { runOpenshell: (args, opts) => runOpenshell(args, opts), appendAuditEntry, log: console.log, + resolveContextWindowForModel, }; } @@ -221,6 +224,7 @@ function buildProviderConfig( existing: ConfigObject, model: string, route: SandboxInferenceConfig, + contextWindow?: number, ): ConfigObject { const firstExistingModel = Array.isArray(existing.models) ? cloneConfigObject(existing.models[0]) @@ -228,6 +232,11 @@ function buildProviderConfig( delete firstExistingModel.compat; firstExistingModel.id = model; firstExistingModel.name = route.primaryModelRef; + // Recompute for the new model rather than inheriting the prior model's window. + // Omitted (undefined) → keep whatever the existing entry had. + if (typeof contextWindow === "number") { + firstExistingModel.contextWindow = contextWindow; + } if (route.inferenceCompat) { firstExistingModel.compat = asConfigObject(route.inferenceCompat); } @@ -246,6 +255,7 @@ export function patchOpenClawInferenceConfig( provider: string, model: string, preferredInferenceApi: string | null = null, + contextWindow?: number, ): { changed: boolean; route: SandboxInferenceConfig } { const before = JSON.stringify(config); const route = getSandboxInferenceConfig(model, provider, preferredInferenceApi); @@ -256,7 +266,7 @@ export function patchOpenClawInferenceConfig( models.mode = "merge"; const providers = ensureObject(models, "providers"); const existingProvider = cloneConfigObject(providers[route.providerKey]); - providers[route.providerKey] = buildProviderConfig(existingProvider, model, route); + providers[route.providerKey] = buildProviderConfig(existingProvider, model, route, contextWindow); return { changed: before !== JSON.stringify(config), route }; } @@ -396,15 +406,29 @@ export async function runInferenceSet( sandboxName, session: deps.loadSession(), }); - const patched = - agentName === "hermes" - ? patchHermesInferenceConfig(config, provider, model, preferredInferenceApi) - : patchOpenClawInferenceConfig( - config, - provider, - model, - preferredInferenceApi || getPreferredInferenceApi(config), - ); + let patched: { changed: boolean; route: SandboxInferenceConfig }; + if (agentName === "hermes") { + patched = patchHermesInferenceConfig(config, provider, model, preferredInferenceApi); + } else { + // Recompute the context window for the model being switched to, so it does + // not inherit the prior model's window (#context-window-on-switch). + const contextWindow = deps.resolveContextWindowForModel(provider, model); + if (contextWindow != null) { + deps.log(` Context window for '${model}': ${contextWindow} tokens`); + } else { + deps.log( + ` Warning: could not determine the context window for '${model}'; keeping the ` + + `existing value. Run '${CLI_NAME} ${sandboxName} rebuild' to re-probe it.`, + ); + } + patched = patchOpenClawInferenceConfig( + config, + provider, + model, + preferredInferenceApi || getPreferredInferenceApi(config), + contextWindow ?? undefined, + ); + } deps.log( agentName === "hermes" diff --git a/src/lib/inference/config.ts b/src/lib/inference/config.ts index 26f3985f961..afb00c320be 100644 --- a/src/lib/inference/config.ts +++ b/src/lib/inference/config.ts @@ -13,6 +13,12 @@ export const INFERENCE_ROUTE_URL = "https://inference.local/v1"; export const NOUS_RECOMMENDED_MODELS_URL = "https://portal.nousresearch.com/api/nous/recommended-models"; export const DEFAULT_CLOUD_MODEL = "nvidia/nemotron-3-super-120b-a12b"; +// Fallback context window used when no per-model value is known. Cloud providers +// have no per-model context metadata today (CLOUD_MODEL_OPTIONS carries only +// id/label), so they fall back to this; matches the onboarding build default in +// scripts/generate-openclaw-config.mts. Per-model cloud accuracy is tracked +// separately (cloud context-window registry). +export const DEFAULT_CONTEXT_WINDOW = 131072; export const HERMES_PROVIDER_MODEL_OPTIONS = [ "moonshotai/kimi-k2.6", "xiaomi/mimo-v2.5-pro", diff --git a/src/lib/inference/context-window.test.ts b/src/lib/inference/context-window.test.ts new file mode 100644 index 00000000000..1b4510e496d --- /dev/null +++ b/src/lib/inference/context-window.test.ts @@ -0,0 +1,68 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { describe, expect, it, vi } from "vitest"; + +// resolveContextWindowForModel takes injected deps, so these mocks only stop the +// real inference stack (and ../runner → ./platform) from loading under vitest; +// the default deps object references them but the tests never exercise it. +vi.mock("./local", () => ({ + getOllamaWarmupCommand: vi.fn(() => ["curl"]), + resolveOllamaRuntimeContextWindow: vi.fn(() => null), +})); +vi.mock("./vllm-runtime-context", () => ({ resolveVllmContextWindowFromModels: vi.fn() })); + +import { type ContextWindowDeps, resolveContextWindowForModel } from "./context-window"; + +function makeDeps(over: Partial = {}): ContextWindowDeps { + return { + warmOllamaModel: vi.fn(), + probeOllamaContextWindow: vi.fn(() => 16384), + probeVllmContextWindow: vi.fn(() => 262144), + defaultCloudContextWindow: vi.fn(() => 131072), + ...over, + }; +} + +describe("resolveContextWindowForModel", () => { + it("ollama-local: warms the model, then returns the probed window", () => { + const deps = makeDeps({ probeOllamaContextWindow: vi.fn(() => 16384) }); + + expect(resolveContextWindowForModel("ollama-local", "qwen2.5:7b", deps)).toBe(16384); + expect(deps.warmOllamaModel).toHaveBeenCalledWith("qwen2.5:7b"); + expect(deps.defaultCloudContextWindow).not.toHaveBeenCalled(); + }); + + it("ollama-local: returns null when the probe cannot read a window", () => { + const deps = makeDeps({ probeOllamaContextWindow: vi.fn(() => null) }); + + expect(resolveContextWindowForModel("ollama-local", "qwen2.5:7b", deps)).toBeNull(); + expect(deps.warmOllamaModel).toHaveBeenCalledTimes(1); + }); + + it("vllm-local: returns the probed max_model_len without warming", () => { + const deps = makeDeps({ probeVllmContextWindow: vi.fn(() => 262144) }); + + expect(resolveContextWindowForModel("vllm-local", "some-model", deps)).toBe(262144); + expect(deps.probeVllmContextWindow).toHaveBeenCalledWith("some-model"); + expect(deps.warmOllamaModel).not.toHaveBeenCalled(); + expect(deps.probeOllamaContextWindow).not.toHaveBeenCalled(); + }); + + it("vllm-local: returns null when the server is unreachable", () => { + const deps = makeDeps({ probeVllmContextWindow: vi.fn(() => null) }); + + expect(resolveContextWindowForModel("vllm-local", "some-model", deps)).toBeNull(); + expect(deps.warmOllamaModel).not.toHaveBeenCalled(); + }); + + it("cloud provider: returns the default window without warming or probing", () => { + const deps = makeDeps({ defaultCloudContextWindow: vi.fn(() => 131072) }); + + expect( + resolveContextWindowForModel("nvidia-prod", "nvidia/nemotron-3-super-120b-a12b", deps), + ).toBe(131072); + expect(deps.warmOllamaModel).not.toHaveBeenCalled(); + expect(deps.probeOllamaContextWindow).not.toHaveBeenCalled(); + }); +}); diff --git a/src/lib/inference/context-window.ts b/src/lib/inference/context-window.ts new file mode 100644 index 00000000000..4f0aa6158e5 --- /dev/null +++ b/src/lib/inference/context-window.ts @@ -0,0 +1,86 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +/** + * Recompute the context window for the model an `inference set` switch targets, + * so the in-sandbox config matches the new model instead of carrying the prior + * model's window. onboard already does this per provider; inference set must + * too, or a switch leaves a stale window (e.g. a 131072 cloud default kept for + * an Ollama model whose runtime window is ~16k → silent overflow; or an Ollama + * window kept for a cloud model → silent under-utilization). + */ + +import { VLLM_PORT } from "../core/ports"; +import { DEFAULT_CONTEXT_WINDOW } from "./config"; +import { + getOllamaWarmupCommand, + type RunCaptureFn, + resolveOllamaRuntimeContextWindow, +} from "./local"; +import { resolveVllmContextWindowFromModels } from "./vllm-runtime-context"; + +export interface ContextWindowDeps { + /** Load the model so the runtime probe can read its effective context length. */ + warmOllamaModel: (model: string) => void; + /** Probe the running Ollama model's context length; null when unavailable. */ + probeOllamaContextWindow: (model: string) => number | null; + /** Read the running vLLM server's max_model_len for the model; null when unavailable. */ + probeVllmContextWindow: (model: string) => number | null; + /** Fallback window for providers without a per-model runtime signal (cloud). */ + defaultCloudContextWindow: () => number; +} + +const defaultContextWindowDeps: ContextWindowDeps = { + warmOllamaModel: (model: string): void => { + // Lazy require: ../runner is CJS and a top-level require fails to resolve + // under the test runner. Runs only for the real (non-injected) deps. + const { runCapture } = require("../runner") as { runCapture: RunCaptureFn }; + runCapture(getOllamaWarmupCommand(model), { ignoreError: true }); + }, + // currentContextWindow = null → always probe (we recompute on every switch + // rather than honoring an unverifiable "user pinned it" guard). + probeOllamaContextWindow: (model: string): number | null => + resolveOllamaRuntimeContextWindow(model, null), + probeVllmContextWindow: (model: string): number | null => { + // Same source onboard uses: GET /v1/models on the host vLLM server and read + // max_model_len (handles both NemoClaw-launched and bring-your-own vLLM). + const { runCapture } = require("../runner") as { runCapture: RunCaptureFn }; + const raw = runCapture(["curl", "-sf", `http://127.0.0.1:${VLLM_PORT}/v1/models`], { + ignoreError: true, + }); + if (!raw) return null; + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + return null; + } + return resolveVllmContextWindowFromModels(parsed, model); + }, + defaultCloudContextWindow: (): number => DEFAULT_CONTEXT_WINDOW, +}; + +/** + * Returns the context window to write for `(provider, model)`, or null when it + * cannot be determined (caller should keep the existing value and warn). + * + * - ollama-local: warm the model, then probe its runtime context length. + * - vllm-local: read the running server's max_model_len from /v1/models (the + * same source onboard uses); null when the server is unreachable. + * - cloud providers: the onboard default. Accuracy is bounded by the missing + * per-model cloud context metadata (tracked as a separate issue). + */ +export function resolveContextWindowForModel( + provider: string, + model: string, + deps: ContextWindowDeps = defaultContextWindowDeps, +): number | null { + if (provider === "ollama-local") { + deps.warmOllamaModel(model); + return deps.probeOllamaContextWindow(model); + } + if (provider === "vllm-local") { + return deps.probeVllmContextWindow(model); + } + return deps.defaultCloudContextWindow(); +} diff --git a/src/lib/inference/vllm-runtime-context.ts b/src/lib/inference/vllm-runtime-context.ts index 490ede519e8..7ed7e6dd330 100644 --- a/src/lib/inference/vllm-runtime-context.ts +++ b/src/lib/inference/vllm-runtime-context.ts @@ -8,22 +8,21 @@ const MAX_AUTODETECTED_VLLM_CONTEXT_WINDOW = 4_194_304; type ModelEntry = { id?: unknown; max_model_len?: unknown }; type ApplyOptions = { env?: NodeJS.ProcessEnv; logger?: Pick }; -export function applyVllmRuntimeContextWindow( +/** + * Extract the runtime context window for `modelId` from a vLLM `/v1/models` + * response (its `max_model_len`), validated against NemoClaw's auto-detect + * ceiling. Returns null when the response is empty/malformed or the value is + * out of range. Pure parse — shared by onboard (applyVllmRuntimeContextWindow) + * and `inference set` so both read the same source. + */ +export function resolveVllmContextWindowFromModels( modelsResponse: unknown, modelId: string | null | undefined, - options: ApplyOptions = {}, -): void { - const env = options.env ?? process.env; - const logger = options.logger ?? console; - - if (hasExplicitContextWindow(env.NEMOCLAW_CONTEXT_WINDOW)) { - logger.log(` ℹ Keeping configured context window: ${env.NEMOCLAW_CONTEXT_WINDOW} tokens`); - return; - } - + logger: Pick = console, +): number | null { const data = (modelsResponse as { data?: unknown } | null | undefined)?.data; const entries = Array.isArray(data) ? (data as ModelEntry[]) : []; - if (entries.length === 0) return; + if (entries.length === 0) return null; const target = String(modelId ?? "").trim(); const entry = @@ -35,7 +34,7 @@ export function applyVllmRuntimeContextWindow( rawMaxModelLen === null || String(rawMaxModelLen).trim() === "" ) { - return; + return null; } const contextLength = parsePositiveInteger(rawMaxModelLen); @@ -44,16 +43,34 @@ export function applyVllmRuntimeContextWindow( ` ⚠ vLLM /v1/models returned a non-positive or malformed max_model_len ` + `(${String(rawMaxModelLen)}); ignoring it.`, ); - return; + return null; } if (contextLength > MAX_AUTODETECTED_VLLM_CONTEXT_WINDOW) { logger.warn( ` ⚠ vLLM /v1/models returned max_model_len=${contextLength}, above NemoClaw's ` + `auto-detect ceiling (${MAX_AUTODETECTED_VLLM_CONTEXT_WINDOW}); ignoring it.`, ); + return null; + } + return contextLength; +} + +export function applyVllmRuntimeContextWindow( + modelsResponse: unknown, + modelId: string | null | undefined, + options: ApplyOptions = {}, +): void { + const env = options.env ?? process.env; + const logger = options.logger ?? console; + + if (hasExplicitContextWindow(env.NEMOCLAW_CONTEXT_WINDOW)) { + logger.log(` ℹ Keeping configured context window: ${env.NEMOCLAW_CONTEXT_WINDOW} tokens`); return; } + const contextLength = resolveVllmContextWindowFromModels(modelsResponse, modelId, logger); + if (contextLength === null) return; + const value = String(contextLength); env.NEMOCLAW_CONTEXT_WINDOW = value; logger.log(` ✓ Using vLLM max_model_len: ${value} tokens`);