diff --git a/src/lib/actions/sandbox/rebuild-inference-preflight.test.ts b/src/lib/actions/sandbox/rebuild-inference-preflight.test.ts index 22d17aa84b9..3f267c24f9e 100644 --- a/src/lib/actions/sandbox/rebuild-inference-preflight.test.ts +++ b/src/lib/actions/sandbox/rebuild-inference-preflight.test.ts @@ -68,21 +68,53 @@ describe("atomic rebuild inference preflight", () => { const command = buildRebuildInferenceProbeCommand({ ...input, model: "gpt-5.4" }); expect(command).toContain("https://inference.local/v1/chat/completions"); - expect(command).toContain('"max_completion_tokens":8'); + expect(command).toContain('"max_completion_tokens":16'); expect(command).not.toContain('"max_tokens"'); }); it("sends max_completion_tokens for an o-series model on the chat completions route", () => { const command = buildRebuildInferenceProbeCommand({ ...input, model: "o3-mini" }); - expect(command).toContain('"max_completion_tokens":8'); + expect(command).toContain('"max_completion_tokens":16'); expect(command).not.toContain('"max_tokens"'); }); it("keeps max_tokens for a model that supports the legacy chat completions field", () => { const command = buildRebuildInferenceProbeCommand({ ...input, model: "nvidia/nemotron" }); - expect(command).toContain('"max_tokens":8'); + expect(command).toContain('"max_tokens":16'); expect(command).not.toContain('"max_completion_tokens"'); }); + + it("sends max_output_tokens on the responses route", () => { + const command = buildRebuildInferenceProbeCommand({ + ...input, + preferredInferenceApi: "openai-responses", + }); + + expect(command).toContain("https://inference.local/v1/responses"); + expect(command).toContain('"max_output_tokens":16'); + }); + + // A hosted endpoint validates the reply budget it is sent, so a budget below + // its floor fails a route that normal inference serves. Every preflight route + // must clear the floor, not just the one the reporter exercised (#7939). + it.each([ + ["chat completions", "nvidia/nemotron", "openai-completions", "max_tokens"], + ["chat completions reasoning", "gpt-5.4", "openai-completions", "max_completion_tokens"], + ["responses", "nvidia/nemotron", "openai-responses", "max_output_tokens"], + ["anthropic messages", "claude-sonnet-4-6", "anthropic-messages", "max_tokens"], + ])("requests a reply budget the endpoint accepts on the %s route (#7939)", (_route, model, preferredInferenceApi, field) => { + const endpointMinimumReplyTokens = 16; + const command = buildRebuildInferenceProbeCommand({ + ...input, + model, + preferredInferenceApi, + }); + + const budget = new RegExp(`"${field}":(\\d+)`).exec(command); + + expect(budget).not.toBeNull(); + expect(Number(budget?.[1])).toBeGreaterThanOrEqual(endpointMinimumReplyTokens); + }); }); diff --git a/src/lib/actions/sandbox/rebuild-inference-preflight.ts b/src/lib/actions/sandbox/rebuild-inference-preflight.ts index e7e92a2b360..739fb82d13a 100644 --- a/src/lib/actions/sandbox/rebuild-inference-preflight.ts +++ b/src/lib/actions/sandbox/rebuild-inference-preflight.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { getSandboxInferenceConfig } from "../../inference/config"; -import { resolveMaxTokensField } from "../../inference/max-tokens-field"; +import { MIN_PROBE_REPLY_TOKENS, resolveMaxTokensField } from "../../inference/max-tokens-field"; import { shellQuote } from "../../runner"; import { executeSandboxExecCommand, type SandboxCommandResult } from "./process-recovery"; @@ -35,7 +35,7 @@ function buildProbeRequest(input: RebuildInferencePreflightInput): { headers: ["anthropic-version: 2023-06-01"], payload: { model: input.model, - max_tokens: 8, + max_tokens: MIN_PROBE_REPLY_TOKENS, messages: [{ role: "user", content: "Reply with OK" }], }, }; @@ -44,7 +44,11 @@ function buildProbeRequest(input: RebuildInferencePreflightInput): { return { endpoint: "https://inference.local/v1/responses", headers: [], - payload: { model: input.model, input: "Reply with OK", max_output_tokens: 8 }, + payload: { + model: input.model, + input: "Reply with OK", + max_output_tokens: MIN_PROBE_REPLY_TOKENS, + }, }; } return { @@ -52,7 +56,7 @@ function buildProbeRequest(input: RebuildInferencePreflightInput): { headers: [], payload: { model: input.model, - [resolveMaxTokensField(input.model)]: 8, + [resolveMaxTokensField(input.model)]: MIN_PROBE_REPLY_TOKENS, messages: [{ role: "user", content: "Reply with OK" }], stream: false, }, diff --git a/src/lib/inference/health.test.ts b/src/lib/inference/health.test.ts index 31f53ca77e2..19d8fefe252 100644 --- a/src/lib/inference/health.test.ts +++ b/src/lib/inference/health.test.ts @@ -112,7 +112,7 @@ describe("inference health", () => { expect(payload).toEqual({ model: "gpt-4o-mini", messages: [{ role: "user", content: "Reply with exactly: OK" }], - max_tokens: 8, + max_tokens: 16, }); }); @@ -228,7 +228,7 @@ describe("inference health", () => { expect(payload).toEqual({ model: "moonshotai/kimi-k2.6", messages: [{ role: "user", content: "Reply with exactly: OK" }], - max_tokens: 8, + max_tokens: 16, chat_template_kwargs: { thinking: false }, }); expect(curlArgValue(capturedArgv, "--connect-timeout")).toBe("3"); @@ -250,7 +250,7 @@ describe("inference health", () => { expect(curlArgValue(capturedArgv, "--connect-timeout")).toBe("3"); expect(curlArgValue(capturedArgv, "--max-time")).toBe("5"); const payload = JSON.parse(capturedArgv[capturedArgv.indexOf("-d") + 1]); - expect(payload).toMatchObject({ max_tokens: 8, stream: true }); + expect(payload).toMatchObject({ max_tokens: 16, stream: true }); }); it.each([ @@ -423,7 +423,7 @@ describe("inference health", () => { const payload = JSON.parse(capturedArgv[capturedArgv.indexOf("-d") + 1]); expect(payload).toEqual({ model: "claude-sonnet-4-6", - max_tokens: 8, + max_tokens: 16, messages: [{ role: "user", content: "Reply with exactly: OK" }], }); }); diff --git a/src/lib/inference/health.ts b/src/lib/inference/health.ts index 493450dd93a..65ee0df86e5 100644 --- a/src/lib/inference/health.ts +++ b/src/lib/inference/health.ts @@ -16,6 +16,7 @@ import { normalizeCredentialValue, resolveProviderCredential } from "../credenti import { getProviderSelectionConfig } from "./config"; import type { LocalProviderHealthProbeOptions } from "./local"; import { probeLocalProviderHealth } from "./local"; +import { MIN_PROBE_REPLY_TOKENS } from "./max-tokens-field"; import { getChatCompletionsProbeCurlArgs } from "./onboard-probes"; import { BUILD_ENDPOINT_URL } from "./provider-models"; @@ -70,7 +71,7 @@ const GEMINI_CHAT_COMPLETIONS_ENDPOINT = "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions"; const ANTHROPIC_MESSAGES_ENDPOINT = "https://api.anthropic.com/v1/messages"; const ANTHROPIC_VERSION_HEADER = "anthropic-version: 2023-06-01"; -const HEALTH_PROBE_MAX_TOKENS = 8; +const HEALTH_PROBE_MAX_TOKENS = MIN_PROBE_REPLY_TOKENS; const CURL_TIMEOUT_STATUS = 28; const NODE_SPAWN_TIMEOUT_STATUS = -110; diff --git a/src/lib/inference/max-tokens-field.ts b/src/lib/inference/max-tokens-field.ts index 561a75616a0..51483b2d1c4 100644 --- a/src/lib/inference/max-tokens-field.ts +++ b/src/lib/inference/max-tokens-field.ts @@ -2,8 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 /** - * Resolves the OpenAI-compatible Chat Completions reply-budget field name for a - * given model. + * Resolves how an inference probe states its reply budget: the OpenAI-compatible + * Chat Completions field name for a given model, and the smallest reply budget + * every probe may request. * * OpenAI's GPT-5 family and the reasoning-model series (o1/o3/o4) reject the * legacy `max_tokens` parameter on `/chat/completions` and require @@ -14,6 +15,20 @@ * share this single resolver rather than each carrying their own model list. */ +/** + * Smallest reply budget an inference probe may request. + * + * Probes only need to read back a short acknowledgement, but the budget is a + * value the endpoint validates, not just a ceiling NemoClaw applies. Some + * supported hosted endpoints reject a budget below 16 with HTTP 400 even when + * discovery succeeds and normal inference works, so a smaller value turns a + * valid endpoint, model, and credential combination into a failed onboarding, + * health, or rebuild preflight check (#7939). The Anthropic Messages probe + * already requests 16; every other probe shares that floor here rather than + * carrying its own literal. + */ +export const MIN_PROBE_REPLY_TOKENS = 16; + // Matched by prefix rather than exact id: Azure OpenAI deployments append // version/suffix segments (e.g. "gpt-5.4", "gpt-5.4-turbo") and callers may or // may not include a provider prefix ("azure/gpt-5.4"). diff --git a/src/lib/inference/onboard-probes.test.ts b/src/lib/inference/onboard-probes.test.ts index 5b1a01c9cab..a6ca3cb6893 100644 --- a/src/lib/inference/onboard-probes.test.ts +++ b/src/lib/inference/onboard-probes.test.ts @@ -310,7 +310,7 @@ describe("OpenAI-compatible inference probes", () => { expect(getChatCompletionsProbePayload("nvidia/nemotron-3-super-120b-a12b")).toEqual({ model: "nvidia/nemotron-3-super-120b-a12b", messages: [{ role: "user", content: "Reply with exactly: OK" }], - max_tokens: 8, + max_tokens: 16, }); }); @@ -318,7 +318,7 @@ describe("OpenAI-compatible inference probes", () => { expect(getChatCompletionsProbePayload("nvidia/nvidia/nemotron-3-ultra")).toEqual({ model: "nvidia/nvidia/nemotron-3-ultra", messages: [{ role: "user", content: "Reply with exactly: OK" }], - max_tokens: 8, + max_tokens: 16, }); }); @@ -327,11 +327,35 @@ describe("OpenAI-compatible inference probes", () => { expect(getChatCompletionsProbePayload(model)).toEqual({ model, messages: [{ role: "user", content: "Reply with exactly: OK" }], - max_completion_tokens: 8, + max_completion_tokens: 16, }); } }); + // Some hosted endpoints reject a reply budget below 16 with HTTP 400 even + // though discovery succeeds and normal inference works, so a bounded probe + // that undershoots that floor fails a valid route. Whichever field a model + // uses, the budget must clear the floor (#7939). + it("requests a reply budget hosted endpoints accept, in whichever field the model uses (#7939)", () => { + const endpointMinimumReplyTokens = 16; + + for (const model of [ + "nvidia/nemotron-3-super-120b-a12b", + "nvidia/nvidia/nemotron-3-ultra", + "openai/openai/gpt-5.6-sol", + "moonshotai/kimi-k2.6", + "deepseek-ai/deepseek-v4-pro", + "gpt-5.4", + "o3-mini", + ]) { + const payload = getChatCompletionsProbePayload(model); + const budget = payload.max_completion_tokens ?? payload.max_tokens; + + expect(typeof budget, `${model} states a reply budget`).toBe("number"); + expect(budget, model).toBeGreaterThanOrEqual(endpointMinimumReplyTokens); + } + }); + it("uses an extended validation budget for DeepSeek V4 Flash", () => { const args = getChatCompletionsProbeCurlArgs({ credentialArgs: FAKE_CREDENTIAL_ARGS, @@ -356,7 +380,7 @@ describe("OpenAI-compatible inference probes", () => { expect(getChatCompletionsProbePayload("moonshotai/kimi-k2.6")).toEqual({ model: "moonshotai/kimi-k2.6", messages: [{ role: "user", content: "Reply with exactly: OK" }], - max_tokens: 8, + max_tokens: 16, chat_template_kwargs: { thinking: false }, }); diff --git a/src/lib/inference/openai-probe-models.ts b/src/lib/inference/openai-probe-models.ts index dccc5ca8962..540fa3fb1f8 100644 --- a/src/lib/inference/openai-probe-models.ts +++ b/src/lib/inference/openai-probe-models.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { resolveMaxTokensField } from "./max-tokens-field"; +import { MIN_PROBE_REPLY_TOKENS, resolveMaxTokensField } from "./max-tokens-field"; export function isDeepSeekV4ProModel(model: unknown): boolean { return String(model || "").toLowerCase() === "deepseek-ai/deepseek-v4-pro"; @@ -16,7 +16,7 @@ export function getChatCompletionsProbePayload(model: string): Record