From 30904bc4926b6c36498d9c2369ad8bbe2bd62e6d Mon Sep 17 00:00:00 2001 From: Shawn Xie Date: Fri, 31 Jul 2026 00:22:00 +0000 Subject: [PATCH] fix(inference): request a reply budget hosted endpoints accept Inference probes hard-coded a bounded reply budget of 8 tokens in onboarding, health, and rebuild-preflight payloads. The budget is a value the endpoint validates, not just a ceiling NemoClaw applies, and some supported hosted endpoints reject a budget below 16 with HTTP 400 even when model discovery succeeds and normal inference works. A valid endpoint, model, and credential combination therefore failed onboarding and rebuild health checks. Share one minimum reply budget of 16 across the probes that carried the literal 8, matching the floor the Anthropic Messages probe already used. The probes stay tightly bounded and no field-name selection changes, so this does not regress the GPT-5 max_completion_tokens contract. Refs: #7939 Signed-off-by: Shawn Xie --- .../rebuild-inference-preflight.test.ts | 38 +++++++++++++++++-- .../sandbox/rebuild-inference-preflight.ts | 12 ++++-- src/lib/inference/health.test.ts | 8 ++-- src/lib/inference/health.ts | 3 +- src/lib/inference/max-tokens-field.ts | 19 +++++++++- src/lib/inference/onboard-probes.test.ts | 32 ++++++++++++++-- src/lib/inference/openai-probe-models.ts | 6 +-- 7 files changed, 97 insertions(+), 21 deletions(-) diff --git a/src/lib/actions/sandbox/rebuild-inference-preflight.test.ts b/src/lib/actions/sandbox/rebuild-inference-preflight.test.ts index 22d17aa84b9..3f267c24f9e 100644 --- a/src/lib/actions/sandbox/rebuild-inference-preflight.test.ts +++ b/src/lib/actions/sandbox/rebuild-inference-preflight.test.ts @@ -68,21 +68,53 @@ describe("atomic rebuild inference preflight", () => { const command = buildRebuildInferenceProbeCommand({ ...input, model: "gpt-5.4" }); expect(command).toContain("https://inference.local/v1/chat/completions"); - expect(command).toContain('"max_completion_tokens":8'); + expect(command).toContain('"max_completion_tokens":16'); expect(command).not.toContain('"max_tokens"'); }); it("sends max_completion_tokens for an o-series model on the chat completions route", () => { const command = buildRebuildInferenceProbeCommand({ ...input, model: "o3-mini" }); - expect(command).toContain('"max_completion_tokens":8'); + expect(command).toContain('"max_completion_tokens":16'); expect(command).not.toContain('"max_tokens"'); }); it("keeps max_tokens for a model that supports the legacy chat completions field", () => { const command = buildRebuildInferenceProbeCommand({ ...input, model: "nvidia/nemotron" }); - expect(command).toContain('"max_tokens":8'); + expect(command).toContain('"max_tokens":16'); expect(command).not.toContain('"max_completion_tokens"'); }); + + it("sends max_output_tokens on the responses route", () => { + const command = buildRebuildInferenceProbeCommand({ + ...input, + preferredInferenceApi: "openai-responses", + }); + + expect(command).toContain("https://inference.local/v1/responses"); + expect(command).toContain('"max_output_tokens":16'); + }); + + // A hosted endpoint validates the reply budget it is sent, so a budget below + // its floor fails a route that normal inference serves. Every preflight route + // must clear the floor, not just the one the reporter exercised (#7939). + it.each([ + ["chat completions", "nvidia/nemotron", "openai-completions", "max_tokens"], + ["chat completions reasoning", "gpt-5.4", "openai-completions", "max_completion_tokens"], + ["responses", "nvidia/nemotron", "openai-responses", "max_output_tokens"], + ["anthropic messages", "claude-sonnet-4-6", "anthropic-messages", "max_tokens"], + ])("requests a reply budget the endpoint accepts on the %s route (#7939)", (_route, model, preferredInferenceApi, field) => { + const endpointMinimumReplyTokens = 16; + const command = buildRebuildInferenceProbeCommand({ + ...input, + model, + preferredInferenceApi, + }); + + const budget = new RegExp(`"${field}":(\\d+)`).exec(command); + + expect(budget).not.toBeNull(); + expect(Number(budget?.[1])).toBeGreaterThanOrEqual(endpointMinimumReplyTokens); + }); }); diff --git a/src/lib/actions/sandbox/rebuild-inference-preflight.ts b/src/lib/actions/sandbox/rebuild-inference-preflight.ts index e7e92a2b360..739fb82d13a 100644 --- a/src/lib/actions/sandbox/rebuild-inference-preflight.ts +++ b/src/lib/actions/sandbox/rebuild-inference-preflight.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import { getSandboxInferenceConfig } from "../../inference/config"; -import { resolveMaxTokensField } from "../../inference/max-tokens-field"; +import { MIN_PROBE_REPLY_TOKENS, resolveMaxTokensField } from "../../inference/max-tokens-field"; import { shellQuote } from "../../runner"; import { executeSandboxExecCommand, type SandboxCommandResult } from "./process-recovery"; @@ -35,7 +35,7 @@ function buildProbeRequest(input: RebuildInferencePreflightInput): { headers: ["anthropic-version: 2023-06-01"], payload: { model: input.model, - max_tokens: 8, + max_tokens: MIN_PROBE_REPLY_TOKENS, messages: [{ role: "user", content: "Reply with OK" }], }, }; @@ -44,7 +44,11 @@ function buildProbeRequest(input: RebuildInferencePreflightInput): { return { endpoint: "https://inference.local/v1/responses", headers: [], - payload: { model: input.model, input: "Reply with OK", max_output_tokens: 8 }, + payload: { + model: input.model, + input: "Reply with OK", + max_output_tokens: MIN_PROBE_REPLY_TOKENS, + }, }; } return { @@ -52,7 +56,7 @@ function buildProbeRequest(input: RebuildInferencePreflightInput): { headers: [], payload: { model: input.model, - [resolveMaxTokensField(input.model)]: 8, + [resolveMaxTokensField(input.model)]: MIN_PROBE_REPLY_TOKENS, messages: [{ role: "user", content: "Reply with OK" }], stream: false, }, diff --git a/src/lib/inference/health.test.ts b/src/lib/inference/health.test.ts index 31f53ca77e2..19d8fefe252 100644 --- a/src/lib/inference/health.test.ts +++ b/src/lib/inference/health.test.ts @@ -112,7 +112,7 @@ describe("inference health", () => { expect(payload).toEqual({ model: "gpt-4o-mini", messages: [{ role: "user", content: "Reply with exactly: OK" }], - max_tokens: 8, + max_tokens: 16, }); }); @@ -228,7 +228,7 @@ describe("inference health", () => { expect(payload).toEqual({ model: "moonshotai/kimi-k2.6", messages: [{ role: "user", content: "Reply with exactly: OK" }], - max_tokens: 8, + max_tokens: 16, chat_template_kwargs: { thinking: false }, }); expect(curlArgValue(capturedArgv, "--connect-timeout")).toBe("3"); @@ -250,7 +250,7 @@ describe("inference health", () => { expect(curlArgValue(capturedArgv, "--connect-timeout")).toBe("3"); expect(curlArgValue(capturedArgv, "--max-time")).toBe("5"); const payload = JSON.parse(capturedArgv[capturedArgv.indexOf("-d") + 1]); - expect(payload).toMatchObject({ max_tokens: 8, stream: true }); + expect(payload).toMatchObject({ max_tokens: 16, stream: true }); }); it.each([ @@ -423,7 +423,7 @@ describe("inference health", () => { const payload = JSON.parse(capturedArgv[capturedArgv.indexOf("-d") + 1]); expect(payload).toEqual({ model: "claude-sonnet-4-6", - max_tokens: 8, + max_tokens: 16, messages: [{ role: "user", content: "Reply with exactly: OK" }], }); }); diff --git a/src/lib/inference/health.ts b/src/lib/inference/health.ts index 493450dd93a..65ee0df86e5 100644 --- a/src/lib/inference/health.ts +++ b/src/lib/inference/health.ts @@ -16,6 +16,7 @@ import { normalizeCredentialValue, resolveProviderCredential } from "../credenti import { getProviderSelectionConfig } from "./config"; import type { LocalProviderHealthProbeOptions } from "./local"; import { probeLocalProviderHealth } from "./local"; +import { MIN_PROBE_REPLY_TOKENS } from "./max-tokens-field"; import { getChatCompletionsProbeCurlArgs } from "./onboard-probes"; import { BUILD_ENDPOINT_URL } from "./provider-models"; @@ -70,7 +71,7 @@ const GEMINI_CHAT_COMPLETIONS_ENDPOINT = "https://generativelanguage.googleapis.com/v1beta/openai/chat/completions"; const ANTHROPIC_MESSAGES_ENDPOINT = "https://api.anthropic.com/v1/messages"; const ANTHROPIC_VERSION_HEADER = "anthropic-version: 2023-06-01"; -const HEALTH_PROBE_MAX_TOKENS = 8; +const HEALTH_PROBE_MAX_TOKENS = MIN_PROBE_REPLY_TOKENS; const CURL_TIMEOUT_STATUS = 28; const NODE_SPAWN_TIMEOUT_STATUS = -110; diff --git a/src/lib/inference/max-tokens-field.ts b/src/lib/inference/max-tokens-field.ts index 561a75616a0..51483b2d1c4 100644 --- a/src/lib/inference/max-tokens-field.ts +++ b/src/lib/inference/max-tokens-field.ts @@ -2,8 +2,9 @@ // SPDX-License-Identifier: Apache-2.0 /** - * Resolves the OpenAI-compatible Chat Completions reply-budget field name for a - * given model. + * Resolves how an inference probe states its reply budget: the OpenAI-compatible + * Chat Completions field name for a given model, and the smallest reply budget + * every probe may request. * * OpenAI's GPT-5 family and the reasoning-model series (o1/o3/o4) reject the * legacy `max_tokens` parameter on `/chat/completions` and require @@ -14,6 +15,20 @@ * share this single resolver rather than each carrying their own model list. */ +/** + * Smallest reply budget an inference probe may request. + * + * Probes only need to read back a short acknowledgement, but the budget is a + * value the endpoint validates, not just a ceiling NemoClaw applies. Some + * supported hosted endpoints reject a budget below 16 with HTTP 400 even when + * discovery succeeds and normal inference works, so a smaller value turns a + * valid endpoint, model, and credential combination into a failed onboarding, + * health, or rebuild preflight check (#7939). The Anthropic Messages probe + * already requests 16; every other probe shares that floor here rather than + * carrying its own literal. + */ +export const MIN_PROBE_REPLY_TOKENS = 16; + // Matched by prefix rather than exact id: Azure OpenAI deployments append // version/suffix segments (e.g. "gpt-5.4", "gpt-5.4-turbo") and callers may or // may not include a provider prefix ("azure/gpt-5.4"). diff --git a/src/lib/inference/onboard-probes.test.ts b/src/lib/inference/onboard-probes.test.ts index 4198ffb8062..fbac549a834 100644 --- a/src/lib/inference/onboard-probes.test.ts +++ b/src/lib/inference/onboard-probes.test.ts @@ -310,7 +310,7 @@ describe("OpenAI-compatible inference probes", () => { expect(getChatCompletionsProbePayload("nvidia/nemotron-3-super-120b-a12b")).toEqual({ model: "nvidia/nemotron-3-super-120b-a12b", messages: [{ role: "user", content: "Reply with exactly: OK" }], - max_tokens: 8, + max_tokens: 16, }); }); @@ -318,7 +318,7 @@ describe("OpenAI-compatible inference probes", () => { expect(getChatCompletionsProbePayload("nvidia/nvidia/nemotron-3-ultra")).toEqual({ model: "nvidia/nvidia/nemotron-3-ultra", messages: [{ role: "user", content: "Reply with exactly: OK" }], - max_tokens: 8, + max_tokens: 16, }); }); @@ -327,11 +327,35 @@ describe("OpenAI-compatible inference probes", () => { expect(getChatCompletionsProbePayload(model)).toEqual({ model, messages: [{ role: "user", content: "Reply with exactly: OK" }], - max_completion_tokens: 8, + max_completion_tokens: 16, }); } }); + // Some hosted endpoints reject a reply budget below 16 with HTTP 400 even + // though discovery succeeds and normal inference works, so a bounded probe + // that undershoots that floor fails a valid route. Whichever field a model + // uses, the budget must clear the floor (#7939). + it("requests a reply budget hosted endpoints accept, in whichever field the model uses (#7939)", () => { + const endpointMinimumReplyTokens = 16; + + for (const model of [ + "nvidia/nemotron-3-super-120b-a12b", + "nvidia/nvidia/nemotron-3-ultra", + "openai/openai/gpt-5.6-sol", + "moonshotai/kimi-k2.6", + "deepseek-ai/deepseek-v4-pro", + "gpt-5.4", + "o3-mini", + ]) { + const payload = getChatCompletionsProbePayload(model); + const budget = payload.max_completion_tokens ?? payload.max_tokens; + + expect(typeof budget, `${model} states a reply budget`).toBe("number"); + expect(budget, model).toBeGreaterThanOrEqual(endpointMinimumReplyTokens); + } + }); + it("uses an extended validation budget for DeepSeek V4 Flash", () => { const args = getChatCompletionsProbeCurlArgs({ credentialArgs: FAKE_CREDENTIAL_ARGS, @@ -356,7 +380,7 @@ describe("OpenAI-compatible inference probes", () => { expect(getChatCompletionsProbePayload("moonshotai/kimi-k2.6")).toEqual({ model: "moonshotai/kimi-k2.6", messages: [{ role: "user", content: "Reply with exactly: OK" }], - max_tokens: 8, + max_tokens: 16, chat_template_kwargs: { thinking: false }, }); diff --git a/src/lib/inference/openai-probe-models.ts b/src/lib/inference/openai-probe-models.ts index dccc5ca8962..540fa3fb1f8 100644 --- a/src/lib/inference/openai-probe-models.ts +++ b/src/lib/inference/openai-probe-models.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import { resolveMaxTokensField } from "./max-tokens-field"; +import { MIN_PROBE_REPLY_TOKENS, resolveMaxTokensField } from "./max-tokens-field"; export function isDeepSeekV4ProModel(model: unknown): boolean { return String(model || "").toLowerCase() === "deepseek-ai/deepseek-v4-pro"; @@ -16,7 +16,7 @@ export function getChatCompletionsProbePayload(model: string): Record