diff --git a/deno.json b/deno.json index 310f81ee1b..f99853647b 100644 --- a/deno.json +++ b/deno.json @@ -1,6 +1,6 @@ { "name": "veryfront", - "version": "0.1.987", + "version": "0.1.988", "license": "Apache-2.0", "nodeModulesDir": "auto", "minimumDependencyAge": { diff --git a/extensions/ext-llm-openai/README.md b/extensions/ext-llm-openai/README.md index d74d7d2138..a25f4a98aa 100644 --- a/extensions/ext-llm-openai/README.md +++ b/extensions/ext-llm-openai/README.md @@ -57,7 +57,7 @@ Any model accessible through the OpenAI Chat Completions, Responses, or Embeddin - **Flagship:** `gpt-4.1`, `gpt-4.1-mini`, `gpt-4.1-nano`, `gpt-4o`, `gpt-4o-mini` - **Frontier:** `gpt-5`, `gpt-5-mini`, `gpt-5-nano` -- **Reasoning:** `o3`, `o4-mini`, `o1`, `o1-mini`, `o3-mini` (sampling parameters are automatically dropped with warnings) +- **Reasoning:** `gpt-5.4-nano`, current `gpt-5`/`gpt-5.x` reasoning models, `o3`, `o4-mini`, `o1`, `o3-mini` (sampling parameters are automatically dropped with warnings) - **Embeddings:** `text-embedding-3-small`, `text-embedding-3-large` - **OpenAI-compatible:** Any third-party model reachable via an OpenAI-compatible endpoint (set `OPENAI_BASE_URL`) @@ -74,7 +74,11 @@ The extension accepts configuration through `LLMProviderConfig` when creating ru ## Model-Specific Behavior -### Reasoning Models (o3, o4-mini, o1) +### Reasoning Models (GPT-5.x, o3, o4-mini, o1) + +Default reasoning params are applied only for native `openai` and `veryfront-cloud` providers. +OpenAI-compatible providers require explicit `reasoning` options. `gpt-5-chat-latest`, +`gpt-5.1`, `o1-mini`, and `o1-preview` are left unmodified by default. Reasoning models automatically: diff --git a/extensions/ext-llm-openai/src/openai-chat-request-builder.test.ts b/extensions/ext-llm-openai/src/openai-chat-request-builder.test.ts index 5a0f0ac5a9..b33dab3a7b 100644 --- a/extensions/ext-llm-openai/src/openai-chat-request-builder.test.ts +++ b/extensions/ext-llm-openai/src/openai-chat-request-builder.test.ts @@ -27,6 +27,102 @@ function createWarningCollector() { } describe("ext-llm-openai/openai-chat-request-builder", () => { + it("sets default reasoning effort for GPT-5.5 chat requests", () => { + const warnings = createWarningCollector(); + + const body = buildOpenAIChatRequest( + "gpt-5.5", + "openai", + { + prompt: [{ role: "user", content: [{ type: "text", text: "Think carefully." }] }], + temperature: 0.2, + }, + true, + warnings, + ); + + assertEquals(body.reasoning_effort, "medium"); + assertEquals(body.temperature, undefined); + assertEquals(warnings.drain().map((warning) => warning.setting), ["temperature"]); + }); + + it("does not set default reasoning effort for GPT-5 chat snapshots", () => { + const warnings = createWarningCollector(); + + const body = buildOpenAIChatRequest( + "gpt-5-chat-latest", + "openai", + { + prompt: [{ role: "user", content: [{ type: "text", text: "Be concise." }] }], + temperature: 0.2, + }, + true, + warnings, + ); + + assertEquals(body.reasoning_effort, undefined); + assertEquals(body.temperature, 0.2); + assertEquals(warnings.drain(), []); + }); + + it("does not set default reasoning effort for legacy o1 chat variants", () => { + const warnings = createWarningCollector(); + + const body = buildOpenAIChatRequest( + "o1-mini", + "openai", + { + prompt: [{ role: "user", content: [{ type: "text", text: "Be concise." }] }], + temperature: 0.2, + }, + true, + warnings, + ); + + assertEquals(body.reasoning_effort, undefined); + assertEquals(body.temperature, undefined); + assertEquals(warnings.drain().map((warning) => warning.setting), ["temperature"]); + }); + + it("does not set default reasoning effort for OpenAI-compatible providers but still drops rejected sampling params", () => { + const warnings = createWarningCollector(); + + const body = buildOpenAIChatRequest( + "gpt-5.5", + "azure", + { + prompt: [{ role: "user", content: [{ type: "text", text: "Be concise." }] }], + temperature: 0.2, + }, + true, + warnings, + ); + + assertEquals(body.reasoning_effort, undefined); + assertEquals(body.temperature, undefined); + assertEquals(warnings.drain().map((warning) => warning.setting), ["temperature"]); + }); + + it("drops rejected sampling params when explicit reasoning is disabled", () => { + const warnings = createWarningCollector(); + + const body = buildOpenAIChatRequest( + "o3-mini", + "openai", + { + prompt: [{ role: "user", content: [{ type: "text", text: "Be concise." }] }], + reasoning: { enabled: false }, + temperature: 0.2, + }, + true, + warnings, + ); + + assertEquals(body.reasoning_effort, undefined); + assertEquals(body.temperature, undefined); + assertEquals(warnings.drain().map((warning) => warning.setting), ["temperature"]); + }); + it("preserves chat request shaping, provider option merge order, and warnings", () => { const prompt: RuntimePromptMessage[] = [ { role: "system", content: "You are concise." }, diff --git a/extensions/ext-llm-openai/src/openai-chat-request-builder.ts b/extensions/ext-llm-openai/src/openai-chat-request-builder.ts index 730c207a39..6fad918a9b 100644 --- a/extensions/ext-llm-openai/src/openai-chat-request-builder.ts +++ b/extensions/ext-llm-openai/src/openai-chat-request-builder.ts @@ -5,14 +5,11 @@ import { unwrapToolInputSchema, } from "veryfront/provider/shared"; import type { OpenAICompatibleChatRequest, RuntimePromptMessage } from "veryfront/provider/shared"; - -type ProviderReasoningEffort = "low" | "medium" | "high" | "max"; - -type ProviderReasoningOption = { - enabled?: boolean; - effort?: ProviderReasoningEffort; - budgetTokens?: number; -}; +import { + type OpenAIProviderReasoningOption, + rejectsOpenAISamplingParams, + resolveOpenAIReasoningConfig, +} from "./openai-reasoning-models.ts"; export type RuntimeToolDefinition = | { @@ -44,7 +41,7 @@ export type OpenAICompatibleLanguageOptions = { providerOptions?: Record; includeRawChunks?: boolean; abortSignal?: AbortSignal; - reasoning?: ProviderReasoningOption; + reasoning?: OpenAIProviderReasoningOption; userId?: string; serviceTier?: "auto" | "default" | "flex" | "scale"; parallelToolCalls?: boolean; @@ -75,10 +72,6 @@ type WarningCollector = { }>; }; -function isOpenAIReasoningModel(modelId: string): boolean { - return /^o[134](-|$)/.test(modelId); -} - function isNativeOpenAIModel(modelId: string): boolean { return /^(gpt-|o[134](-|$)|chatgpt-)/.test(modelId); } @@ -87,24 +80,6 @@ function isFixedSamplingModel(modelId: string): boolean { return /^kimi-k2\.5/.test(modelId); } -function resolveOpenAIReasoningEffort( - option: ProviderReasoningOption | undefined, -): "low" | "medium" | "high" | undefined { - if (!option || option.enabled !== true) { - return undefined; - } - switch (option.effort) { - case "low": - return "low"; - case "high": - case "max": - return "high"; - case "medium": - default: - return "medium"; - } -} - export function buildOpenAIChatRequest( modelId: string, providerName: string, @@ -112,11 +87,11 @@ export function buildOpenAIChatRequest( stream: boolean, warnings: WarningCollector, ): OpenAICompatibleChatRequest { - const isReasoningModel = isOpenAIReasoningModel(modelId); - const reasoningEffort = resolveOpenAIReasoningEffort(options.reasoning); - const reasoningEnabled = isReasoningModel || reasoningEffort !== undefined; + const reasoning = resolveOpenAIReasoningConfig(modelId, providerName, options.reasoning); + const reasoningEnabled = reasoning !== undefined; + const samplingRejected = rejectsOpenAISamplingParams(modelId); const fixedSampling = isFixedSamplingModel(modelId); - const dropSamplingParams = reasoningEnabled || fixedSampling; + const dropSamplingParams = reasoningEnabled || samplingRejected || fixedSampling; // OpenAI Chat Completions has no top_k surface. if (options.topK !== undefined) { @@ -128,7 +103,7 @@ export function buildOpenAIChatRequest( }); } - // Reasoning models (o1 / o3 / o4) and models with fixed sampling params + // Reasoning models and models with fixed sampling params // reject sampling params outright. Emit warnings. if (dropSamplingParams) { const dropped: Array<[keyof typeof options, string]> = [ @@ -145,7 +120,9 @@ export function buildOpenAIChatRequest( setting: key, details: fixedSampling ? `Dropped because this model uses fixed sampling parameters.` - : `Dropped because OpenAI reasoning models reject ${openaiName}. Reasoning was active for this request.`, + : samplingRejected + ? `Dropped because this model rejects ${openaiName}.` + : `Dropped because reasoning was active for this request and OpenAI rejects ${openaiName} with reasoning.`, }); } } @@ -178,7 +155,7 @@ export function buildOpenAIChatRequest( ...(!dropSamplingParams && options.frequencyPenalty !== undefined ? { frequency_penalty: options.frequencyPenalty } : {}), - ...(reasoningEffort !== undefined ? { reasoning_effort: reasoningEffort } : {}), + ...(reasoning !== undefined ? { reasoning_effort: reasoning.effort } : {}), ...(typeof options.userId === "string" && options.userId.length > 0 ? { user: options.userId } : {}), diff --git a/extensions/ext-llm-openai/src/openai-reasoning-models.test.ts b/extensions/ext-llm-openai/src/openai-reasoning-models.test.ts new file mode 100644 index 0000000000..6963a7cdab --- /dev/null +++ b/extensions/ext-llm-openai/src/openai-reasoning-models.test.ts @@ -0,0 +1,59 @@ +import { assertEquals } from "#veryfront/testing/assert.ts"; +import { describe, it } from "#veryfront/testing/bdd.ts"; +import { + getDefaultOpenAIReasoningEffort, + rejectsOpenAISamplingParams, +} from "./openai-reasoning-models.ts"; + +describe("ext-llm-openai/openai-reasoning-models", () => { + it("defaults known reasoning models while excluding chat snapshots and legacy o1 variants", () => { + const cases: Array<[string, "medium" | undefined]> = [ + ["gpt-5", "medium"], + ["gpt-5-mini", "medium"], + ["gpt-5.4-nano", "medium"], + ["gpt-5.5", "medium"], + ["gpt-5.1", undefined], + ["gpt-5-chat-latest", undefined], + ["o1", "medium"], + ["o1-2024-12-17", "medium"], + ["o1-mini", undefined], + ["o1-preview", undefined], + ["o3-mini", "medium"], + ["o4-mini", "medium"], + ]; + + for (const [modelId, expected] of cases) { + assertEquals(getDefaultOpenAIReasoningEffort(modelId), expected, modelId); + } + }); + + it("only enables default reasoning params for native OpenAI providers", () => { + assertEquals(getDefaultOpenAIReasoningEffort("gpt-5.4-nano", "openai"), "medium"); + assertEquals(getDefaultOpenAIReasoningEffort("gpt-5.4-nano", "veryfront-cloud"), "medium"); + assertEquals(getDefaultOpenAIReasoningEffort("gpt-5.4-nano", "azure"), undefined); + assertEquals(getDefaultOpenAIReasoningEffort("gpt-5.4-nano", "moonshot"), undefined); + }); + + it("detects models that reject sampling params separately from default reasoning params", () => { + const cases: Array<[string, boolean]> = [ + ["gpt-5", true], + ["gpt-5-mini", true], + ["gpt-5.4-nano", true], + ["gpt-5.5", true], + ["gpt-5.1", false], + ["gpt-5-chat-latest", false], + ["o1", true], + ["o1-2024-12-17", true], + ["o1-mini", true], + ["o1-preview", true], + ["o1-pro", true], + ["o3-mini", true], + ["o4-mini", true], + ["gpt-4o-mini", false], + ]; + + for (const [modelId, expected] of cases) { + assertEquals(rejectsOpenAISamplingParams(modelId), expected, modelId); + } + }); +}); diff --git a/extensions/ext-llm-openai/src/openai-reasoning-models.ts b/extensions/ext-llm-openai/src/openai-reasoning-models.ts new file mode 100644 index 0000000000..3f10b992aa --- /dev/null +++ b/extensions/ext-llm-openai/src/openai-reasoning-models.ts @@ -0,0 +1,115 @@ +export type OpenAIReasoningEffort = "low" | "medium" | "high"; + +export type OpenAIProviderReasoningEffort = OpenAIReasoningEffort | "max"; + +export type OpenAIProviderReasoningOption = { + enabled?: boolean; + effort?: OpenAIProviderReasoningEffort; + budgetTokens?: number; +}; + +export type ResolvedOpenAIReasoning = { + effort: OpenAIReasoningEffort; + source: "default" | "explicit"; +}; + +const DEFAULT_REASONING_EFFORT: OpenAIReasoningEffort = "medium"; + +function supportsDefaultReasoningParams(providerName: string): boolean { + return providerName === "openai" || providerName === "veryfront-cloud"; +} + +function isGpt5ChatSnapshot(modelId: string): boolean { + return /^gpt-5-chat($|-)/.test(modelId); +} + +function isGpt51(modelId: string): boolean { + return /^gpt-5\.1($|-)/.test(modelId); +} + +function isReasoningCapableGpt5(modelId: string): boolean { + if (isGpt5ChatSnapshot(modelId) || isGpt51(modelId)) { + return false; + } + + if (/^gpt-5(-|$)/.test(modelId)) { + return true; + } + + const gpt5Version = /^gpt-5\.(\d+)(-|$)/.exec(modelId)?.[1]; + return gpt5Version !== undefined && Number.parseInt(gpt5Version, 10) >= 2; +} + +export function getDefaultOpenAIReasoningEffort( + modelId: string, + providerName = "openai", +): OpenAIReasoningEffort | undefined { + const normalized = modelId.toLowerCase(); + const normalizedProvider = providerName.toLowerCase(); + + if (!supportsDefaultReasoningParams(normalizedProvider)) { + return undefined; + } + + if (isGpt5ChatSnapshot(normalized)) { + return undefined; + } + + // GPT-5.1 defaults upstream reasoning to none unless callers opt in explicitly. + if (isGpt51(normalized)) { + return undefined; + } + + if (/^o1($|-\d)/.test(normalized) || /^o[34](-|$)/.test(normalized)) { + return DEFAULT_REASONING_EFFORT; + } + + if (isReasoningCapableGpt5(normalized)) { + return DEFAULT_REASONING_EFFORT; + } + + return undefined; +} + +export function resolveOpenAIReasoningConfig( + modelId: string, + providerName: string, + option: OpenAIProviderReasoningOption | undefined, +): ResolvedOpenAIReasoning | undefined { + if (!option) { + const effort = getDefaultOpenAIReasoningEffort(modelId, providerName); + return effort === undefined ? undefined : { effort, source: "default" }; + } + + if (option.enabled !== true) { + return undefined; + } + + switch (option.effort) { + case "low": + return { effort: "low", source: "explicit" }; + case "high": + case "max": + return { effort: "high", source: "explicit" }; + case "medium": + default: + return { effort: "medium", source: "explicit" }; + } +} + +export function shouldRequestOpenAIReasoningSummary( + providerName: string, + reasoning: ResolvedOpenAIReasoning, +): boolean { + return reasoning.source === "explicit" || providerName.toLowerCase() === "veryfront-cloud"; +} + +export function isOpenAIReasoningModel(modelId: string, providerName = "openai"): boolean { + return getDefaultOpenAIReasoningEffort(modelId, providerName) !== undefined; +} + +export function rejectsOpenAISamplingParams(modelId: string): boolean { + const normalized = modelId.toLowerCase(); + + return /^o[134]($|-)/.test(normalized) || isReasoningCapableGpt5(normalized); +} diff --git a/extensions/ext-llm-openai/src/openai-responses-request-builder.test.ts b/extensions/ext-llm-openai/src/openai-responses-request-builder.test.ts index e06baac264..8874bf3d39 100644 --- a/extensions/ext-llm-openai/src/openai-responses-request-builder.test.ts +++ b/extensions/ext-llm-openai/src/openai-responses-request-builder.test.ts @@ -27,6 +27,97 @@ function createWarningCollector() { } describe("ext-llm-openai/openai-responses-request-builder", () => { + it("requests reasoning summaries by default for Veryfront Cloud GPT-5.5 Responses models", () => { + const warnings = createWarningCollector(); + + const body = buildOpenAIResponsesRequest( + "gpt-5.5", + "veryfront-cloud", + { + prompt: [{ role: "user", content: [{ type: "text", text: "Think carefully." }] }], + temperature: 0.2, + }, + true, + warnings, + ); + + assertEquals(body.reasoning, { effort: "medium", summary: "auto" }); + assertEquals(body.temperature, undefined); + assertEquals(warnings.drain().map((warning) => warning.setting), ["temperature"]); + }); + + it("omits reasoning summaries for direct OpenAI default reasoning requests", () => { + const warnings = createWarningCollector(); + + const body = buildOpenAIResponsesRequest( + "gpt-5.5", + "openai", + { + prompt: [{ role: "user", content: [{ type: "text", text: "Think carefully." }] }], + }, + true, + warnings, + ); + + assertEquals(body.reasoning, { effort: "medium" }); + }); + + it("keeps explicit reasoning summaries for direct OpenAI requests", () => { + const warnings = createWarningCollector(); + + const body = buildOpenAIResponsesRequest( + "gpt-5.5", + "openai", + { + prompt: [{ role: "user", content: [{ type: "text", text: "Think carefully." }] }], + reasoning: { enabled: true, effort: "high" }, + }, + true, + warnings, + ); + + assertEquals(body.reasoning, { effort: "high", summary: "auto" }); + }); + + it("does not set default reasoning for OpenAI-compatible providers but still drops rejected sampling params", () => { + const warnings = createWarningCollector(); + + const body = buildOpenAIResponsesRequest( + "gpt-5.5", + "azure", + { + prompt: [{ role: "user", content: [{ type: "text", text: "Think carefully." }] }], + temperature: 0.2, + }, + true, + warnings, + ); + + assertEquals(body.reasoning, undefined); + assertEquals(body.temperature, undefined); + assertEquals(warnings.drain().map((warning) => warning.setting), ["temperature"]); + }); + + it("drops rejected sampling params when explicit reasoning is disabled", () => { + const warnings = createWarningCollector(); + + const body = buildOpenAIResponsesRequest( + "o3-mini", + "openai", + { + prompt: [{ role: "user", content: [{ type: "text", text: "Think carefully." }] }], + reasoning: { enabled: false }, + temperature: 0.2, + }, + true, + warnings, + ); + + assertEquals(body.reasoning, undefined); + assertEquals(body.temperature, undefined); + assertEquals(warnings.drain().map((warning) => warning.setting), ["temperature"]); + }); + it("preserves Responses request shaping, provider option merge order, and warnings", () => { const prompt: RuntimePromptMessage[] = [ { role: "system", content: "You are concise." }, diff --git a/extensions/ext-llm-openai/src/openai-responses-request-builder.ts b/extensions/ext-llm-openai/src/openai-responses-request-builder.ts index 3c8437dd20..531fa56ed5 100644 --- a/extensions/ext-llm-openai/src/openai-responses-request-builder.ts +++ b/extensions/ext-llm-openai/src/openai-responses-request-builder.ts @@ -8,8 +8,11 @@ import type { OpenAICompatibleLanguageOptions, RuntimeToolDefinition, } from "./openai-chat-request-builder.ts"; - -type ProviderReasoningOption = OpenAICompatibleLanguageOptions["reasoning"]; +import { + rejectsOpenAISamplingParams, + resolveOpenAIReasoningConfig, + shouldRequestOpenAIReasoningSummary, +} from "./openai-reasoning-models.ts"; export type OpenAIResponsesInputItem = Record; @@ -47,28 +50,6 @@ type WarningCollector = { }>; }; -function isOpenAIReasoningModel(modelId: string): boolean { - return /^o[134](-|$)/.test(modelId); -} - -function resolveOpenAIReasoningEffort( - option: ProviderReasoningOption | undefined, -): "low" | "medium" | "high" | undefined { - if (!option || option.enabled !== true) { - return undefined; - } - switch (option.effort) { - case "low": - return "low"; - case "high": - case "max": - return "high"; - case "medium": - default: - return "medium"; - } -} - function toSnakeCaseRecord(record: Record): Record { return Object.fromEntries( Object.entries(record).map(([key, value]) => [ @@ -215,9 +196,10 @@ export function buildOpenAIResponsesRequest( stream: boolean, warnings: WarningCollector, ): OpenAIResponsesRequest { - const isReasoningModel = isOpenAIReasoningModel(modelId); - const reasoningEffort = resolveOpenAIReasoningEffort(options.reasoning); - const reasoningEnabled = isReasoningModel || reasoningEffort !== undefined; + const reasoning = resolveOpenAIReasoningConfig(modelId, providerName, options.reasoning); + const reasoningEnabled = reasoning !== undefined; + const samplingRejected = rejectsOpenAISamplingParams(modelId); + const dropSamplingParams = reasoningEnabled || samplingRejected; if (options.topK !== undefined) { warnings.push({ @@ -227,7 +209,7 @@ export function buildOpenAIResponsesRequest( details: "OpenAI Responses API does not expose top_k; the value was dropped.", }); } - if (reasoningEnabled) { + if (dropSamplingParams) { const dropped: Array<[keyof typeof options, string]> = [ ["temperature", "temperature"], ["topP", "top_p"], @@ -240,8 +222,9 @@ export function buildOpenAIResponsesRequest( type: "unsupported-setting", provider: "openai", setting: key, - details: - `Dropped because OpenAI reasoning models reject ${openaiName}. Reasoning was active for this request.`, + details: samplingRejected + ? `Dropped because this model rejects ${openaiName}.` + : `Dropped because reasoning was active for this request and OpenAI rejects ${openaiName} with reasoning.`, }); } } @@ -258,14 +241,21 @@ export function buildOpenAIResponsesRequest( ...(options.maxOutputTokens !== undefined ? { max_output_tokens: options.maxOutputTokens } : {}), - ...(!reasoningEnabled && options.temperature !== undefined + ...(!dropSamplingParams && options.temperature !== undefined ? { temperature: options.temperature } : {}), - ...(!reasoningEnabled && options.topP !== undefined ? { top_p: options.topP } : {}), + ...(!dropSamplingParams && options.topP !== undefined ? { top_p: options.topP } : {}), ...(responsesTools ? { tools: responsesTools } : {}), ...(options.toolChoice !== undefined ? { tool_choice: options.toolChoice } : {}), - ...(reasoningEffort !== undefined - ? { reasoning: { effort: reasoningEffort, summary: "auto" } } + ...(reasoning !== undefined + ? { + reasoning: { + effort: reasoning.effort, + ...(shouldRequestOpenAIReasoningSummary(providerName, reasoning) + ? { summary: "auto" } + : {}), + }, + } : {}), ...(typeof options.userId === "string" && options.userId.length > 0 ? { user: options.userId } diff --git a/src/utils/version-constant.ts b/src/utils/version-constant.ts index e8f43e5089..3ffb7841cf 100644 --- a/src/utils/version-constant.ts +++ b/src/utils/version-constant.ts @@ -1,4 +1,4 @@ // Keep in sync with deno.json version. // scripts/release.ts updates this constant during releases. /** Shared version value. */ -export const VERSION = "0.1.987"; +export const VERSION = "0.1.988";