diff --git a/packages/ai/CHANGELOG.md b/packages/ai/CHANGELOG.md index 7c001c1a4dc..b1fab8c5abb 100644 --- a/packages/ai/CHANGELOG.md +++ b/packages/ai/CHANGELOG.md @@ -4,6 +4,8 @@ ### Fixed +- Fixed OpenAI Codex cached WebSocket continuations after grammar tool calls to send only the real tool-result delta. +- Fixed constrained tool sampling across Google, Amazon Bedrock, Mistral, and Azure OpenAI Responses adapters, including model-aware strict-tool capabilities, grammar configuration validation, and malformed grammar-call replay errors. - Fixed `cacheRetention: "none"` to disable implicit prompt-cache writes for supported OpenAI models and session-based caching for OpenAI Codex ([#6618](https://github.com/earendil-works/pi/pull/6618) by [@tmustier](https://github.com/tmustier)). - Fixed OpenAI and Anthropic provider retry waits to honor abort signals and configured delay limits ([#6911](https://github.com/earendil-works/pi/issues/6911)). - Fixed OpenRouter Anthropic cache breakpoints to advance through tool results and enabled cache control for `~anthropic/*-latest` aliases ([#6941](https://github.com/earendil-works/pi/pull/6941) by [@mteam88](https://github.com/mteam88)). @@ -169,6 +171,7 @@ ### Added - Added OpenAI GPT-5.6 model metadata for `gpt-5.6`, `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna`, plus verified `openai-codex` support for `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna`. +- Added provider-side constrained sampling for tools via `Tool.constrainedSampling`: strict JSON-schema enforcement for OpenAI and Anthropic tool calls, and OpenAI custom grammar tools (Lark/regex). Grammar tool capability comes from the model catalog's `supportsGrammarTools` compat flag, enabled for GPT-5+ models on OpenAI, OpenAI Codex, Azure OpenAI, GitHub Copilot, opencode, and Cloudflare AI Gateway ([#6341](https://github.com/earendil-works/pi/pull/6341)). - Refreshed generated model catalogs from models.dev, adding newly listed models including Kimi K2.7 Code for GitHub Copilot and Fable 5 to several providers ([#6256](https://github.com/earendil-works/pi/issues/6256)). - Added Claude Sonnet 5 to the GitHub Copilot model catalog ([#6200](https://github.com/earendil-works/pi/issues/6200)). - Added zstd request-body compression for the OpenAI Codex Responses SSE transport. Requests are sent with `Content-Encoding: zstd` when Node/Bun zstd support is available; the WebSocket transport is unchanged. diff --git a/packages/ai/README.md b/packages/ai/README.md index b20ebd17566..7ff6a091732 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -478,6 +478,40 @@ const bookMeetingTool: Tool = { }; ``` +### Constrained Sampling for Tools + +Tools can opt in to provider-side constrained sampling. For JSON-schema tools, `strict: 'prefer'` uses provider-side strict schema enforcement when supported and otherwise falls back to normal tool calling. `strict: 'require'` fails the request when the active provider/model cannot honor it. Set `constrainedSampling: false` to explicitly opt out; it behaves the same as omitting the field. + +```typescript +const strictTool: Tool = { + name: 'edit_file', + description: 'Edit a file', + parameters: Type.Object({ + path: Type.String(), + content: Type.String() + }, { additionalProperties: false }), + constrainedSampling: { type: 'json_schema', strict: 'prefer' } +}; +``` + +Strict JSON-schema constrained sampling is supported for OpenAI, Anthropic, supported Amazon Bedrock Converse models, Mistral, and Gemini 3 tool calls through the Google Generative AI and Vertex adapters. Google uses `VALIDATED` function-calling mode (or `ANY` when explicitly requested); earlier Gemini versions fall back for `strict: 'prefer'` and reject `strict: 'require'` because they do not enforce required parameters. Bedrock strict-tool capability is generated from model structured-output metadata; custom Bedrock models can override `compat.supportsStrictMode`. OpenAI Responses and Chat Completions can also emit grammar-constrained custom tools with OpenAI Lark or regex grammar variants. If multiple OpenAI variants are supplied, Lark is preferred over regex. Grammar constraints are enforced when the active model supports grammar tools; otherwise the tool falls back to normal function/JSON-schema handling. Grammar tool capability is model metadata: the generated catalog sets `compat.supportsOpenAIGrammarTools` for GPT-5+ models on endpoints that pass OpenAI custom tools through (OpenAI, OpenAI Codex, Azure OpenAI Responses, GitHub Copilot, opencode, and Cloudflare AI Gateway). OpenAI rejects `type: "custom"` tools for pre-GPT-5 models, and gateways that normalize tool schemas (e.g. OpenRouter) mangle them, so the flag stays off elsewhere. Custom model definitions can opt in via `compat`. Grammar-capable models reject grammar configurations without a non-empty supported variant. Native grammar tools must have an object parameter schema with exactly one required string property: + +```typescript +const patchTool: Tool = { + name: 'apply_patch', + description: 'Apply a patch', + parameters: Type.Object({ + input: Type.String() + }, { additionalProperties: false }), + constrainedSampling: { + type: 'grammar', + variants: { + openai_lark: 'start: /.+/s' + } + } +}; +``` + ### Handling Tool Calls Tool results use content blocks and can include both text and images: @@ -1124,6 +1158,7 @@ interface OpenAICompletionsCompat { supportsReasoningEffort?: boolean; // Whether provider supports `reasoning_effort` (default: true) supportsUsageInStreaming?: boolean; // Whether provider supports `stream_options: { include_usage: true }` (default: true) supportsStrictMode?: boolean; // Whether provider supports `strict` in tool definitions (default: true) + supportsOpenAIGrammarTools?: boolean; // Whether to emit OpenAI custom Lark/regex grammar tools; false falls back to normal function tools (default: false; the generated catalog enables it for capable models) sendSessionAffinityHeaders?: boolean; // Send session-affinity data from `sessionId` (default: false) sessionAffinityFormat?: 'openai' | 'openai-nosession' | 'openrouter'; // Format for session affinity: 'openai' uses `prompt_cache_key`, `session_id`, `x-client-request-id`, and `x-session-affinity`; 'openai-nosession' uses `prompt_cache_key`, `x-client-request-id`, and `x-session-affinity`; 'openrouter' uses `x-session-id` (default: auto-detected) maxTokensField?: 'max_completion_tokens' | 'max_tokens'; // Which field name to use (default: max_completion_tokens) @@ -1142,6 +1177,8 @@ interface OpenAIResponsesCompat { supportsDeveloperRole?: boolean; // Whether provider supports `developer` role vs `system` (default: true) sessionAffinityFormat?: 'openai' | 'openai-nosession' | 'openrouter'; // Session-affinity header format: 'openai' sends `session_id` and `x-client-request-id`; 'openai-nosession' sends `x-client-request-id`; 'openrouter' sends `x-session-id`. Does not affect the `prompt_cache_key` body param (default: auto-detected) supportsLongCacheRetention?: boolean; // Whether provider supports `prompt_cache_retention: "24h"` (default: true) + supportsStrictMode?: boolean; // Whether provider supports strict JSON-schema function tools (default: false; enabled in metadata for built-in OpenAI models) + supportsOpenAIGrammarTools?: boolean; // Whether to emit OpenAI custom Lark/regex grammar tools; false falls back to normal function tools (default: false; the generated catalog enables it for capable models) } ``` diff --git a/packages/ai/scripts/generate-models.ts b/packages/ai/scripts/generate-models.ts index 8b3e58a2a2e..a88168bb04a 100644 --- a/packages/ai/scripts/generate-models.ts +++ b/packages/ai/scripts/generate-models.ts @@ -83,6 +83,7 @@ interface ModelsDevModel { id: string; name: string; tool_call?: boolean; + structured_output?: boolean; reasoning?: boolean; reasoning_options?: ModelsDevReasoningOption[]; limit?: { @@ -514,6 +515,7 @@ const OPENAI_COMPLETIONS_DEFAULT_COMPAT = { chatTemplateKwargs: {}, zaiToolStream: false, supportsStrictMode: true, + supportsOpenAIGrammarTools: false, sendSessionAffinityHeaders: false, supportsLongCacheRetention: true, } satisfies Required> & { @@ -602,6 +604,7 @@ function detectOpenAICompletionsCompat(model: Model<"openai-completions">): Open chatTemplateKwargs: {}, zaiToolStream: false, supportsStrictMode: !isMoonshot && !isTogether && !isCloudflareAiGateway && !isNvidia, + supportsOpenAIGrammarTools: false, ...(cacheControlFormat ? { cacheControlFormat } : {}), sendSessionAffinityHeaders: false, supportsLongCacheRetention: !( @@ -643,6 +646,39 @@ function applyOpenAICompletionsCompatMetadata(model: Model): void { } } +function applyStrictToolCompatMetadata(model: Model): void { + if (model.provider === "openai" && model.api === "openai-responses") { + model.compat = { ...(model.compat as OpenAIResponsesCompat | undefined), supportsStrictMode: true }; + } else if (model.provider === "anthropic" && model.api === "anthropic-messages") { + mergeAnthropicMessagesCompat(model, { supportsStrictTools: true }); + } +} + +// Responses endpoints verified (OpenAI, ChatGPT Codex backend, GitHub Copilot, +// opencode zen) or documented (Azure OpenAI, Cloudflare AI Gateway) to pass +// OpenAI custom grammar tools through. OpenAI rejects `type: "custom"` tools +// for pre-GPT-5 models (gpt-4.x, gpt-4o, o-series). +const OPENAI_GRAMMAR_TOOL_PROVIDERS = new Set([ + "openai", + "openai-codex", + "azure-openai-responses", + "github-copilot", + "opencode", + "cloudflare-ai-gateway", +]); +const OPENAI_GRAMMAR_TOOL_APIS = new Set([ + "openai-responses", + "azure-openai-responses", + "openai-codex-responses", +]); + +function applyOpenAIGrammarToolCompatMetadata(model: Model): void { + if (!OPENAI_GRAMMAR_TOOL_APIS.has(model.api) || !OPENAI_GRAMMAR_TOOL_PROVIDERS.has(model.provider)) return; + const match = /^gpt-(\d+)/.exec(model.id); + if (!match || Number(match[1]) < 5) return; + model.compat = { ...(model.compat as OpenAIResponsesCompat | undefined), supportsOpenAIGrammarTools: true }; +} + function applyOpenAIToolSearchMetadata(model: Model): void { const isOpenAIResponses = model.provider === "openai" && model.api === "openai-responses"; const isOpenAICodex = model.provider === "openai-codex" && model.api === "openai-codex-responses"; @@ -1045,6 +1081,7 @@ async function loadModelsDevData(): Promise[]> { }, contextWindow: m.limit?.context || 4096, maxTokens: m.limit?.output || 4096, + ...(m.structured_output === true && { compat: { supportsStrictMode: true } }), }); recordModelsDevReasoningOptions("amazon-bedrock" as const, id, m); } @@ -2467,6 +2504,8 @@ async function generateModels() { applyOpenAICompletionsCompatMetadata(model); applyModelsDevReasoningOptionMetadata(model); applyThinkingLevelMetadata(model); + applyStrictToolCompatMetadata(model); + applyOpenAIGrammarToolCompatMetadata(model); applyOpenAIToolSearchMetadata(model); applyOpenAIExplicitPromptCacheMetadata(model); } diff --git a/packages/ai/src/api/anthropic-messages.ts b/packages/ai/src/api/anthropic-messages.ts index 68c5a15707e..99f698b4002 100644 --- a/packages/ai/src/api/anthropic-messages.ts +++ b/packages/ai/src/api/anthropic-messages.ts @@ -37,6 +37,7 @@ import { getProviderEnvValue } from "../utils/provider-env.ts"; import { retryProviderRequest } from "../utils/provider-retry.ts"; import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts"; +import { resolveJsonSchemaStrictSampling } from "./constrained-sampling.ts"; import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts"; import { adjustMaxTokensForThinking, buildBaseOptions, clampMaxTokensToContext } from "./simple-options.ts"; import { transformMessages } from "./transform-messages.ts"; @@ -179,6 +180,7 @@ function getAnthropicCompat( supportsCacheControlOnTools: model.compat?.supportsCacheControlOnTools ?? true, supportsTemperature: model.compat?.supportsTemperature ?? true, allowEmptySignature: model.compat?.allowEmptySignature ?? false, + supportsStrictTools: model.compat?.supportsStrictTools ?? false, supportsToolReferences: model.compat?.supportsToolReferences ?? defaultSupportsToolReferences(model), }; } @@ -999,9 +1001,17 @@ function buildParams( immediateTools, isOAuthToken, compat.supportsEagerToolInputStreaming, + compat.supportsStrictTools, compat.supportsCacheControlOnTools ? cacheControl : undefined, ), - ...convertTools(deferredTools, isOAuthToken, compat.supportsEagerToolInputStreaming, undefined, true), + ...convertTools( + deferredTools, + isOAuthToken, + compat.supportsEagerToolInputStreaming, + compat.supportsStrictTools, + undefined, + true, + ), ]; } @@ -1269,23 +1279,34 @@ function convertTools( tools: Tool[], isOAuthToken: boolean, supportsEagerToolInputStreaming: boolean, + supportsStrictTools: boolean, cacheControl?: CacheControlEphemeral, deferLoading = false, ): Anthropic.Messages.Tool[] { if (!tools) return []; return tools.map((tool, index) => { + const strict = resolveJsonSchemaStrictSampling(tool, supportsStrictTools); const schema = tool.parameters as { properties?: unknown; required?: string[] }; + const legacyInputSchema = { + type: "object" as const, + properties: schema.properties ?? {}, + required: schema.required ?? [], + }; + const inputSchema = + strict === true + ? { + ...(tool.parameters as Record), + ...legacyInputSchema, + } + : legacyInputSchema; return { name: isOAuthToken ? toClaudeCodeName(tool.name) : tool.name, description: tool.description, ...(supportsEagerToolInputStreaming ? { eager_input_streaming: true } : {}), - input_schema: { - type: "object", - properties: schema.properties ?? {}, - required: schema.required ?? [], - }, + ...(strict === true ? { strict: true } : {}), + input_schema: inputSchema, ...(deferLoading ? { defer_loading: true } : {}), ...(cacheControl && index === tools.length - 1 ? { cache_control: cacheControl } : {}), }; diff --git a/packages/ai/src/api/azure-openai-responses.ts b/packages/ai/src/api/azure-openai-responses.ts index 3910c2219f8..1e409650a8f 100644 --- a/packages/ai/src/api/azure-openai-responses.ts +++ b/packages/ai/src/api/azure-openai-responses.ts @@ -15,6 +15,7 @@ import { AssistantMessageEventStream } from "../utils/event-stream.ts"; import { headersToRecord } from "../utils/headers.ts"; import { getProviderEnvValue } from "../utils/provider-env.ts"; import { retryProviderRequest } from "../utils/provider-retry.ts"; +import { createGrammarToolInputProperties } from "./constrained-sampling.ts"; import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts"; import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.ts"; import { buildBaseOptions } from "./simple-options.ts"; @@ -100,7 +101,11 @@ export const stream: StreamFunction<"azure-openai-responses", AzureOpenAIRespons throw new Error(`No API key for provider: ${model.provider}`); } const client = createClient(model, apiKey, options); - let params = buildParams(model, context, options, deploymentName); + const grammarToolInputProperties = createGrammarToolInputProperties( + context.tools, + model.compat?.supportsOpenAIGrammarTools ?? false, + ); + let params = buildParams(model, context, options, deploymentName, grammarToolInputProperties); const nextParams = await options?.onPayload?.(params, model); if (nextParams !== undefined) { params = nextParams as ResponseCreateParamsStreaming; @@ -121,7 +126,7 @@ export const stream: StreamFunction<"azure-openai-responses", AzureOpenAIRespons await options?.onResponse?.({ status: response.status, headers: headersToRecord(response.headers) }, model); stream.push({ type: "start", partial: output }); - await processResponsesStream(openaiStream, output, stream, model); + await processResponsesStream(openaiStream, output, stream, model, { grammarToolInputProperties }); if (options?.signal?.aborted) { throw new Error("Request was aborted"); @@ -136,8 +141,9 @@ export const stream: StreamFunction<"azure-openai-responses", AzureOpenAIRespons } catch (error) { for (const block of output.content) { delete (block as { index?: number }).index; - // partialJson is only a streaming scratch buffer; never persist it. + // Streaming scratch buffers are only used during parsing; never persist them. delete (block as { partialJson?: string }).partialJson; + delete (block as { customInput?: unknown }).customInput; } output.stopReason = options?.signal?.aborted ? "aborted" : "error"; output.errorMessage = formatAzureOpenAIError(error); @@ -262,8 +268,14 @@ function buildParams( context: Context, options: AzureOpenAIResponsesOptions | undefined, deploymentName: string, + grammarToolInputProperties: ReadonlyMap = createGrammarToolInputProperties( + context.tools, + model.compat?.supportsOpenAIGrammarTools ?? false, + ), ) { - const messages = convertResponsesMessages(model, context, AZURE_TOOL_CALL_PROVIDERS); + const messages = convertResponsesMessages(model, context, AZURE_TOOL_CALL_PROVIDERS, { + grammarToolInputProperties, + }); const params: ResponseCreateParamsStreaming = { model: deploymentName, @@ -282,7 +294,10 @@ function buildParams( } if (context.tools && context.tools.length > 0) { - params.tools = convertResponsesTools(context.tools); + params.tools = convertResponsesTools(context.tools, { + supportsStrictMode: model.compat?.supportsStrictMode ?? true, + supportsOpenAIGrammarTools: model.compat?.supportsOpenAIGrammarTools ?? false, + }); } if (model.reasoning) { diff --git a/packages/ai/src/api/bedrock-converse-stream.ts b/packages/ai/src/api/bedrock-converse-stream.ts index d0af8131508..4c10bc923af 100644 --- a/packages/ai/src/api/bedrock-converse-stream.ts +++ b/packages/ai/src/api/bedrock-converse-stream.ts @@ -54,6 +54,7 @@ import { parseStreamingJson } from "../utils/json-parse.ts"; import { resolveHttpProxyUrlForTarget } from "../utils/node-http-proxy.ts"; import { getProviderEnvValue } from "../utils/provider-env.ts"; import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts"; +import { resolveJsonSchemaStrictSampling } from "./constrained-sampling.ts"; import { adjustMaxTokensForThinking, buildBaseOptions, @@ -228,7 +229,7 @@ export const stream: StreamFunction<"bedrock-converse-stream", BedrockOptions> = ...(inferenceMaxTokens !== undefined && { maxTokens: inferenceMaxTokens }), ...(options.temperature !== undefined && { temperature: options.temperature }), }, - toolConfig: convertToolConfig(context.tools, options.toolChoice), + toolConfig: convertToolConfig(context.tools, options.toolChoice, model.compat?.supportsStrictMode ?? false), additionalModelRequestFields: buildAdditionalModelRequestFields(model, options), ...(options.requestMetadata !== undefined && { requestMetadata: options.requestMetadata }), }; @@ -908,16 +909,22 @@ function convertMessages( function convertToolConfig( tools: Tool[] | undefined, toolChoice: BedrockOptions["toolChoice"], + supportsStrictMode: boolean, ): ToolConfiguration | undefined { - if (!tools?.length || toolChoice === "none") return undefined; - - const bedrockTools: BedrockTool[] = tools.map((tool) => ({ - toolSpec: { - name: tool.name, - description: tool.description, - inputSchema: { json: tool.parameters as unknown as DocumentType }, - }, - })); + if (!tools?.length) return undefined; + if (toolChoice === "none") return undefined; + + const bedrockTools: BedrockTool[] = tools.map((tool) => { + const strict = resolveJsonSchemaStrictSampling(tool, supportsStrictMode); + return { + toolSpec: { + name: tool.name, + description: tool.description, + inputSchema: { json: tool.parameters as unknown as DocumentType }, + ...(strict === true ? { strict: true } : {}), + }, + }; + }); let bedrockToolChoice: ToolChoice | undefined; switch (toolChoice) { diff --git a/packages/ai/src/api/constrained-sampling.ts b/packages/ai/src/api/constrained-sampling.ts new file mode 100644 index 00000000000..ec961a12399 --- /dev/null +++ b/packages/ai/src/api/constrained-sampling.ts @@ -0,0 +1,148 @@ +import type { Tool } from "../types.ts"; + +interface JsonSchemaObject { + type?: unknown; + properties?: Record; + required?: unknown; +} + +export interface GrammarConstrainedSampling { + format: "lark" | "regex"; + definition: string; + inputProperty: string; +} + +export interface GrammarToolInputJsonBuffer { + input: string; + started: boolean; + closed: boolean; +} + +export function getGrammarToolInput( + toolName: string, + arguments_: Record, + inputProperty: string, +): string { + const input = arguments_[inputProperty]; + if (typeof input !== "string") { + throw new Error(`Grammar tool call "${toolName}" requires argument "${inputProperty}" to be a string.`); + } + return input; +} + +export function appendGrammarToolInputJsonDelta( + buffer: GrammarToolInputJsonBuffer, + inputProperty: string, + nextInput: string, + close: boolean, +): string | undefined { + if (buffer.closed) { + if (close && nextInput === buffer.input) return undefined; + throw new Error(`grammar tool input for property "${inputProperty}" changed after it was closed`); + } + if (!nextInput.startsWith(buffer.input)) { + throw new Error(`grammar tool input for property "${inputProperty}" changed non-monotonically`); + } + + const inputDelta = nextInput.slice(buffer.input.length); + if (!close && inputDelta.length === 0) return undefined; + + let delta = ""; + if (!buffer.started) { + delta += `{${JSON.stringify(inputProperty)}:"`; + buffer.started = true; + } + delta += JSON.stringify(inputDelta).slice(1, -1); + buffer.input = nextInput; + + if (close) { + delta += '"}'; + buffer.closed = true; + } + return delta; +} + +function inferGrammarInputProperty(tool: Tool): string { + const schema = tool.parameters as JsonSchemaObject; + if (schema.type !== "object") { + throw new Error("grammar constrained sampling requires an object parameter schema"); + } + if (!Array.isArray(schema.required) || schema.required.length !== 1 || typeof schema.required[0] !== "string") { + throw new Error("grammar constrained sampling requires exactly one required string property"); + } + + const inputProperty = schema.required[0]; + if (!schema.properties?.[inputProperty]) { + throw new Error(`grammar constrained sampling requires a properties entry for ${inputProperty}`); + } + if (schema.properties[inputProperty]?.type !== "string") { + throw new Error(`grammar constrained sampling property ${inputProperty} must have type string`); + } + return inputProperty; +} + +export function resolveJsonSchemaStrictSampling(tool: Tool, supportsStrictMode: boolean): boolean | undefined { + const config = tool.constrainedSampling; + if (!config || config.type !== "json_schema") { + return undefined; + } + + if (supportsStrictMode) { + return true; + } + if (config.strict === "require") { + throw new Error( + `Tool "${tool.name}" requires JSON-schema constrained sampling, but strict tools are unsupported.`, + ); + } + return undefined; +} + +export function resolveGrammarConstrainedSampling( + tool: Tool, + supportsOpenAIGrammarTools: boolean, +): GrammarConstrainedSampling | undefined { + const config = tool.constrainedSampling; + if (!config || config.type !== "grammar") { + return undefined; + } + + if (!supportsOpenAIGrammarTools) { + return undefined; + } + + const larkDefinition = config.variants.openai_lark; + const regexDefinition = config.variants.openai_regex; + const hasLarkDefinition = typeof larkDefinition === "string" && larkDefinition.trim().length > 0; + const hasRegexDefinition = typeof regexDefinition === "string" && regexDefinition.trim().length > 0; + if (!hasLarkDefinition && !hasRegexDefinition) { + throw new Error( + `Tool "${tool.name}" cannot use grammar constrained sampling: no supported grammar variant was provided.`, + ); + } + + try { + return { + format: hasLarkDefinition ? "lark" : "regex", + definition: hasLarkDefinition ? larkDefinition : regexDefinition!, + inputProperty: inferGrammarInputProperty(tool), + }; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Tool "${tool.name}" cannot use grammar constrained sampling: ${message}.`); + } +} + +export function createGrammarToolInputProperties( + tools: Tool[] | undefined, + supportsOpenAIGrammarTools: boolean, +): ReadonlyMap { + const properties = new Map(); + for (const tool of tools ?? []) { + const grammar = resolveGrammarConstrainedSampling(tool, supportsOpenAIGrammarTools); + if (grammar) { + properties.set(tool.name, grammar.inputProperty); + } + } + return properties; +} diff --git a/packages/ai/src/api/google-generative-ai.ts b/packages/ai/src/api/google-generative-ai.ts index b971c219c58..c925cf885d8 100644 --- a/packages/ai/src/api/google-generative-ai.ts +++ b/packages/ai/src/api/google-generative-ai.ts @@ -30,8 +30,9 @@ import { convertTools, isThinkingPart, mapStopReason, - mapToolChoice, + resolveGoogleFunctionCallingMode, retainThoughtSignature, + supportsGoogleStrictToolSampling, } from "./google-shared.ts"; import { buildBaseOptions } from "./simple-options.ts"; @@ -355,22 +356,18 @@ function buildParams( generationConfig.maxOutputTokens = options.maxTokens; } + const functionCallingMode = context.tools?.length + ? resolveGoogleFunctionCallingMode(context.tools, options.toolChoice, supportsGoogleStrictToolSampling(model.id)) + : undefined; const config: GenerateContentConfig = { ...(Object.keys(generationConfig).length > 0 && generationConfig), ...(context.systemPrompt && { systemInstruction: sanitizeSurrogates(context.systemPrompt) }), ...(context.tools && context.tools.length > 0 && { tools: convertTools(context.tools) }), + ...(functionCallingMode !== undefined && { + toolConfig: { functionCallingConfig: { mode: functionCallingMode } }, + }), }; - if (context.tools && context.tools.length > 0 && options.toolChoice) { - config.toolConfig = { - functionCallingConfig: { - mode: mapToolChoice(options.toolChoice), - }, - }; - } else { - config.toolConfig = undefined; - } - if (options.thinking?.enabled && model.reasoning) { const thinkingConfig: ThinkingConfig = { includeThoughts: true }; if (options.thinking.level !== undefined) { diff --git a/packages/ai/src/api/google-shared.ts b/packages/ai/src/api/google-shared.ts index 1559b20b78c..345bcfbbde0 100644 --- a/packages/ai/src/api/google-shared.ts +++ b/packages/ai/src/api/google-shared.ts @@ -5,6 +5,7 @@ import { type Content, FinishReason, FunctionCallingConfigMode, type Part } from "@google/genai"; import type { Context, ImageContent, Model, StopReason, TextContent, Tool } from "../types.ts"; import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts"; +import { resolveJsonSchemaStrictSampling } from "./constrained-sampling.ts"; import { transformMessages } from "./transform-messages.ts"; type GoogleApiType = "google-generative-ai" | "google-vertex"; @@ -287,9 +288,13 @@ export function convertTools( ]; } -/** - * Map tool choice string to Gemini FunctionCallingConfigMode. - */ +/** Gemini 3+ enforces required function parameters in validated tool-calling modes. */ +export function supportsGoogleStrictToolSampling(modelId: string): boolean { + const majorVersion = getGeminiMajorVersion(modelId); + return majorVersion !== undefined && majorVersion >= 3; +} + +/** Map tool choice string to Gemini FunctionCallingConfigMode. */ export function mapToolChoice(choice: string): FunctionCallingConfigMode { switch (choice) { case "auto": @@ -303,6 +308,21 @@ export function mapToolChoice(choice: string): FunctionCallingConfigMode { } } +export function resolveGoogleFunctionCallingMode( + tools: Tool[], + toolChoice: string | undefined, + supportsStrictMode: boolean, +): FunctionCallingConfigMode | undefined { + const useStrictMode = tools.some((tool) => resolveJsonSchemaStrictSampling(tool, supportsStrictMode) === true); + if (toolChoice === "none" || toolChoice === "any") { + return mapToolChoice(toolChoice); + } + if (useStrictMode) { + return FunctionCallingConfigMode.VALIDATED; + } + return toolChoice ? mapToolChoice(toolChoice) : undefined; +} + /** * Map Gemini FinishReason to our StopReason. */ diff --git a/packages/ai/src/api/google-vertex.ts b/packages/ai/src/api/google-vertex.ts index e6a10043b7c..24ee3c95f29 100644 --- a/packages/ai/src/api/google-vertex.ts +++ b/packages/ai/src/api/google-vertex.ts @@ -35,8 +35,9 @@ import { convertTools, isThinkingPart, mapStopReason, - mapToolChoice, + resolveGoogleFunctionCallingMode, retainThoughtSignature, + supportsGoogleStrictToolSampling, } from "./google-shared.ts"; import { buildBaseOptions } from "./simple-options.ts"; @@ -454,22 +455,18 @@ function buildParams( generationConfig.maxOutputTokens = options.maxTokens; } + const functionCallingMode = context.tools?.length + ? resolveGoogleFunctionCallingMode(context.tools, options.toolChoice, supportsGoogleStrictToolSampling(model.id)) + : undefined; const config: GenerateContentConfig = { ...(Object.keys(generationConfig).length > 0 && generationConfig), ...(context.systemPrompt && { systemInstruction: sanitizeSurrogates(context.systemPrompt) }), ...(context.tools && context.tools.length > 0 && { tools: convertTools(context.tools) }), + ...(functionCallingMode !== undefined && { + toolConfig: { functionCallingConfig: { mode: functionCallingMode } }, + }), }; - if (context.tools && context.tools.length > 0 && options.toolChoice) { - config.toolConfig = { - functionCallingConfig: { - mode: mapToolChoice(options.toolChoice), - }, - }; - } else { - config.toolConfig = undefined; - } - if (options.thinking?.enabled && model.reasoning) { const thinkingConfig: ThinkingConfig = { includeThoughts: true }; if (options.thinking.level !== undefined) { diff --git a/packages/ai/src/api/mistral-conversations.ts b/packages/ai/src/api/mistral-conversations.ts index f1e4fc4c485..3a5e59dd7f9 100644 --- a/packages/ai/src/api/mistral-conversations.ts +++ b/packages/ai/src/api/mistral-conversations.ts @@ -25,6 +25,7 @@ import { AssistantMessageEventStream } from "../utils/event-stream.ts"; import { shortHash } from "../utils/hash.ts"; import { parseStreamingJson } from "../utils/json-parse.ts"; import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts"; +import { resolveJsonSchemaStrictSampling } from "./constrained-sampling.ts"; import { buildBaseOptions } from "./simple-options.ts"; import { transformMessages } from "./transform-messages.ts"; @@ -483,15 +484,18 @@ async function consumeChatStream( } function toFunctionTools(tools: Tool[]): Array { - return tools.map((tool) => ({ - type: "function", - function: { - name: tool.name, - description: tool.description, - parameters: stripSymbolKeys(tool.parameters) as Record, - strict: false, - }, - })); + return tools.map((tool) => { + const strict = resolveJsonSchemaStrictSampling(tool, true); + return { + type: "function", + function: { + name: tool.name, + description: tool.description, + parameters: stripSymbolKeys(tool.parameters) as Record, + strict: strict ?? false, + }, + }; + }); } function stripSymbolKeys(value: unknown): unknown { diff --git a/packages/ai/src/api/openai-codex-responses.ts b/packages/ai/src/api/openai-codex-responses.ts index 0dcf26d2c83..cf89ee03430 100644 --- a/packages/ai/src/api/openai-codex-responses.ts +++ b/packages/ai/src/api/openai-codex-responses.ts @@ -47,6 +47,7 @@ import { AssistantMessageEventStream } from "../utils/event-stream.ts"; import { headersToRecord } from "../utils/headers.ts"; import { resolveHttpProxyUrlForTarget } from "../utils/node-http-proxy.ts"; import { uuidv7 } from "../utils/uuid.ts"; +import { createGrammarToolInputProperties } from "./constrained-sampling.ts"; import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts"; import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.ts"; import { buildBaseOptions } from "./simple-options.ts"; @@ -262,9 +263,13 @@ export const stream: StreamFunction<"openai-codex-responses", OpenAICodexRespons } const accountId = extractAccountId(apiKey); + const grammarToolInputProperties = createGrammarToolInputProperties( + context.tools, + model.compat?.supportsOpenAIGrammarTools ?? false, + ); const cacheSessionId = options?.cacheRetention === "none" ? undefined : options?.sessionId; const codexSessionId = clampOpenAIPromptCacheKey(cacheSessionId); - let body = buildRequestBody(model, context, options, codexSessionId); + let body = buildRequestBody(model, context, options, codexSessionId, grammarToolInputProperties); const nextBody = await options?.onPayload?.(body, model); if (nextBody !== undefined) { body = nextBody as RequestBody; @@ -312,6 +317,7 @@ export const stream: StreamFunction<"openai-codex-responses", OpenAICodexRespons httpTimeoutMs, websocketConnectTimeoutMs, cacheSessionId, + grammarToolInputProperties, options, ); @@ -459,7 +465,7 @@ export const stream: StreamFunction<"openai-codex-responses", OpenAICodexRespons startEmitted = true; stream.push({ type: "start", partial: output }); } - await processStream(response, output, stream, model, options); + await processStream(response, output, stream, model, grammarToolInputProperties, options); if (options?.signal?.aborted) { throw new Error("Request was aborted"); @@ -469,8 +475,9 @@ export const stream: StreamFunction<"openai-codex-responses", OpenAICodexRespons stream.end(); } catch (error) { for (const block of output.content) { - // partialJson is only a streaming scratch buffer; never persist it. + // Streaming scratch buffers are only used during parsing; never persist them. delete (block as { partialJson?: string }).partialJson; + delete (block as { customInput?: unknown }).customInput; } output.stopReason = options?.signal?.aborted ? "aborted" : "error"; output.errorMessage = formatProviderError(normalizeProviderError(error)); @@ -511,11 +518,23 @@ function buildRequestBody( context: Context, options: OpenAICodexResponsesOptions | undefined, cacheSessionId: string | undefined, + grammarToolInputProperties: ReadonlyMap = createGrammarToolInputProperties( + context.tools, + model.compat?.supportsOpenAIGrammarTools ?? false, + ), ): RequestBody { + const supportsStrictMode = model.compat?.supportsStrictMode ?? true; + const supportsOpenAIGrammarTools = model.compat?.supportsOpenAIGrammarTools ?? false; const toolPlacement = splitDeferredTools(context, model.compat?.supportsToolSearch ?? false); const messages = convertResponsesMessages(model, context, CODEX_TOOL_CALL_PROVIDERS, { includeSystemPrompt: false, + grammarToolInputProperties, deferredTools: toolPlacement.deferred, + toolOptions: { + strict: null, + supportsStrictMode, + supportsOpenAIGrammarTools, + }, }); const body: RequestBody = { @@ -540,7 +559,11 @@ function buildRequestBody( } if (toolPlacement.immediate.length > 0) { - body.tools = convertResponsesTools(toolPlacement.immediate, { strict: null }); + body.tools = convertResponsesTools(toolPlacement.immediate, { + strict: null, + supportsStrictMode, + supportsOpenAIGrammarTools, + }); } if (options?.reasoningEffort !== undefined) { @@ -622,10 +645,12 @@ async function processStream( output: AssistantMessage, stream: AssistantMessageEventStream, model: Model<"openai-codex-responses">, + grammarToolInputProperties: ReadonlyMap, options?: OpenAICodexResponsesOptions, ): Promise { await processResponsesStream(mapCodexEvents(parseSSE(response, options?.signal)), output, stream, model, { serviceTier: options?.serviceTier, + grammarToolInputProperties, resolveServiceTier: resolveCodexServiceTier, applyServiceTierPricing: (usage, serviceTier) => applyServiceTierPricing(usage, serviceTier, model), }); @@ -1412,6 +1437,7 @@ async function processWebSocketStream( idleTimeoutMs: number | undefined, websocketConnectTimeoutMs: number | undefined, cacheSessionId: string | undefined, + grammarToolInputProperties: ReadonlyMap, options?: OpenAICodexResponsesOptions, ): Promise { const { socket, entry, reused, release } = await acquireWebSocket( @@ -1458,6 +1484,7 @@ async function processWebSocketStream( model, { serviceTier: options?.serviceTier, + grammarToolInputProperties, resolveServiceTier: resolveCodexServiceTier, applyServiceTierPricing: (usage, serviceTier) => applyServiceTierPricing(usage, serviceTier, model), }, @@ -1467,7 +1494,8 @@ async function processWebSocketStream( } else if (useCachedContext && entry && output.responseId) { const responseItems = convertResponsesMessages(model, { messages: [output] }, CODEX_TOOL_CALL_PROVIDERS, { includeSystemPrompt: false, - }).filter((item) => item.type !== "function_call_output"); + grammarToolInputProperties, + }).filter((item) => item.type !== "function_call_output" && item.type !== "custom_tool_call_output"); entry.continuation = { lastRequestBody: fullBody, lastResponseId: output.responseId, diff --git a/packages/ai/src/api/openai-completions.ts b/packages/ai/src/api/openai-completions.ts index 35b83d405fd..29bb064f9bb 100644 --- a/packages/ai/src/api/openai-completions.ts +++ b/packages/ai/src/api/openai-completions.ts @@ -7,6 +7,7 @@ import type { ChatCompletionContentPartText, ChatCompletionDeveloperMessageParam, ChatCompletionMessageParam, + ChatCompletionMessageToolCall, ChatCompletionSystemMessageParam, ChatCompletionToolMessageParam, } from "openai/resources/chat/completions.js"; @@ -40,6 +41,14 @@ import { parseStreamingJson } from "../utils/json-parse.ts"; import { getProviderEnvValue } from "../utils/provider-env.ts"; import { retryProviderRequest } from "../utils/provider-retry.ts"; import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts"; +import { + appendGrammarToolInputJsonDelta, + createGrammarToolInputProperties, + type GrammarToolInputJsonBuffer, + getGrammarToolInput, + resolveGrammarConstrainedSampling, + resolveJsonSchemaStrictSampling, +} from "./constrained-sampling.ts"; import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts"; import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts"; import { buildBaseOptions } from "./simple-options.ts"; @@ -130,10 +139,14 @@ function isEncryptedReasoningDetail(detail: unknown): detail is OpenAIEncryptedR } export interface OpenAICompletionsOptions extends StreamOptions { - toolChoice?: "auto" | "none" | "required" | { type: "function"; function: { name: string } }; + toolChoice?: OpenAI.Chat.Completions.ChatCompletionToolChoiceOption; reasoningEffort?: "minimal" | "low" | "medium" | "high" | "xhigh" | "max"; } +export interface ConvertCompletionsMessagesOptions { + grammarToolInputProperties?: ReadonlyMap; +} + interface OpenAICompatCacheControl { type: "ephemeral"; ttl?: string; @@ -209,10 +222,14 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio try { const apiKey = getClientApiKey(model.provider, options?.apiKey, options?.headers); const compat = getCompat(model); + const grammarToolInputProperties = createGrammarToolInputProperties( + context.tools, + compat.supportsOpenAIGrammarTools, + ); const cacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env); const cacheSessionId = cacheRetention === "none" ? undefined : options?.sessionId; const client = createClient(model, context, apiKey, options?.headers, cacheSessionId, compat); - let params = buildParams(model, context, options, compat, cacheRetention); + let params = buildParams(model, context, options, compat, cacheRetention, grammarToolInputProperties); const nextParams = await options?.onPayload?.(params, model); if (nextParams !== undefined) { params = nextParams as OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming; @@ -235,10 +252,20 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio interface StreamingToolCallBlock extends ToolCall { partialArgs?: string; + customInput?: { + property: string; + jsonBuffer: GrammarToolInputJsonBuffer; + }; streamIndex?: number; } type StreamingBlock = TextContent | ThinkingContent | StreamingToolCallBlock; - type StreamingToolCallDelta = NonNullable[number]; + type StreamingToolCallDelta = { + index?: number; + id?: string; + type?: string; + function?: { name?: string; arguments?: string }; + custom?: { name?: string; input?: string }; + }; let textBlock: TextContent | null = null; let thinkingBlock: ThinkingContent | null = null; @@ -248,6 +275,28 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio const pendingReasoningDetailsByToolCallId = new Map(); const blocks = output.content as StreamingBlock[]; const getContentIndex = (block: StreamingBlock) => blocks.indexOf(block); + const getCustomToolCallInput = (block: StreamingToolCallBlock): string => { + const property = block.customInput?.property; + if (property === undefined) return ""; + const value = block.arguments[property]; + return typeof value === "string" ? value : ""; + }; + const appendCustomToolCallInput = ( + block: StreamingToolCallBlock, + nextInput: string, + close: boolean, + ): string | undefined => { + const customInput = block.customInput; + if (!customInput) return undefined; + const delta = appendGrammarToolInputJsonDelta( + customInput.jsonBuffer, + customInput.property, + nextInput, + close, + ); + block.arguments = { [customInput.property]: nextInput }; + return delta; + }; const finishBlock = (block: StreamingBlock) => { const contentIndex = getContentIndex(block); if (contentIndex === -1) { @@ -268,10 +317,23 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio partial: output, }); } else if (block.type === "toolCall") { - block.arguments = parseStreamingJson(block.partialArgs); + if (block.customInput) { + const delta = appendCustomToolCallInput(block, getCustomToolCallInput(block), true); + if (delta !== undefined) { + stream.push({ + type: "toolcall_delta", + contentIndex, + delta, + partial: output, + }); + } + } else { + block.arguments = parseStreamingJson(block.partialArgs); + } // Finalize in-place and strip the scratch buffers so replay only // carries parsed arguments. delete block.partialArgs; + delete block.customInput; delete block.streamIndex; stream.push({ type: "toolcall_end", @@ -313,17 +375,27 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio }; const ensureToolCallBlock = (toolCall: StreamingToolCallDelta) => { const streamIndex = typeof toolCall.index === "number" ? toolCall.index : undefined; + const name = toolCall.function?.name ?? toolCall.custom?.name ?? ""; let block = streamIndex !== undefined ? toolCallBlocksByIndex.get(streamIndex) : undefined; if (!block && toolCall.id) { block = toolCallBlocksById.get(toolCall.id); } if (!block) { + // Note: the "input" fallback here should/must not be taken. in case the LLM makes up + // a tool we don't knwo about, we at least have a place to stash our stuff. + const customInputProperty = toolCall.custom + ? (grammarToolInputProperties.get(name) ?? "input") + : undefined; + const hasCustomInput = customInputProperty !== undefined; block = { type: "toolCall", id: toolCall.id || "", - name: toolCall.function?.name || "", - arguments: {}, - partialArgs: "", + name, + arguments: hasCustomInput ? { [customInputProperty]: "" } : {}, + partialArgs: hasCustomInput ? undefined : "", + customInput: hasCustomInput + ? { property: customInputProperty, jsonBuffer: { input: "", started: false, closed: false } } + : undefined, streamIndex, }; if (streamIndex !== undefined) { @@ -346,6 +418,18 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio if (toolCall.id) { toolCallBlocksById.set(toolCall.id, block); } + if (!block.name && name) { + block.name = name; + } + if (toolCall.custom && !block.customInput) { + const customInputProperty = grammarToolInputProperties.get(block.name) ?? "input"; + block.arguments = { [customInputProperty]: "" }; + block.customInput = { + property: customInputProperty, + jsonBuffer: { input: "", started: false, closed: false }, + }; + delete block.partialArgs; + } applyPendingReasoningDetail(block); return block; }; @@ -431,14 +515,15 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio } if (choice?.delta?.tool_calls) { - for (const toolCall of choice.delta.tool_calls) { + for (const toolCall of choice.delta.tool_calls as StreamingToolCallDelta[]) { const block = ensureToolCallBlock(toolCall); if (!block.id && toolCall.id) { block.id = toolCall.id; toolCallBlocksById.set(toolCall.id, block); } - if (!block.name && toolCall.function?.name) { - block.name = toolCall.function.name; + const name = toolCall.function?.name ?? toolCall.custom?.name; + if (!block.name && name) { + block.name = name; } let delta = ""; @@ -446,6 +531,9 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio delta = toolCall.function.arguments; block.partialArgs = (block.partialArgs ?? "") + toolCall.function.arguments; block.arguments = parseStreamingJson(block.partialArgs); + } else if (toolCall.custom?.input) { + const nextInput = getCustomToolCallInput(block) + toolCall.custom.input; + delta = appendCustomToolCallInput(block, nextInput, false) ?? ""; } stream.push({ type: "toolcall_delta", @@ -497,6 +585,7 @@ export const stream: StreamFunction<"openai-completions", OpenAICompletionsOptio delete (block as { index?: number }).index; // Streaming scratch buffers are only used during parsing; never persist them. delete (block as { partialArgs?: string }).partialArgs; + delete (block as { customInput?: unknown }).customInput; delete (block as { streamIndex?: number }).streamIndex; } output.stopReason = options?.signal?.aborted ? "aborted" : "error"; @@ -585,8 +674,12 @@ function buildParams( options?: OpenAICompletionsOptions, compat: ResolvedOpenAICompletionsCompat = getCompat(model), cacheRetention: CacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env), + grammarToolInputProperties: ReadonlyMap = createGrammarToolInputProperties( + context.tools, + compat.supportsOpenAIGrammarTools, + ), ) { - const messages = convertMessages(model, context, compat); + const messages = convertMessages(model, context, compat, { grammarToolInputProperties }); const cacheControl = getCompatCacheControl(compat, cacheRetention); const params: OpenAI.Chat.Completions.ChatCompletionCreateParamsStreaming = { @@ -895,6 +988,7 @@ export function convertMessages( model: Model<"openai-completions">, context: Context, compat: ResolvedOpenAICompletionsCompat, + options?: ConvertCompletionsMessagesOptions, ): ChatCompletionMessageParam[] { const params: ChatCompletionMessageParam[] = []; @@ -1032,14 +1126,27 @@ export function convertMessages( const toolCalls = msg.content.filter(isToolCallBlock); if (toolCalls.length > 0) { - assistantMsg.tool_calls = toolCalls.map((tc) => ({ - id: tc.id, - type: "function" as const, - function: { - name: tc.name, - arguments: JSON.stringify(tc.arguments), - }, - })); + assistantMsg.tool_calls = toolCalls.map((tc): ChatCompletionMessageToolCall => { + const customInputProperty = options?.grammarToolInputProperties?.get(tc.name); + if (customInputProperty !== undefined) { + return { + id: tc.id, + type: "custom", + custom: { + name: tc.name, + input: sanitizeSurrogates(getGrammarToolInput(tc.name, tc.arguments, customInputProperty)), + }, + }; + } + return { + id: tc.id, + type: "function", + function: { + name: tc.name, + arguments: JSON.stringify(tc.arguments), + }, + }; + }); const reasoningDetails = toolCalls .filter((tc) => tc.thoughtSignature) .map((tc) => { @@ -1172,16 +1279,37 @@ function convertTools( tools: Tool[], compat: ResolvedOpenAICompletionsCompat, ): OpenAI.Chat.Completions.ChatCompletionTool[] { - return tools.map((tool) => ({ - type: "function", - function: { - name: tool.name, - description: tool.description, - parameters: tool.parameters as any, // TypeBox already generates JSON Schema - // Only include strict if provider supports it. Some reject unknown fields. - ...(compat.supportsStrictMode !== false && { strict: false }), - }, - })); + return tools.map((tool) => { + const grammar = resolveGrammarConstrainedSampling(tool, compat.supportsOpenAIGrammarTools); + if (grammar) { + return { + type: "custom", + custom: { + name: tool.name, + description: tool.description, + format: { + type: "grammar", + grammar: { + syntax: grammar.format, + definition: grammar.definition, + }, + }, + }, + }; + } + + const strict = resolveJsonSchemaStrictSampling(tool, compat.supportsStrictMode !== false); + return { + type: "function", + function: { + name: tool.name, + description: tool.description, + parameters: tool.parameters as Record, // TypeBox already generates JSON Schema + // Only include strict if provider supports it. Some reject unknown fields. + ...(compat.supportsStrictMode !== false && { strict: strict ?? false }), + }, + }; + }); } function parseChunkUsage( @@ -1324,6 +1452,7 @@ function detectCompat(model: Model<"openai-completions">): ResolvedOpenAIComplet chatTemplateKwargs: {}, zaiToolStream: false, supportsStrictMode: !isMoonshot && !isTogether && !isCloudflareAiGateway && !isNvidia, + supportsOpenAIGrammarTools: false, cacheControlFormat, sendSessionAffinityHeaders: false, deferredToolsMode: undefined, @@ -1365,6 +1494,7 @@ function getCompat(model: Model<"openai-completions">): ResolvedOpenAICompletion chatTemplateKwargs: model.compat.chatTemplateKwargs ?? detected.chatTemplateKwargs, zaiToolStream: model.compat.zaiToolStream ?? detected.zaiToolStream, supportsStrictMode: model.compat.supportsStrictMode ?? detected.supportsStrictMode, + supportsOpenAIGrammarTools: model.compat.supportsOpenAIGrammarTools ?? detected.supportsOpenAIGrammarTools, cacheControlFormat: model.compat.cacheControlFormat ?? detected.cacheControlFormat, sendSessionAffinityHeaders: model.compat.sendSessionAffinityHeaders ?? detected.sendSessionAffinityHeaders, deferredToolsMode: model.compat.deferredToolsMode ?? detected.deferredToolsMode, diff --git a/packages/ai/src/api/openai-responses-shared.ts b/packages/ai/src/api/openai-responses-shared.ts index ddb3e734611..de98cd48dea 100644 --- a/packages/ai/src/api/openai-responses-shared.ts +++ b/packages/ai/src/api/openai-responses-shared.ts @@ -2,7 +2,6 @@ import type OpenAI from "openai"; import type { Tool as OpenAITool, ResponseCreateParamsStreaming, - ResponseFunctionCallOutputItemList, ResponseInput, ResponseInputContent, ResponseInputImage, @@ -33,6 +32,13 @@ import type { AssistantMessageEventStream } from "../utils/event-stream.ts"; import { shortHash } from "../utils/hash.ts"; import { parseStreamingJson } from "../utils/json-parse.ts"; import { sanitizeSurrogates } from "../utils/sanitize-unicode.ts"; +import { + appendGrammarToolInputJsonDelta, + type GrammarToolInputJsonBuffer, + getGrammarToolInput, + resolveGrammarConstrainedSampling, + resolveJsonSchemaStrictSampling, +} from "./constrained-sampling.ts"; import { transformMessages } from "./transform-messages.ts"; // ============================================================================= @@ -65,8 +71,40 @@ function parseTextSignature( return { id: signature }; } +type ToolResultOutputContent = Array; + +function convertToolResultOutput( + model: Model, + content: readonly (TextContent | ImageContent)[], +): string | ToolResultOutputContent { + const textResult = content + .filter((c): c is TextContent => c.type === "text") + .map((c) => c.text) + .join("\n"); + const images = content.filter((c): c is ImageContent => c.type === "image"); + const hasText = textResult.length > 0; + + if (images.length === 0 || !model.input.includes("image")) { + return sanitizeSurrogates(hasText ? textResult : images.length > 0 ? "(see attached image)" : "(no tool output)"); + } + + const output: ToolResultOutputContent = []; + if (hasText) { + output.push({ type: "input_text", text: sanitizeSurrogates(textResult) }); + } + for (const image of images) { + output.push({ + type: "input_image", + detail: "auto", + image_url: `data:${image.mimeType};base64,${image.data}`, + }); + } + return output; +} + export interface OpenAIResponsesStreamOptions { serviceTier?: ResponseCreateParamsStreaming["service_tier"]; + grammarToolInputProperties?: ReadonlyMap; resolveServiceTier?: ( responseServiceTier: ResponseCreateParamsStreaming["service_tier"] | undefined, requestServiceTier: ResponseCreateParamsStreaming["service_tier"] | undefined, @@ -79,16 +117,18 @@ export interface OpenAIResponsesStreamOptions { export interface ConvertResponsesMessagesOptions { includeSystemPrompt?: boolean; + grammarToolInputProperties?: ReadonlyMap; deferredTools?: ReadonlyMap; + toolOptions?: ConvertResponsesToolsOptions; } export interface ConvertResponsesToolsOptions { strict?: boolean | null; + supportsStrictMode?: boolean; + supportsOpenAIGrammarTools?: boolean; deferLoading?: boolean; } -type OpenAIFunctionTool = Extract; - // ============================================================================= // Message conversion // ============================================================================= @@ -206,67 +246,62 @@ export function convertResponsesMessages( } else if (block.type === "toolCall") { const toolCall = block as ToolCall; const [callId, itemIdRaw] = toolCall.id.split("|"); + const customInputProperty = options?.grammarToolInputProperties?.get(toolCall.name); let itemId: string | undefined = itemIdRaw; // For different-model messages, set id to undefined to avoid pairing validation. // OpenAI tracks which fc_xxx IDs were paired with rs_xxx reasoning items. // By omitting the id, we avoid triggering that validation (like cross-provider does). - if (isDifferentModel && itemId?.startsWith("fc_")) { + // When replaying custom-tool calls as a function_call, also drop non-fc_* ids such as + // ctc_* custom-tool ids because function_call item ids must be fc_*. + if ( + (isDifferentModel && itemId?.startsWith("fc_")) || + (customInputProperty === undefined && !itemId?.startsWith("fc_")) + ) { itemId = undefined; } - output.push({ - type: "function_call", - id: itemId, - call_id: callId, - name: toolCall.name, - arguments: JSON.stringify(toolCall.arguments), - }); + if (customInputProperty !== undefined) { + output.push({ + type: "custom_tool_call", + id: itemId, + call_id: callId, + name: toolCall.name, + input: sanitizeSurrogates( + getGrammarToolInput(toolCall.name, toolCall.arguments, customInputProperty), + ), + } satisfies ResponseOutputItem); + } else { + output.push({ + type: "function_call", + id: itemId, + call_id: callId, + name: toolCall.name, + arguments: JSON.stringify(toolCall.arguments), + }); + } } } if (output.length === 0) continue; messages.push(...output); } else if (msg.role === "toolResult") { - const textResult = msg.content - .filter((c): c is TextContent => c.type === "text") - .map((c) => c.text) - .join("\n"); - const hasImages = msg.content.some((c): c is ImageContent => c.type === "image"); - const hasText = textResult.length > 0; const [callId] = msg.toolCallId.split("|"); + const output = convertToolResultOutput(model, msg.content); - let output: string | ResponseFunctionCallOutputItemList; - if (hasImages && model.input.includes("image")) { - const contentParts: ResponseFunctionCallOutputItemList = []; - - if (hasText) { - contentParts.push({ - type: "input_text", - text: sanitizeSurrogates(textResult), - }); - } - - for (const block of msg.content) { - if (block.type === "image") { - contentParts.push({ - type: "input_image", - detail: "auto", - image_url: `data:${block.mimeType};base64,${block.data}`, - }); - } - } - - output = contentParts; + if (options?.grammarToolInputProperties?.has(msg.toolName)) { + messages.push({ + type: "custom_tool_call_output", + call_id: callId, + output, + }); } else { - output = sanitizeSurrogates(hasText ? textResult : hasImages ? "(see attached image)" : "(no tool output)"); + messages.push({ + type: "function_call_output", + call_id: callId, + output, + }); } - messages.push({ - type: "function_call_output", - call_id: callId, - output, - }); - const deferredTools: Tool[] = []; for (const name of msg.addedToolNames ?? []) { const tool = options?.deferredTools?.get(name); @@ -289,7 +324,10 @@ export function convertResponsesMessages( call_id: searchCallId, execution: "client", status: "completed", - tools: convertResponsesTools(deferredTools, { deferLoading: true }), + tools: convertResponsesTools(deferredTools, { + ...options?.toolOptions, + deferLoading: true, + }), } satisfies ResponseToolSearchOutputItemParam); } } @@ -304,30 +342,77 @@ export function convertResponsesMessages( // ============================================================================= export function convertResponsesTools(tools: readonly Tool[], options?: ConvertResponsesToolsOptions): OpenAITool[] { - const strict = options?.strict === undefined ? false : options.strict; - return tools.map( - (tool): OpenAIFunctionTool => ({ + const defaultStrict = options?.strict === undefined ? false : options.strict; + const supportsStrictMode = options?.supportsStrictMode ?? true; + const supportsOpenAIGrammarTools = options?.supportsOpenAIGrammarTools ?? false; + + return tools.map((tool) => { + const grammar = resolveGrammarConstrainedSampling(tool, supportsOpenAIGrammarTools); + if (grammar) { + return { + type: "custom", + name: tool.name, + description: tool.description, + format: { + type: "grammar", + syntax: grammar.format, + definition: grammar.definition, + }, + ...(options?.deferLoading ? { defer_loading: true } : {}), + } satisfies OpenAITool; + } + + const constrainedStrict = resolveJsonSchemaStrictSampling(tool, supportsStrictMode); + const functionTool: Omit, "strict"> & { + strict?: Extract["strict"]; + } = { type: "function", name: tool.name, description: tool.description, parameters: tool.parameters as Record, // TypeBox already generates JSON Schema - strict, ...(options?.deferLoading ? { defer_loading: true } : {}), - }), - ); + }; + if (supportsStrictMode) { + functionTool.strict = constrainedStrict ?? defaultStrict; + } + return functionTool as OpenAITool; + }); } // ============================================================================= // Stream processing // ============================================================================= -type StreamingToolCall = ToolCall & { partialJson: string }; +type StreamingToolCall = ToolCall & { + partialJson?: string; + customInput?: { + property: string; + jsonBuffer: GrammarToolInputJsonBuffer; + }; +}; + +function getCustomToolCallInput(block: StreamingToolCall): string { + const property = block.customInput?.property; + if (property === undefined) return ""; + const value = block.arguments[property]; + return typeof value === "string" ? value : ""; +} + +function appendCustomToolCallInput(block: StreamingToolCall, nextInput: string, close: boolean): string | undefined { + const customInput = block.customInput; + if (!customInput) return undefined; + const delta = appendGrammarToolInputJsonDelta(customInput.jsonBuffer, customInput.property, nextInput, close); + block.arguments = { [customInput.property]: nextInput }; + return delta; +} type ResponsesOutputSlot = | { type: "thinking"; block: ThinkingContent; contentIndex: number } | { type: "text"; block: TextContent; contentIndex: number } | { type: "toolCall"; block: StreamingToolCall; contentIndex: number }; +type ToolCallOutputSlot = Extract; + export async function processResponsesStream( openaiStream: AsyncIterable, output: AssistantMessage, @@ -345,6 +430,15 @@ export async function processResponsesStream( const slot = outputSlots.get(outputIndex); return slot?.type === type ? (slot as Extract) : undefined; }; + const pushToolCallDelta = (slot: ToolCallOutputSlot, delta: string | undefined): void => { + if (delta === undefined) return; + stream.push({ + type: "toolcall_delta", + contentIndex: slot.contentIndex, + delta, + partial: output, + }); + }; const createSlot = (outputIndex: number, item: ResponseOutputItem): ResponsesOutputSlot | undefined => { if (item.type === "reasoning") { const block: ThinkingContent = { type: "thinking", thinking: "" }; @@ -384,6 +478,29 @@ export async function processResponsesStream( stream.push({ type: "toolcall_start", contentIndex: slot.contentIndex, partial: output }); return slot; } + if (item.type === "custom_tool_call") { + const inputProperty = options?.grammarToolInputProperties?.get(item.name) ?? "input"; + const input = item.input || ""; + const block: StreamingToolCall = { + type: "toolCall", + id: `${item.call_id}|${item.id}`, + name: item.name, + arguments: { [inputProperty]: input }, + customInput: { + property: inputProperty, + jsonBuffer: { input: "", started: false, closed: false }, + }, + }; + output.content.push(block); + const slot = { + type: "toolCall", + block, + contentIndex: output.content.length - 1, + } satisfies ResponsesOutputSlot; + outputSlots.set(outputIndex, slot); + stream.push({ type: "toolcall_start", contentIndex: slot.contentIndex, partial: output }); + return slot; + } return undefined; }; const getOrCreateSlot = (outputIndex: number, item: ResponseOutputItem): ResponsesOutputSlot | undefined => { @@ -503,33 +620,32 @@ export async function processResponsesStream( }); } else if (event.type === "response.function_call_arguments.delta") { const slot = getSlot(event.output_index, "toolCall"); - if (!slot) continue; + if (!slot || slot.block.partialJson === undefined) continue; slot.block.partialJson += event.delta; slot.block.arguments = parseStreamingJson(slot.block.partialJson); - stream.push({ - type: "toolcall_delta", - contentIndex: slot.contentIndex, - delta: event.delta, - partial: output, - }); + pushToolCallDelta(slot, event.delta); } else if (event.type === "response.function_call_arguments.done") { const slot = getSlot(event.output_index, "toolCall"); - if (!slot) continue; + if (!slot || slot.block.partialJson === undefined) continue; const previousPartialJson = slot.block.partialJson; slot.block.partialJson = event.arguments; slot.block.arguments = parseStreamingJson(slot.block.partialJson); if (event.arguments.startsWith(previousPartialJson)) { const delta = event.arguments.slice(previousPartialJson.length); - if (delta.length > 0) { - stream.push({ - type: "toolcall_delta", - contentIndex: slot.contentIndex, - delta, - partial: output, - }); - } + if (delta.length > 0) pushToolCallDelta(slot, delta); } + } else if (event.type === "response.custom_tool_call_input.delta") { + const slot = getSlot(event.output_index, "toolCall"); + if (!slot || !slot.block.customInput) continue; + pushToolCallDelta( + slot, + appendCustomToolCallInput(slot.block, getCustomToolCallInput(slot.block) + event.delta, false), + ); + } else if (event.type === "response.custom_tool_call_input.done") { + const slot = getSlot(event.output_index, "toolCall"); + if (!slot || !slot.block.customInput) continue; + pushToolCallDelta(slot, appendCustomToolCallInput(slot.block, event.input, true)); } else if (event.type === "response.output_item.done") { const item = event.item; const slot = getOrCreateSlot(event.output_index, item); @@ -557,11 +673,28 @@ export async function processResponsesStream( partial: output, }); outputSlots.delete(event.output_index); - } else if (item.type === "function_call" && slot?.type === "toolCall") { + } else if ( + item.type === "function_call" && + slot?.type === "toolCall" && + slot.block.partialJson !== undefined + ) { slot.block.arguments = parseStreamingJson(item.arguments || slot.block.partialJson || "{}"); // Finalize in-place and strip the scratch buffer so replay only // carries parsed arguments. - delete (slot.block as { partialJson?: string }).partialJson; + delete slot.block.partialJson; + stream.push({ + type: "toolcall_end", + contentIndex: slot.contentIndex, + toolCall: slot.block, + partial: output, + }); + outputSlots.delete(event.output_index); + } else if (item.type === "custom_tool_call" && slot?.type === "toolCall" && slot.block.customInput) { + pushToolCallDelta( + slot, + appendCustomToolCallInput(slot.block, item.input ?? getCustomToolCallInput(slot.block), true), + ); + delete slot.block.customInput; stream.push({ type: "toolcall_end", contentIndex: slot.contentIndex, diff --git a/packages/ai/src/api/openai-responses.ts b/packages/ai/src/api/openai-responses.ts index ad19046dcae..59f92a25cd8 100644 --- a/packages/ai/src/api/openai-responses.ts +++ b/packages/ai/src/api/openai-responses.ts @@ -21,6 +21,7 @@ import { AssistantMessageEventStream } from "../utils/event-stream.ts"; import { headersToRecord } from "../utils/headers.ts"; import { getProviderEnvValue } from "../utils/provider-env.ts"; import { retryProviderRequest } from "../utils/provider-retry.ts"; +import { createGrammarToolInputProperties } from "./constrained-sampling.ts"; import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.ts"; import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.ts"; import { convertResponsesMessages, convertResponsesTools, processResponsesStream } from "./openai-responses-shared.ts"; @@ -68,6 +69,8 @@ function getCompat(model: Model<"openai-responses">): Required const apiKey = getClientApiKey(model.provider, options?.apiKey, options?.headers); const cacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env); const cacheSessionId = cacheRetention === "none" ? undefined : options?.sessionId; + const compat = getCompat(model); + const grammarToolInputProperties = createGrammarToolInputProperties( + context.tools, + compat.supportsOpenAIGrammarTools, + ); const client = createClient(model, context, apiKey, options?.headers, cacheSessionId); - let params = buildParams(model, context, options); + let params = buildParams(model, context, options, compat, grammarToolInputProperties); const nextParams = await options?.onPayload?.(params, model); if (nextParams !== undefined) { params = nextParams as ResponseCreateParamsStreaming; @@ -151,6 +159,7 @@ export const stream: StreamFunction<"openai-responses", OpenAIResponsesOptions> await processResponsesStream(openaiStream, output, stream, model, { serviceTier: options?.serviceTier, + grammarToolInputProperties, applyServiceTierPricing: (usage, serviceTier) => applyServiceTierPricing(usage, serviceTier, model), }); @@ -167,8 +176,9 @@ export const stream: StreamFunction<"openai-responses", OpenAIResponsesOptions> } catch (error) { for (const block of output.content) { delete (block as { index?: number }).index; - // partialJson is only a streaming scratch buffer; never persist it. + // Streaming scratch buffers are only used during parsing; never persist them. delete (block as { partialJson?: string }).partialJson; + delete (block as { customInput?: unknown }).customInput; } output.stopReason = options?.signal?.aborted ? "aborted" : "error"; output.errorMessage = formatOpenAIResponsesError(error); @@ -239,11 +249,24 @@ function createClient( }); } -function buildParams(model: Model<"openai-responses">, context: Context, options?: OpenAIResponsesOptions) { - const compat = getCompat(model); +function buildParams( + model: Model<"openai-responses">, + context: Context, + options: OpenAIResponsesOptions | undefined, + compat: Required = getCompat(model), + grammarToolInputProperties: ReadonlyMap = createGrammarToolInputProperties( + context.tools, + compat.supportsOpenAIGrammarTools, + ), +) { const toolPlacement = splitDeferredTools(context, compat.supportsToolSearch); const messages = convertResponsesMessages(model, context, OPENAI_TOOL_CALL_PROVIDERS, { + grammarToolInputProperties, deferredTools: toolPlacement.deferred, + toolOptions: { + supportsStrictMode: compat.supportsStrictMode, + supportsOpenAIGrammarTools: compat.supportsOpenAIGrammarTools, + }, }); const cacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env); @@ -271,7 +294,10 @@ function buildParams(model: Model<"openai-responses">, context: Context, options } if (toolPlacement.immediate.length > 0) { - params.tools = convertResponsesTools(toolPlacement.immediate); + params.tools = convertResponsesTools(toolPlacement.immediate, { + supportsStrictMode: compat.supportsStrictMode, + supportsOpenAIGrammarTools: compat.supportsOpenAIGrammarTools, + }); } if (options?.toolChoice !== undefined) { diff --git a/packages/ai/src/types.ts b/packages/ai/src/types.ts index 36ace9d0a74..1d8fcfd3af2 100644 --- a/packages/ai/src/types.ts +++ b/packages/ai/src/types.ts @@ -445,10 +445,33 @@ export interface AssistantImages { import type { TSchema } from "typebox"; +/** OpenAI grammar variants for constrained sampling. */ +export type GrammarFormat = "openai_lark" | "openai_regex"; + +export type GrammarVariants = Partial>; + +/** + * Optional provider-side constrained sampling configs for a tool. + * + * The `json_schema` value roughly maps to the concept of `strict` in APIs which is + * implemented as json-schema constrained sampling by APIs. Grammar variants let + * callers provide provider-specific encodings of the same intended language. + */ +export type ConstrainedSamplingConfig = + | { + type: "json_schema"; + strict: "prefer" | "require"; + } + | { + type: "grammar"; + variants: GrammarVariants; + }; + export interface Tool { name: string; description: string; parameters: TParameters; + constrainedSampling?: false | ConstrainedSamplingConfig; } export interface Context { @@ -522,6 +545,8 @@ export interface OpenAICompletionsCompat { vercelGatewayRouting?: VercelGatewayRouting; /** Whether z.ai supports top-level `tool_stream: true` for streaming tool call deltas. Default: false. */ zaiToolStream?: boolean; + /** Whether the provider supports OpenAI custom tools with Lark/regex grammar formats. When false, grammar-constrained tools fall back to normal function tools. Default: false; the generated model catalog enables it for capable models. */ + supportsOpenAIGrammarTools?: boolean; /** Whether the provider supports the `strict` field in tool definitions. Default: true. */ supportsStrictMode?: boolean; /** Cache control convention for prompt caching. "anthropic" applies Anthropic-style `cache_control` markers to the system prompt, last tool definition, and last user, assistant, or tool-result text content. */ @@ -544,6 +569,10 @@ export interface OpenAIResponsesCompat { sessionAffinityFormat?: SessionAffinityFormat; /** Whether the provider supports `prompt_cache_retention: "24h"`. Default: true. */ supportsLongCacheRetention?: boolean; + /** Whether the provider supports strict JSON-schema function tools. Defaults are API-specific; generated OpenAI models enable it explicitly. */ + supportsStrictMode?: boolean; + /** Whether to emit OpenAI custom tools with Lark/regex grammar formats. When false, grammar-constrained tools fall back to normal function tools. Default: false; the generated model catalog enables it for capable models. */ + supportsOpenAIGrammarTools?: boolean; /** Whether the model supports client-executed tool search for deferred tools. Default: false. */ supportsToolSearch?: boolean; /** Whether the model accepts `prompt_cache_options` (OpenAI GPT-5.6+ explicit prompt caching). Older OpenAI models reject the parameter. Default: false. */ @@ -596,6 +625,8 @@ export interface AnthropicMessagesCompat { forceAdaptiveThinking?: boolean; /** Whether to replay empty thinking signatures as `signature: ""` instead of converting thinking to text. Default: false. */ allowEmptySignature?: boolean; + /** Whether the provider supports Anthropic strict tool schemas. Default: false; generated Anthropic models enable it explicitly. */ + supportsStrictTools?: boolean; /** * Whether the provider supports deferred tools loaded by `tool_reference` * blocks in tool results. Default: true for first-party Anthropic models @@ -604,6 +635,12 @@ export interface AnthropicMessagesCompat { supportsToolReferences?: boolean; } +/** Compatibility settings for Amazon Bedrock models. */ +export interface BedrockCompat { + /** Whether the model supports Bedrock strict tool schemas. Default: false. */ + supportsStrictMode?: boolean; +} + /** * OpenRouter provider routing preferences. * Controls which upstream providers OpenRouter routes requests to. @@ -729,11 +766,13 @@ export interface Model { /** Compatibility overrides for OpenAI-compatible APIs. If not set, auto-detected from baseUrl. */ compat?: TApi extends "openai-completions" ? OpenAICompletionsCompat - : TApi extends "openai-responses" | "openai-codex-responses" + : TApi extends "openai-responses" | "azure-openai-responses" | "openai-codex-responses" ? OpenAIResponsesCompat : TApi extends "anthropic-messages" ? AnthropicMessagesCompat - : never; + : TApi extends "bedrock-converse-stream" + ? BedrockCompat + : never; } export interface ImagesModel diff --git a/packages/ai/test/anthropic-eager-tool-input-compat.test.ts b/packages/ai/test/anthropic-eager-tool-input-compat.test.ts index c53c9d6b2b9..39be3a9a1b4 100644 --- a/packages/ai/test/anthropic-eager-tool-input-compat.test.ts +++ b/packages/ai/test/anthropic-eager-tool-input-compat.test.ts @@ -32,6 +32,17 @@ const tool: Tool = { parameters: Type.Object({ value: Type.String() }), }; +const schemaCompatibilityTool: Tool = { + ...tool, + parameters: Type.Object({ value: Type.String() }, { additionalProperties: false, title: "LookupInput" }), +}; + +const strictTool: Tool = { + ...tool, + parameters: Type.Object({ value: Type.String() }, { additionalProperties: false, title: "StrictLookupInput" }), + constrainedSampling: { type: "json_schema", strict: "prefer" }, +}; + function createContext(tools: Tool[] = [tool]): Context { return { messages: [{ role: "user", content: "Use the tool", timestamp: Date.now() }], @@ -98,6 +109,14 @@ function getFirstTool(body: Record): Record { return tools[0] as Record; } +function getFirstToolInputSchema(body: Record): Record { + const inputSchema = getFirstTool(body).input_schema; + if (typeof inputSchema !== "object" || inputSchema === null || Array.isArray(inputSchema)) { + throw new Error("Expected first tool input schema in request body"); + } + return inputSchema as Record; +} + describe("Anthropic eager tool input streaming compatibility", () => { it("sends per-tool eager_input_streaming by default", async () => { const request = await captureAnthropicRequest(undefined, createContext()); @@ -119,4 +138,24 @@ describe("Anthropic eager tool input streaming compatibility", () => { expect(request.body.tools).toBeUndefined(); expect(request.headers["anthropic-beta"]).toBeUndefined(); }); + + it("only sends the full input schema for strict JSON-schema tools", async () => { + const legacyRequest = await captureAnthropicRequest( + { supportsStrictTools: true }, + createContext([schemaCompatibilityTool]), + ); + const parameters = schemaCompatibilityTool.parameters as { properties?: unknown; required?: unknown }; + expect(getFirstToolInputSchema(legacyRequest.body)).toEqual({ + type: "object", + properties: parameters.properties, + required: parameters.required, + }); + + const strictRequest = await captureAnthropicRequest({ supportsStrictTools: true }, createContext([strictTool])); + expect(getFirstTool(strictRequest.body).strict).toBe(true); + expect(getFirstToolInputSchema(strictRequest.body)).toMatchObject({ + additionalProperties: false, + title: "StrictLookupInput", + }); + }); }); diff --git a/packages/ai/test/azure-openai-base-url.test.ts b/packages/ai/test/azure-openai-base-url.test.ts index 5908ad3828f..12e3fb25e6a 100644 --- a/packages/ai/test/azure-openai-base-url.test.ts +++ b/packages/ai/test/azure-openai-base-url.test.ts @@ -1,7 +1,8 @@ +import { Type } from "typebox"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { stream as streamAzureOpenAIResponses } from "../src/api/azure-openai-responses.ts"; import { getModel } from "../src/compat.ts"; -import type { Context } from "../src/types.ts"; +import type { Context, Model } from "../src/types.ts"; interface CapturedAzureClientOptions { apiKey: string; @@ -14,6 +15,7 @@ interface CapturedAzureClientOptions { interface CapturedAzureResponsesPayload { prompt_cache_key?: string; store?: boolean; + tools?: Array<{ strict?: boolean }>; } const azureMock = vi.hoisted(() => ({ @@ -165,6 +167,32 @@ describe("azure-openai-responses base URL normalization", () => { expect(azureMock.lastParams?.store).toBe(false); }); + it("honors supportsStrictMode: false", async () => { + const baseModel = getModel("azure-openai-responses", "gpt-4o-mini"); + const model: Model<"azure-openai-responses"> = { + ...baseModel, + compat: { ...baseModel.compat, supportsStrictMode: false }, + }; + + await streamAzureOpenAIResponses( + model, + { + ...context, + tools: [ + { + name: "preferred", + description: "Preferred constrained tool", + parameters: Type.Object({ value: Type.String() }), + constrainedSampling: { type: "json_schema", strict: "prefer" }, + }, + ], + }, + { apiKey: "test-api-key", azureBaseUrl: "https://my-resource.openai.azure.com" }, + ).result(); + + expect(azureMock.lastParams?.tools?.[0]).not.toHaveProperty("strict"); + }); + it("builds correct default URL from AZURE_OPENAI_RESOURCE_NAME", async () => { process.env.AZURE_OPENAI_RESOURCE_NAME = "my-resource"; const model = getModel("azure-openai-responses", "gpt-4o-mini"); diff --git a/packages/ai/test/bedrock-convert-messages.test.ts b/packages/ai/test/bedrock-convert-messages.test.ts index c43f7978de8..5d5b7c319e0 100644 --- a/packages/ai/test/bedrock-convert-messages.test.ts +++ b/packages/ai/test/bedrock-convert-messages.test.ts @@ -1,3 +1,4 @@ +import { Type } from "typebox"; import { describe, expect, it, vi } from "vitest"; const bedrockMock = vi.hoisted(() => ({ @@ -50,9 +51,9 @@ import type { Context, Message } from "../src/types.ts"; const baseModel = getModel("amazon-bedrock", "us.anthropic.claude-sonnet-4-5-20250929-v1:0"); -async function capturePayload(context: Context): Promise { +async function capturePayload(context: Context, model = baseModel): Promise { let capturedPayload: unknown; - const s = streamBedrock(baseModel, context, { + const s = streamBedrock(model, context, { cacheRetention: "none", signal: AbortSignal.abort(), onPayload: (payload) => { @@ -66,6 +67,34 @@ async function capturePayload(context: Context): Promise { return capturedPayload; } +describe("Bedrock constrained sampling", () => { + it("gates native strict tool use by model capability", async () => { + const context: Context = { + messages: [{ role: "user", content: "Use the tool", timestamp: Date.now() }], + tools: [ + { + name: "lookup", + description: "Look up a value", + parameters: Type.Object({ value: Type.String() }), + constrainedSampling: { type: "json_schema", strict: "require" }, + }, + ], + }; + const payload = await capturePayload(context); + const toolConfig = (payload as { toolConfig: { tools: Array<{ toolSpec: { strict?: boolean } }> } }).toolConfig; + expect(toolConfig.tools[0].toolSpec.strict).toBe(true); + + context.tools![0].constrainedSampling = { type: "json_schema", strict: "prefer" }; + const novaPayload = await capturePayload(context, getModel("amazon-bedrock", "amazon.nova-lite-v1:0")); + const novaToolConfig = ( + novaPayload as { + toolConfig: { tools: Array<{ toolSpec: { strict?: boolean } }> }; + } + ).toolConfig; + expect(novaToolConfig.tools[0].toolSpec.strict).toBeUndefined(); + }); +}); + describe("bedrock convertMessages skips unknown content types", () => { it("skips unknown user content blocks instead of throwing", async () => { const messages: Message[] = [ diff --git a/packages/ai/test/constrained-sampling.test.ts b/packages/ai/test/constrained-sampling.test.ts new file mode 100644 index 00000000000..244bcbcb75b --- /dev/null +++ b/packages/ai/test/constrained-sampling.test.ts @@ -0,0 +1,229 @@ +import type { ResponseStreamEvent } from "openai/resources/responses/responses.js"; +import { Type } from "typebox"; +import { describe, expect, it } from "vitest"; +import { appendGrammarToolInputJsonDelta } from "../src/api/constrained-sampling.ts"; +import { + convertResponsesMessages, + convertResponsesTools, + processResponsesStream, +} from "../src/api/openai-responses-shared.ts"; +import type { AssistantMessage, Context, Model, Tool, ToolCall } from "../src/types.ts"; +import { AssistantMessageEventStream } from "../src/utils/event-stream.ts"; + +function makeModel(): Model<"openai-responses"> { + return { + id: "gpt-test", + name: "GPT Test", + api: "openai-responses", + provider: "openai", + baseUrl: "https://api.openai.com/v1", + reasoning: false, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128000, + maxTokens: 4096, + }; +} + +function makeUsage(): AssistantMessage["usage"] { + return { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }; +} + +function makeOutput(): AssistantMessage { + return { + role: "assistant", + content: [], + api: "openai-responses", + provider: "openai", + model: "gpt-test", + usage: makeUsage(), + stopReason: "stop", + timestamp: Date.now(), + }; +} + +async function* iterateEvents(events: ResponseStreamEvent[]): AsyncGenerator { + yield* events; +} + +function makeTool(overrides: Partial = {}): Tool { + return { + name: "sample_tool", + description: "Sample tool", + parameters: Type.Object({ payload: Type.String() }, { additionalProperties: false }), + ...overrides, + }; +} + +function captureToolCallDeltas(stream: AssistantMessageEventStream): string[] { + const deltas: string[] = []; + const originalPush = stream.push.bind(stream); + stream.push = (event) => { + if (event.type === "toolcall_delta") { + deltas.push(event.delta); + } + originalPush(event); + }; + return deltas; +} + +describe("constrained tool sampling", () => { + it("converts supported constraints and falls back when unsupported", () => { + expect( + convertResponsesTools([makeTool({ constrainedSampling: { type: "json_schema", strict: "prefer" } })])[0], + ).toMatchObject({ type: "function", name: "sample_tool", strict: true }); + + expect(() => + convertResponsesTools([makeTool({ constrainedSampling: { type: "json_schema", strict: "require" } })], { + supportsStrictMode: false, + }), + ).toThrow('Tool "sample_tool" requires JSON-schema constrained sampling'); + + const grammarTool = makeTool({ + constrainedSampling: { type: "grammar", variants: { openai_lark: "start: /[a-z]+/" } }, + }); + expect(convertResponsesTools([grammarTool], { supportsOpenAIGrammarTools: true })[0]).toMatchObject({ + type: "custom", + name: "sample_tool", + format: { type: "grammar", syntax: "lark", definition: "start: /[a-z]+/" }, + }); + expect(() => + convertResponsesTools([makeTool({ constrainedSampling: { type: "grammar", variants: {} } })], { + supportsOpenAIGrammarTools: true, + }), + ).toThrow( + 'Tool "sample_tool" cannot use grammar constrained sampling: no supported grammar variant was provided', + ); + + const fallback = convertResponsesTools([grammarTool], { + supportsOpenAIGrammarTools: false, + supportsStrictMode: false, + })[0]; + expect(fallback).toMatchObject({ type: "function", name: "sample_tool" }); + expect("strict" in (fallback as object)).toBe(false); + + expect(convertResponsesTools([makeTool({ constrainedSampling: false })])).toEqual( + convertResponsesTools([makeTool()]), + ); + }); + + it("replays grammar calls as custom Responses items", () => { + const replayedToolCall: ToolCall = { + type: "toolCall", + id: "call_1|ctc_1", + name: "sample_tool", + arguments: { payload: "abc" }, + }; + const context: Context = { + messages: [ + { + role: "assistant", + api: "openai-responses", + provider: "openai", + model: "gpt-test", + content: [replayedToolCall], + usage: makeUsage(), + stopReason: "toolUse", + timestamp: Date.now(), + }, + { + role: "toolResult", + toolCallId: "call_1|ctc_1", + toolName: "sample_tool", + content: [{ type: "text", text: "done" }], + isError: false, + timestamp: Date.now(), + }, + ], + }; + for (const invalidArguments of [{}, { payload: 42 }]) { + replayedToolCall.arguments = invalidArguments; + expect(() => + convertResponsesMessages(makeModel(), context, new Set(["openai"]), { + grammarToolInputProperties: new Map([["sample_tool", "payload"]]), + }), + ).toThrow('Grammar tool call "sample_tool" requires argument "payload" to be a string'); + } + + replayedToolCall.arguments = { payload: "abc" }; + const messages = convertResponsesMessages(makeModel(), context, new Set(["openai"]), { + grammarToolInputProperties: new Map([["sample_tool", "payload"]]), + }); + + expect(messages).toContainEqual({ + type: "custom_tool_call", + id: "ctc_1", + call_id: "call_1", + name: "sample_tool", + input: "abc", + }); + expect(messages).toContainEqual({ + type: "custom_tool_call_output", + call_id: "call_1", + output: "done", + }); + }); + + it("keeps grammar input JSON deltas append-only", () => { + const buffer = { input: "", started: false, closed: false }; + const first = appendGrammarToolInputJsonDelta(buffer, "payload", 'a"', false); + const second = appendGrammarToolInputJsonDelta(buffer, "payload", 'a"\nb', true); + + expect(JSON.parse(`${first}${second}`)).toEqual({ payload: 'a"\nb' }); + expect(appendGrammarToolInputJsonDelta(buffer, "payload", 'a"\nb', true)).toBeUndefined(); + expect(() => appendGrammarToolInputJsonDelta(buffer, "payload", "changed", true)).toThrow( + 'grammar tool input for property "payload" changed after it was closed', + ); + }); + + it("streams custom Responses tool calls as string arguments", async () => { + const output = makeOutput(); + const stream = new AssistantMessageEventStream(); + const deltas = captureToolCallDeltas(stream); + const events = [ + { + type: "response.output_item.added", + output_index: 0, + item: { type: "custom_tool_call", call_id: "call_1", id: "ctc_1", name: "sample_tool", input: "" }, + }, + { + type: "response.custom_tool_call_input.delta", + output_index: 0, + item_id: "ctc_1", + delta: "ab", + }, + { + type: "response.custom_tool_call_input.done", + output_index: 0, + item_id: "ctc_1", + input: "abc", + }, + { + type: "response.output_item.done", + output_index: 0, + item: { type: "custom_tool_call", call_id: "call_1", id: "ctc_1", name: "sample_tool", input: "abc" }, + }, + { + type: "response.completed", + response: { status: "completed", usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 } }, + }, + ] as ResponseStreamEvent[]; + + await processResponsesStream(iterateEvents(events), output, stream, makeModel(), { + grammarToolInputProperties: new Map([["sample_tool", "payload"]]), + }); + + expect(output.stopReason).toBe("toolUse"); + expect(output.content).toEqual([ + { type: "toolCall", id: "call_1|ctc_1", name: "sample_tool", arguments: { payload: "abc" } }, + ]); + expect(JSON.parse(deltas.join(""))).toEqual({ payload: "abc" }); + }); +}); diff --git a/packages/ai/test/deferred-tools.test.ts b/packages/ai/test/deferred-tools.test.ts index 095c2fb95fc..9d6404c07c9 100644 --- a/packages/ai/test/deferred-tools.test.ts +++ b/packages/ai/test/deferred-tools.test.ts @@ -369,6 +369,7 @@ describe("deferred tools", () => { chatTemplateKwargs: {}, zaiToolStream: false, supportsStrictMode: false, + supportsOpenAIGrammarTools: false, cacheControlFormat: undefined, sendSessionAffinityHeaders: false, deferredToolsMode: "kimi", diff --git a/packages/ai/test/google-shared-convert-tools.test.ts b/packages/ai/test/google-shared-convert-tools.test.ts index d91bcfa0f8c..82876e5fe53 100644 --- a/packages/ai/test/google-shared-convert-tools.test.ts +++ b/packages/ai/test/google-shared-convert-tools.test.ts @@ -1,5 +1,9 @@ import { describe, expect, it } from "vitest"; -import { convertTools } from "../src/api/google-shared.ts"; +import { + convertTools, + resolveGoogleFunctionCallingMode, + supportsGoogleStrictToolSampling, +} from "../src/api/google-shared.ts"; import type { Tool } from "../src/types.ts"; function makeTool(parameters: Record): Tool { @@ -180,6 +184,18 @@ describe("google-shared convertTools", () => { }); }); + it("uses validated function calling for strict tools on Gemini 3", () => { + const tool = makeTool({ type: "object", properties: {} }); + tool.constrainedSampling = { type: "json_schema", strict: "require" }; + + expect(supportsGoogleStrictToolSampling("gemini-3.1-pro-preview")).toBe(true); + expect(supportsGoogleStrictToolSampling("gemini-2.5-pro")).toBe(false); + expect(resolveGoogleFunctionCallingMode([tool], undefined, true)).toBe("VALIDATED"); + expect(() => resolveGoogleFunctionCallingMode([tool], undefined, false)).toThrow( + 'Tool "test_tool" requires JSON-schema constrained sampling', + ); + }); + it("returns undefined for empty tool list", () => { expect(convertTools([])).toBeUndefined(); expect(convertTools([], true)).toBeUndefined(); diff --git a/packages/ai/test/mistral-tool-schema.test.ts b/packages/ai/test/mistral-tool-schema.test.ts index 7691775a97c..6f3bac16e7e 100644 --- a/packages/ai/test/mistral-tool-schema.test.ts +++ b/packages/ai/test/mistral-tool-schema.test.ts @@ -9,6 +9,7 @@ interface MistralToolPayload { function: { name: string; parameters: Record; + strict?: boolean; }; }>; } @@ -31,6 +32,7 @@ describe("Mistral tool schema serialization", () => { name: "inspect_schema", description: "Inspect the schema", parameters, + constrainedSampling: { type: "json_schema", strict: "require" }, }, ], }; @@ -45,6 +47,7 @@ describe("Mistral tool schema serialization", () => { }); expect(capturedPayload?.tools).toHaveLength(1); + expect(capturedPayload?.tools?.[0]?.function.strict).toBe(true); const payloadParameters = capturedPayload?.tools?.[0]?.function.parameters; expect(payloadParameters).toBeDefined(); expect(Object.getOwnPropertySymbols(payloadParameters ?? {})).toHaveLength(0); diff --git a/packages/ai/test/openai-codex-stream.test.ts b/packages/ai/test/openai-codex-stream.test.ts index 45e8696daf8..582ec9a1c5a 100644 --- a/packages/ai/test/openai-codex-stream.test.ts +++ b/packages/ai/test/openai-codex-stream.test.ts @@ -855,6 +855,75 @@ describe("openai-codex streaming", () => { expect(requestedToolChoice).toBe("required"); }); + it("sets Codex strict mode explicitly and honors constrained sampling", async () => { + const token = mockToken(); + const encoder = new TextEncoder(); + const sse = buildSSEPayload({ status: "completed" }); + let requestedTools: Array<{ type?: string; name?: string; strict?: boolean | null }> | undefined; + + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode(sse)); + controller.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + ), + ), + ); + + const model: Model<"openai-codex-responses"> = { + id: "gpt-5.5", + name: "GPT-5.5", + api: "openai-codex-responses", + provider: "openai-codex", + baseUrl: "https://chatgpt.com/backend-api", + reasoning: true, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 400000, + maxTokens: 128000, + }; + + await streamOpenAICodexResponses( + model, + { + messages: [{ role: "user", content: "Use a tool", timestamp: Date.now() }], + tools: [ + { + name: "optional", + description: "Optional constrained sampling", + parameters: Type.Object({ value: Type.String() }), + constrainedSampling: false, + }, + { + name: "strict", + description: "Strict constrained sampling", + parameters: Type.Object({ value: Type.String() }, { additionalProperties: false }), + constrainedSampling: { type: "json_schema", strict: "prefer" }, + }, + ], + }, + { + apiKey: token, + transport: "sse", + onPayload: (payload) => { + requestedTools = (payload as { tools?: typeof requestedTools }).tools; + }, + }, + ).result(); + + expect(requestedTools).toMatchObject([ + { type: "function", name: "optional", strict: null }, + { type: "function", name: "strict", strict: true }, + ]); + }); + it.each(["gpt-5.3-codex", "gpt-5.4", "gpt-5.5"])("clamps %s minimal reasoning effort to low", async (modelId) => { const tempDir = mkdtempSync(join(tmpdir(), "pi-codex-stream-")); process.env.PI_CODING_AGENT_DIR = tempDir; @@ -1803,10 +1872,6 @@ describe("openai-codex streaming", () => { it("sends only response input deltas in websocket-cached mode", async () => { const token = mockToken(); const sentBodies: unknown[] = []; - const responses = [ - { responseId: "resp_1", messageId: "msg_1", text: "Hello" }, - { responseId: "resp_2", messageId: "msg_2", text: "Done" }, - ]; class MockWebSocket { static OPEN = 1; @@ -1832,36 +1897,41 @@ describe("openai-codex streaming", () => { send(data: string): void { sentBodies.push(JSON.parse(data)); - const response = responses.shift(); - if (!response) throw new Error("unexpected websocket request"); + const responseId = `resp_${sentBodies.length}`; + const outputEvents = + sentBodies.length === 1 + ? [ + { + type: "response.output_item.added", + item: { + type: "custom_tool_call", + id: "ctc_1", + call_id: "call_1", + name: "sample_tool", + input: "", + }, + }, + { type: "response.custom_tool_call_input.delta", item_id: "ctc_1", delta: "abc" }, + { type: "response.custom_tool_call_input.done", item_id: "ctc_1", input: "abc" }, + { + type: "response.output_item.done", + item: { + type: "custom_tool_call", + id: "ctc_1", + call_id: "call_1", + name: "sample_tool", + input: "abc", + }, + }, + ] + : []; const events = [ - { type: "response.created", response: { id: response.responseId } }, - { - type: "response.output_item.added", - item: { - type: "message", - id: response.messageId, - role: "assistant", - status: "in_progress", - content: [], - }, - }, - { type: "response.content_part.added", part: { type: "output_text", text: "" } }, - { type: "response.output_text.delta", delta: response.text }, - { - type: "response.output_item.done", - item: { - type: "message", - id: response.messageId, - role: "assistant", - status: "completed", - content: [{ type: "output_text", text: response.text }], - }, - }, + { type: "response.created", response: { id: responseId } }, + ...outputEvents, { type: "response.completed", response: { - id: response.responseId, + id: responseId, status: "completed", usage: { input_tokens: 5, @@ -1903,10 +1973,19 @@ describe("openai-codex streaming", () => { cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, contextWindow: 400000, maxTokens: 128000, + compat: { supportsOpenAIGrammarTools: true }, }; const firstContext: Context = { systemPrompt: "You are a helpful assistant.", - messages: [{ role: "user", content: "Say hello", timestamp: 1 }], + messages: [{ role: "user", content: "Use the tool", timestamp: 1 }], + tools: [ + { + name: "sample_tool", + description: "Sample tool", + parameters: Type.Object({ payload: Type.String() }), + constrainedSampling: { type: "grammar", variants: { openai_lark: "start: /[a-z]+/" } }, + }, + ], }; const first = await streamOpenAICodexResponses(model, firstContext, { @@ -1916,8 +1995,20 @@ describe("openai-codex streaming", () => { }).result(); const secondContext: Context = { - systemPrompt: "You are a helpful assistant.", - messages: [...firstContext.messages, first, { role: "user", content: "Now finish", timestamp: 2 }], + ...firstContext, + messages: [ + ...firstContext.messages, + first, + { + role: "toolResult", + toolCallId: "call_1|ctc_1", + toolName: "sample_tool", + content: [{ type: "text", text: "real result" }], + isError: false, + timestamp: 2, + }, + { role: "user", content: "Now finish", timestamp: 3 }, + ], }; await streamOpenAICodexResponses(model, secondContext, { apiKey: token, @@ -1930,10 +2021,13 @@ describe("openai-codex streaming", () => { const secondBody = sentBodies[1] as { input: unknown[]; previous_response_id?: string; store?: boolean }; expect(firstBody.store).toBe(false); expect(firstBody.previous_response_id).toBeUndefined(); - expect(firstBody.input).toEqual([{ role: "user", content: [{ type: "input_text", text: "Say hello" }] }]); + expect(firstBody.input).toEqual([{ role: "user", content: [{ type: "input_text", text: "Use the tool" }] }]); expect(secondBody.store).toBe(false); expect(secondBody.previous_response_id).toBe("resp_1"); - expect(secondBody.input).toEqual([{ role: "user", content: [{ type: "input_text", text: "Now finish" }] }]); + expect(secondBody.input).toEqual([ + { type: "custom_tool_call_output", call_id: "call_1", output: "real result" }, + { role: "user", content: [{ type: "input_text", text: "Now finish" }] }, + ]); expect(getOpenAICodexWebSocketDebugStats("session-1")).toMatchObject({ requests: 2, connectionsCreated: 1, @@ -1942,7 +2036,7 @@ describe("openai-codex streaming", () => { storeTrueRequests: 0, fullContextRequests: 1, deltaRequests: 1, - lastDeltaInputItems: 1, + lastDeltaInputItems: 2, lastPreviousResponseId: "resp_1", }); }); diff --git a/packages/ai/test/openai-completions-thinking-as-text.test.ts b/packages/ai/test/openai-completions-thinking-as-text.test.ts index 6a2b125c0e4..1d581abecf9 100644 --- a/packages/ai/test/openai-completions-thinking-as-text.test.ts +++ b/packages/ai/test/openai-completions-thinking-as-text.test.ts @@ -37,6 +37,7 @@ const compat = { chatTemplateKwargs: {}, zaiToolStream: false, supportsStrictMode: true, + supportsOpenAIGrammarTools: false, cacheControlFormat: undefined, sendSessionAffinityHeaders: false, sessionAffinityFormat: "openai", diff --git a/packages/ai/test/openai-completions-tool-choice.test.ts b/packages/ai/test/openai-completions-tool-choice.test.ts index a74849d8a0f..b585bb431d8 100644 --- a/packages/ai/test/openai-completions-tool-choice.test.ts +++ b/packages/ai/test/openai-completions-tool-choice.test.ts @@ -1258,6 +1258,7 @@ describe("openai-completions tool_choice", () => { chatTemplateKwargs: {}, zaiToolStream: false, supportsStrictMode: true, + supportsOpenAIGrammarTools: false, sendSessionAffinityHeaders: false, sessionAffinityFormat: "openai", supportsLongCacheRetention: true, diff --git a/packages/ai/test/openai-completions-tool-result-images.test.ts b/packages/ai/test/openai-completions-tool-result-images.test.ts index 4c3f8a9f1d4..7cde69dd14b 100644 --- a/packages/ai/test/openai-completions-tool-result-images.test.ts +++ b/packages/ai/test/openai-completions-tool-result-images.test.ts @@ -37,6 +37,7 @@ const compat: Omit, "deferredToolsMode"> & { chatTemplateKwargs: {}, zaiToolStream: false, supportsStrictMode: true, + supportsOpenAIGrammarTools: false, cacheControlFormat: "anthropic", sendSessionAffinityHeaders: false, sessionAffinityFormat: "openai", diff --git a/packages/ai/test/providers.test.ts b/packages/ai/test/providers.test.ts index e034ebecde8..e3a7a938751 100644 --- a/packages/ai/test/providers.test.ts +++ b/packages/ai/test/providers.test.ts @@ -3,7 +3,7 @@ import { envApiKeyAuth } from "../src/auth/helpers.ts"; import type { AuthContext, AuthEvent } from "../src/auth/types.ts"; import { createModels, createProvider } from "../src/models.ts"; import { InMemoryModelsStore, type ModelsStoreEntry } from "../src/models-store.ts"; -import { builtinModels, builtinProviders } from "../src/providers/all.ts"; +import { builtinModels, builtinProviders, getBuiltinModel } from "../src/providers/all.ts"; import { amazonBedrockProvider } from "../src/providers/amazon-bedrock.ts"; import { anthropicProvider } from "../src/providers/anthropic.ts"; import { cloudflareAIGatewayProvider } from "../src/providers/cloudflare-ai-gateway.ts"; @@ -44,6 +44,17 @@ describe("builtin providers", () => { } }); + it("stores native constrained-sampling capabilities in model metadata", () => { + const gpt4o = getBuiltinModel("openai", "gpt-4o"); + expect(gpt4o.compat?.supportsStrictMode).toBe(true); + expect(gpt4o.compat?.supportsOpenAIGrammarTools).toBeUndefined(); + expect(getBuiltinModel("openai", "gpt-5.4").compat).toMatchObject({ + supportsStrictMode: true, + supportsOpenAIGrammarTools: true, + }); + expect(getBuiltinModel("anthropic", "claude-haiku-4-5").compat?.supportsStrictTools).toBe(true); + }); + it("uses official Kimi K3 pricing for Moonshot providers", () => { const models = builtinModels(); for (const provider of ["moonshotai", "moonshotai-cn"]) { diff --git a/packages/coding-agent/docs/custom-provider.md b/packages/coding-agent/docs/custom-provider.md index 82625904402..83b9cff663c 100644 --- a/packages/coding-agent/docs/custom-provider.md +++ b/packages/coding-agent/docs/custom-provider.md @@ -737,6 +737,8 @@ interface ProviderModelConfig { supportsDeveloperRole?: boolean; supportsReasoningEffort?: boolean; supportsUsageInStreaming?: boolean; + supportsStrictMode?: boolean; + supportsOpenAIGrammarTools?: boolean; // openai-completions/openai-responses; false falls back to normal function tools maxTokensField?: "max_completion_tokens" | "max_tokens"; requiresToolResultName?: boolean; requiresAssistantAfterToolResult?: boolean; @@ -755,6 +757,7 @@ interface ProviderModelConfig { supportsCacheControlOnTools?: boolean; forceAdaptiveThinking?: boolean; allowEmptySignature?: boolean; + supportsStrictTools?: boolean; }; } ``` diff --git a/packages/coding-agent/docs/models.md b/packages/coding-agent/docs/models.md index 04066c4b126..f2493b2d67a 100644 --- a/packages/coding-agent/docs/models.md +++ b/packages/coding-agent/docs/models.md @@ -375,6 +375,8 @@ Some Anthropic models require adaptive thinking (`thinking.type: "adaptive"` plu Some Anthropic-compatible providers emit thinking blocks with empty signatures and still expect them on replay. Set `allowEmptySignature` to `true` only for those providers; real Anthropic rejects empty thinking signatures. +Built-in Anthropic models enable `supportsStrictTools` in their model metadata. Custom Anthropic-compatible models must set it to `true` when their endpoint accepts strict JSON-schema tool definitions. + ```json { "providers": { @@ -408,6 +410,7 @@ Some Anthropic-compatible providers emit thinking blocks with empty signatures a | `supportsCacheControlOnTools` | Whether the provider accepts Anthropic-style `cache_control` markers on tool definitions. Default: `true`. | | `forceAdaptiveThinking` | Whether to send adaptive thinking (`thinking.type: "adaptive"` plus `output_config.effort`) for this model. Built-in adaptive models set this automatically. Default: `false`. | | `allowEmptySignature` | Whether to replay empty thinking signatures as `signature: ""` instead of converting thinking to text. Default: `false`. | +| `supportsStrictTools` | Whether the provider accepts strict JSON-schema tool definitions. Default: `false`; built-in Anthropic models enable it in generated metadata. | ## OpenAI Compatibility @@ -448,7 +451,8 @@ For providers with partial OpenAI compatibility, use the `compat` field. | `cacheControlFormat` | Use Anthropic-style `cache_control` markers on the system prompt, last tool definition, and last user, assistant, or tool-result text content. Currently only `anthropic` is supported. | | `sendSessionAffinityHeaders` | For `openai-completions`, send session-affinity headers from the session id when caching is enabled. Default: `false`. | | `sessionAffinityFormat` | For `openai-completions` and `openai-responses`, the session-affinity header format: `openai` sends `session_id`/`x-client-request-id` (completions also `x-session-affinity`), `openai-nosession` omits the underscore-containing `session_id` header, `openrouter` sends `x-session-id`. Does not affect the `prompt_cache_key` body param. Default: auto-detected. | -| `supportsStrictMode` | Include the `strict` field in tool definitions | +| `supportsStrictMode` | Whether the provider accepts strict JSON-schema function tool definitions. Defaults depend on the API; built-in OpenAI models carry explicit capability metadata. | +| `supportsOpenAIGrammarTools` | Whether OpenAI-compatible APIs emit custom Lark/regex grammar tools. When `false`, grammar-constrained tools fall back to normal function tools. Default: `false`; the built-in model catalog enables it for GPT-5+ models on OpenAI, OpenAI Codex, Azure OpenAI, GitHub Copilot, opencode, and Cloudflare AI Gateway. | | `deferredToolsMode` | Use provider-specific deferred tool serialization. Currently only `"kimi"` is supported for Kimi's OpenAI-compatible Chat Completions format. | | `supportsLongCacheRetention` | Whether the provider accepts long cache retention when cache retention is `long`: `prompt_cache_retention: "24h"` for OpenAI prompt caching, or `cache_control.ttl: "1h"` when `cacheControlFormat` is `anthropic`. Default: `true`. | | `openRouterRouting` | OpenRouter provider routing preferences. This object is sent as-is in the `provider` field of the [OpenRouter API request](https://openrouter.ai/docs/guides/routing/provider-selection). | diff --git a/packages/coding-agent/src/core/extensions/types.ts b/packages/coding-agent/src/core/extensions/types.ts index 286172969a3..22b386ba8e3 100644 --- a/packages/coding-agent/src/core/extensions/types.ts +++ b/packages/coding-agent/src/core/extensions/types.ts @@ -19,6 +19,7 @@ import type { Api, AssistantMessageEvent, AssistantMessageEventStream, + ConstrainedSamplingConfig, Context, ImageContent, Model, @@ -452,6 +453,8 @@ export interface ToolDefinition( label: definition.label, description: definition.description, parameters: definition.parameters, + constrainedSampling: definition.constrainedSampling, prepareArguments: definition.prepareArguments, executionMode: definition.executionMode, execute: (toolCallId, params, signal, onUpdate, ctx?: ExtensionContext) => @@ -38,6 +39,7 @@ export function createToolDefinitionFromAgentTool(tool: AgentTool): ToolDef label: tool.label, description: tool.description, parameters: tool.parameters as any, + constrainedSampling: tool.constrainedSampling, prepareArguments: tool.prepareArguments, executionMode: tool.executionMode, execute: async (toolCallId, params, signal, onUpdate) => tool.execute(toolCallId, params, signal, onUpdate),