From cd7ddae92e43bc79124d7b0beb1b695da3b42a5b Mon Sep 17 00:00:00 2001 From: Luca Steeb Date: Fri, 10 Jul 2026 18:51:20 +0100 Subject: [PATCH] fix(gateway): support web_search tool in responses api The /v1/responses handler dropped all non-function tools when converting to the internal chat completions request, so web_search never reached the provider even though chat completions supports it natively. Pass web_search through, forward url citations in both the non-streaming and streaming converters (flattened to the Responses API annotation shape), and accept allowed_domains/blocked_domains on the responses web_search tool schema for parity with chat completions. Co-Authored-By: Claude Fable 5 --- apps/gateway/src/chat-websearch.e2e.ts | 65 +++++++++++++++++++ apps/gateway/src/responses/responses.ts | 8 ++- apps/gateway/src/responses/schemas.ts | 2 + .../tools/convert-chat-to-responses.ts | 28 +++++++- .../tools/convert-streaming-to-responses.ts | 36 ++++++++-- 5 files changed, 132 insertions(+), 7 deletions(-) diff --git a/apps/gateway/src/chat-websearch.e2e.ts b/apps/gateway/src/chat-websearch.e2e.ts index ed8062b7df..f3cac9e3d9 100644 --- a/apps/gateway/src/chat-websearch.e2e.ts +++ b/apps/gateway/src/chat-websearch.e2e.ts @@ -121,6 +121,71 @@ describeWebSearch("e2e web search", getConcurrentTestOptions(), () => { }, ); + test.each(webSearchModels)( + "web search responses api $model", + { timeout: 300000 }, // Increase timeout for web search + async ({ model }) => { + const requestId = generateTestRequestId(); + const res = await app.request("/v1/responses", { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-request-id": requestId, + "x-no-fallback": "true", + Authorization: `Bearer real-token`, + }, + body: JSON.stringify({ + model: model, + input: + "Search the web for the latest news about artificial intelligence from today. What are the top stories?", + tools: [ + { + type: "web_search", + }, + ], + }), + }); + + const json = await res.json(); + if (logMode) { + console.log( + "web search responses api response:", + JSON.stringify(json, null, 2), + ); + } + + expect(res.status).toBe(200); + expect(json).toHaveProperty("output"); + expect(Array.isArray(json.output)).toBe(true); + + const message = json.output.find( + (item: { type: string }) => item.type === "message", + ); + expect(message).toBeDefined(); + const text = (message.content ?? []) + .filter( + (c: { type: string; text?: string }) => c.type === "output_text", + ) + .map((c: { text?: string }) => c.text ?? "") + .join(""); + expect(text.length).toBeGreaterThan(0); + + // Validate logs + const log = await validateLogByRequestId(requestId); + + // Verify web search was used and cost is tracked + expect(log).toHaveProperty("webSearchCost"); + expect(typeof log.webSearchCost).toBe("number"); + expect(log.webSearchCost).toBeGreaterThan(0); + + if (logMode) { + console.log( + `Web search was used for ${model} via responses api, cost: ${log.webSearchCost}`, + ); + } + }, + ); + test.each(streamingWebSearchModels)( "web search streaming $model", { timeout: 180000 }, // Increase timeout for web search diff --git a/apps/gateway/src/responses/responses.ts b/apps/gateway/src/responses/responses.ts index f196ef906a..7268835bfa 100644 --- a/apps/gateway/src/responses/responses.ts +++ b/apps/gateway/src/responses/responses.ts @@ -261,8 +261,9 @@ responses.post("/", async (c) => { // Convert tools format: Responses API has name/description/parameters at top level, // chat completions nests under function. - // Only forward user-defined function tools to chat completions. - // Built-in tool types (web_search, computer_use, code_interpreter, shell, etc.) + // web_search passes through unchanged — the chat completions layer resolves + // it to the provider's native web search / grounding. + // Other built-in tool types (computer_use, code_interpreter, shell, etc.) // are OpenAI-native capabilities that cannot be proxied through the gateway's // provider routing, so they are dropped here. const tools = req.tools @@ -277,6 +278,9 @@ responses.post("/", async (c) => { }, }; } + if (tool.type === "web_search") { + return tool; + } return null; }) .filter((t): t is NonNullable => t !== null); diff --git a/apps/gateway/src/responses/schemas.ts b/apps/gateway/src/responses/schemas.ts index 64af96eb14..4f37e99fe8 100644 --- a/apps/gateway/src/responses/schemas.ts +++ b/apps/gateway/src/responses/schemas.ts @@ -185,6 +185,8 @@ export const responsesRequestSchema = z.object({ .optional(), search_context_size: z.enum(["low", "medium", "high"]).optional(), max_uses: z.number().optional(), + allowed_domains: z.array(z.string()).optional(), + blocked_domains: z.array(z.string()).optional(), }), // catch-all for unknown tool types (e.g. computer_use, code_interpreter) z.record(z.any()), diff --git a/apps/gateway/src/responses/tools/convert-chat-to-responses.ts b/apps/gateway/src/responses/tools/convert-chat-to-responses.ts index dd6c048431..76a0e742b1 100644 --- a/apps/gateway/src/responses/tools/convert-chat-to-responses.ts +++ b/apps/gateway/src/responses/tools/convert-chat-to-responses.ts @@ -211,6 +211,32 @@ function resolveServedServiceTier( return undefined; } +/** + * Normalize chat-completions-style annotations to the Responses API shape: + * chat nests citation fields under `url_citation`, the Responses API flattens + * them onto the annotation object. + */ +export function normalizeAnnotationsToResponses( + annotations: Array> | undefined, +): Array> { + if (!annotations?.length) { + return []; + } + return annotations.map((annotation) => { + if ( + annotation.type === "url_citation" && + annotation.url_citation && + typeof annotation.url_citation === "object" + ) { + return { + type: "url_citation", + ...(annotation.url_citation as Record), + }; + } + return annotation; + }); +} + /** * Converts a chat completions response to Responses API format. */ @@ -262,7 +288,7 @@ export function convertChatResponseToResponses( { type: "output_text", text: message.content, - annotations: message.annotations ?? [], + annotations: normalizeAnnotationsToResponses(message.annotations), }, ]; diff --git a/apps/gateway/src/responses/tools/convert-streaming-to-responses.ts b/apps/gateway/src/responses/tools/convert-streaming-to-responses.ts index ec3f0a3216..64ee6be80d 100644 --- a/apps/gateway/src/responses/tools/convert-streaming-to-responses.ts +++ b/apps/gateway/src/responses/tools/convert-streaming-to-responses.ts @@ -1,6 +1,9 @@ import { shortid } from "@llmgateway/db"; -import { normalizeEchoedTools } from "./convert-chat-to-responses.js"; +import { + normalizeAnnotationsToResponses, + normalizeEchoedTools, +} from "./convert-chat-to-responses.js"; import type { ResponsesEchoRequest } from "./convert-chat-to-responses.js"; @@ -15,6 +18,7 @@ interface StreamingState { reasoningId: string; fullContent: string[]; fullReasoning: string[]; + annotations: Record[]; reasoningStarted: boolean; finishReason: string | null; sequenceNumber: number; @@ -72,6 +76,7 @@ export function createStreamingState( reasoningId: `rs_${shortid(24)}`, fullContent: [], fullReasoning: [], + annotations: [], reasoningStarted: false, finishReason: null, sequenceNumber: 0, @@ -214,6 +219,7 @@ export function processStreamChunk( delta?: { content?: string | null; reasoning?: string | null; + annotations?: Array>; tool_calls?: Array<{ index: number; id?: string; @@ -390,6 +396,28 @@ export function processStreamChunk( ); } + // Handle annotations delta (url citations from native web search) + if (delta.annotations?.length) { + for (const annotation of normalizeAnnotationsToResponses( + delta.annotations, + )) { + const annotationIndex = state.annotations.length; + state.annotations.push(annotation); + if (state.contentPartStarted) { + events.push( + emitEvent(state, "response.output_text.annotation.added", { + type: "response.output_text.annotation.added", + item_id: state.messageId, + output_index: state.outputItemIndex, + content_index: 0, + annotation_index: annotationIndex, + annotation, + }), + ); + } + } + } + // Check for usage in the chunk if (chunk.usage) { const usage = chunk.usage as Record; @@ -453,7 +481,7 @@ export function createCompletionEvents(state: StreamingState): SSEEvent[] { part: { type: "output_text", text: state.fullContent.join(""), - annotations: [], + annotations: state.annotations, }, }), ); @@ -473,7 +501,7 @@ export function createCompletionEvents(state: StreamingState): SSEEvent[] { { type: "output_text", text: state.fullContent.join(""), - annotations: [], + annotations: state.annotations, }, ], status: "completed", @@ -542,7 +570,7 @@ export function createCompletionEvents(state: StreamingState): SSEEvent[] { { type: "output_text", text: state.fullContent.join(""), - annotations: [], + annotations: state.annotations, }, ], status: "completed",