diff --git a/apps/api/src/routes/chats.ts b/apps/api/src/routes/chats.ts index 445f5304e8..e00df0c848 100644 --- a/apps/api/src/routes/chats.ts +++ b/apps/api/src/routes/chats.ts @@ -47,6 +47,7 @@ const messageSchema = z.object({ content: z.string().nullable(), images: z.string().nullable(), // JSON string audios: z.string().nullable(), // JSON string of audio attachments + documents: z.string().nullable().optional(), // JSON string of document attachments reasoning: z.string().nullable(), // Reasoning content tools: z.string().nullable(), // JSON string of tool parts metadata: z.record(z.unknown()).nullable(), @@ -85,6 +86,7 @@ const orgShareSchema = z.object({ content: z.string().nullable(), images: z.string().nullable(), audios: z.string().nullable().optional(), + documents: z.string().nullable().optional(), reasoning: z.string().nullable(), tools: z.string().nullable(), metadata: z.record(z.unknown()).nullable().optional(), @@ -101,6 +103,7 @@ const sharedMessageSnapshotSchema = z.array( content: z.string().nullable(), images: z.string().nullable(), audios: z.string().nullable().optional(), + documents: z.string().nullable().optional(), reasoning: z.string().nullable(), tools: z.string().nullable(), metadata: z.record(z.unknown()).nullable().optional(), @@ -135,6 +138,7 @@ const createMessageSchema = z content: z.string().optional(), images: z.string().optional(), // JSON string audios: z.string().optional(), // JSON string of audio attachments + documents: z.string().optional(), // JSON string of document attachments reasoning: z.string().optional(), // Reasoning content tools: z.string().optional(), // Tool parts JSON metadata: z.record(z.unknown()).optional(), @@ -144,10 +148,11 @@ const createMessageSchema = z data.content ?? data.images ?? data.audios ?? + data.documents ?? data.reasoning ?? data.tools, { - message: "Either content, images, or audios must be provided", + message: "Either content, images, audios, or documents must be provided", }, ); @@ -712,7 +717,8 @@ chats.openapi(getChat, async (c) => { role: message.role as "user" | "assistant" | "system", content: message.content, images: message.images, - audios: (message as any).audios ?? null, + audios: message.audios ?? null, + documents: message.documents ?? null, reasoning: message.reasoning, tools: message.tools ?? null, metadata: message.metadata ?? null, @@ -969,6 +975,7 @@ chats.openapi(shareChat, async (c) => { content: tables.message.content, images: tables.message.images, audios: tables.message.audios, + documents: tables.message.documents, reasoning: tables.message.reasoning, tools: tables.message.tools, metadata: tables.message.metadata, @@ -993,6 +1000,7 @@ chats.openapi(shareChat, async (c) => { content: message.content, images: message.images, audios: message.audios, + documents: message.documents, reasoning: message.reasoning, tools: message.tools, metadata: message.metadata, @@ -1433,6 +1441,7 @@ chats.openapi(forkSharedChat, async (c) => { content: message.content, images: message.images, audios: message.audios ?? null, + documents: message.documents ?? null, reasoning: message.reasoning, tools: message.tools, metadata: message.metadata ?? null, @@ -1724,6 +1733,7 @@ chats.openapi(addMessage, async (c) => { content: body.content ?? null, images: body.images ?? null, audios: body.audios ?? null, + documents: body.documents ?? null, reasoning: body.reasoning ?? null, tools: body.tools ?? null, metadata: body.metadata ?? null, @@ -1744,7 +1754,7 @@ chats.openapi(addMessage, async (c) => { role: newMessage.role as "user" | "assistant" | "system", content: newMessage.content, images: newMessage.images, - audios: (newMessage as any).audios ?? null, + audios: newMessage.audios ?? null, reasoning: newMessage.reasoning, tools: newMessage.tools ?? null, metadata: newMessage.metadata ?? null, diff --git a/apps/api/src/routes/internal-models.ts b/apps/api/src/routes/internal-models.ts index 7a179690c3..353f623001 100644 --- a/apps/api/src/routes/internal-models.ts +++ b/apps/api/src/routes/internal-models.ts @@ -61,6 +61,7 @@ const modelProviderMappingSchema = z.object({ streaming: z.boolean(), vision: z.boolean().nullable(), audio: z.boolean().nullable(), + document: z.boolean().nullable(), reasoning: z.boolean().nullable(), reasoningOutput: z.string().nullable(), tools: z.boolean().nullable(), @@ -214,6 +215,7 @@ internalModels.openapi(getModelsRoute, async (c) => { ...mapping, discount: effectiveDiscount, audio: sharedMapping?.audio ?? null, + document: sharedMapping?.document ?? null, imageOutputPrice: sharedMapping?.imageOutputPrice !== undefined ? String(sharedMapping.imageOutputPrice) diff --git a/apps/api/src/routes/public-chat-shares.ts b/apps/api/src/routes/public-chat-shares.ts index 6b34be54e1..e631543510 100644 --- a/apps/api/src/routes/public-chat-shares.ts +++ b/apps/api/src/routes/public-chat-shares.ts @@ -12,6 +12,7 @@ const sharedMessageSchema = z.object({ content: z.string().nullable(), images: z.string().nullable(), audios: z.string().nullable().optional(), + documents: z.string().nullable().optional(), reasoning: z.string().nullable(), tools: z.string().nullable(), metadata: z.record(z.unknown()).nullable().optional(), diff --git a/apps/code/src/lib/api/v1.d.ts b/apps/code/src/lib/api/v1.d.ts index 1ca84ce051..e6dd582a49 100644 --- a/apps/code/src/lib/api/v1.d.ts +++ b/apps/code/src/lib/api/v1.d.ts @@ -571,6 +571,7 @@ export interface paths { content: string | null; images: string | null; audios?: string | null; + documents?: string | null; reasoning: string | null; tools: string | null; metadata?: { @@ -9207,6 +9208,7 @@ export interface paths { content: string | null; images: string | null; audios: string | null; + documents?: string | null; reasoning: string | null; tools: string | null; metadata: { @@ -9476,6 +9478,7 @@ export interface paths { content: string | null; images: string | null; audios?: string | null; + documents?: string | null; reasoning: string | null; tools: string | null; metadata?: { @@ -9674,6 +9677,7 @@ export interface paths { content?: string; images?: string; audios?: string; + documents?: string; reasoning?: string; tools?: string; metadata?: { @@ -9697,6 +9701,7 @@ export interface paths { content: string | null; images: string | null; audios: string | null; + documents?: string | null; reasoning: string | null; tools: string | null; metadata: { @@ -9764,6 +9769,7 @@ export interface paths { content: string | null; images: string | null; audios: string | null; + documents?: string | null; reasoning: string | null; tools: string | null; metadata: { @@ -11608,6 +11614,7 @@ export interface operations { streaming: boolean; vision: boolean | null; audio: boolean | null; + document: boolean | null; reasoning: boolean | null; reasoningOutput: string | null; tools: boolean | null; diff --git a/apps/docs/content/features/documents.mdx b/apps/docs/content/features/documents.mdx new file mode 100644 index 0000000000..e1a38396a3 --- /dev/null +++ b/apps/docs/content/features/documents.mdx @@ -0,0 +1,137 @@ +--- +title: Document Reading +description: Learn how to send PDFs and other document data to document-capable models. +icon: FileText +--- + +import { Callout } from "fumadocs-ui/components/callout"; + +# Document Reading + +LLMGateway supports sending documents (PDFs and other file types) to document-capable models using OpenAI's `file` content block format. The gateway forwards the document to the underlying provider so the model can read and reason over its contents. + +## Document-Capable Models + +Document input is currently supported on Google Gemini models via Google AI Studio. You can find document-capable models on the [models page with the document filter](https://llmgateway.io/models?filters=1&document=true). + +## Sending a Document + +Add a `file` content block to a user message. The `file_data` field must be a base64-encoded data URL that includes the document's MIME type. + +```bash +curl -X POST "https://api.llmgateway.io/v1/chat/completions" \ + -H "Authorization: Bearer $LLM_GATEWAY_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gemini-2.5-flash", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Summarize this document." + }, + { + "type": "file", + "file": { + "filename": "report.pdf", + "file_data": "data:application/pdf;base64,JVBERi0xLjQKJ..." + } + } + ] + } + ] + }' +``` + +### Content Block Fields + +- **`type`**: must be `"file"`. +- **`file.filename`** _(optional)_: original filename, shown in the playground and forwarded for context. +- **`file.file_data`**: base64-encoded data URL of the form `data:;base64,`. + + + The `file.file_id` field (for referencing files uploaded via a provider's + Files API) is accepted by the schema but not currently supported by the Google + transform. Use `file_data` with an inline base64 data URL. + + +## Supported File Types + +The accepted MIME types depend on the target model. Gemini models commonly support: + +- `application/pdf` +- `text/plain` +- `text/html` +- `text/css` +- `text/javascript` +- `text/csv` +- `text/markdown` +- `text/xml` + +If the upstream provider rejects the MIME type, the gateway surfaces a `400` error including the unsupported MIME type and the provider it was sent to. To use a different file type, encode the file with the matching MIME type in the data URL prefix. + +## Encoding a File as a Data URL + +Any tool that can produce base64 output works. For example, in a shell: + +```bash +DATA=$(base64 -i report.pdf | tr -d '\n') +echo "data:application/pdf;base64,$DATA" +``` + +Or in JavaScript: + +```javascript +import { readFileSync } from "node:fs"; + +const buffer = readFileSync("report.pdf"); +const fileData = `data:application/pdf;base64,${buffer.toString("base64")}`; +``` + +Then pass `fileData` as the `file.file_data` value in your request. + +## Multiple Documents + +You can include multiple `file` blocks in a single message, optionally mixed with text and image content: + +```bash +curl -X POST "https://api.llmgateway.io/v1/chat/completions" \ + -H "Authorization: Bearer $LLM_GATEWAY_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gemini-2.5-pro", + "messages": [ + { + "role": "user", + "content": [ + { "type": "text", "text": "Compare these two reports." }, + { + "type": "file", + "file": { + "filename": "q1.pdf", + "file_data": "data:application/pdf;base64,JVBERi0x..." + } + }, + { + "type": "file", + "file": { + "filename": "q2.pdf", + "file_data": "data:application/pdf;base64,JVBERi0x..." + } + } + ] + } + ] + }' +``` + +## Error Handling + +The gateway returns `400` for the following document-related errors: + +- The selected model does not support document input. +- The `file` block is missing both `file_data` and `file_id`. +- `file_data` is not a valid base64 data URL. +- The upstream provider rejects the document's MIME type for the selected model. diff --git a/apps/gateway/src/app.ts b/apps/gateway/src/app.ts index a4d75ebe21..56f155b689 100644 --- a/apps/gateway/src/app.ts +++ b/apps/gateway/src/app.ts @@ -7,7 +7,11 @@ import { cors } from "hono/cors"; import { HTTPException } from "hono/http-exception"; import { z } from "zod"; -import { UnsupportedAudioFormatError } from "@llmgateway/actions"; +import { + InvalidFileContentError, + UnsupportedAudioFormatError, + UnsupportedDocumentFormatError, +} from "@llmgateway/actions"; import { redisClient } from "@llmgateway/cache"; import { db } from "@llmgateway/db"; import { @@ -130,6 +134,36 @@ app.onError((error, c) => { ); } + if (error instanceof InvalidFileContentError) { + logger.warn("Invalid file content", { message: error.message }); + return c.json( + { + error: true, + status: 400, + message: error.message, + }, + 400, + ); + } + + if (error instanceof UnsupportedDocumentFormatError) { + logger.warn("Unsupported document format", { + message: error.message, + mimeType: error.mimeType, + providerTarget: error.providerTarget, + }); + return c.json( + { + error: true, + status: 400, + message: error.message, + mimeType: error.mimeType, + providerTarget: error.providerTarget, + }, + 400, + ); + } + if (error instanceof HTTPException) { const status = error.status; diff --git a/apps/gateway/src/chat/chat.ts b/apps/gateway/src/chat/chat.ts index 8dc1fa1cf0..90970f56d3 100644 --- a/apps/gateway/src/chat/chat.ts +++ b/apps/gateway/src/chat/chat.ts @@ -67,7 +67,11 @@ import { getProviderHeaders, getProviderSelectionPrice, googleProviderSupportsAudioFormat, + InvalidFileContentError, + parseGoogleUpstreamDocumentError, prepareRequestBody, + UnsupportedAudioFormatError, + UnsupportedDocumentFormatError, type RoutingMetadata, } from "@llmgateway/actions"; import { @@ -147,6 +151,7 @@ import { getAudioFormatsFromMessages, messagesContainAudio, } from "./tools/messages-contain-audio.js"; +import { messagesContainDocuments } from "./tools/messages-contain-documents.js"; import { messagesContainImages } from "./tools/messages-contain-images.js"; import { mightBeCompleteJson } from "./tools/might-be-complete-json.js"; import { normalizeStreamingError } from "./tools/normalize-streaming-error.js"; @@ -368,6 +373,7 @@ function filterEligibleModelProviders( hasImages: boolean; hasAudio: boolean; audioFormats?: string[]; + hasDocuments: boolean; maxTokens?: number; reasoningEffort?: string; }, @@ -426,6 +432,10 @@ function filterEligibleModelProviders( return false; } + if (options.hasDocuments && provider.document !== true) { + return false; + } + if ( options.maxTokens !== undefined && provider.maxOutput !== undefined && @@ -1173,6 +1183,7 @@ chat.openapi(completions, async (c) => { const audioFormats = hasAudio ? getAudioFormatsFromMessages(messages as BaseMessage[]) : []; + const hasDocuments = messagesContainDocuments(messages as BaseMessage[]); // Extract web_search tool from tools array if present // The web_search tool is a special tool that enables native web search for providers that support it @@ -1678,7 +1689,7 @@ chat.openapi(completions, async (c) => { } } - // Validate model capabilities (JSON output, reasoning, tools, web search) + // Validate model capabilities (JSON output, reasoning, tools, web search, documents) validateModelCapabilities(modelInfo, requestedModel, requestedProvider, { response_format, reasoning_effort, @@ -1687,6 +1698,7 @@ chat.openapi(completions, async (c) => { tool_choice, webSearchTool, hasImages, + hasDocuments, }); let usedProvider = requestedProvider; @@ -1837,7 +1849,11 @@ chat.openapi(completions, async (c) => { if (!("free" in modelDef && modelDef.free)) { continue; } - } else if (!allowedAutoModels.includes(modelDef.id) && !hasAudio) { + } else if ( + !allowedAutoModels.includes(modelDef.id) && + !hasAudio && + !hasDocuments + ) { continue; } else if ( estimatedInputTokens > 10_000 && @@ -1969,6 +1985,10 @@ chat.openapi(completions, async (c) => { return false; } + if (hasDocuments && provider.document !== true) { + return false; + } + if ( max_tokens !== undefined && provider.maxOutput !== undefined && @@ -2157,6 +2177,16 @@ chat.openapi(completions, async (c) => { }); } } + if (hasDocuments) { + sameProviderMappings = sameProviderMappings.filter( + (p) => p.document === true, + ); + if (sameProviderMappings.length === 0) { + throw new HTTPException(400, { + message: `Provider ${usedProvider} does not support document input for model ${modelInfo.id}.`, + }); + } + } const sameProviderRegionalMappings = sameProviderMappings.filter( (p) => p.region, ); @@ -2204,6 +2234,7 @@ chat.openapi(completions, async (c) => { hasImages, hasAudio, audioFormats, + hasDocuments, maxTokens: max_tokens, reasoningEffort: reasoning_effort, }, @@ -2378,6 +2409,9 @@ chat.openapi(completions, async (c) => { ) { return false; } + if (hasDocuments && provider.document !== true) { + return false; + } return true; }); @@ -2522,6 +2556,7 @@ chat.openapi(completions, async (c) => { hasImages, hasAudio, audioFormats, + hasDocuments, maxTokens: max_tokens, reasoningEffort: reasoning_effort, }, @@ -2703,6 +2738,7 @@ chat.openapi(completions, async (c) => { hasImages, hasAudio, audioFormats, + hasDocuments, maxTokens: max_tokens, reasoningEffort: reasoning_effort, }, @@ -2949,6 +2985,7 @@ chat.openapi(completions, async (c) => { hasImages, hasAudio, audioFormats, + hasDocuments, maxTokens: max_tokens, reasoningEffort: reasoning_effort, }, @@ -4354,34 +4391,114 @@ chat.openapi(completions, async (c) => { } } - let requestBody: ProviderRequestBody | FormData = await prepareRequestBody( - usedProvider, - upstreamModelName, - messages as BaseMessage[], - effectiveStream, - temperature, - max_tokens, - top_p, - frequency_penalty, - presence_penalty, - response_format, - tools, - tool_choice, - reasoning_effort, - supportsReasoning, - process.env.NODE_ENV === "production", - maxImageSizeMB, - userPlan, - sensitive_word_check, - image_config, - effort, - isImageGeneration, - webSearchTool, - reasoning_max_tokens, - useResponsesApi, - prompt_cache_key, - prompt_cache_retention, - ); + let requestBody: ProviderRequestBody | FormData; + try { + requestBody = await prepareRequestBody( + usedProvider, + upstreamModelName, + messages as BaseMessage[], + effectiveStream, + temperature, + max_tokens, + top_p, + frequency_penalty, + presence_penalty, + response_format, + tools, + tool_choice, + reasoning_effort, + supportsReasoning, + process.env.NODE_ENV === "production", + maxImageSizeMB, + userPlan, + sensitive_word_check, + image_config, + effort, + isImageGeneration, + webSearchTool, + reasoning_max_tokens, + useResponsesApi, + prompt_cache_key, + prompt_cache_retention, + ); + } catch (e) { + // Surface typed pre-upstream input errors in the activity feed as a + // client_error. Without this, app.onError returns a 400 but no log row + // is written, so the user never sees the rejected request in history. + if ( + e instanceof InvalidFileContentError || + e instanceof UnsupportedAudioFormatError || + e instanceof UnsupportedDocumentFormatError + ) { + try { + await insertLogEntry({ + ...createLogEntry( + requestId, + project, + apiKey, + undefined, + upstreamModelName, + undefined, + usedProvider, + requestedModel, + requestedProvider, + messages as any[], + temperature, + max_tokens, + top_p, + frequency_penalty, + presence_penalty, + reasoning_effort, + reasoning_max_tokens, + effort as "low" | "medium" | "high" | undefined, + response_format, + tools, + tool_choice, + source, + customHeaders, + debugMode, + userAgent, + ), + content: null, + responseSize: 0, + finishReason: "client_error", + promptTokens: null, + completionTokens: null, + totalTokens: null, + reasoningTokens: null, + cachedTokens: null, + hasError: true, + streamed: !!stream, + canceled: false, + errorDetails: { + statusCode: 400, + statusText: "Bad Request", + responseText: e.message, + cause: e.constructor.name, + }, + duration: 0, + timeToFirstToken: null, + inputCost: 0, + outputCost: 0, + cachedInputCost: 0, + requestCost: 0, + webSearchCost: 0, + imageInputTokens: null, + imageOutputTokens: null, + imageInputCost: null, + imageOutputCost: null, + cost: 0, + estimatedCost: false, + discount: null, + pricingTier: null, + dataStorageCost: "0", + }); + } catch { + // Silently ignore logging failures + } + } + throw e; + } if (forceImageStreamUpstream) { requestBody = injectImageStreamParams(requestBody); @@ -5502,6 +5619,16 @@ chat.openapi(completions, async (c) => { ? extractAwsBedrockHttpError(res, rawErrorResponseText) : rawErrorResponseText; + // If the upstream Google provider rejected the document MIME, + // surface a typed error event so streaming clients see the same + // clean shape as the non-streaming path does (via app.onError). + const documentErr = hasDocuments + ? parseGoogleUpstreamDocumentError( + errorResponseText, + usedProvider, + ) + : null; + // Determine the finish reason for error handling const finishReason = getFinishReasonFromError( res.status, @@ -5766,7 +5893,18 @@ chat.openapi(completions, async (c) => { } else { // For client errors, return the original provider error response let errorData; - if (finishReason === "client_error") { + if (documentErr) { + errorData = { + error: { + message: documentErr.message, + type: "invalid_request_error", + param: null, + code: "unsupported_document_format", + mimeType: documentErr.mimeType, + providerTarget: documentErr.providerTarget, + }, + }; + } else if (finishReason === "client_error") { try { errorData = JSON.parse(errorResponseText); } catch { @@ -9098,6 +9236,19 @@ chat.openapi(completions, async (c) => { throw bodyError; } + // If the upstream Google provider rejected the request because the + // document MIME isn't supported by that specific model, re-emit as a + // typed error so app.ts:onError returns a clean 400. + if (hasDocuments) { + const documentErr = parseGoogleUpstreamDocumentError( + errorResponseText, + usedProvider, + ); + if (documentErr) { + throw documentErr; + } + } + // Determine the finish reason first const finishReason = getFinishReasonFromError( res.status, diff --git a/apps/gateway/src/chat/schemas/completions.ts b/apps/gateway/src/chat/schemas/completions.ts index 464937d481..1e5b2a3545 100644 --- a/apps/gateway/src/chat/schemas/completions.ts +++ b/apps/gateway/src/chat/schemas/completions.ts @@ -53,6 +53,25 @@ export const completionsRequestSchema = z.object({ ]), }), }), + z.object({ + type: z.literal("file"), + file: z + .object({ + filename: z.string().optional(), + file_data: z.string().optional().openapi({ + description: + "Base64-encoded data URL with MIME prefix, e.g. 'data:application/pdf;base64,'.", + }), + file_id: z.string().optional().openapi({ + description: + "Reference to a file uploaded via the provider's Files API.", + }), + }) + .refine((file) => Boolean(file.file_data || file.file_id), { + message: + "file.file_data or file.file_id is required for file content", + }), + }), ]), ), ]) diff --git a/apps/gateway/src/chat/tools/messages-contain-documents.spec.ts b/apps/gateway/src/chat/tools/messages-contain-documents.spec.ts new file mode 100644 index 0000000000..694ac316fe --- /dev/null +++ b/apps/gateway/src/chat/tools/messages-contain-documents.spec.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vitest"; + +import { models } from "@llmgateway/models"; + +import { messagesContainDocuments } from "./messages-contain-documents.js"; + +import type { BaseMessage, ProviderModelMapping } from "@llmgateway/models"; + +describe("messagesContainDocuments", () => { + it("returns false for plain text-only messages", () => { + const msgs: BaseMessage[] = [{ role: "user", content: "hello" }]; + expect(messagesContainDocuments(msgs)).toBe(false); + }); + + it("returns false for image/audio-only multipart content", () => { + const msgs: BaseMessage[] = [ + { + role: "user", + content: [ + { type: "text", text: "describe this" }, + { type: "image_url", image_url: { url: "data:image/png;base64,A" } }, + { + type: "input_audio", + input_audio: { data: "AAAA", format: "wav" }, + }, + ], + }, + ]; + expect(messagesContainDocuments(msgs)).toBe(false); + }); + + it("returns true when any part is a file block", () => { + const msgs: BaseMessage[] = [ + { + role: "user", + content: [ + { type: "text", text: "summarize" }, + { + type: "file", + file: { + filename: "doc.pdf", + file_data: "data:application/pdf;base64,AAAA", + }, + }, + ], + }, + ]; + expect(messagesContainDocuments(msgs)).toBe(true); + }); +}); + +describe("Gemini document capability flag", () => { + const expectedDocumentModelIds = [ + "gemini-2.5-pro", + "gemini-2.5-flash", + "gemini-2.5-flash-lite", + ]; + + for (const id of expectedDocumentModelIds) { + it(`marks google-ai-studio variant of ${id} as document: true`, () => { + const model = models.find((m) => m.id === id); + expect(model, `Expected model ${id} to exist`).toBeDefined(); + const providers = model!.providers as readonly ProviderModelMapping[]; + const aiStudio = providers.find( + (p) => p.providerId === "google-ai-studio", + ); + expect(aiStudio).toBeDefined(); + expect(aiStudio?.document).toBe(true); + }); + } +}); diff --git a/apps/gateway/src/chat/tools/messages-contain-documents.ts b/apps/gateway/src/chat/tools/messages-contain-documents.ts new file mode 100644 index 0000000000..6356475054 --- /dev/null +++ b/apps/gateway/src/chat/tools/messages-contain-documents.ts @@ -0,0 +1,24 @@ +import type { BaseMessage } from "@llmgateway/models"; + +/** + * Checks if any messages contain `file` content blocks. Used to filter + * providers/models that do not accept document input when the router is + * selecting a target (model: "auto") or when validating an explicit selection. + */ +export function messagesContainDocuments(messages: BaseMessage[]): boolean { + for (const message of messages) { + if (Array.isArray(message.content)) { + for (const part of message.content) { + if ( + typeof part === "object" && + part !== null && + "type" in part && + part.type === "file" + ) { + return true; + } + } + } + } + return false; +} diff --git a/apps/gateway/src/chat/tools/validate-model-capabilities.ts b/apps/gateway/src/chat/tools/validate-model-capabilities.ts index 4e0b72de67..585f5a5e98 100644 --- a/apps/gateway/src/chat/tools/validate-model-capabilities.ts +++ b/apps/gateway/src/chat/tools/validate-model-capabilities.ts @@ -19,6 +19,7 @@ export interface ValidateModelCapabilitiesOptions { tool_choice?: unknown; webSearchTool?: WebSearchTool; hasImages?: boolean; + hasDocuments?: boolean; } /** @@ -43,6 +44,7 @@ export function validateModelCapabilities( tool_choice, webSearchTool, hasImages, + hasDocuments, } = options; if ( @@ -77,6 +79,32 @@ export function validateModelCapabilities( } } + // Validate document capability when the request contains `file` content blocks. + // Skip for "auto" and "custom" models (router/transform handle dynamic resolution). + if ( + hasDocuments && + requestedModel !== "auto" && + requestedModel !== "custom" + ) { + const providersToCheck = requestedProvider + ? modelInfo.providers.filter( + (p) => (p as ProviderModelMapping).providerId === requestedProvider, + ) + : modelInfo.providers; + + const supportsDocuments = providersToCheck.some( + (provider) => (provider as ProviderModelMapping).document === true, + ); + + if (!supportsDocuments) { + throw new HTTPException(400, { + message: requestedProvider + ? `Provider ${requestedProvider} does not support document input for model ${requestedModel}. Remove the file content or use a document-capable model.` + : `Model ${requestedModel} does not support document input. Remove the file content or use a document-capable model.`, + }); + } + } + // Validate JSON object output capability if (response_format?.type === "json_object") { const providersToCheck = requestedProvider diff --git a/apps/playground/src/app/share/[shareId]/page.tsx b/apps/playground/src/app/share/[shareId]/page.tsx index feba304c83..55def5bde3 100644 --- a/apps/playground/src/app/share/[shareId]/page.tsx +++ b/apps/playground/src/app/share/[shareId]/page.tsx @@ -16,6 +16,7 @@ interface SharedMessage { content: string | null; images: string | null; audios: string | null; + documents: string | null; reasoning: string | null; tools: string | null; metadata?: unknown; @@ -30,6 +31,13 @@ interface StoredAudioPart { name?: string; } +interface StoredDocumentPart { + type?: string; + url?: string; + mediaType?: string; + name?: string; +} + interface SharedChatResponse { share: { id: string; @@ -333,6 +341,27 @@ function toUiMessage(message: SharedMessage): UIMessage { } } + if (message.documents) { + try { + const parsedDocuments = JSON.parse(message.documents) as unknown; + if (Array.isArray(parsedDocuments)) { + for (const document of parsedDocuments.filter(isStoredDocumentPart)) { + if (!document.url) { + continue; + } + parts.push({ + type: "file", + mediaType: document.mediaType ?? "application/octet-stream", + url: document.url, + ...(document.name ? { name: document.name } : {}), + }); + } + } + } catch { + // Ignore malformed legacy document payloads in public snapshots. + } + } + if (message.tools) { try { const parsedTools = JSON.parse(message.tools) as unknown; @@ -382,6 +411,15 @@ function isStoredAudioPart(value: unknown): value is StoredAudioPart { ); } +function isStoredDocumentPart(value: unknown): value is StoredDocumentPart { + return ( + typeof value === "object" && + value !== null && + "url" in value && + typeof (value as { url?: unknown }).url === "string" + ); +} + function isToolUiPart(value: unknown): value is UIMessage["parts"][number] { return ( typeof value === "object" && diff --git a/apps/playground/src/components/playground/chat-page-client.tsx b/apps/playground/src/components/playground/chat-page-client.tsx index 27098909f6..ba4446732c 100644 --- a/apps/playground/src/components/playground/chat-page-client.tsx +++ b/apps/playground/src/components/playground/chat-page-client.tsx @@ -648,6 +648,23 @@ export default function ChatPageClient({ return !!model?.audio; }, [availableModels, selectedModel]); + const supportsDocuments = useMemo(() => { + if (!selectedModel) { + return false; + } + const { providerId, modelId, providerModelName } = + parseModelSelectorValue(selectedModel); + const def = models.find((m) => m.id === modelId); + if (!def) { + return false; + } + if (!providerId) { + return def.mappings.some((p: ApiModelProviderMapping) => p.document); + } + const mapping = getSelectedMapping(def, providerId, providerModelName); + return !!mapping?.document; + }, [models, selectedModel]); + const supportsImageGen = useMemo(() => { if (!selectedModel) { return false; @@ -983,6 +1000,27 @@ export default function ChatPageClient({ } } + if ((msg as any).documents) { + try { + const parsedDocuments = JSON.parse((msg as any).documents); + if (Array.isArray(parsedDocuments)) { + for (const d of parsedDocuments) { + if (!d?.url) { + continue; + } + parts.push({ + type: "file", + mediaType: d.mediaType ?? "application/octet-stream", + url: d.url, + ...(d.name ? { name: d.name } : {}), + }); + } + } + } catch (error) { + toast.error("Failed to parse documents: " + getErrorMessage(error)); + } + } + if ((msg as any).tools) { try { const parsedTools = JSON.parse((msg as any).tools); @@ -1108,6 +1146,12 @@ export default function ChatPageClient({ mediaType: string; name?: string; }>, + documents?: Array<{ + type: "file"; + url: string; + mediaType: string; + name?: string; + }>, ) => { if (selectedOrganization && Number(selectedOrganization.credits) <= 0) { setShowTopUp(true); @@ -1122,6 +1166,7 @@ export default function ChatPageClient({ model: selectedModel, has_images: !!images?.length, has_audio: !!audio?.length, + has_documents: !!documents?.length, web_search: webSearchEnabled, }); errorOccurredRef.current = false; @@ -1162,6 +1207,9 @@ export default function ChatPageClient({ ...(content.trim() ? { content } : {}), ...(images?.length ? { images: JSON.stringify(images) } : {}), ...(audio?.length ? { audios: JSON.stringify(audio) } : {}), + ...(documents?.length + ? { documents: JSON.stringify(documents) } + : {}), }, }); savedUserMessage = savedMessage.message; @@ -1183,6 +1231,9 @@ export default function ChatPageClient({ ...(content.trim() ? { content } : {}), ...(images?.length ? { images: JSON.stringify(images) } : {}), ...(audio?.length ? { audios: JSON.stringify(audio) } : {}), + ...(documents?.length + ? { documents: JSON.stringify(documents) } + : {}), }, }); setIsLoading(false); @@ -1788,6 +1839,7 @@ export default function ChatPageClient({ messages={messages} supportsImages={supportsImages} supportsAudio={supportsAudio} + supportsDocuments={supportsDocuments} supportsImageGen={supportsImageGen} sendMessage={sendMessageWithHeaders} selectedModel={selectedModel} @@ -1843,6 +1895,7 @@ export default function ChatPageClient({ messages={messages} supportsImages={supportsImages} supportsAudio={supportsAudio} + supportsDocuments={supportsDocuments} supportsImageGen={supportsImageGen} sendMessage={sendMessageWithHeaders} selectedModel={selectedModel} @@ -2173,6 +2226,23 @@ function ExtraChatPanel({ return !!model?.audio; }, [availableModels, selectedModel]); + const supportsDocuments = useMemo(() => { + if (!selectedModel) { + return false; + } + const { providerId, modelId, providerModelName } = + parseModelSelectorValue(selectedModel); + const def = models.find((m) => m.id === modelId); + if (!def) { + return false; + } + if (!providerId) { + return def.mappings.some((p: ApiModelProviderMapping) => p.document); + } + const mapping = getSelectedMapping(def, providerId, providerModelName); + return !!mapping?.document; + }, [models, selectedModel]); + const supportsReasoning = useMemo(() => { if (!selectedModel) { return false; @@ -2518,6 +2588,7 @@ function ExtraChatPanel({ messages={messages} supportsImages={supportsImages} supportsAudio={supportsAudio} + supportsDocuments={supportsDocuments} supportsImageGen={supportsImageGen} sendMessage={sendMessageWithHeaders} selectedModel={selectedModel} diff --git a/apps/playground/src/components/playground/chat-ui.tsx b/apps/playground/src/components/playground/chat-ui.tsx index 48475d7e3a..602d7ffa19 100644 --- a/apps/playground/src/components/playground/chat-ui.tsx +++ b/apps/playground/src/components/playground/chat-ui.tsx @@ -6,6 +6,7 @@ import { Brain, GlobeIcon, AlertTriangle, + FileText, Info, GitFork, Loader2, @@ -158,6 +159,7 @@ interface ChatUIProps { messages: UIMessage[]; supportsImages: boolean; supportsAudio: boolean; + supportsDocuments: boolean; supportsImageGen: boolean; sendMessage: ( message: UIMessage, @@ -233,6 +235,12 @@ interface ChatUIProps { mediaType: string; name?: string; }>, + documents?: Array<{ + type: "file"; + url: string; + mediaType: string; + name?: string; + }>, ) => Promise<{ id: string } | undefined>; onEditUserMessage?: (message: UIMessage, content: string) => Promise; isLoading?: boolean; @@ -266,15 +274,37 @@ interface ExtractedParts { textParts: string[]; imageParts: any[]; audioParts: any[]; + documentParts: any[]; toolParts: any[]; reasoningContent: string; sourceParts: any[]; } +function isDocumentMediaType(mediaType: string | null | undefined): boolean { + // A "document" is anything that isn't an image or audio. We forward the + // MIME to the gateway verbatim and let the provider reject it if it's not + // supported — `UnsupportedDocumentFormatError` surfaces those rejections + // as a clean 400 with the actual MIME the provider refused. + if (!mediaType) { + return true; + } + if (mediaType.startsWith("image/") || mediaType.startsWith("audio/")) { + return false; + } + return true; +} + +function getDocumentMediaType(mediaType: string | null | undefined): string { + return mediaType && isDocumentMediaType(mediaType) + ? mediaType + : "application/octet-stream"; +} + function extractMessageParts(parts: any[]): ExtractedParts { const textParts: string[] = []; const imageParts: any[] = []; const audioParts: any[] = []; + const documentParts: any[] = []; const toolParts: any[] = []; const reasoningParts: string[] = []; const sourceParts: any[] = []; @@ -296,6 +326,8 @@ function extractMessageParts(parts: any[]): ExtractedParts { imageParts.push(p); } else if (p.type === "file" && p.mediaType?.startsWith("audio/")) { audioParts.push(p); + } else if (p.type === "file" && isDocumentMediaType(p.mediaType)) { + documentParts.push(p); } } @@ -303,6 +335,7 @@ function extractMessageParts(parts: any[]): ExtractedParts { textParts, imageParts, audioParts, + documentParts, toolParts, reasoningContent: reasoningParts.join(""), sourceParts, @@ -745,7 +778,7 @@ const UserMessage = memo( onEditCancel?: () => void; onEditConfirm?: (content: string) => Promise; }) => { - const { textParts, imageParts, audioParts } = useMemo( + const { textParts, imageParts, audioParts, documentParts } = useMemo( () => extractMessageParts(message.parts), [message.parts], ); @@ -868,6 +901,28 @@ const UserMessage = memo( ))} )} + {documentParts.length > 0 && ( +
+ {documentParts.map((part: any, idx: number) => { + const name = part.name ?? part.filename ?? "Document"; + const mediaType: string = part.mediaType ?? ""; + return ( + + + {name} + + ); + })} +
+ )} {canEdit && !isEditing ? ( @@ -1009,6 +1064,7 @@ export const ChatUI = ({ messages, supportsImages, supportsAudio, + supportsDocuments, supportsImageGen, sendMessage, selectedModel, @@ -1215,6 +1271,18 @@ export const ChatUI = ({ })) : undefined; + const documentsToSave = + supportsDocuments && files?.length + ? files + .filter((f) => f.url && isDocumentMediaType(f.mediaType)) + .map((f) => ({ + type: "file" as const, + url: f.url!, + mediaType: getDocumentMediaType(f.mediaType), + ...(f.filename ? { name: f.filename } : {}), + })) + : undefined; + if (content.trim()) { parts.push({ type: "text", text: content }); } @@ -1247,6 +1315,19 @@ export const ChatUI = ({ } } + if (supportsDocuments && files?.length) { + for (const file of files) { + if (file.url && isDocumentMediaType(file.mediaType)) { + parts.push({ + type: "file", + url: file.url, + mediaType: getDocumentMediaType(file.mediaType), + name: file.filename, + }); + } + } + } + if (parts.length === 0) { return; } @@ -1258,11 +1339,18 @@ export const ChatUI = ({ // Otherwise `onFinish` may run before `chatIdRef` is set, and we can't save the AI response. if ( onUserMessage && - (content.trim() || imagesToSave?.length || audioToSave?.length) + (content.trim() || + imagesToSave?.length || + audioToSave?.length || + documentsToSave?.length) ) { savedMessage = - (await onUserMessage(content, imagesToSave, audioToSave)) ?? - undefined; + (await onUserMessage( + content, + imagesToSave, + audioToSave, + documentsToSave, + )) ?? undefined; } // If a persistent chat was expected (onUserMessage provided) but persistence @@ -1540,23 +1628,24 @@ export const ChatUI = ({ transition={{ duration: 0.18, ease: "easeOut" }} > { void handlePromptSubmit(message.text ?? "", message.files); @@ -1616,18 +1705,31 @@ export const ChatUI = ({ - {(supportsImages || supportsAudio) && ( + {(supportsImages || supportsAudio || supportsDocuments) && ( { + const parts: string[] = []; + if (supportsImages) { + parts.push("photos"); + } + if (supportsAudio) { + parts.push("audio"); + } + if (supportsDocuments) { + parts.push("documents"); + } + if (parts.length === 0) { + return undefined; + } + if (parts.length === 1) { + return `Add ${parts[0]}`; + } + const last = parts.pop(); + return `Add ${parts.join(", ")} or ${last}`; + })()} /> diff --git a/apps/playground/src/lib/api/v1.d.ts b/apps/playground/src/lib/api/v1.d.ts index 1ca84ce051..e6dd582a49 100644 --- a/apps/playground/src/lib/api/v1.d.ts +++ b/apps/playground/src/lib/api/v1.d.ts @@ -571,6 +571,7 @@ export interface paths { content: string | null; images: string | null; audios?: string | null; + documents?: string | null; reasoning: string | null; tools: string | null; metadata?: { @@ -9207,6 +9208,7 @@ export interface paths { content: string | null; images: string | null; audios: string | null; + documents?: string | null; reasoning: string | null; tools: string | null; metadata: { @@ -9476,6 +9478,7 @@ export interface paths { content: string | null; images: string | null; audios?: string | null; + documents?: string | null; reasoning: string | null; tools: string | null; metadata?: { @@ -9674,6 +9677,7 @@ export interface paths { content?: string; images?: string; audios?: string; + documents?: string; reasoning?: string; tools?: string; metadata?: { @@ -9697,6 +9701,7 @@ export interface paths { content: string | null; images: string | null; audios: string | null; + documents?: string | null; reasoning: string | null; tools: string | null; metadata: { @@ -9764,6 +9769,7 @@ export interface paths { content: string | null; images: string | null; audios: string | null; + documents?: string | null; reasoning: string | null; tools: string | null; metadata: { @@ -11608,6 +11614,7 @@ export interface operations { streaming: boolean; vision: boolean | null; audio: boolean | null; + document: boolean | null; reasoning: boolean | null; reasoningOutput: string | null; tools: boolean | null; diff --git a/apps/playground/src/lib/fetch-models.ts b/apps/playground/src/lib/fetch-models.ts index b0464d2882..22ded944c9 100644 --- a/apps/playground/src/lib/fetch-models.ts +++ b/apps/playground/src/lib/fetch-models.ts @@ -35,6 +35,7 @@ export interface ApiModelProviderMapping { streaming: boolean; vision: boolean | null; audio: boolean | null; + document: boolean | null; reasoning: boolean | null; reasoningOutput: string | null; tools: boolean | null; diff --git a/apps/ui/public/changelog/document-reading.png b/apps/ui/public/changelog/document-reading.png new file mode 100644 index 0000000000..e82005ae92 Binary files /dev/null and b/apps/ui/public/changelog/document-reading.png differ diff --git a/apps/ui/src/content/changelog/2026-05-26-document-reading-support.md b/apps/ui/src/content/changelog/2026-05-26-document-reading-support.md new file mode 100644 index 0000000000..c2124b1a08 --- /dev/null +++ b/apps/ui/src/content/changelog/2026-05-26-document-reading-support.md @@ -0,0 +1,40 @@ +--- +id: "49" +slug: "document-reading-support" +date: "2026-05-26" +title: "Document Reading (PDFs & more)" +summary: "Send PDFs and text-family documents to Gemini models via the OpenAI-compatible `file` content block." +image: + src: "/changelog/document-reading.png" + alt: "LLM Gateway now supports document attachments on chat completions" + width: 1024 + height: 1024 +--- + +LLM Gateway now accepts **document attachments** on chat completions through OpenAI's `file` content block. Send a PDF (or other supported file type) as base64 `file_data` and the gateway forwards it to the model. + +```bash +curl -X POST "https://api.llmgateway.io/v1/chat/completions" \ + -H "Authorization: Bearer $LLM_GATEWAY_API_KEY" \ + -H "Content-Type: application/json" \ + -d '{ + "model": "gemini-2.5-flash", + "messages": [{ + "role": "user", + "content": [ + { "type": "text", "text": "Summarize this document." }, + { "type": "file", "file": { + "filename": "report.pdf", + "file_data": "data:application/pdf;base64,JVBERi0xLjQK..." + }} + ] + }] + }' +``` + +- Initial support on **Google Gemini** via Google AI Studio +- New `document` capability flag on models — filter at [/models?filters=1&document=true](https://llmgateway.io/models?filters=1&document=true) +- Playground accepts file uploads and persists them across sessions and shares +- Clean `400` errors when a model doesn't accept the MIME type, instead of opaque upstream failures + +**[Read the docs →](https://docs.llmgateway.io/features/documents)** diff --git a/apps/ui/src/lib/api/v1.d.ts b/apps/ui/src/lib/api/v1.d.ts index 1ca84ce051..e6dd582a49 100644 --- a/apps/ui/src/lib/api/v1.d.ts +++ b/apps/ui/src/lib/api/v1.d.ts @@ -571,6 +571,7 @@ export interface paths { content: string | null; images: string | null; audios?: string | null; + documents?: string | null; reasoning: string | null; tools: string | null; metadata?: { @@ -9207,6 +9208,7 @@ export interface paths { content: string | null; images: string | null; audios: string | null; + documents?: string | null; reasoning: string | null; tools: string | null; metadata: { @@ -9476,6 +9478,7 @@ export interface paths { content: string | null; images: string | null; audios?: string | null; + documents?: string | null; reasoning: string | null; tools: string | null; metadata?: { @@ -9674,6 +9677,7 @@ export interface paths { content?: string; images?: string; audios?: string; + documents?: string; reasoning?: string; tools?: string; metadata?: { @@ -9697,6 +9701,7 @@ export interface paths { content: string | null; images: string | null; audios: string | null; + documents?: string | null; reasoning: string | null; tools: string | null; metadata: { @@ -9764,6 +9769,7 @@ export interface paths { content: string | null; images: string | null; audios: string | null; + documents?: string | null; reasoning: string | null; tools: string | null; metadata: { @@ -11608,6 +11614,7 @@ export interface operations { streaming: boolean; vision: boolean | null; audio: boolean | null; + document: boolean | null; reasoning: boolean | null; reasoningOutput: string | null; tools: boolean | null; diff --git a/ee/admin/src/lib/api/v1.d.ts b/ee/admin/src/lib/api/v1.d.ts index 1ca84ce051..e6dd582a49 100644 --- a/ee/admin/src/lib/api/v1.d.ts +++ b/ee/admin/src/lib/api/v1.d.ts @@ -571,6 +571,7 @@ export interface paths { content: string | null; images: string | null; audios?: string | null; + documents?: string | null; reasoning: string | null; tools: string | null; metadata?: { @@ -9207,6 +9208,7 @@ export interface paths { content: string | null; images: string | null; audios: string | null; + documents?: string | null; reasoning: string | null; tools: string | null; metadata: { @@ -9476,6 +9478,7 @@ export interface paths { content: string | null; images: string | null; audios?: string | null; + documents?: string | null; reasoning: string | null; tools: string | null; metadata?: { @@ -9674,6 +9677,7 @@ export interface paths { content?: string; images?: string; audios?: string; + documents?: string; reasoning?: string; tools?: string; metadata?: { @@ -9697,6 +9701,7 @@ export interface paths { content: string | null; images: string | null; audios: string | null; + documents?: string | null; reasoning: string | null; tools: string | null; metadata: { @@ -9764,6 +9769,7 @@ export interface paths { content: string | null; images: string | null; audios: string | null; + documents?: string | null; reasoning: string | null; tools: string | null; metadata: { @@ -11608,6 +11614,7 @@ export interface operations { streaming: boolean; vision: boolean | null; audio: boolean | null; + document: boolean | null; reasoning: boolean | null; reasoningOutput: string | null; tools: boolean | null; diff --git a/packages/actions/src/transform-google-messages.spec.ts b/packages/actions/src/transform-google-messages.spec.ts index 77be22d006..4396d87771 100644 --- a/packages/actions/src/transform-google-messages.spec.ts +++ b/packages/actions/src/transform-google-messages.spec.ts @@ -2,8 +2,10 @@ import { describe, expect, it } from "vitest"; import { googleProviderSupportsAudioFormat, + parseGoogleUpstreamDocumentError, transformGoogleMessages, UnsupportedAudioFormatError, + UnsupportedDocumentFormatError, } from "./transform-google-messages.js"; import type { BaseMessage } from "@llmgateway/models"; @@ -172,3 +174,207 @@ describe("googleProviderSupportsAudioFormat", () => { expect(googleProviderSupportsAudioFormat(undefined, "wav")).toBe(true); }); }); + +function fileMessage(mime: string, data = "ZHVtbXk="): BaseMessage[] { + return [ + { + role: "user", + content: [ + { type: "text", text: "summarize" }, + { + type: "file", + file: { + filename: "doc", + file_data: `data:${mime};base64,${data}`, + }, + }, + ], + }, + ]; +} + +describe("transformGoogleMessages — document file blocks", () => { + it("passes the supplied MIME through to inline_data verbatim", async () => { + const out = await transformGoogleMessages( + fileMessage("application/pdf"), + false, + 20, + null, + undefined, + "google-ai-studio", + ); + const filePart = out[0].parts.find((p) => p.inline_data); + expect(filePart?.inline_data?.mime_type).toBe("application/pdf"); + expect(filePart?.inline_data?.data).toBe("ZHVtbXk="); + }); + + it("does not pre-validate MIMEs — Google's API is authoritative", async () => { + // Pass a MIME Gemini would reject (.docx). The transform still builds the + // request body; the upstream call rejects, and parseGoogleUpstreamDocumentError + // re-emits the typed error from Google's response. + const out = await transformGoogleMessages( + fileMessage( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ), + false, + 20, + null, + undefined, + "google-ai-studio", + ); + const filePart = out[0].parts.find((p) => p.inline_data); + expect(filePart?.inline_data?.mime_type).toBe( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ); + }); + + it("preserves a mixed-case mime verbatim", async () => { + const out = await transformGoogleMessages( + fileMessage("Application/PDF"), + false, + 20, + null, + undefined, + "google-ai-studio", + ); + const filePart = out[0].parts.find((p) => p.inline_data); + expect(filePart?.inline_data?.mime_type).toBe("Application/PDF"); + }); + + it("accepts RFC 2397 MIME parameters and strips them", async () => { + const out = await transformGoogleMessages( + [ + { + role: "user", + content: [ + { + type: "file", + file: { + filename: "doc", + file_data: "data:text/plain;charset=utf-8;base64,SGVsbG8=", + }, + }, + ], + }, + ], + false, + 20, + null, + undefined, + "google-ai-studio", + ); + const filePart = out[0].parts.find((p) => p.inline_data); + expect(filePart?.inline_data?.mime_type).toBe("text/plain"); + expect(filePart?.inline_data?.data).toBe("SGVsbG8="); + }); + + it("throws a non-typed Error when file_data isn't a base64 data URL", async () => { + await expect( + transformGoogleMessages( + [ + { + role: "user", + content: [ + { + type: "file", + file: { filename: "x", file_data: "not-a-data-url" }, + }, + ], + }, + ], + false, + 20, + null, + undefined, + "google-ai-studio", + ), + ).rejects.toThrow(/data URL/); + }); + + it("throws a non-typed Error when only file_id is provided", async () => { + await expect( + transformGoogleMessages( + [ + { + role: "user", + content: [{ type: "file", file: { file_id: "file-abc123" } }], + }, + ], + false, + 20, + null, + undefined, + "google-ai-studio", + ), + ).rejects.toThrow(/file_data/); + }); +}); + +describe("parseGoogleUpstreamDocumentError", () => { + const aiStudioUnsupportedMime = JSON.stringify({ + error: { + code: 400, + status: "INVALID_ARGUMENT", + message: "Unsupported MIME type: application/msword", + }, + }); + + it("returns a typed UnsupportedDocumentFormatError for the canonical message", () => { + const result = parseGoogleUpstreamDocumentError( + aiStudioUnsupportedMime, + "google-ai-studio", + ); + expect(result).toBeInstanceOf(UnsupportedDocumentFormatError); + expect(result?.mimeType).toBe("application/msword"); + expect(result?.providerTarget).toBe("Google AI Studio"); + }); + + it("maps Vertex provider to the right target string", () => { + const result = parseGoogleUpstreamDocumentError( + aiStudioUnsupportedMime, + "google-vertex", + ); + expect(result?.providerTarget).toBe("Vertex AI"); + }); + + it("handles a trailing period in Google's message", () => { + const body = JSON.stringify({ + error: { message: "Unsupported MIME type: application/zip." }, + }); + const result = parseGoogleUpstreamDocumentError(body, "google-ai-studio"); + expect(result?.mimeType).toBe("application/zip"); + }); + + it("returns null for unrelated Google errors", () => { + const body = JSON.stringify({ + error: { + code: 400, + status: "INVALID_ARGUMENT", + message: "Request contains an invalid argument.", + }, + }); + expect( + parseGoogleUpstreamDocumentError(body, "google-ai-studio"), + ).toBeNull(); + }); + + it("returns null for the RTF 500 case (Google internal error)", () => { + const body = JSON.stringify({ + error: { + code: 500, + status: "INTERNAL", + message: "An internal error has occurred. Please retry.", + }, + }); + expect( + parseGoogleUpstreamDocumentError(body, "google-ai-studio"), + ).toBeNull(); + }); + + it("returns null for non-JSON garbage", () => { + expect( + parseGoogleUpstreamDocumentError("503", "google-ai-studio"), + ).toBeNull(); + expect(parseGoogleUpstreamDocumentError("", "google-ai-studio")).toBeNull(); + }); +}); diff --git a/packages/actions/src/transform-google-messages.ts b/packages/actions/src/transform-google-messages.ts index aabd92608d..ae99832d6c 100644 --- a/packages/actions/src/transform-google-messages.ts +++ b/packages/actions/src/transform-google-messages.ts @@ -1,5 +1,6 @@ import { type BaseMessage, + isFileContent, isImageUrlContent, isInputAudioContent, isTextContent, @@ -89,6 +90,107 @@ export class UnsupportedAudioFormatError extends Error { } } +/** + * Thrown when a `file` content block is structurally invalid for Google + * providers (e.g. missing `file_data`, or `file_data` that isn't a base64 + * data URL). The gateway maps this to HTTP 400 so the client gets an + * actionable validation error instead of a generic 500. + */ +export class InvalidFileContentError extends Error { + constructor(message: string) { + super(message); + this.name = "InvalidFileContentError"; + } +} + +/** + * Thrown when an upstream Google provider rejects the request because the + * document MIME we passed isn't supported by that specific model. We don't + * pre-validate the MIME on our side — Gemini's per-model support varies and + * Google's API is authoritative. Instead we parse Google's own error response + * after the fact (see `parseGoogleUpstreamDocumentError`) and re-emit as this + * typed error so the client sees a clean 400 with a consistent shape. + */ +export class UnsupportedDocumentFormatError extends Error { + readonly mimeType: string; + readonly providerTarget: string; + constructor(mimeType: string, providerTarget: string) { + super( + `Document MIME type "${mimeType}" is not supported by ${providerTarget}.`, + ); + this.name = "UnsupportedDocumentFormatError"; + this.mimeType = mimeType; + this.providerTarget = providerTarget; + } +} + +/** + * Parses an upstream error response body from a Google provider. If the body + * matches Google's "Unsupported MIME type: " pattern (HTTP 400 with + * status `INVALID_ARGUMENT`), returns a typed `UnsupportedDocumentFormatError` + * the gateway can throw to surface a clean 400 to the client. Returns null + * for any other error shape — the caller falls back to its normal error path. + * + * Empirically verified against Google AI Studio generateContent — the error + * shape is: + * { "error": { "code": 400, "status": "INVALID_ARGUMENT", + * "message": "Unsupported MIME type: application/msword" } } + */ +export function parseGoogleUpstreamDocumentError( + errorBody: string, + providerId: ProviderId | string | undefined, +): UnsupportedDocumentFormatError | null { + let parsed: unknown; + try { + parsed = JSON.parse(errorBody); + } catch { + return null; + } + if (!parsed || typeof parsed !== "object") { + return null; + } + const err = (parsed as { error?: unknown }).error; + if (!err || typeof err !== "object") { + return null; + } + const message = (err as { message?: unknown }).message; + if (typeof message !== "string") { + return null; + } + const match = message.match(/^Unsupported MIME type:\s*(.+?)\.?\s*$/i); + if (!match) { + return null; + } + return new UnsupportedDocumentFormatError( + match[1].trim(), + resolveGoogleProviderTarget(providerId), + ); +} + +/** + * Parses a `data:[;param=value]*;base64,` URL into its parts. + * Returns null when the value isn't a base64 data URL. Optional RFC 2397 + * MIME parameters (e.g. `;charset=utf-8`) are accepted but stripped, since + * Google's `inline_data.mime_type` expects a bare type/subtype. + */ +function parseFileDataUrl( + fileData: string, +): { mimeType: string; data: string } | null { + const match = fileData.match( + /^data:([^;,]+)((?:;[^;,]+=[^;,]*)*);base64,(.*)$/i, + ); + if (!match) { + return null; + } + return { mimeType: match[1], data: match[3] }; +} + +function resolveGoogleProviderTarget( + providerId: ProviderId | string | undefined, +): string { + return VERTEX_FAMILY.has(providerId ?? "") ? "Vertex AI" : "Google AI Studio"; +} + function resolveGoogleAudioMime( format: GoogleAudioFormat, providerId: ProviderId | string | undefined, @@ -287,6 +389,29 @@ export async function transformGoogleMessages( data: content.input_audio.data, }, }); + } else if (isFileContent(content)) { + if (!content.file.file_data) { + throw new InvalidFileContentError( + "Google providers require base64 file_data on `file` content blocks; file_id references are not supported.", + ); + } + const parsed = parseFileDataUrl(content.file.file_data); + if (!parsed) { + throw new InvalidFileContentError( + "Invalid file_data: expected a base64-encoded data URL (e.g. 'data:application/pdf;base64,...').", + ); + } + // MIME support varies across Gemini models, so we don't pre-validate; + // Google's API is authoritative. If it rejects with "Unsupported MIME + // type: X", `parseGoogleUpstreamDocumentError` (called by the gateway + // after the upstream call) re-emits it as a typed + // UnsupportedDocumentFormatError -> clean HTTP 400 for the client. + parts.push({ + inline_data: { + mime_type: parsed.mimeType, + data: parsed.data, + }, + }); } else { throw new Error( `Not supported content type yet: ${(content as any).type}`, diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 80f0aabb80..04f1c7a551 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -1122,6 +1122,7 @@ export const message = pgTable( content: text(), // Made nullable to support image-only messages images: text(), // JSON string to store images array audios: text(), // JSON string to store audio attachments array + documents: text(), // JSON string to store document attachments array reasoning: text(), // Reasoning content from AI models tools: text(), // JSON string to store tool call parts metadata: jsonb().$type>(), diff --git a/packages/models/src/models.ts b/packages/models/src/models.ts index 557c1696c0..3db3df5a11 100644 --- a/packages/models/src/models.ts +++ b/packages/models/src/models.ts @@ -292,6 +292,15 @@ export interface ProviderModelMapping { * audio content. */ audio?: boolean; + /** + * Whether this specific model accepts document inputs (`file` content + * blocks carrying PDF or text-family MIME types) for this provider. Used by + * the `model: "auto"` router and capability validator to avoid selecting + * providers that would fail upstream when the request contains document + * content. Per-provider MIME allowlists live in the provider transform + * modules (e.g. transform-google-messages.ts). + */ + document?: boolean; /** * Whether this model supports reasoning mode */ diff --git a/packages/models/src/models/google.ts b/packages/models/src/models/google.ts index 5074763b4c..f4055f5cc7 100644 --- a/packages/models/src/models/google.ts +++ b/packages/models/src/models/google.ts @@ -39,6 +39,7 @@ export const googleModels = [ streaming: true, vision: true, audio: true, + document: true, tools: true, webSearch: true, webSearchPrice: "0.035", // $35 per 1000 prompts @@ -118,6 +119,7 @@ export const googleModels = [ streaming: true, vision: true, audio: true, + document: true, tools: true, jsonOutput: true, jsonOutputSchema: true, @@ -193,6 +195,7 @@ export const googleModels = [ streaming: true, vision: true, audio: true, + document: true, tools: true, jsonOutput: true, jsonOutputSchema: true, @@ -253,6 +256,7 @@ export const googleModels = [ streaming: true, vision: true, audio: true, + document: true, tools: true, jsonOutput: true, jsonOutputSchema: true, @@ -298,6 +302,7 @@ export const googleModels = [ streaming: true, vision: true, audio: true, + document: true, tools: true, jsonOutput: true, jsonOutputSchema: true, @@ -345,6 +350,7 @@ export const googleModels = [ streaming: true, vision: true, audio: true, + document: true, tools: true, jsonOutput: true, jsonOutputSchema: true, @@ -393,6 +399,7 @@ export const googleModels = [ streaming: true, vision: true, audio: true, + document: true, tools: true, webSearch: true, webSearchPrice: "0.035", // $35 per 1000 prompts @@ -445,6 +452,7 @@ export const googleModels = [ streaming: true, vision: true, audio: true, + document: true, tools: true, jsonOutput: true, jsonOutputSchema: true, @@ -489,6 +497,7 @@ export const googleModels = [ streaming: true, vision: true, audio: true, + document: true, tools: true, jsonOutput: true, jsonOutputSchema: true, @@ -550,6 +559,7 @@ export const googleModels = [ streaming: true, vision: true, audio: true, + document: true, tools: true, webSearch: true, webSearchPrice: "0.014", @@ -597,6 +607,7 @@ export const googleModels = [ streaming: true, vision: true, audio: true, + document: true, tools: true, webSearch: true, webSearchPrice: "0.014", // $14 per 1000 queries for Gemini 3 @@ -685,6 +696,7 @@ export const googleModels = [ streaming: true, vision: true, audio: true, + document: true, tools: true, webSearch: true, webSearchPrice: "0.014", @@ -791,6 +803,7 @@ export const googleModels = [ streaming: true, vision: true, audio: true, + document: true, tools: true, reasoning: true, reasoningMaxTokens: true, @@ -844,6 +857,7 @@ export const googleModels = [ streaming: true, vision: true, audio: true, + document: true, tools: true, webSearch: true, reasoning: true, @@ -902,6 +916,7 @@ export const googleModels = [ streaming: true, vision: true, audio: true, + document: true, tools: true, webSearch: true, jsonOutput: true, @@ -1212,6 +1227,7 @@ export const googleModels = [ streaming: true, vision: true, audio: true, + document: true, tools: true, webSearch: true, webSearchPrice: "0.014", // $14 per 1000 queries for Gemini 3 @@ -1495,6 +1511,7 @@ export const googleModels = [ streaming: true, vision: true, audio: true, + document: true, tools: true, jsonOutput: true, jsonOutputSchema: true, @@ -1540,6 +1557,7 @@ export const googleModels = [ streaming: true, vision: true, audio: true, + document: true, tools: true, jsonOutput: true, jsonOutputSchema: true, @@ -1585,6 +1603,7 @@ export const googleModels = [ streaming: true, vision: true, audio: true, + document: true, tools: true, jsonOutput: true, jsonOutputSchema: true, @@ -1630,6 +1649,7 @@ export const googleModels = [ streaming: true, vision: false, audio: true, + document: true, tools: true, jsonOutput: true, jsonOutputSchema: true, @@ -1673,6 +1693,7 @@ export const googleModels = [ streaming: true, vision: false, audio: true, + document: true, tools: true, jsonOutput: true, jsonOutputSchema: true, @@ -1719,6 +1740,7 @@ export const googleModels = [ streaming: true, vision: false, audio: true, + document: true, tools: true, jsonOutput: true, jsonOutputSchema: true, diff --git a/packages/models/src/types.ts b/packages/models/src/types.ts index a6efc9bf85..437f4c1a9c 100644 --- a/packages/models/src/types.ts +++ b/packages/models/src/types.ts @@ -51,6 +51,15 @@ export interface InputAudioContent { }; } +export interface FileContent { + type: "file"; + file: { + filename?: string; + file_data?: string; + file_id?: string; + }; +} + export interface ToolUseContent { type: "tool_use"; id: string; @@ -69,6 +78,7 @@ export type MessageContent = | ImageUrlContent | ImageContent | InputAudioContent + | FileContent | ToolUseContent | ToolResultContent; @@ -442,6 +452,10 @@ export function isInputAudioContent( return content.type === "input_audio"; } +export function isFileContent(content: MessageContent): content is FileContent { + return content.type === "file"; +} + export function isToolUseContent( content: MessageContent, ): content is ToolUseContent {