From 8f20ca97b4f65605cd7442bce320ca31c2d7f865 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Wed, 26 Aug 2026 06:55:33 +0900 Subject: [PATCH] fix(google): carry chat video input onto Gemini wire (#2458) --- src/adapters/anthropic.ts | 1 + src/adapters/command-code.ts | 3 +- src/adapters/cursor/protobuf-request.ts | 1 + src/adapters/google.ts | 9 ++++- src/adapters/image.ts | 2 +- src/chat/inbound.ts | 15 +++++++- src/responses/parser.ts | 4 ++ src/responses/schema.ts | 6 ++- src/server/responses/input-admission.ts | 4 +- src/types/request.ts | 10 ++++- tests/google-adapter.test.ts | 49 +++++++++++++++++++++++++ 11 files changed, 96 insertions(+), 8 deletions(-) diff --git a/src/adapters/anthropic.ts b/src/adapters/anthropic.ts index 25e6467ad0b..b8312eb453d 100644 --- a/src/adapters/anthropic.ts +++ b/src/adapters/anthropic.ts @@ -37,6 +37,7 @@ function toAnthropicContentPart(p: OcxContentPart): unknown { ? { type: "image", source: { type: "base64", media_type: data.mediaType, data: data.base64 } } : { type: "image", source: { type: "url", url: p.imageUrl } }; } + if (p.type === "video") return { type: "text", text: "[video]" }; return { type: "text", text: p.text }; } diff --git a/src/adapters/command-code.ts b/src/adapters/command-code.ts index 49f8bcf2a39..df6843ca20f 100644 --- a/src/adapters/command-code.ts +++ b/src/adapters/command-code.ts @@ -144,7 +144,8 @@ function wireMessages(messages: OcxMessage[]): Array> { if (typeof message.content === "string") content.push({ type: "text", text: message.content }); else for (const part of message.content) { if (part.type === "text") content.push({ type: "text", text: part.text }); - else content.push(wireImagePart(part.imageUrl)); + else if (part.type === "image") content.push(wireImagePart(part.imageUrl)); + else content.push({ type: "text", text: "[video]" }); } out.push({ role: "user", content }); } diff --git a/src/adapters/cursor/protobuf-request.ts b/src/adapters/cursor/protobuf-request.ts index 18d5957eab1..74e64bd5b57 100644 --- a/src/adapters/cursor/protobuf-request.ts +++ b/src/adapters/cursor/protobuf-request.ts @@ -436,6 +436,7 @@ function decodeResultParts(message: OcxToolResultMessage): DecodedResultPart[] | if (typeof content === "string") return undefined; return content.map((part): DecodedResultPart => { if (part.type === "text") return { kind: "text", text: part.text }; + if (part.type === "video") return { kind: "text", text: "[video]" }; const decoded = decodeInlineImage(part.imageUrl); return decoded ? { kind: "image", ...decoded } : { kind: "undecodable" }; }); diff --git a/src/adapters/google.ts b/src/adapters/google.ts index 23361f7735f..78354e7fe51 100644 --- a/src/adapters/google.ts +++ b/src/adapters/google.ts @@ -178,7 +178,7 @@ function geminiTextPart(text: unknown): { text: string } | undefined { */ function geminiToolResultText(content: string | OcxContentPart[]): string { if (typeof content === "string") return content || GEMINI_EMPTY_TOOL_OUTPUT_PLACEHOLDER; - const hasContent = content.some(p => p.type === "image" || (typeof p.text === "string" && p.text.length > 0)); + const hasContent = content.some(p => p.type !== "text" || p.text.length > 0); return hasContent ? contentPartsToText(content) : GEMINI_EMPTY_TOOL_OUTPUT_PLACEHOLDER; } @@ -262,6 +262,13 @@ function messagesToGeminiFormat( parts.push(data ? { inline_data: { mime_type: data.mediaType, data: data.base64 } } : { text: `[image: ${p.imageUrl}]` }); continue; } + if (p.type === "video") { + const data = parseDataUrl(p.videoUrl); + // Gemini accepts inline video bytes in the same Part union as images. Arbitrary + // remote URLs are not valid fileData references, so retain only a short marker. + parts.push(data ? { inline_data: { mime_type: data.mediaType, data: data.base64 } } : { text: `[video: ${p.videoUrl}]` }); + continue; + } // Drop empty/malformed text instead of emitting `{ text: "" }` or a bare `{}` part. const textPart = geminiTextPart(p.text); if (textPart) parts.push(textPart); diff --git a/src/adapters/image.ts b/src/adapters/image.ts index 0b2bcbf6fe6..39c5d683e50 100644 --- a/src/adapters/image.ts +++ b/src/adapters/image.ts @@ -18,6 +18,6 @@ export function parseDataUrl(url: string): { mediaType: string; base64: string } */ export function contentPartsToText(content: string | OcxContentPart[]): string { if (typeof content === "string") return content; - const text = content.map(p => (p.type === "text" ? p.text : "[image]")).join(""); + const text = content.map(p => p.type === "text" ? p.text : p.type === "image" ? "[image]" : "[video]").join(""); return text || "[image]"; } diff --git a/src/chat/inbound.ts b/src/chat/inbound.ts index 46268f13b45..db3b41d12ee 100644 --- a/src/chat/inbound.ts +++ b/src/chat/inbound.ts @@ -53,6 +53,14 @@ function imageUrlFromPart(part: Rec): string | null { return null; } +function videoUrlFromPart(part: Rec): string | null { + if (part.type !== "video_url") return null; + const videoUrl = part.video_url; + if (typeof videoUrl === "string" && videoUrl.length > 0) return videoUrl; + if (isRec(videoUrl) && typeof videoUrl.url === "string" && videoUrl.url.length > 0) return videoUrl.url; + return null; +} + function userContentToBlocks(content: unknown): Rec[] { if (typeof content === "string") { return content.length > 0 ? [{ type: "input_text", text: content }] : []; @@ -70,7 +78,12 @@ function userContentToBlocks(content: unknown): Rec[] { continue; } const imageUrl = imageUrlFromPart(raw); - if (imageUrl) blocks.push({ type: "input_image", image_url: imageUrl }); + if (imageUrl) { + blocks.push({ type: "input_image", image_url: imageUrl }); + continue; + } + const videoUrl = videoUrlFromPart(raw); + if (videoUrl) blocks.push({ type: "input_video", video_url: videoUrl }); } return blocks; } diff --git a/src/responses/parser.ts b/src/responses/parser.ts index 28b7c8c2bba..33de5ebeae8 100644 --- a/src/responses/parser.ts +++ b/src/responses/parser.ts @@ -45,6 +45,7 @@ type InputBlock = | { type: "input_text"; text: string } | { type: "text"; text: string } | { type: "input_image"; image_url?: string; file_id?: string; detail?: string } + | { type: "input_video"; video_url?: string } | { type: "input_file"; file_id?: string; filename?: string; file_data?: string }; /** A usable reference string, or undefined. Empty strings and non-strings are not references. */ @@ -80,6 +81,9 @@ function inputContentParts(blocks: unknown): string | OcxContentPart[] { } // No usable reference: omit the block. A "[image: ?]" marker would claim an attachment // the request never carried, which is worse than dropping malformed input. + } else if (block.type === "input_video") { + const videoUrl = nonEmptyString(block.video_url); + if (videoUrl) parts.push({ type: "video", videoUrl }); } else if (block.type === "input_file") { const b = block as { file_id?: string; filename?: string; file_data?: string }; const fileId = nonEmptyString(b.file_id); diff --git a/src/responses/schema.ts b/src/responses/schema.ts index 5e1f8762363..bc29734c8ae 100644 --- a/src/responses/schema.ts +++ b/src/responses/schema.ts @@ -11,6 +11,10 @@ const inputImageBlockSchema = z.object({ }).refine(v => typeof v.image_url === "string" || typeof v.file_id === "string", { message: "input_image requires at least one of image_url or file_id", }); +const inputVideoBlockSchema = z.object({ + type: z.literal("input_video"), + video_url: z.string().min(1), +}); const inputFileBlockSchema = z.object({ type: z.literal("input_file"), file_id: z.string().optional(), @@ -24,7 +28,7 @@ const reasoningTextSchema = z.object({ type: z.literal("reasoning_text"), text: // codex-rs FunctionCallOutputContentItem (protocol/src/models.rs): input_text | input_image | encrypted_content. const encryptedContentBlockSchema = z.object({ type: z.literal("encrypted_content"), encrypted_content: z.string() }); -const inputContentBlockSchema = z.union([inputTextSchema, plainTextSchema, inputImageBlockSchema, inputFileBlockSchema]); +const inputContentBlockSchema = z.union([inputTextSchema, plainTextSchema, inputImageBlockSchema, inputVideoBlockSchema, inputFileBlockSchema]); const outputContentBlockSchema = z.union([outputTextSchema, plainTextSchema, outputRefusalSchema]); // Tool outputs on the wire mix codex-rs FunctionCallOutputContentItem with legacy output blocks. const toolOutputContentBlockSchema = z.union([ diff --git a/src/server/responses/input-admission.ts b/src/server/responses/input-admission.ts index a2a06e5b737..2bd01a67804 100644 --- a/src/server/responses/input-admission.ts +++ b/src/server/responses/input-admission.ts @@ -68,7 +68,9 @@ function imageTokens(imageUrl: string): number { } function contentPartTokens(part: OcxContentPart, modelId: string): number { - return part.type === "image" ? imageTokens(part.imageUrl) : estimateTokens(part.text, modelId); + if (part.type === "image") return imageTokens(part.imageUrl); + if (part.type === "video") return imageTokens(part.videoUrl); + return estimateTokens(part.text, modelId); } function contentTokens(content: string | readonly OcxContentPart[], modelId: string): number { diff --git a/src/types/request.ts b/src/types/request.ts index efe2164e373..28f37ae6669 100644 --- a/src/types/request.ts +++ b/src/types/request.ts @@ -190,8 +190,14 @@ export interface OcxImageContent { detail?: string; } -/** A user/developer message content part: text or an image (vision). */ -export type OcxContentPart = OcxTextContent | OcxImageContent; +export interface OcxVideoContent { + type: "video"; + /** A base64 `data:` URL from an OpenAI-compatible `video_url` part. */ + videoUrl: string; +} + +/** A user/developer message content part: text or native media. */ +export type OcxContentPart = OcxTextContent | OcxImageContent | OcxVideoContent; export interface OcxThinkingContent { type: "thinking"; diff --git a/tests/google-adapter.test.ts b/tests/google-adapter.test.ts index 15af3a264ae..d067af9589b 100644 --- a/tests/google-adapter.test.ts +++ b/tests/google-adapter.test.ts @@ -1,5 +1,7 @@ import { describe, expect, test } from "bun:test"; import { createGoogleAdapter } from "../src/adapters/google"; +import { chatCompletionsToResponsesBody } from "../src/chat/inbound"; +import { parseRequest } from "../src/responses/parser"; import type { OcxParsedRequest } from "../src/types"; const provider = { adapter: "google", baseUrl: "https://generativelanguage.googleapis.com", apiKey: "key" }; @@ -82,6 +84,53 @@ describe("google adapter — tool result images", () => { }); }); +describe("google adapter — Chat Completions video input", () => { + test("carries an inline video through Chat translation onto Gemini inline_data", async () => { + const responsesBody = chatCompletionsToResponsesBody({ + model: "google-antigravity/gemini-3.7-flash", + messages: [{ + role: "user", + content: [ + { type: "text", text: "Summarize this video" }, + { type: "video_url", video_url: { url: "data:video/mp4;base64,aGVsbG8=" } }, + ], + }], + }); + const parsed = parseRequest(responsesBody); + parsed.modelId = "gemini-3.7-flash"; + + const contents = await geminiContents(parsed); + + expect(contents).toContainEqual({ + role: "user", + parts: [ + { text: "Summarize this video" }, + { inline_data: { mime_type: "video/mp4", data: "aGVsbG8=" } }, + ], + }); + }); + + test("does not mislabel an arbitrary remote video URL as Gemini file_data", async () => { + const responsesBody = chatCompletionsToResponsesBody({ + model: "google-antigravity/gemini-3.7-flash", + messages: [{ + role: "user", + content: [{ type: "video_url", video_url: { url: "https://example.test/video.mp4" } }], + }], + }); + const parsed = parseRequest(responsesBody); + parsed.modelId = "gemini-3.7-flash"; + + const contents = await geminiContents(parsed); + + expect(contents).toContainEqual({ + role: "user", + parts: [{ text: "[video: https://example.test/video.mp4]" }], + }); + expect(JSON.stringify(contents)).not.toContain("file_data"); + }); +}); + describe("google adapter — tool-call ids on the wire", () => { test("v2 collaboration encrypted marker never reaches functionDeclarations (issue #85)", async () => { // Codex Desktop v2 stamps `encrypted: true` on collaboration message properties; CCA/Gemini