diff --git a/.gitignore b/.gitignore index 705216c80c5..4501689258e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,7 @@ !.gemini/config.yaml !.gemini/commands/ -# Note: .gemini-clipboard/ is NOT in gitignore so Gemini can access pasted images +# Note: .qwen-clipboard/ is NOT in gitignore so Gemini can access pasted images # Dependency directory node_modules diff --git a/packages/cli/src/ui/components/InputPrompt.test.tsx b/packages/cli/src/ui/components/InputPrompt.test.tsx index 584dc15f696..0ddeee83e03 100644 --- a/packages/cli/src/ui/components/InputPrompt.test.tsx +++ b/packages/cli/src/ui/components/InputPrompt.test.tsx @@ -381,7 +381,7 @@ describe('InputPrompt', () => { it('should handle Ctrl+V when clipboard has an image', async () => { vi.mocked(clipboardUtils.clipboardHasImage).mockResolvedValue(true); vi.mocked(clipboardUtils.saveClipboardImage).mockResolvedValue( - '/test/.gemini-clipboard/clipboard-123.png', + '/test/.qwen-clipboard/clipboard-123.png', ); const { stdin, unmount } = renderWithProviders( @@ -441,7 +441,7 @@ describe('InputPrompt', () => { it('should insert image path at cursor position with proper spacing', async () => { const imagePath = path.join( 'test', - '.gemini-clipboard', + '.qwen-clipboard', 'clipboard-456.png', ); vi.mocked(clipboardUtils.clipboardHasImage).mockResolvedValue(true); diff --git a/packages/cli/src/ui/utils/clipboardUtils.ts b/packages/cli/src/ui/utils/clipboardUtils.ts index 9ccca7b6ce9..42910af4711 100644 --- a/packages/cli/src/ui/utils/clipboardUtils.ts +++ b/packages/cli/src/ui/utils/clipboardUtils.ts @@ -52,7 +52,7 @@ export async function saveClipboardImage( // Create a temporary directory for clipboard images within the target directory // This avoids security restrictions on paths outside the target directory const baseDir = targetDir || process.cwd(); - const tempDir = path.join(baseDir, '.gemini-clipboard'); + const tempDir = path.join(baseDir, '.qwen-clipboard'); await fs.mkdir(tempDir, { recursive: true }); // Generate a unique filename with timestamp @@ -130,7 +130,7 @@ export async function cleanupOldClipboardImages( ): Promise { try { const baseDir = targetDir || process.cwd(); - const tempDir = path.join(baseDir, '.gemini-clipboard'); + const tempDir = path.join(baseDir, '.qwen-clipboard'); const files = await fs.readdir(tempDir); const oneHourAgo = Date.now() - 60 * 60 * 1000; diff --git a/packages/core/src/core/anthropicContentGenerator/converter.test.ts b/packages/core/src/core/anthropicContentGenerator/converter.test.ts index f2ab79411ae..7d7f3166227 100644 --- a/packages/core/src/core/anthropicContentGenerator/converter.test.ts +++ b/packages/core/src/core/anthropicContentGenerator/converter.test.ts @@ -208,6 +208,385 @@ describe('AnthropicContentConverter', () => { ], }); }); + + it('converts function response with inlineData image parts into tool_result with images', () => { + const { messages } = converter.convertGeminiRequestToAnthropic({ + model: 'models/test', + contents: [ + { + role: 'user', + parts: [ + { + functionResponse: { + id: 'call-1', + name: 'Read', + response: { output: 'Image content' }, + parts: [ + { + inlineData: { + mimeType: 'image/png', + data: 'base64encodeddata', + }, + }, + ], + }, + }, + ], + }, + ], + }); + + expect(messages).toEqual([ + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'call-1', + content: [ + { type: 'text', text: 'Image content' }, + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: 'base64encodeddata', + }, + }, + ], + }, + ], + }, + ]); + }); + + it('renders non-image inlineData as a text block (avoids invalid image media_type)', () => { + const { messages } = converter.convertGeminiRequestToAnthropic({ + model: 'models/test', + contents: [ + { + role: 'user', + parts: [ + { + functionResponse: { + id: 'call-1', + name: 'Read', + response: { output: 'Audio content' }, + parts: [ + { + inlineData: { + mimeType: 'audio/mpeg', + data: 'base64encodedaudiodata', + }, + }, + ], + }, + }, + ], + }, + ], + }); + + expect(messages).toHaveLength(1); + expect(messages[0]?.role).toBe('user'); + + const toolResult = messages[0]?.content?.[0] as { + type: string; + content: Array<{ type: string; text?: string }>; + }; + expect(toolResult.type).toBe('tool_result'); + expect(Array.isArray(toolResult.content)).toBe(true); + expect(toolResult.content[0]).toEqual({ + type: 'text', + text: 'Audio content', + }); + expect(toolResult.content[1]?.type).toBe('text'); + expect(toolResult.content[1]?.text).toContain( + 'Unsupported inline media type', + ); + expect(toolResult.content[1]?.text).toContain('audio/mpeg'); + }); + + it('converts inlineData with PDF into document block', () => { + const { messages } = converter.convertGeminiRequestToAnthropic({ + model: 'models/test', + contents: [ + { + role: 'user', + parts: [ + { + functionResponse: { + id: 'call-1', + name: 'Read', + response: { output: 'PDF content' }, + parts: [ + { + inlineData: { + mimeType: 'application/pdf', + data: 'pdfbase64data', + }, + }, + ], + }, + }, + ], + }, + ], + }); + + expect(messages).toEqual([ + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'call-1', + content: [ + { type: 'text', text: 'PDF content' }, + { + type: 'document', + source: { + type: 'base64', + media_type: 'application/pdf', + data: 'pdfbase64data', + }, + }, + ], + }, + ], + }, + ]); + }); + + it('converts fileData with image into image url block', () => { + const { messages } = converter.convertGeminiRequestToAnthropic({ + model: 'models/test', + contents: [ + { + role: 'user', + parts: [ + { + functionResponse: { + id: 'call-1', + name: 'Read', + response: { output: 'Image content' }, + parts: [ + { + fileData: { + mimeType: 'image/jpeg', + fileUri: + 'https://upload.wikimedia.org/wikipedia/commons/a/a7/Camponotus_flavomarginatus_ant.jpg', + }, + }, + ], + }, + }, + ], + }, + ], + }); + + expect(messages).toEqual([ + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'call-1', + content: [ + { type: 'text', text: 'Image content' }, + { + type: 'image', + source: { + type: 'url', + url: 'https://upload.wikimedia.org/wikipedia/commons/a/a7/Camponotus_flavomarginatus_ant.jpg', + }, + }, + ], + }, + ], + }, + ]); + }); + + it('converts fileData with PDF into document url block', () => { + const { messages } = converter.convertGeminiRequestToAnthropic({ + model: 'models/test', + contents: [ + { + role: 'user', + parts: [ + { + functionResponse: { + id: 'call-1', + name: 'Read', + response: { output: 'PDF content' }, + parts: [ + { + fileData: { + mimeType: 'application/pdf', + fileUri: + 'https://assets.anthropic.com/m/1cd9d098ac3e6467/original/Claude-3-Model-Card-October-Addendum.pdf', + }, + }, + ], + }, + }, + ], + }, + ], + }); + + expect(messages).toEqual([ + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'call-1', + content: [ + { type: 'text', text: 'PDF content' }, + { + type: 'document', + source: { + type: 'url', + url: 'https://assets.anthropic.com/m/1cd9d098ac3e6467/original/Claude-3-Model-Card-October-Addendum.pdf', + }, + }, + ], + }, + ], + }, + ]); + }); + + it('renders unsupported fileData as a text block', () => { + const { messages } = converter.convertGeminiRequestToAnthropic({ + model: 'models/test', + contents: [ + { + role: 'user', + parts: [ + { + functionResponse: { + id: 'call-1', + name: 'Read', + response: { output: 'File content' }, + parts: [ + { + fileData: { + mimeType: 'application/zip', + fileUri: 'https://example.com/archive.zip', + displayName: 'archive.zip', + }, + }, + ], + }, + }, + ], + }, + ], + }); + + const toolResult = messages[0]?.content?.[0] as { + type: string; + content: Array<{ type: string; text?: string }>; + }; + expect(toolResult.type).toBe('tool_result'); + expect(toolResult.content[0]).toEqual({ + type: 'text', + text: 'File content', + }); + expect(toolResult.content[1]?.type).toBe('text'); + expect(toolResult.content[1]?.text).toContain( + 'Unsupported file media type', + ); + expect(toolResult.content[1]?.text).toContain('application/zip'); + expect(toolResult.content[1]?.text).toContain('archive.zip'); + }); + + it('associates each image with its preceding functionResponse', () => { + const { messages } = converter.convertGeminiRequestToAnthropic({ + model: 'models/test', + contents: [ + { + role: 'user', + parts: [ + // Tool 1 with image 1 + { + functionResponse: { + id: 'call-1', + name: 'Read', + response: { output: 'File 1' }, + parts: [ + { + inlineData: { + mimeType: 'image/png', + data: 'image1data', + }, + }, + ], + }, + }, + // Tool 2 with image 2 + { + functionResponse: { + id: 'call-2', + name: 'Read', + response: { output: 'File 2' }, + parts: [ + { + inlineData: { + mimeType: 'image/jpeg', + data: 'image2data', + }, + }, + ], + }, + }, + ], + }, + ], + }); + + // Multiple tool_result blocks are emitted in order + expect(messages).toHaveLength(1); + expect(messages[0]).toEqual({ + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'call-1', + content: [ + { type: 'text', text: 'File 1' }, + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: 'image1data', + }, + }, + ], + }, + { + type: 'tool_result', + tool_use_id: 'call-2', + content: [ + { type: 'text', text: 'File 2' }, + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/jpeg', + data: 'image2data', + }, + }, + ], + }, + ], + }); + }); }); describe('convertGeminiToolsToAnthropic', () => { diff --git a/packages/core/src/core/anthropicContentGenerator/converter.ts b/packages/core/src/core/anthropicContentGenerator/converter.ts index 2fb9b7feeb6..4aade511b61 100644 --- a/packages/core/src/core/anthropicContentGenerator/converter.ts +++ b/packages/core/src/core/anthropicContentGenerator/converter.ts @@ -10,7 +10,6 @@ import type { Content, ContentListUnion, ContentUnion, - FunctionCall, FunctionResponse, GenerateContentParameters, Part, @@ -30,15 +29,6 @@ type AnthropicMessageParam = Anthropic.MessageParam; type AnthropicToolParam = Anthropic.Tool; type AnthropicContentBlockParam = Anthropic.ContentBlockParam; -type ThoughtPart = { text: string; signature?: string }; - -interface ParsedParts { - thoughtParts: ThoughtPart[]; - contentParts: string[]; - functionCalls: FunctionCall[]; - functionResponses: FunctionResponse[]; -} - export class AnthropicContentConverter { private model: string; private schemaCompliance: SchemaComplianceMode; @@ -228,127 +218,189 @@ export class AnthropicContentConverter { } if (!this.isContentObject(content)) return; + const parts = content.parts || []; + const role = content.role === 'model' ? 'assistant' : 'user'; + const contentBlocks: AnthropicContentBlockParam[] = []; + let toolCallIndex = 0; - const parsed = this.parseParts(content.parts || []); - - if (parsed.functionResponses.length > 0) { - for (const response of parsed.functionResponses) { - messages.push({ - role: 'user', - content: [ - { - type: 'tool_result', - tool_use_id: response.id || '', - content: this.extractFunctionResponseContent(response.response), - }, - ], - }); + for (const part of parts) { + if (typeof part === 'string') { + contentBlocks.push({ type: 'text', text: part }); + continue; } - return; - } - if (content.role === 'model' && parsed.functionCalls.length > 0) { - const thinkingBlocks: AnthropicContentBlockParam[] = - parsed.thoughtParts.map((part) => { + if ('text' in part && 'thought' in part && part.thought) { + if (role === 'assistant') { const thinkingBlock: unknown = { type: 'thinking', - thinking: part.text, + thinking: part.text || '', }; - if (part.signature) { + if ( + 'thoughtSignature' in part && + typeof part.thoughtSignature === 'string' + ) { (thinkingBlock as { signature?: string }).signature = - part.signature; + part.thoughtSignature; } - return thinkingBlock as AnthropicContentBlockParam; - }); - const toolUses: AnthropicContentBlockParam[] = parsed.functionCalls.map( - (call, index) => ({ - type: 'tool_use', - id: call.id || `tool_${index}`, - name: call.name || '', - input: (call.args as Record) || {}, - }), - ); - - const textBlocks: AnthropicContentBlockParam[] = parsed.contentParts.map( - (text) => ({ - type: 'text' as const, - text, - }), - ); + contentBlocks.push(thinkingBlock as AnthropicContentBlockParam); + } + } - messages.push({ - role: 'assistant', - content: [...thinkingBlocks, ...textBlocks, ...toolUses], - }); - return; + if ('text' in part && part.text && !('thought' in part && part.thought)) { + contentBlocks.push({ type: 'text', text: part.text }); + } + + const mediaBlock = this.createMediaBlockFromPart(part); + if (mediaBlock) { + contentBlocks.push(mediaBlock); + } + + if ('functionCall' in part && part.functionCall) { + if (role === 'assistant') { + contentBlocks.push({ + type: 'tool_use', + id: part.functionCall.id || `tool_${toolCallIndex}`, + name: part.functionCall.name || '', + input: (part.functionCall.args as Record) || {}, + }); + toolCallIndex += 1; + } + } + + if (part.functionResponse) { + const toolResultBlock = this.createToolResultBlock( + part.functionResponse, + ); + if (toolResultBlock && role === 'user') { + contentBlocks.push(toolResultBlock); + } + } } - const role = content.role === 'model' ? 'assistant' : 'user'; - const thinkingBlocks: AnthropicContentBlockParam[] = - role === 'assistant' - ? parsed.thoughtParts.map((part) => { - const thinkingBlock: unknown = { - type: 'thinking', - thinking: part.text, - }; - if (part.signature) { - (thinkingBlock as { signature?: string }).signature = - part.signature; - } - return thinkingBlock as AnthropicContentBlockParam; - }) - : []; - const textBlocks: AnthropicContentBlockParam[] = [ - ...thinkingBlocks, - ...parsed.contentParts.map((text) => ({ - type: 'text' as const, - text, - })), - ]; - if (textBlocks.length > 0) { - messages.push({ role, content: textBlocks }); + if (contentBlocks.length > 0) { + messages.push({ role, content: contentBlocks }); } } - private parseParts(parts: Part[]): ParsedParts { - const thoughtParts: ThoughtPart[] = []; - const contentParts: string[] = []; - const functionCalls: FunctionCall[] = []; - const functionResponses: FunctionResponse[] = []; + private createToolResultBlock( + response: FunctionResponse, + ): Anthropic.ToolResultBlockParam | null { + const textContent = this.extractFunctionResponseContent(response.response); - for (const part of parts) { - if (typeof part === 'string') { - contentParts.push(part); - } else if ( - 'text' in part && - part.text && - !('thought' in part && part.thought) - ) { - contentParts.push(part.text); - } else if ('text' in part && 'thought' in part && part.thought) { - thoughtParts.push({ - text: part.text || '', - signature: - 'thoughtSignature' in part && - typeof part.thoughtSignature === 'string' - ? part.thoughtSignature - : undefined, - }); - } else if ('functionCall' in part && part.functionCall) { - functionCalls.push(part.functionCall); - } else if ('functionResponse' in part && part.functionResponse) { - functionResponses.push(part.functionResponse); + type ToolResultContent = Anthropic.ToolResultBlockParam['content']; + const partBlocks: AnthropicContentBlockParam[] = []; + + for (const part of response.parts || []) { + const block = this.createMediaBlockFromPart(part); + if (block) { + partBlocks.push(block); + } + } + + let content: ToolResultContent; + if (partBlocks.length > 0) { + const blocks: AnthropicContentBlockParam[] = []; + if (textContent) { + blocks.push({ type: 'text', text: textContent }); } + blocks.push(...partBlocks); + content = blocks as unknown as ToolResultContent; + } else { + content = textContent; } return { - thoughtParts, - contentParts, - functionCalls, - functionResponses, + type: 'tool_result', + tool_use_id: response.id || '', + content, }; } + private createMediaBlockFromPart( + part: Part, + ): AnthropicContentBlockParam | null { + if (part.inlineData?.mimeType && part.inlineData?.data) { + if (this.isSupportedAnthropicImageMimeType(part.inlineData.mimeType)) { + return { + type: 'image', + source: { + type: 'base64', + media_type: part.inlineData.mimeType as + | 'image/jpeg' + | 'image/png' + | 'image/gif' + | 'image/webp', + data: part.inlineData.data, + }, + }; + } + + if (part.inlineData.mimeType === 'application/pdf') { + return { + type: 'document', + source: { + type: 'base64', + media_type: 'application/pdf', + data: part.inlineData.data, + }, + }; + } + + const displayName = part.inlineData.displayName + ? ` (${part.inlineData.displayName})` + : ''; + return { + type: 'text', + text: `Unsupported inline media type: ${part.inlineData.mimeType}${displayName}.`, + }; + } + + if (part.fileData?.mimeType && part.fileData?.fileUri) { + const displayName = part.fileData.displayName + ? ` (${part.fileData.displayName})` + : ''; + const fileUri = part.fileData.fileUri; + + if (this.isSupportedAnthropicImageMimeType(part.fileData.mimeType)) { + return { + type: 'image', + source: { + type: 'url', + url: fileUri, + }, + } as unknown as AnthropicContentBlockParam; + } + + if (part.fileData.mimeType === 'application/pdf') { + return { + type: 'document', + source: { + type: 'url', + url: fileUri, + }, + } as unknown as AnthropicContentBlockParam; + } + + return { + type: 'text', + text: `Unsupported file media type: ${part.fileData.mimeType}${displayName}.`, + }; + } + + return null; + } + + private isSupportedAnthropicImageMimeType( + mimeType: string, + ): mimeType is 'image/jpeg' | 'image/png' | 'image/gif' | 'image/webp' { + return ( + mimeType === 'image/jpeg' || + mimeType === 'image/png' || + mimeType === 'image/gif' || + mimeType === 'image/webp' + ); + } + private extractTextFromContentUnion(contentUnion: unknown): string { if (typeof contentUnion === 'string') { return contentUnion; diff --git a/packages/core/src/core/coreToolScheduler.test.ts b/packages/core/src/core/coreToolScheduler.test.ts index 9ccbf926306..fff05f3b94a 100644 --- a/packages/core/src/core/coreToolScheduler.test.ts +++ b/packages/core/src/core/coreToolScheduler.test.ts @@ -860,11 +860,11 @@ describe('convertToFunctionResponse', () => { name: toolName, id: callId, response: { - output: 'Binary content of type image/png was processed.', + output: '', }, + parts: [{ inlineData: { mimeType: 'image/png', data: 'base64...' } }], }, }, - llmContent, ]); }); @@ -879,11 +879,15 @@ describe('convertToFunctionResponse', () => { name: toolName, id: callId, response: { - output: 'Binary content of type application/pdf was processed.', + output: '', }, + parts: [ + { + fileData: { mimeType: 'application/pdf', fileUri: 'gs://...' }, + }, + ], }, }, - llmContent, ]); }); @@ -917,11 +921,13 @@ describe('convertToFunctionResponse', () => { name: toolName, id: callId, response: { - output: 'Binary content of type image/gif was processed.', + output: '', }, + parts: [ + { inlineData: { mimeType: 'image/gif', data: 'gifdata...' } }, + ], }, }, - ...llmContent, ]); }); diff --git a/packages/core/src/core/coreToolScheduler.ts b/packages/core/src/core/coreToolScheduler.ts index 2187be2e721..3fefacbc6d6 100644 --- a/packages/core/src/core/coreToolScheduler.ts +++ b/packages/core/src/core/coreToolScheduler.ts @@ -31,8 +31,13 @@ import { InputFormat, SkillTool, } from '../index.js'; +import type { + FunctionResponse, + FunctionResponsePart, + Part, + PartListUnion, +} from '@google/genai'; import { ToolNames } from '../tools/tool-names.js'; -import type { Part, PartListUnion } from '@google/genai'; import { getResponseTextFromParts } from '../utils/generateContentResponseUtilities.js'; import type { ModifyContext } from '../tools/modifiable-tool.js'; import { @@ -153,13 +158,17 @@ function createFunctionResponsePart( callId: string, toolName: string, output: string, + mediaParts?: FunctionResponsePart[], ): Part { + const functionResponse: FunctionResponse = { + id: callId, + name: toolName, + response: { output }, + ...(mediaParts && mediaParts.length > 0 ? { parts: mediaParts } : {}), + }; + return { - functionResponse: { - id: callId, - name: toolName, - response: { output }, - }, + functionResponse, }; } @@ -200,16 +209,21 @@ export function convertToFunctionResponse( } if (contentToProcess.inlineData || contentToProcess.fileData) { - const mimeType = - contentToProcess.inlineData?.mimeType || - contentToProcess.fileData?.mimeType || - 'unknown'; + const mediaParts: FunctionResponsePart[] = []; + if (contentToProcess.inlineData) { + mediaParts.push({ inlineData: contentToProcess.inlineData }); + } + if (contentToProcess.fileData) { + mediaParts.push({ fileData: contentToProcess.fileData }); + } + const functionResponse = createFunctionResponsePart( callId, toolName, - `Binary content of type ${mimeType} was processed.`, + '', + mediaParts, ); - return [functionResponse, contentToProcess]; + return [functionResponse]; } if (contentToProcess.text !== undefined) { diff --git a/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.test.ts b/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.test.ts index bdf9bfb9979..992d35483d7 100644 --- a/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.test.ts +++ b/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.test.ts @@ -205,4 +205,167 @@ describe('GeminiContentGenerator', () => { }), ); }); + + it('should strip displayName from inlineData and fileData before sending to API', async () => { + const request = { + model: 'gemini-1.5-flash', + contents: [ + { + role: 'user' as const, + parts: [ + { + inlineData: { + mimeType: 'image/png', + data: 'base64data', + displayName: 'image.png', + }, + }, + { + inlineData: { + mimeType: 'application/pdf', + data: 'base64pdfdata', + displayName: 'document.pdf', + }, + }, + { + fileData: { + mimeType: 'application/pdf', + fileUri: 'gs://bucket/file.pdf', + displayName: 'document.pdf', + }, + }, + ], + }, + ], + }; + + await generator.generateContent(request, 'prompt-id'); + + const calledWith = mockGoogleGenAI.models.generateContent.mock.calls[0][0]; + + // Verify displayName is stripped from inlineData + expect(calledWith.contents[0].parts[0].inlineData).toEqual({ + mimeType: 'image/png', + data: 'base64data', + }); + expect( + calledWith.contents[0].parts[0].inlineData.displayName, + ).toBeUndefined(); + + expect(calledWith.contents[0].parts[1].inlineData).toEqual({ + mimeType: 'application/pdf', + data: 'base64pdfdata', + }); + expect( + calledWith.contents[0].parts[1].inlineData.displayName, + ).toBeUndefined(); + + // Verify displayName is stripped from fileData + expect(calledWith.contents[0].parts[2].fileData).toEqual({ + mimeType: 'application/pdf', + fileUri: 'gs://bucket/file.pdf', + }); + expect( + calledWith.contents[0].parts[2].fileData.displayName, + ).toBeUndefined(); + }); + + it('should strip displayName from functionResponse parts', async () => { + const request = { + model: 'gemini-1.5-flash', + contents: [ + { + role: 'user' as const, + parts: [ + { + functionResponse: { + id: 'call-1', + name: 'Read', + response: { output: 'content' }, + parts: [ + { + inlineData: { + mimeType: 'image/png', + data: 'base64data', + displayName: 'screenshot.png', + }, + }, + ], + }, + }, + ], + }, + ], + }; + + await generator.generateContent(request, 'prompt-id'); + + const calledWith = mockGoogleGenAI.models.generateContent.mock.calls[0][0]; + const functionResponseParts = + calledWith.contents[0].parts[0].functionResponse.parts; + + // Verify displayName is stripped from nested inlineData + expect(functionResponseParts[0].inlineData).toEqual({ + mimeType: 'image/png', + data: 'base64data', + }); + expect(functionResponseParts[0].inlineData.displayName).toBeUndefined(); + }); + + it('should convert audio and video to text in functionResponse parts', async () => { + const request = { + model: 'gemini-1.5-flash', + contents: [ + { + role: 'user' as const, + parts: [ + { + functionResponse: { + id: 'call-1', + name: 'Read', + response: { output: 'content' }, + parts: [ + { + inlineData: { + mimeType: 'image/png', + data: 'imagedata', + }, + }, + { + inlineData: { + mimeType: 'audio/wav', + data: 'audiodata', + displayName: 'recording.wav', + }, + }, + { + inlineData: { + mimeType: 'video/mp4', + data: 'videodata', + }, + }, + ], + }, + }, + ], + }, + ], + }; + + await generator.generateContent(request, 'prompt-id'); + + const calledWith = mockGoogleGenAI.models.generateContent.mock.calls[0][0]; + const functionResponseParts = + calledWith.contents[0].parts[0].functionResponse.parts; + + // All parts should remain, but audio/video converted to text + expect(functionResponseParts).toHaveLength(3); + expect(functionResponseParts[0].inlineData.mimeType).toBe('image/png'); + expect(functionResponseParts[1].text).toBe( + 'Unsupported media type for Gemini: audio/wav (recording.wav).', + ); + expect(functionResponseParts[2].text).toBe( + 'Unsupported media type for Gemini: video/mp4.', + ); + }); }); diff --git a/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.ts b/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.ts index 33819cd7f07..17a14b5a957 100644 --- a/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.ts +++ b/packages/core/src/core/geminiContentGenerator/geminiContentGenerator.ts @@ -13,6 +13,8 @@ import type { GenerateContentResponse, GenerateContentConfig, ThinkingLevel, + Content, + Part, } from '@google/genai'; import { GoogleGenAI } from '@google/genai'; import type { @@ -146,6 +148,7 @@ export class GeminiContentGenerator implements ContentGenerator { ): Promise { const finalRequest = { ...request, + contents: this.stripUnsupportedFields(request.contents), config: this.buildGenerateContentConfig(request), }; return this.googleGenAI.models.generateContent(finalRequest); @@ -157,11 +160,131 @@ export class GeminiContentGenerator implements ContentGenerator { ): Promise> { const finalRequest = { ...request, + contents: this.stripUnsupportedFields(request.contents), config: this.buildGenerateContentConfig(request), }; return this.googleGenAI.models.generateContentStream(finalRequest); } + /** + * Strip fields not supported by Gemini API (e.g., displayName in inlineData/fileData) + */ + private stripUnsupportedFields( + contents: GenerateContentParameters['contents'], + ): GenerateContentParameters['contents'] { + if (!contents) return contents; + + if (typeof contents === 'string') return contents; + + if (Array.isArray(contents)) { + return contents.map((content) => + this.stripContentFields(content), + ) as GenerateContentParameters['contents']; + } + + return this.stripContentFields( + contents, + ) as GenerateContentParameters['contents']; + } + + private stripContentFields( + content: Content | Part | string, + ): Content | Part | string { + if (typeof content === 'string') { + return content; + } + + // Handle Part directly (for arrays of parts) + if (!('role' in content) && !('parts' in content)) { + return this.stripPartFields(content as Part); + } + + // Handle Content object + const contentObj = content as Content; + if (!contentObj.parts) return contentObj; + + return { + ...contentObj, + parts: contentObj.parts.map((part) => this.stripPartFields(part)), + }; + } + + private stripPartFields(part: Part): Part { + if (typeof part === 'string') { + return part; + } + + const result = { ...part }; + + // Strip displayName from inlineData + if (result.inlineData) { + const { displayName: _, ...inlineDataWithoutDisplayName } = + result.inlineData as { displayName?: string; [key: string]: unknown }; + result.inlineData = inlineDataWithoutDisplayName as Part['inlineData']; + } + + // Strip displayName from fileData + if (result.fileData) { + const { displayName: _, ...fileDataWithoutDisplayName } = + result.fileData as { displayName?: string; [key: string]: unknown }; + result.fileData = fileDataWithoutDisplayName as Part['fileData']; + } + + // Handle functionResponse parts (which may contain nested media parts) + // Convert unsupported media types (audio, video) to text for Gemini API + if (result.functionResponse?.parts) { + const processedParts = result.functionResponse.parts.map((p) => { + // First convert unsupported media to text (before stripping displayName) + const converted = this.convertUnsupportedMediaToText(p); + // Then strip unsupported fields from remaining parts + return this.stripPartFields(converted); + }); + + result.functionResponse = { + ...result.functionResponse, + parts: processedParts, + }; + } + + return result; + } + + /** + * Convert unsupported media types (audio, video) to explanatory text for Gemini API + */ + private convertUnsupportedMediaToText(part: Part): Part { + if (typeof part === 'string') return part; + + const inlineMimeType = part.inlineData?.mimeType || ''; + const fileMimeType = part.fileData?.mimeType || ''; + + if ( + inlineMimeType.startsWith('audio/') || + inlineMimeType.startsWith('video/') + ) { + const displayName = (part.inlineData as { displayName?: string }) + ?.displayName; + const displayNameText = displayName ? ` (${displayName})` : ''; + return { + text: `Unsupported media type for Gemini: ${inlineMimeType}${displayNameText}.`, + }; + } + + if ( + fileMimeType.startsWith('audio/') || + fileMimeType.startsWith('video/') + ) { + const displayName = (part.fileData as { displayName?: string }) + ?.displayName; + const displayNameText = displayName ? ` (${displayName})` : ''; + return { + text: `Unsupported media type for Gemini: ${fileMimeType}${displayNameText}.`, + }; + } + + return part; + } + async countTokens( request: CountTokensParameters, ): Promise { diff --git a/packages/core/src/core/nonInteractiveToolExecutor.test.ts b/packages/core/src/core/nonInteractiveToolExecutor.test.ts index 5b319deda3f..cbc4c145a73 100644 --- a/packages/core/src/core/nonInteractiveToolExecutor.test.ts +++ b/packages/core/src/core/nonInteractiveToolExecutor.test.ts @@ -309,11 +309,13 @@ describe('executeToolCall', () => { name: 'testTool', id: 'call6', response: { - output: 'Binary content of type image/png was processed.', + output: '', }, + parts: [ + { inlineData: { mimeType: 'image/png', data: 'base64data' } }, + ], }, }, - imageDataPart, ], }); }); diff --git a/packages/core/src/core/openaiContentGenerator/converter.test.ts b/packages/core/src/core/openaiContentGenerator/converter.test.ts index c896cb9b7f9..03f7dda486b 100644 --- a/packages/core/src/core/openaiContentGenerator/converter.test.ts +++ b/packages/core/src/core/openaiContentGenerator/converter.test.ts @@ -122,7 +122,13 @@ describe('OpenAIContentConverter', () => { const toolMessage = messages.find((message) => message.role === 'tool'); expect(toolMessage).toBeDefined(); - expect(toolMessage?.content).toBe('Raw output text'); + expect(Array.isArray(toolMessage?.content)).toBe(true); + const contentArray = toolMessage?.content as Array<{ + type: string; + text?: string; + }>; + expect(contentArray[0].type).toBe('text'); + expect(contentArray[0].text).toBe('Raw output text'); }); it('should prioritize error field when present', () => { @@ -134,7 +140,13 @@ describe('OpenAIContentConverter', () => { const toolMessage = messages.find((message) => message.role === 'tool'); expect(toolMessage).toBeDefined(); - expect(toolMessage?.content).toBe('Command failed'); + expect(Array.isArray(toolMessage?.content)).toBe(true); + const contentArray = toolMessage?.content as Array<{ + type: string; + text?: string; + }>; + expect(contentArray[0].type).toBe('text'); + expect(contentArray[0].text).toBe('Command failed'); }); it('should stringify non-string responses', () => { @@ -146,7 +158,677 @@ describe('OpenAIContentConverter', () => { const toolMessage = messages.find((message) => message.role === 'tool'); expect(toolMessage).toBeDefined(); - expect(toolMessage?.content).toBe('{"data":{"value":42}}'); + expect(Array.isArray(toolMessage?.content)).toBe(true); + const contentArray = toolMessage?.content as Array<{ + type: string; + text?: string; + }>; + expect(contentArray[0].type).toBe('text'); + expect(contentArray[0].text).toBe('{"data":{"value":42}}'); + }); + + it('should convert function responses with inlineData to tool message with embedded image_url', () => { + const request: GenerateContentParameters = { + model: 'models/test', + contents: [ + { + role: 'model', + parts: [ + { + functionCall: { + id: 'call_1', + name: 'Read', + args: {}, + }, + }, + ], + }, + { + role: 'user', + parts: [ + { + functionResponse: { + id: 'call_1', + name: 'Read', + response: { output: 'Image content' }, + parts: [ + { + inlineData: { + mimeType: 'image/png', + data: 'base64encodedimagedata', + }, + }, + ], + }, + }, + ], + }, + ], + }; + + const messages = converter.convertGeminiRequestToOpenAI(request); + + // Should have tool message with both text and image content + const toolMessage = messages.find((message) => message.role === 'tool'); + expect(toolMessage).toBeDefined(); + expect((toolMessage as { tool_call_id?: string }).tool_call_id).toBe( + 'call_1', + ); + expect(Array.isArray(toolMessage?.content)).toBe(true); + const contentArray = toolMessage?.content as Array<{ + type: string; + text?: string; + image_url?: { url: string }; + }>; + expect(contentArray).toHaveLength(2); + expect(contentArray[0].type).toBe('text'); + expect(contentArray[0].text).toBe('Image content'); + expect(contentArray[1].type).toBe('image_url'); + expect(contentArray[1].image_url?.url).toBe( + 'data:image/png;base64,base64encodedimagedata', + ); + + // No separate user message should be created + const userMessage = messages.find((message) => message.role === 'user'); + expect(userMessage).toBeUndefined(); + }); + + it('should convert function responses with fileData to tool message with embedded image_url', () => { + const request: GenerateContentParameters = { + model: 'models/test', + contents: [ + { + role: 'model', + parts: [ + { + functionCall: { + id: 'call_1', + name: 'Read', + args: {}, + }, + }, + ], + }, + { + role: 'user', + parts: [ + { + functionResponse: { + id: 'call_1', + name: 'Read', + response: { output: 'File content' }, + parts: [ + { + fileData: { + mimeType: 'image/jpeg', + fileUri: 'base64imagedata', + }, + }, + ], + }, + }, + ], + }, + ], + }; + + const messages = converter.convertGeminiRequestToOpenAI(request); + + // Should have tool message with both text and image content + const toolMessage = messages.find((message) => message.role === 'tool'); + expect(toolMessage).toBeDefined(); + expect(Array.isArray(toolMessage?.content)).toBe(true); + const contentArray = toolMessage?.content as Array<{ + type: string; + text?: string; + image_url?: { url: string }; + }>; + expect(contentArray).toHaveLength(2); + expect(contentArray[0].type).toBe('text'); + expect(contentArray[0].text).toBe('File content'); + expect(contentArray[1].type).toBe('image_url'); + expect(contentArray[1].image_url?.url).toBe('base64imagedata'); + + // No separate user message should be created + const userMessage = messages.find((message) => message.role === 'user'); + expect(userMessage).toBeUndefined(); + }); + + it('should convert PDF inlineData to tool message with embedded input_file', () => { + const request: GenerateContentParameters = { + model: 'models/test', + contents: [ + { + role: 'model', + parts: [ + { + functionCall: { + id: 'call_1', + name: 'Read', + args: {}, + }, + }, + ], + }, + { + role: 'user', + parts: [ + { + functionResponse: { + id: 'call_1', + name: 'Read', + response: { output: 'PDF content' }, + parts: [ + { + inlineData: { + mimeType: 'application/pdf', + data: 'base64pdfdata', + displayName: 'document.pdf', + }, + }, + ], + }, + }, + ], + }, + ], + }; + + const messages = converter.convertGeminiRequestToOpenAI(request); + + // Should have tool message with both text and file content + const toolMessage = messages.find((message) => message.role === 'tool'); + expect(toolMessage).toBeDefined(); + expect(Array.isArray(toolMessage?.content)).toBe(true); + const contentArray = toolMessage?.content as Array<{ + type: string; + text?: string; + file?: { filename: string; file_data: string }; + }>; + expect(contentArray).toHaveLength(2); + expect(contentArray[0].type).toBe('text'); + expect(contentArray[0].text).toBe('PDF content'); + expect(contentArray[1].type).toBe('file'); + expect(contentArray[1].file?.filename).toBe('document.pdf'); + expect(contentArray[1].file?.file_data).toBe( + 'data:application/pdf;base64,base64pdfdata', + ); + + // No separate user message should be created + const userMessage = messages.find((message) => message.role === 'user'); + expect(userMessage).toBeUndefined(); + }); + + it('should convert audio parts to tool message with embedded input_audio', () => { + const request: GenerateContentParameters = { + model: 'models/test', + contents: [ + { + role: 'model', + parts: [ + { + functionCall: { + id: 'call_1', + name: 'Record', + args: {}, + }, + }, + ], + }, + { + role: 'user', + parts: [ + { + functionResponse: { + id: 'call_1', + name: 'Record', + response: { output: 'Audio recorded' }, + parts: [ + { + inlineData: { + mimeType: 'audio/wav', + data: 'audiobase64data', + }, + }, + ], + }, + }, + ], + }, + ], + }; + + const messages = converter.convertGeminiRequestToOpenAI(request); + + // Should have tool message with both text and audio content + const toolMessage = messages.find((message) => message.role === 'tool'); + expect(toolMessage).toBeDefined(); + expect(Array.isArray(toolMessage?.content)).toBe(true); + const contentArray = toolMessage?.content as Array<{ + type: string; + text?: string; + input_audio?: { data: string; format: string }; + }>; + expect(contentArray).toHaveLength(2); + expect(contentArray[0].type).toBe('text'); + expect(contentArray[0].text).toBe('Audio recorded'); + expect(contentArray[1].type).toBe('input_audio'); + expect(contentArray[1].input_audio?.data).toBe( + 'data:audio/wav;base64,audiobase64data', + ); + expect(contentArray[1].input_audio?.format).toBe('wav'); + + // No separate user message should be created + const userMessage = messages.find((message) => message.role === 'user'); + expect(userMessage).toBeUndefined(); + }); + + it('should convert image fileData URL to tool message with embedded image_url', () => { + const request: GenerateContentParameters = { + model: 'models/test', + contents: [ + { + role: 'model', + parts: [ + { + functionCall: { + id: 'call_1', + name: 'Read', + args: {}, + }, + }, + ], + }, + { + role: 'user', + parts: [ + { + functionResponse: { + id: 'call_1', + name: 'Read', + response: { output: 'Image content' }, + parts: [ + { + fileData: { + mimeType: 'image/jpeg', + fileUri: + 'https://upload.wikimedia.org/wikipedia/commons/a/a7/Camponotus_flavomarginatus_ant.jpg', + displayName: 'ant.jpg', + }, + }, + ], + }, + }, + ], + }, + ], + }; + + const messages = converter.convertGeminiRequestToOpenAI(request); + + const toolMessage = messages.find((message) => message.role === 'tool'); + expect(toolMessage).toBeDefined(); + expect(Array.isArray(toolMessage?.content)).toBe(true); + const contentArray = toolMessage?.content as Array<{ + type: string; + text?: string; + image_url?: { url: string }; + }>; + expect(contentArray).toHaveLength(2); + expect(contentArray[0].type).toBe('text'); + expect(contentArray[0].text).toBe('Image content'); + expect(contentArray[1].type).toBe('image_url'); + expect(contentArray[1].image_url?.url).toBe( + 'https://upload.wikimedia.org/wikipedia/commons/a/a7/Camponotus_flavomarginatus_ant.jpg', + ); + }); + + it('should convert PDF fileData URL to tool message with embedded file', () => { + const request: GenerateContentParameters = { + model: 'models/test', + contents: [ + { + role: 'model', + parts: [ + { + functionCall: { + id: 'call_1', + name: 'Read', + args: {}, + }, + }, + ], + }, + { + role: 'user', + parts: [ + { + functionResponse: { + id: 'call_1', + name: 'Read', + response: { output: 'PDF content' }, + parts: [ + { + fileData: { + mimeType: 'application/pdf', + fileUri: + 'https://assets.anthropic.com/m/1cd9d098ac3e6467/original/Claude-3-Model-Card-October-Addendum.pdf', + displayName: 'document.pdf', + }, + }, + ], + }, + }, + ], + }, + ], + }; + + const messages = converter.convertGeminiRequestToOpenAI(request); + + const toolMessage = messages.find((message) => message.role === 'tool'); + expect(toolMessage).toBeDefined(); + expect(Array.isArray(toolMessage?.content)).toBe(true); + const contentArray = toolMessage?.content as Array<{ + type: string; + text?: string; + file?: { filename: string; file_data: string }; + }>; + expect(contentArray).toHaveLength(2); + expect(contentArray[0].type).toBe('text'); + expect(contentArray[0].text).toBe('PDF content'); + expect(contentArray[1].type).toBe('file'); + expect(contentArray[1].file?.filename).toBe('document.pdf'); + expect(contentArray[1].file?.file_data).toBe( + 'https://assets.anthropic.com/m/1cd9d098ac3e6467/original/Claude-3-Model-Card-October-Addendum.pdf', + ); + }); + + it('should convert video inlineData to tool message with embedded video_url', () => { + const request: GenerateContentParameters = { + model: 'models/test', + contents: [ + { + role: 'model', + parts: [ + { + functionCall: { + id: 'call_1', + name: 'Read', + args: {}, + }, + }, + ], + }, + { + role: 'user', + parts: [ + { + functionResponse: { + id: 'call_1', + name: 'Read', + response: { output: 'Video content' }, + parts: [ + { + inlineData: { + mimeType: 'video/mp4', + data: 'videobase64data', + displayName: 'recording.mp4', + }, + }, + ], + }, + }, + ], + }, + ], + }; + + const messages = converter.convertGeminiRequestToOpenAI(request); + + // Should have tool message with both text and video content + const toolMessage = messages.find((message) => message.role === 'tool'); + expect(toolMessage).toBeDefined(); + expect(Array.isArray(toolMessage?.content)).toBe(true); + const contentArray = toolMessage?.content as Array<{ + type: string; + text?: string; + video_url?: { url: string }; + }>; + expect(contentArray).toHaveLength(2); + expect(contentArray[0].type).toBe('text'); + expect(contentArray[0].text).toBe('Video content'); + expect(contentArray[1].type).toBe('video_url'); + expect(contentArray[1].video_url?.url).toBe( + 'data:video/mp4;base64,videobase64data', + ); + + // No separate user message should be created + const userMessage = messages.find((message) => message.role === 'user'); + expect(userMessage).toBeUndefined(); + }); + + it('should convert video fileData URL to tool message with embedded video_url', () => { + const request: GenerateContentParameters = { + model: 'models/test', + contents: [ + { + role: 'model', + parts: [ + { + functionCall: { + id: 'call_1', + name: 'Read', + args: {}, + }, + }, + ], + }, + { + role: 'user', + parts: [ + { + functionResponse: { + id: 'call_1', + name: 'Read', + response: { output: 'Video content' }, + parts: [ + { + fileData: { + mimeType: 'video/mp4', + fileUri: 'https://www.youtube.com/watch?v=dQw4w9WgXcQ', + displayName: 'recording.mp4', + }, + }, + ], + }, + }, + ], + }, + ], + }; + + const messages = converter.convertGeminiRequestToOpenAI(request); + + const toolMessage = messages.find((message) => message.role === 'tool'); + expect(toolMessage).toBeDefined(); + expect(Array.isArray(toolMessage?.content)).toBe(true); + const contentArray = toolMessage?.content as Array<{ + type: string; + text?: string; + video_url?: { url: string }; + }>; + expect(contentArray).toHaveLength(2); + expect(contentArray[0].type).toBe('text'); + expect(contentArray[0].text).toBe('Video content'); + expect(contentArray[1].type).toBe('video_url'); + expect(contentArray[1].video_url?.url).toBe( + 'https://www.youtube.com/watch?v=dQw4w9WgXcQ', + ); + }); + + it('should render unsupported inlineData file types as a text block', () => { + const request: GenerateContentParameters = { + model: 'models/test', + contents: [ + { + role: 'model', + parts: [ + { + functionCall: { + id: 'call_1', + name: 'Read', + args: {}, + }, + }, + ], + }, + { + role: 'user', + parts: [ + { + functionResponse: { + id: 'call_1', + name: 'Read', + response: { output: 'File content' }, + parts: [ + { + inlineData: { + mimeType: 'application/zip', + data: 'base64zipdata', + displayName: 'archive.zip', + }, + }, + ], + }, + }, + ], + }, + ], + }; + + const messages = converter.convertGeminiRequestToOpenAI(request); + + const toolMessage = messages.find((message) => message.role === 'tool'); + expect(toolMessage).toBeDefined(); + expect(Array.isArray(toolMessage?.content)).toBe(true); + const contentArray = toolMessage?.content as Array<{ + type: string; + text?: string; + }>; + expect(contentArray).toHaveLength(2); + expect(contentArray[0].type).toBe('text'); + expect(contentArray[0].text).toBe('File content'); + expect(contentArray[1].type).toBe('text'); + expect(contentArray[1].text).toContain('Unsupported inline media type'); + expect(contentArray[1].text).toContain('application/zip'); + expect(contentArray[1].text).toContain('archive.zip'); + }); + + it('should render unsupported fileData types (including audio) as a text block', () => { + const request: GenerateContentParameters = { + model: 'models/test', + contents: [ + { + role: 'model', + parts: [ + { + functionCall: { + id: 'call_1', + name: 'Read', + args: {}, + }, + }, + ], + }, + { + role: 'user', + parts: [ + { + functionResponse: { + id: 'call_1', + name: 'Read', + response: { output: 'File content' }, + parts: [ + { + fileData: { + mimeType: 'audio/mpeg', + fileUri: 'https://example.com/audio.mp3', + displayName: 'audio.mp3', + }, + }, + ], + }, + }, + ], + }, + ], + }; + + const messages = converter.convertGeminiRequestToOpenAI(request); + + const toolMessage = messages.find((message) => message.role === 'tool'); + expect(toolMessage).toBeDefined(); + expect(Array.isArray(toolMessage?.content)).toBe(true); + const contentArray = toolMessage?.content as Array<{ + type: string; + text?: string; + }>; + expect(contentArray).toHaveLength(2); + expect(contentArray[0].type).toBe('text'); + expect(contentArray[0].text).toBe('File content'); + expect(contentArray[1].type).toBe('text'); + expect(contentArray[1].text).toContain('Unsupported file media type'); + expect(contentArray[1].text).toContain('audio/mpeg'); + expect(contentArray[1].text).toContain('audio.mp3'); + }); + + it('should create tool message with text-only content when no media parts', () => { + const request = createRequestWithFunctionResponse({ + output: 'Plain text output', + }); + + const messages = converter.convertGeminiRequestToOpenAI(request); + const toolMessage = messages.find((message) => message.role === 'tool'); + + expect(toolMessage).toBeDefined(); + expect(Array.isArray(toolMessage?.content)).toBe(true); + const contentArray = toolMessage?.content as Array<{ + type: string; + text?: string; + }>; + expect(contentArray).toHaveLength(1); + expect(contentArray[0].type).toBe('text'); + expect(contentArray[0].text).toBe('Plain text output'); + + // No user message should be created when there's no media + const userMessage = messages.find((message) => message.role === 'user'); + expect(userMessage).toBeUndefined(); + }); + + it('should skip empty function responses with no media and no text', () => { + const request: GenerateContentParameters = { + model: 'models/test', + contents: [ + { + role: 'user', + parts: [ + { + functionResponse: { + id: 'call_1', + name: 'Empty', + response: { output: '' }, + }, + }, + ], + }, + ], + }; + + const messages = converter.convertGeminiRequestToOpenAI(request); + + // Should have no messages for empty response + expect(messages).toHaveLength(0); }); }); @@ -180,6 +862,35 @@ describe('OpenAIContentConverter', () => { ); }); + it('should convert reasoning to a thought part for non-streaming responses', () => { + const response = converter.convertOpenAIResponseToGemini({ + object: 'chat.completion', + id: 'chatcmpl-2', + created: 123, + model: 'gpt-test', + choices: [ + { + index: 0, + message: { + role: 'assistant', + content: 'final answer', + reasoning: 'chain-of-thought', + }, + finish_reason: 'stop', + logprobs: null, + }, + ], + } as unknown as OpenAI.Chat.ChatCompletion); + + const parts = response.candidates?.[0]?.content?.parts; + expect(parts?.[0]).toEqual( + expect.objectContaining({ thought: true, text: 'chain-of-thought' }), + ); + expect(parts?.[1]).toEqual( + expect.objectContaining({ text: 'final answer' }), + ); + }); + it('should convert streaming reasoning_content delta to a thought part', () => { const chunk = converter.convertOpenAIChunkToGemini({ object: 'chat.completion.chunk', @@ -208,6 +919,34 @@ describe('OpenAIContentConverter', () => { ); }); + it('should convert streaming reasoning delta to a thought part', () => { + const chunk = converter.convertOpenAIChunkToGemini({ + object: 'chat.completion.chunk', + id: 'chunk-1b', + created: 456, + choices: [ + { + index: 0, + delta: { + content: 'visible text', + reasoning: 'thinking...', + }, + finish_reason: 'stop', + logprobs: null, + }, + ], + model: 'gpt-test', + } as unknown as OpenAI.Chat.ChatCompletionChunk); + + const parts = chunk.candidates?.[0]?.content?.parts; + expect(parts?.[0]).toEqual( + expect.objectContaining({ thought: true, text: 'thinking...' }), + ); + expect(parts?.[1]).toEqual( + expect.objectContaining({ text: 'visible text' }), + ); + }); + it('should not throw when streaming chunk has no delta', () => { const chunk = converter.convertOpenAIChunkToGemini({ object: 'chat.completion.chunk', @@ -584,11 +1323,7 @@ describe('OpenAIContentConverter', () => { expect(messages).toHaveLength(1); expect(messages[0].role).toBe('assistant'); - const content = messages[0] - .content as OpenAI.Chat.ChatCompletionContentPart[]; - expect(content).toHaveLength(2); - expect(content[0]).toEqual({ type: 'text', text: 'First part' }); - expect(content[1]).toEqual({ type: 'text', text: 'Second part' }); + expect(messages[0].content).toBe('First partSecond part'); }); it('should merge multiple consecutive assistant messages', () => { @@ -614,9 +1349,7 @@ describe('OpenAIContentConverter', () => { expect(messages).toHaveLength(1); expect(messages[0].role).toBe('assistant'); - const content = messages[0] - .content as OpenAI.Chat.ChatCompletionContentPart[]; - expect(content).toHaveLength(3); + expect(messages[0].content).toBe('Part 1Part 2Part 3'); }); it('should merge tool_calls from consecutive assistant messages', () => { @@ -674,7 +1407,9 @@ describe('OpenAIContentConverter', () => { ], }; - const messages = converter.convertGeminiRequestToOpenAI(request); + const messages = converter.convertGeminiRequestToOpenAI(request, { + cleanOrphanToolCalls: false, + }); // Should have: assistant (tool_call_1), tool (result_1), assistant (tool_call_2), tool (result_2) expect(messages).toHaveLength(4); @@ -729,10 +1464,7 @@ describe('OpenAIContentConverter', () => { const messages = converter.convertGeminiRequestToOpenAI(request); expect(messages).toHaveLength(1); - const content = messages[0] - .content as OpenAI.Chat.ChatCompletionContentPart[]; - expect(Array.isArray(content)).toBe(true); - expect(content).toHaveLength(2); + expect(messages[0].content).toBe('Text partAnother text'); }); it('should merge empty content correctly', () => { @@ -758,11 +1490,7 @@ describe('OpenAIContentConverter', () => { // Empty messages should be filtered out expect(messages).toHaveLength(1); - const content = messages[0] - .content as OpenAI.Chat.ChatCompletionContentPart[]; - expect(content).toHaveLength(2); - expect(content[0]).toEqual({ type: 'text', text: 'First' }); - expect(content[1]).toEqual({ type: 'text', text: 'Second' }); + expect(messages[0].content).toBe('FirstSecond'); }); }); }); diff --git a/packages/core/src/core/openaiContentGenerator/converter.ts b/packages/core/src/core/openaiContentGenerator/converter.ts index 690751a2a9d..2ebc88f1767 100644 --- a/packages/core/src/core/openaiContentGenerator/converter.ts +++ b/packages/core/src/core/openaiContentGenerator/converter.ts @@ -11,7 +11,6 @@ import type { Tool, ToolListUnion, CallableTool, - FunctionCall, FunctionResponse, ContentListUnion, ContentUnion, @@ -47,11 +46,13 @@ type ExtendedChatCompletionMessageParam = export interface ExtendedCompletionMessage extends OpenAI.Chat.ChatCompletionMessage { reasoning_content?: string | null; + reasoning?: string | null; } export interface ExtendedCompletionChunkDelta extends OpenAI.Chat.ChatCompletionChunk.Choice.Delta { reasoning_content?: string | null; + reasoning?: string | null; } /** @@ -63,21 +64,27 @@ export interface ToolCallAccumulator { arguments: string; } -/** - * Parsed parts from Gemini content, categorized by type - */ -interface ParsedParts { - thoughtParts: string[]; - contentParts: string[]; - functionCalls: FunctionCall[]; - functionResponses: FunctionResponse[]; - mediaParts: Array<{ - type: 'image' | 'audio' | 'file'; - data: string; - mimeType: string; - fileUri?: string; - }>; -} +type OpenAIContentPartVideoUrl = { + type: 'video_url'; + video_url: { + url: string; + }; +}; + +type OpenAIContentPartFile = { + type: 'file'; + file: { + filename: string; + file_data: string; + }; +}; + +type OpenAIContentPart = + | OpenAI.Chat.ChatCompletionContentPartText + | OpenAI.Chat.ChatCompletionContentPartImage + | OpenAI.Chat.ChatCompletionContentPartInputAudio + | OpenAIContentPartVideoUrl + | OpenAIContentPartFile; /** * Converter class for transforming data between Gemini and OpenAI formats @@ -271,28 +278,48 @@ export class OpenAIContentConverter { ): OpenAI.Chat.ChatCompletion { const candidate = response.candidates?.[0]; const parts = (candidate?.content?.parts || []) as Part[]; - const parsedParts = this.parseParts(parts); + + // Parse parts inline + const thoughtParts: string[] = []; + const contentParts: string[] = []; + const toolCalls: OpenAI.Chat.ChatCompletionMessageToolCall[] = []; + let toolCallIndex = 0; + + for (const part of parts) { + if (typeof part === 'string') { + contentParts.push(part); + } else if ('text' in part && part.text) { + if ('thought' in part && part.thought) { + thoughtParts.push(part.text); + } else { + contentParts.push(part.text); + } + } else if ('functionCall' in part && part.functionCall) { + toolCalls.push({ + id: part.functionCall.id || `call_${toolCallIndex}`, + type: 'function' as const, + function: { + name: part.functionCall.name || '', + arguments: JSON.stringify(part.functionCall.args || {}), + }, + }); + toolCallIndex += 1; + } + } const message: ExtendedCompletionMessage = { role: 'assistant', - content: parsedParts.contentParts.join('') || null, + content: contentParts.join('') || null, refusal: null, }; - const reasoningContent = parsedParts.thoughtParts.join(''); + const reasoningContent = thoughtParts.join(''); if (reasoningContent) { message.reasoning_content = reasoningContent; } - if (parsedParts.functionCalls.length > 0) { - message.tool_calls = parsedParts.functionCalls.map((call, index) => ({ - id: call.id || `call_${index}`, - type: 'function' as const, - function: { - name: call.name || '', - arguments: JSON.stringify(call.args || {}), - }, - })); + if (toolCalls.length > 0) { + message.tool_calls = toolCalls; } const finishReason = this.mapGeminiFinishReasonToOpenAI( @@ -390,40 +417,82 @@ export class OpenAIContentConverter { } if (!this.isContentObject(content)) return; + const parts = content.parts || []; + const role = content.role === 'model' ? 'assistant' : 'user'; + + const contentParts: OpenAIContentPart[] = []; + const reasoningParts: string[] = []; + const toolCalls: OpenAI.Chat.ChatCompletionMessageToolCall[] = []; + let toolCallIndex = 0; - const parsedParts = this.parseParts(content.parts || []); + for (const part of parts) { + if (typeof part === 'string') { + contentParts.push({ type: 'text' as const, text: part }); + continue; + } - // Handle function responses (tool results) first - if (parsedParts.functionResponses.length > 0) { - for (const funcResponse of parsedParts.functionResponses) { - messages.push({ - role: 'tool' as const, - tool_call_id: funcResponse.id || '', - content: this.extractFunctionResponseContent(funcResponse.response), + if ('text' in part && 'thought' in part && part.thought) { + if (role === 'assistant' && part.text) { + reasoningParts.push(part.text); + } + } + + if ('text' in part && part.text && !('thought' in part && part.thought)) { + contentParts.push({ type: 'text' as const, text: part.text }); + } + + const mediaPart = this.createMediaContentPart(part); + if (mediaPart && role === 'user') { + contentParts.push(mediaPart); + } + + if ('functionCall' in part && part.functionCall && role === 'assistant') { + toolCalls.push({ + id: part.functionCall.id || `call_${toolCallIndex}`, + type: 'function' as const, + function: { + name: part.functionCall.name || '', + arguments: JSON.stringify(part.functionCall.args || {}), + }, }); + toolCallIndex += 1; + } + + if (part.functionResponse && role === 'user') { + // Create tool message for the function response (with embedded media) + const toolMessage = this.createToolMessage(part.functionResponse); + if (toolMessage) { + messages.push(toolMessage); + } } - return; } - // Handle model messages with function calls - if (content.role === 'model' && parsedParts.functionCalls.length > 0) { - const toolCalls = parsedParts.functionCalls.map((fc, index) => ({ - id: fc.id || `call_${index}`, - type: 'function' as const, - function: { - name: fc.name || '', - arguments: JSON.stringify(fc.args || {}), - }, - })); + if (role === 'assistant') { + if ( + contentParts.length === 0 && + toolCalls.length === 0 && + reasoningParts.length === 0 + ) { + return; + } + const assistantTextContent = contentParts + .filter( + (part): part is OpenAI.Chat.ChatCompletionContentPartText => + part.type === 'text', + ) + .map((part) => part.text) + .join(''); const assistantMessage: ExtendedChatCompletionAssistantMessageParam = { - role: 'assistant' as const, - content: parsedParts.contentParts.join('') || null, - tool_calls: toolCalls, + role: 'assistant', + content: assistantTextContent || null, }; - // Only include reasoning_content if it has actual content - const reasoningContent = parsedParts.thoughtParts.join(''); + if (toolCalls.length > 0) { + assistantMessage.tool_calls = toolCalls; + } + + const reasoningContent = reasoningParts.join(''); if (reasoningContent) { assistantMessage.reasoning_content = reasoningContent; } @@ -432,77 +501,13 @@ export class OpenAIContentConverter { return; } - // Handle regular messages with multimodal content - const role = content.role === 'model' ? 'assistant' : 'user'; - const openAIMessage = this.createMultimodalMessage(role, parsedParts); - - if (openAIMessage) { - messages.push(openAIMessage); - } - } - - /** - * Parse Gemini parts into categorized components - */ - private parseParts(parts: Part[]): ParsedParts { - const thoughtParts: string[] = []; - const contentParts: string[] = []; - const functionCalls: FunctionCall[] = []; - const functionResponses: FunctionResponse[] = []; - const mediaParts: Array<{ - type: 'image' | 'audio' | 'file'; - data: string; - mimeType: string; - fileUri?: string; - }> = []; - - for (const part of parts) { - if (typeof part === 'string') { - contentParts.push(part); - } else if ( - 'text' in part && - part.text && - !('thought' in part && part.thought) - ) { - contentParts.push(part.text); - } else if ( - 'text' in part && - part.text && - 'thought' in part && - part.thought - ) { - thoughtParts.push(part.text); - } else if ('functionCall' in part && part.functionCall) { - functionCalls.push(part.functionCall); - } else if ('functionResponse' in part && part.functionResponse) { - functionResponses.push(part.functionResponse); - } else if ('inlineData' in part && part.inlineData) { - const { data, mimeType } = part.inlineData; - if (data && mimeType) { - const mediaType = this.getMediaType(mimeType); - mediaParts.push({ type: mediaType, data, mimeType }); - } - } else if ('fileData' in part && part.fileData) { - const { fileUri, mimeType } = part.fileData; - if (fileUri && mimeType) { - const mediaType = this.getMediaType(mimeType); - mediaParts.push({ - type: mediaType, - data: '', - mimeType, - fileUri, - }); - } - } + if (contentParts.length > 0) { + messages.push({ + role: 'user', + content: + contentParts as unknown as OpenAI.Chat.ChatCompletionContentPart[], + }); } - - return { - thoughtParts, - contentParts, - functionCalls, - functionResponses, - mediaParts, - }; } private extractFunctionResponseContent(response: unknown): string { @@ -536,91 +541,151 @@ export class OpenAIContentConverter { } /** - * Determine media type from MIME type + * Create a tool message from function response (with embedded media parts) */ - private getMediaType(mimeType: string): 'image' | 'audio' | 'file' { - if (mimeType.startsWith('image/')) return 'image'; - if (mimeType.startsWith('audio/')) return 'audio'; - return 'file'; - } + private createToolMessage( + response: FunctionResponse, + ): OpenAI.Chat.ChatCompletionToolMessageParam | null { + const textContent = this.extractFunctionResponseContent(response.response); + const contentParts: OpenAIContentPart[] = []; + + // Add text content first if present + if (textContent) { + contentParts.push({ type: 'text' as const, text: textContent }); + } - /** - * Create multimodal OpenAI message from parsed parts - */ - private createMultimodalMessage( - role: 'user' | 'assistant', - parsedParts: Pick< - ParsedParts, - 'contentParts' | 'mediaParts' | 'thoughtParts' - >, - ): ExtendedChatCompletionMessageParam | null { - const { contentParts, mediaParts, thoughtParts } = parsedParts; - const reasoningContent = thoughtParts.join(''); - const content = contentParts.map((text) => ({ - type: 'text' as const, - text, - })); - - // If no media parts, return simple text message - if (mediaParts.length === 0) { - if (content.length === 0) return null; - const message: ExtendedChatCompletionMessageParam = { role, content }; - // Only include reasoning_content if it has actual content - if (reasoningContent) { - ( - message as ExtendedChatCompletionAssistantMessageParam - ).reasoning_content = reasoningContent; + // Add media parts from function response + for (const part of response.parts || []) { + const mediaPart = this.createMediaContentPart(part); + if (mediaPart) { + contentParts.push(mediaPart); } - return message; } - // For assistant messages with media, convert to text only - // since OpenAI assistant messages don't support media content arrays - if (role === 'assistant') { - return content.length > 0 - ? { role: 'assistant' as const, content } - : null; + // Tool messages require content, so skip if empty + if (contentParts.length === 0) { + return null; } - const contentArray: OpenAI.Chat.ChatCompletionContentPart[] = [...content]; + // Cast to OpenAI type - some OpenAI-compatible APIs support richer content in tool messages + return { + role: 'tool' as const, + tool_call_id: response.id || '', + content: contentParts as unknown as + | string + | OpenAI.Chat.ChatCompletionContentPartText[], + }; + } - // Add media content - for (const mediaPart of mediaParts) { - if (mediaPart.type === 'image') { - if (mediaPart.fileUri) { - // For file URIs, use the URI directly - contentArray.push({ - type: 'image_url' as const, - image_url: { url: mediaPart.fileUri }, - }); - } else if (mediaPart.data) { - // For inline data, create data URL - const dataUrl = `data:${mediaPart.mimeType};base64,${mediaPart.data}`; - contentArray.push({ - type: 'image_url' as const, - image_url: { url: dataUrl }, - }); - } - } else if (mediaPart.type === 'audio' && mediaPart.data) { - // Convert audio format from MIME type - const format = this.getAudioFormat(mediaPart.mimeType); + /** + * Create OpenAI media content part from Gemini part + */ + private createMediaContentPart(part: Part): OpenAIContentPart | null { + if (part.inlineData?.mimeType && part.inlineData?.data) { + const mimeType = part.inlineData.mimeType; + const mediaType = this.getMediaType(mimeType); + if (mediaType === 'image') { + const dataUrl = `data:${mimeType};base64,${part.inlineData.data}`; + return { + type: 'image_url' as const, + image_url: { url: dataUrl }, + }; + } + + if (mimeType === 'application/pdf') { + const filename = part.inlineData.displayName || 'document.pdf'; + return { + type: 'file' as const, + file: { + filename, + file_data: `data:${mimeType};base64,${part.inlineData.data}`, + }, + }; + } + + if (mediaType === 'audio') { + const format = this.getAudioFormat(mimeType); if (format) { - contentArray.push({ + return { type: 'input_audio' as const, input_audio: { - data: mediaPart.data, - format: format as 'wav' | 'mp3', + data: `data:${mimeType};base64,${part.inlineData.data}`, + format, }, - }); + }; } } - // Note: File type is not directly supported in OpenAI's current API - // Could be extended in the future or handled as text description + + if (mediaType === 'video') { + return { + type: 'video_url' as const, + video_url: { + url: `data:${mimeType};base64,${part.inlineData.data}`, + }, + }; + } + + const displayName = part.inlineData.displayName + ? ` (${part.inlineData.displayName})` + : ''; + return { + type: 'text' as const, + text: `Unsupported inline media type: ${mimeType}${displayName}.`, + }; } - return contentArray.length > 0 - ? { role: 'user' as const, content: contentArray } - : null; + if (part.fileData?.mimeType && part.fileData?.fileUri) { + const filename = part.fileData.displayName || 'file'; + const fileUri = part.fileData.fileUri; + const mimeType = part.fileData.mimeType; + const mediaType = this.getMediaType(mimeType); + + if (mediaType === 'image') { + return { + type: 'image_url' as const, + image_url: { url: fileUri }, + }; + } + + if (mimeType === 'application/pdf') { + return { + type: 'file' as const, + file: { + filename, + file_data: fileUri, + }, + }; + } + + if (mediaType === 'video') { + return { + type: 'video_url' as const, + video_url: { + url: fileUri, + }, + }; + } + + const displayName = part.fileData.displayName + ? ` (${part.fileData.displayName})` + : ''; + return { + type: 'text' as const, + text: `Unsupported file media type: ${mimeType}${displayName}.`, + }; + } + + return null; + } + + /** + * Determine media type from MIME type + */ + private getMediaType(mimeType: string): 'image' | 'audio' | 'video' | 'file' { + if (mimeType.startsWith('image/')) return 'image'; + if (mimeType.startsWith('audio/')) return 'audio'; + if (mimeType.startsWith('video/')) return 'video'; + return 'file'; } /** @@ -693,8 +758,9 @@ export class OpenAIContentConverter { const parts: Part[] = []; // Handle reasoning content (thoughts) - const reasoningText = (choice.message as ExtendedCompletionMessage) - .reasoning_content; + const reasoningText = + (choice.message as ExtendedCompletionMessage).reasoning_content ?? + (choice.message as ExtendedCompletionMessage).reasoning; if (reasoningText) { parts.push({ text: reasoningText, thought: true }); } @@ -798,8 +864,9 @@ export class OpenAIContentConverter { if (choice) { const parts: Part[] = []; - const reasoningText = (choice.delta as ExtendedCompletionChunkDelta) - ?.reasoning_content; + const reasoningText = + (choice.delta as ExtendedCompletionChunkDelta)?.reasoning_content ?? + (choice.delta as ExtendedCompletionChunkDelta)?.reasoning; if (reasoningText) { parts.push({ text: reasoningText, thought: true }); } @@ -1130,6 +1197,10 @@ export class OpenAIContentConverter { // If the last message is also an assistant message, merge them if (lastMessage.role === 'assistant') { + const lastToolCalls = + 'tool_calls' in lastMessage ? lastMessage.tool_calls || [] : []; + const currentToolCalls = + 'tool_calls' in message ? message.tool_calls || [] : []; // Combine content const lastContent = lastMessage.content; const currentContent = message.content; @@ -1171,10 +1242,6 @@ export class OpenAIContentConverter { } // Combine tool calls - const lastToolCalls = - 'tool_calls' in lastMessage ? lastMessage.tool_calls || [] : []; - const currentToolCalls = - 'tool_calls' in message ? message.tool_calls || [] : []; const combinedToolCalls = [...lastToolCalls, ...currentToolCalls]; // Update the last message with combined data diff --git a/packages/core/src/core/openaiContentGenerator/pipeline.ts b/packages/core/src/core/openaiContentGenerator/pipeline.ts index 0f00ecb30cd..0ee0f1e2505 100644 --- a/packages/core/src/core/openaiContentGenerator/pipeline.ts +++ b/packages/core/src/core/openaiContentGenerator/pipeline.ts @@ -320,13 +320,15 @@ export class ContentGenerationPipeline { 'frequency_penalty', 'frequencyPenalty', ), - ...this.buildReasoningConfig(), + ...this.buildReasoningConfig(request), }; return params; } - private buildReasoningConfig(): Record { + private buildReasoningConfig( + request: GenerateContentParameters, + ): Record { // Reasoning configuration for OpenAI-compatible endpoints is highly fragmented. // For example, across common providers and models: // @@ -336,13 +338,21 @@ export class ContentGenerationPipeline { // - gpt-5.x series — thinking is enabled by default; can be disabled via `reasoning.effort` // - qwen3 series — model-dependent; can be manually disabled via `extra_body.enable_thinking` // - // Given this inconsistency, we choose not to set any reasoning config here and - // instead rely on each model’s default behavior. + // Given this inconsistency, we avoid mapping values and only pass through the + // configured reasoning object when explicitly enabled. This keeps provider- and + // model-specific semantics intact while honoring request-level opt-out. + + if (request.config?.thinkingConfig?.includeThoughts === false) { + return {}; + } + + const reasoning = this.contentGeneratorConfig.reasoning; - // We plan to introduce provider- and model-specific settings to enable more - // fine-grained control over reasoning configuration. + if (reasoning === false || reasoning === undefined) { + return {}; + } - return {}; + return { reasoning }; } /** diff --git a/packages/core/src/core/openaiContentGenerator/provider/dashscope.test.ts b/packages/core/src/core/openaiContentGenerator/provider/dashscope.test.ts index 09f4c83ca83..9deebc20d75 100644 --- a/packages/core/src/core/openaiContentGenerator/provider/dashscope.test.ts +++ b/packages/core/src/core/openaiContentGenerator/provider/dashscope.test.ts @@ -624,7 +624,7 @@ describe('DashScopeOpenAICompatibleProvider', () => { }); }); - it('should add empty text item with cache control if last item is not text for streaming requests', () => { + it('should add cache control to last item even if not text for streaming requests', () => { const requestWithNonTextLast: OpenAI.Chat.ChatCompletionCreateParams = { model: 'qwen-max', stream: true, // This will trigger cache control on last message @@ -649,12 +649,12 @@ describe('DashScopeOpenAICompatibleProvider', () => { const content = result.messages[0] .content as OpenAI.Chat.ChatCompletionContentPart[]; - expect(content).toHaveLength(3); + expect(content).toHaveLength(2); - // Should add empty text item with cache control - expect(content[2]).toEqual({ - type: 'text', - text: '', + // Cache control should be added to the last item (image) + expect(content[1]).toEqual({ + type: 'image_url', + image_url: { url: 'https://example.com/image.jpg' }, cache_control: { type: 'ephemeral' }, }); }); @@ -725,13 +725,8 @@ describe('DashScopeOpenAICompatibleProvider', () => { const content = result.messages[0] .content as OpenAI.Chat.ChatCompletionContentPart[]; - expect(content).toEqual([ - { - type: 'text', - text: '', - cache_control: { type: 'ephemeral' }, - }, - ]); + // Empty content array should remain empty + expect(content).toEqual([]); }); }); diff --git a/packages/core/src/core/openaiContentGenerator/provider/dashscope.ts b/packages/core/src/core/openaiContentGenerator/provider/dashscope.ts index 0a8458e0af7..f971e3ed6d7 100644 --- a/packages/core/src/core/openaiContentGenerator/provider/dashscope.ts +++ b/packages/core/src/core/openaiContentGenerator/provider/dashscope.ts @@ -265,31 +265,15 @@ export class DashScopeOpenAICompatibleProvider contentArray: ChatCompletionContentPartWithCache[], ): ChatCompletionContentPartWithCache[] { if (contentArray.length === 0) { - return [ - { - type: 'text', - text: '', - cache_control: { type: 'ephemeral' }, - } as ChatCompletionContentPartTextWithCache, - ]; + return contentArray; } + // Add cache_control to the last text item const lastItem = contentArray[contentArray.length - 1]; - - if (lastItem.type === 'text') { - // Add cache_control to the last text item - contentArray[contentArray.length - 1] = { - ...lastItem, - cache_control: { type: 'ephemeral' }, - } as ChatCompletionContentPartTextWithCache; - } else { - // If the last item is not text, add a new text item with cache_control - contentArray.push({ - type: 'text', - text: '', - cache_control: { type: 'ephemeral' }, - } as ChatCompletionContentPartTextWithCache); - } + contentArray[contentArray.length - 1] = { + ...lastItem, + cache_control: { type: 'ephemeral' }, + } as ChatCompletionContentPartTextWithCache; return contentArray; } diff --git a/packages/core/src/subagents/subagent.test.ts b/packages/core/src/subagents/subagent.test.ts index 67b9f375ecb..d3dea2dc0a0 100644 --- a/packages/core/src/subagents/subagent.test.ts +++ b/packages/core/src/subagents/subagent.test.ts @@ -784,7 +784,9 @@ describe('subagent.ts', () => { const promptConfig: PromptConfig = { systemPrompt: 'Execute task.' }; // Helper to create a mock stream that yields specific parts - const createMockStreamWithParts = (parts: Part[]) => vi.fn().mockImplementation(async () => (async function* () { + const createMockStreamWithParts = (parts: Part[]) => + vi.fn().mockImplementation(async () => + (async function* () { yield { type: 'chunk', value: { @@ -795,7 +797,8 @@ describe('subagent.ts', () => { ], }, }; - })()); + })(), + ); it('should emit STREAM_TEXT events with thought flag', async () => { const { config } = await createMockConfig(); diff --git a/packages/core/src/tools/read-file.test.ts b/packages/core/src/tools/read-file.test.ts index 01568eed9e0..4972f26e737 100644 --- a/packages/core/src/tools/read-file.test.ts +++ b/packages/core/src/tools/read-file.test.ts @@ -283,6 +283,7 @@ describe('ReadFileTool', () => { inlineData: { data: pngHeader.toString('base64'), mimeType: 'image/png', + displayName: 'image.png', }, }); expect(result.returnDisplay).toBe('Read image file: image.png'); @@ -304,6 +305,7 @@ describe('ReadFileTool', () => { inlineData: { data: pdfHeader.toString('base64'), mimeType: 'application/pdf', + displayName: 'document.pdf', }, }); expect(result.returnDisplay).toBe('Read pdf file: document.pdf'); diff --git a/packages/core/src/tools/read-many-files.test.ts b/packages/core/src/tools/read-many-files.test.ts index 758fb5d6a3e..f755abeccf2 100644 --- a/packages/core/src/tools/read-many-files.test.ts +++ b/packages/core/src/tools/read-many-files.test.ts @@ -383,6 +383,7 @@ describe('ReadManyFilesTool', () => { 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, ]).toString('base64'), mimeType: 'image/png', + displayName: 'image.png', }, }, '\n--- End of content ---', @@ -407,6 +408,7 @@ describe('ReadManyFilesTool', () => { 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, ]).toString('base64'), mimeType: 'image/png', + displayName: 'myExactImage.png', }, }, '\n--- End of content ---', @@ -444,6 +446,7 @@ describe('ReadManyFilesTool', () => { inlineData: { data: Buffer.from('%PDF-1.4...').toString('base64'), mimeType: 'application/pdf', + displayName: 'important.pdf', }, }, '\n--- End of content ---', @@ -460,6 +463,7 @@ describe('ReadManyFilesTool', () => { inlineData: { data: Buffer.from('%PDF-1.4...').toString('base64'), mimeType: 'application/pdf', + displayName: 'report-final.pdf', }, }, '\n--- End of content ---', diff --git a/packages/core/src/utils/fileUtils.test.ts b/packages/core/src/utils/fileUtils.test.ts index 92af55e427a..da9f257fdf1 100644 --- a/packages/core/src/utils/fileUtils.test.ts +++ b/packages/core/src/utils/fileUtils.test.ts @@ -731,6 +731,10 @@ describe('fileUtils', () => { expect( (result.llmContent as { inlineData: { data: string } }).inlineData.data, ).toBe(fakePngData.toString('base64')); + expect( + (result.llmContent as { inlineData: { displayName?: string } }) + .inlineData.displayName, + ).toBe('image.png'); expect(result.returnDisplay).toContain('Read image file: image.png'); }); @@ -752,6 +756,10 @@ describe('fileUtils', () => { expect( (result.llmContent as { inlineData: { data: string } }).inlineData.data, ).toBe(fakePdfData.toString('base64')); + expect( + (result.llmContent as { inlineData: { displayName?: string } }) + .inlineData.displayName, + ).toBe('document.pdf'); expect(result.returnDisplay).toContain('Read pdf file: document.pdf'); }); diff --git a/packages/core/src/utils/fileUtils.ts b/packages/core/src/utils/fileUtils.ts index 940e9794d94..76b8acc09c7 100644 --- a/packages/core/src/utils/fileUtils.ts +++ b/packages/core/src/utils/fileUtils.ts @@ -351,6 +351,7 @@ export async function processSingleFileContent( .relative(rootDirectory, filePath) .replace(/\\/g, '/'); + const displayName = path.basename(filePath); switch (fileType) { case 'binary': { return { @@ -456,9 +457,9 @@ export async function processSingleFileContent( }; } case 'image': - case 'pdf': case 'audio': - case 'video': { + case 'video': + case 'pdf': { const contentBuffer = await fs.promises.readFile(filePath); const base64Data = contentBuffer.toString('base64'); return { @@ -466,6 +467,7 @@ export async function processSingleFileContent( inlineData: { data: base64Data, mimeType: mime.getType(filePath) || 'application/octet-stream', + displayName, }, }, returnDisplay: `Read ${fileType} file: ${relativePathForDisplay}`, diff --git a/packages/core/src/utils/pathReader.test.ts b/packages/core/src/utils/pathReader.test.ts index fd6ff224582..5de10765b2b 100644 --- a/packages/core/src/utils/pathReader.test.ts +++ b/packages/core/src/utils/pathReader.test.ts @@ -113,6 +113,7 @@ describe('readPathFromWorkspace', () => { inlineData: { mimeType: 'image/png', data: imageData.toString('base64'), + displayName: 'image.png', }, }, ]); @@ -263,6 +264,7 @@ describe('readPathFromWorkspace', () => { inlineData: { mimeType: 'image/png', data: imageData.toString('base64'), + displayName: 'photo.png', }, }); });