diff --git a/packages/core/src/tools/read-file.test.ts b/packages/core/src/tools/read-file.test.ts index bae1bb0d8f9..1c5d6596386 100644 --- a/packages/core/src/tools/read-file.test.ts +++ b/packages/core/src/tools/read-file.test.ts @@ -12,6 +12,7 @@ import path from 'node:path'; import os from 'node:os'; import fs from 'node:fs'; import fsp from 'node:fs/promises'; +import sharp from 'sharp'; import type { Config } from '../config/config.js'; import { Storage } from '../config/storage.js'; import { FileDiscoveryService } from '../services/fileDiscoveryService.js'; @@ -545,11 +546,16 @@ describe('ReadFileTool', () => { it('should handle image file and return appropriate content', async () => { const imagePath = path.join(tempRootDir, 'image.png'); - // Minimal PNG header - const pngHeader = Buffer.from([ - 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, - ]); - await fsp.writeFile(imagePath, pngHeader); + await sharp({ + create: { + width: 20, + height: 10, + channels: 3, + background: '#306090', + }, + }) + .png() + .toFile(imagePath); const params: ReadFileToolParams = { file_path: imagePath }; const invocation = tool.build(params) as ToolInvocation< ReadFileToolParams, @@ -557,13 +563,20 @@ describe('ReadFileTool', () => { >; const result = await invocation.execute(abortSignal); - expect(result.llmContent).toEqual({ - inlineData: { - data: pngHeader.toString('base64'), - mimeType: 'image/png', - displayName: 'image.png', + expect(result.llmContent).toEqual([ + { + text: expect.stringMatching( + /Image overview: 20x10; oriented source: 20x10.*tool_search.*zoom_image.*0 to 1000/, + ), }, - }); + { + inlineData: { + data: expect.any(String), + mimeType: 'image/jpeg', + displayName: 'image.png', + }, + }, + ]); expect(result.returnDisplay).toBe('Read image file: image.png'); }); @@ -1526,17 +1539,23 @@ describe('ReadFileTool', () => { it('does not return the placeholder for image files', async () => { const imagePath = path.join(tempRootDir, 'pic.png'); - const pngHeader = Buffer.from([ - 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, - ]); - await fsp.writeFile(imagePath, pngHeader); + await sharp({ + create: { + width: 8, + height: 8, + channels: 3, + background: '#306090', + }, + }) + .png() + .toFile(imagePath); const first = await read({ file_path: imagePath }); - // Image returns a Part, not a string. + // Image returns Parts, not a string. expect(typeof first.llmContent).not.toBe('string'); const second = await read({ file_path: imagePath }); - // Must remain a Part — never collapsed to a string placeholder. + // Must remain Parts — never collapsed to a string placeholder. expect(typeof second.llmContent).not.toBe('string'); }); diff --git a/packages/core/src/tools/zoom-image.ts b/packages/core/src/tools/zoom-image.ts index 619615af265..34fc314ebd9 100644 --- a/packages/core/src/tools/zoom-image.ts +++ b/packages/core/src/tools/zoom-image.ts @@ -5,15 +5,17 @@ */ import path from 'node:path'; -import fs from 'node:fs/promises'; import type { Part } from '@google/genai'; -import type { Metadata } from 'sharp'; import type { Config } from '../config/config.js'; import type { PermissionDecision } from '../permissions/types.js'; import { logFileOperation } from '../telemetry/loggers.js'; import { FileOperation } from '../telemetry/metrics.js'; import { FileOperationEvent } from '../telemetry/types.js'; import { getSpecificMimeType } from '../utils/fileUtils.js'; +import { + ImageViewError, + renderNormalizedImageCrop, +} from '../utils/image-view.js'; import { makeRelative, shortenPath, unescapePath } from '../utils/paths.js'; import { getFileReadDefaultPermission } from './file-read-permission.js'; import { ToolErrorType } from './tool-error.js'; @@ -21,15 +23,6 @@ import { ToolDisplayNames, ToolNames } from './tool-names.js'; import type { ToolInvocation, ToolLocation, ToolResult } from './tools.js'; import { BaseDeclarativeTool, BaseToolInvocation, Kind } from './tools.js'; -const IMAGE_VIEW_MAX_EDGE = 1568; -const IMAGE_VIEW_MAX_PATCHES = 1568; -const IMAGE_PATCH_SIZE = 28; -const IMAGE_MAX_UPSCALE = 8; -const IMAGE_JPEG_QUALITY = 92; -const IMAGE_MAX_SOURCE_BYTES = 100 * 1024 * 1024; -const IMAGE_MAX_OUTPUT_BYTES = 9 * 1024 * 1024; -const SUPPORTED_IMAGE_FORMATS = new Set(['jpeg', 'png', 'webp']); - export interface ZoomImageParams { file_path: string; x1: number; @@ -38,11 +31,6 @@ export interface ZoomImageParams { y2: number; } -interface ImageSize { - width: number; - height: number; -} - function failureResult(message: string, type: ToolErrorType): ToolResult { return { llmContent: message, @@ -51,48 +39,6 @@ function failureResult(message: string, type: ToolErrorType): ToolResult { }; } -function fitsVisualBudget({ width, height }: ImageSize): boolean { - return ( - width <= IMAGE_VIEW_MAX_EDGE && - height <= IMAGE_VIEW_MAX_EDGE && - Math.ceil(width / IMAGE_PATCH_SIZE) * - Math.ceil(height / IMAGE_PATCH_SIZE) <= - IMAGE_VIEW_MAX_PATCHES - ); -} - -function magnifiedSize(width: number, height: number): ImageSize { - const widthIsLongEdge = width >= height; - const maxLongEdge = Math.min( - IMAGE_VIEW_MAX_EDGE, - Math.max(width, height) * IMAGE_MAX_UPSCALE, - ); - let low = 1; - let high = maxLongEdge; - let best: ImageSize = { width: 1, height: 1 }; - - while (low <= high) { - const longEdge = Math.floor((low + high) / 2); - const candidate = widthIsLongEdge - ? { - width: longEdge, - height: Math.max(1, Math.round((height / width) * longEdge)), - } - : { - width: Math.max(1, Math.round((width / height) * longEdge)), - height: longEdge, - }; - if (fitsVisualBudget(candidate)) { - best = candidate; - low = longEdge + 1; - } else { - high = longEdge - 1; - } - } - - return best; -} - class ZoomImageInvocation extends BaseToolInvocation< ZoomImageParams, ToolResult @@ -130,145 +76,47 @@ class ZoomImageInvocation extends BaseToolInvocation< ToolErrorType.READ_CONTENT_FAILURE, ); } - let sharp: typeof import('sharp'); + let view: Awaited>; try { - // sharp is a CJS `export =` module: at runtime the dynamic-import - // namespace carries the callable on `.default`, which the NodeNext types - // collapse away, so unwrap it explicitly (cf. utils/iconvHelper.ts). - sharp = ( - (await import('sharp')) as unknown as { - default: typeof import('sharp'); - } - ).default; - } catch { - return failureResult( - 'zoom_image is unavailable because the "sharp" image module could not be loaded.', - ToolErrorType.READ_CONTENT_FAILURE, + view = await renderNormalizedImageCrop( + this.params.file_path, + this.params, + signal, ); - } - let stats: Awaited>; - try { - stats = await fs.stat(this.params.file_path); } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') { - return failureResult( - `Image file not found: ${this.params.file_path}`, - ToolErrorType.FILE_NOT_FOUND, - ); - } - throw error; - } - if (stats.isDirectory()) { - return failureResult( - `Image path is a directory: ${this.params.file_path}`, - ToolErrorType.TARGET_IS_DIRECTORY, - ); - } - if (!stats.isFile()) { - return failureResult( - `Image path is not a regular file: ${this.params.file_path}`, - ToolErrorType.TARGET_NOT_REGULAR_FILE, - ); - } - if (stats.size > IMAGE_MAX_SOURCE_BYTES) { - return failureResult( - `Image file exceeds the 100 MB source limit: ${this.params.file_path}`, - ToolErrorType.FILE_TOO_LARGE, - ); - } - let metadata: Metadata; - try { - metadata = await sharp(this.params.file_path, { - failOn: 'error', - limitInputPixels: true, - }).metadata(); - } catch { - return failureResult( - `Unsupported image. zoom_image accepts static PNG, JPEG, or WebP files: ${this.params.file_path}`, - ToolErrorType.READ_CONTENT_FAILURE, - ); - } - signal.throwIfAborted(); - if (!SUPPORTED_IMAGE_FORMATS.has(metadata.format)) { - return failureResult( - `Unsupported image. zoom_image accepts static PNG, JPEG, or WebP files: ${this.params.file_path}`, - ToolErrorType.READ_CONTENT_FAILURE, - ); - } - if ((metadata.pages ?? 1) > 1) { - return failureResult( - `zoom_image accepts static images only: ${this.params.file_path}`, - ToolErrorType.READ_CONTENT_FAILURE, - ); - } - - const sourceWidth = metadata.autoOrient.width; - const sourceHeight = metadata.autoOrient.height; - const left = Math.min( - sourceWidth - 1, - Math.max(0, Math.floor((this.params.x1 / 1000) * sourceWidth)), - ); - const top = Math.min( - sourceHeight - 1, - Math.max(0, Math.floor((this.params.y1 / 1000) * sourceHeight)), - ); - const right = Math.min( - sourceWidth, - Math.max(left + 1, Math.ceil((this.params.x2 / 1000) * sourceWidth)), - ); - const bottom = Math.min( - sourceHeight, - Math.max(top + 1, Math.ceil((this.params.y2 / 1000) * sourceHeight)), - ); - const cropWidth = right - left; - const cropHeight = bottom - top; - const outputSize = magnifiedSize(cropWidth, cropHeight); - - let output: Buffer; - try { - output = await sharp(this.params.file_path, { - autoOrient: true, - failOn: 'error', - limitInputPixels: true, - }) - .extract({ left, top, width: cropWidth, height: cropHeight }) - .resize(outputSize.width, outputSize.height, { - fit: 'fill', - kernel: sharp.kernel.lanczos3, - }) - .flatten({ background: '#ffffff' }) - .jpeg({ - quality: IMAGE_JPEG_QUALITY, - chromaSubsampling: '4:4:4', - }) - .toBuffer(); - } catch { signal.throwIfAborted(); - return failureResult( - `Failed to decode image: ${this.params.file_path}`, - ToolErrorType.READ_CONTENT_FAILURE, - ); - } - signal.throwIfAborted(); - if (output.length > IMAGE_MAX_OUTPUT_BYTES) { - return failureResult( - `Zoomed image exceeds the 9 MB output limit: ${this.params.file_path}`, - ToolErrorType.FILE_TOO_LARGE, - ); + const message = + error instanceof Error ? error.message : 'Failed to decode image.'; + let errorType = ToolErrorType.READ_CONTENT_FAILURE; + if (error instanceof ImageViewError) { + if (error.code === 'file_not_found') { + errorType = ToolErrorType.FILE_NOT_FOUND; + } else if (error.code === 'target_is_directory') { + errorType = ToolErrorType.TARGET_IS_DIRECTORY; + } else if (error.code === 'target_not_regular_file') { + errorType = ToolErrorType.TARGET_NOT_REGULAR_FILE; + } else if ( + error.code === 'source_too_large' || + error.code === 'output_too_large' + ) { + errorType = ToolErrorType.FILE_TOO_LARGE; + } + } + return failureResult(message, errorType); } const text = `Zoomed normalized region (${this.params.x1},${this.params.y1})-` + `(${this.params.x2},${this.params.y2}) from ${this.params.file_path}. ` + - `Oriented source: ${sourceWidth}x${sourceHeight}; source crop: ` + - `${cropWidth}x${cropHeight}; returned view: ` + - `${outputSize.width}x${outputSize.height}.`; + `Oriented source: ${view.sourceWidth}x${view.sourceHeight}; source crop: ` + + `${view.selectedWidth}x${view.selectedHeight}; returned view: ` + + `${view.outputWidth}x${view.outputHeight}.`; const llmContent: Part[] = [ { text }, { inlineData: { - mimeType: 'image/jpeg', - data: output.toString('base64'), + mimeType: view.mimeType, + data: view.bytes.toString('base64'), }, }, ]; diff --git a/packages/core/src/utils/fileUtils.test.ts b/packages/core/src/utils/fileUtils.test.ts index 865bc46340e..286e68ee484 100644 --- a/packages/core/src/utils/fileUtils.test.ts +++ b/packages/core/src/utils/fileUtils.test.ts @@ -22,6 +22,7 @@ import path from 'node:path'; import os from 'node:os'; import mime from 'mime/lite'; import type { Part } from '@google/genai'; +import sharp from 'sharp'; import { isWithinRoot, @@ -1179,30 +1180,234 @@ describe('fileUtils', () => { }); it('should process an image file', async () => { - const fakePngData = Buffer.from('fake png data'); - actualNodeFs.writeFileSync(testImageFilePath, fakePngData); + await sharp({ + create: { + width: 20, + height: 10, + channels: 3, + background: '#306090', + }, + }) + .png() + .toFile(testImageFilePath); mockMimeGetType.mockReturnValue('image/png'); const result = await processSingleFileContent( testImageFilePath, mockConfig, ); - expect( - (result.llmContent as { inlineData: unknown }).inlineData, - ).toBeDefined(); - expect( - (result.llmContent as { inlineData: { mimeType: string } }).inlineData - .mimeType, - ).toBe('image/png'); - expect( - (result.llmContent as { inlineData: { data: string } }).inlineData.data, - ).toBe(fakePngData.toString('base64')); - expect( - (result.llmContent as { inlineData: { displayName?: string } }) - .inlineData.displayName, - ).toBe('image.png'); + const parts = result.llmContent as Part[]; + expect(parts[0]).toEqual({ + text: + 'Image overview: 20x10; oriented source: 20x10. ' + + 'If details are too small, use tool_search for "zoom image", then ' + + 'call zoom_image with coordinates normalized from 0 to 1000.', + }); + expect(parts[1]).toEqual({ + inlineData: { + mimeType: 'image/jpeg', + data: expect.any(String), + displayName: 'image.png', + }, + }); + const metadata = await sharp( + Buffer.from(parts[1]!.inlineData!.data!, 'base64'), + ).metadata(); + expect(metadata).toMatchObject({ width: 20, height: 10 }); expect(result.returnDisplay).toContain('Read image file: image.png'); }); + it('returns a bounded overview when a PNG exceeds the old data URI limit', async () => { + const largeImagePath = path.join(tempRootDir, 'large.png'); + await sharp({ + create: { + width: 2200, + height: 1800, + channels: 3, + background: '#306090', + }, + }) + .png({ compressionLevel: 0 }) + .toFile(largeImagePath); + expect(actualNodeFs.statSync(largeImagePath).size).toBeGreaterThan( + 9.9 * 1024 * 1024, + ); + mockMimeGetType.mockReturnValue('image/png'); + + const result = await processSingleFileContent(largeImagePath, mockConfig); + + expect(result.error).toBeUndefined(); + const parts = result.llmContent as Part[]; + const overview = parts[1]!.inlineData!; + const metadata = await sharp( + Buffer.from(overview.data!, 'base64'), + ).metadata(); + expect(overview.mimeType).toBe('image/jpeg'); + expect(Buffer.from(overview.data!, 'base64').length).toBeLessThanOrEqual( + 9 * 1024 * 1024, + ); + expect(Math.max(metadata.width!, metadata.height!)).toBeLessThanOrEqual( + 1568, + ); + expect( + Math.ceil(metadata.width! / 28) * Math.ceil(metadata.height! / 28), + ).toBeLessThanOrEqual(1568); + }); + + it('rejects canonical image sources above 100 MB before decoding', async () => { + const oversizedPath = path.join(tempRootDir, 'oversized.png'); + const handle = await fsPromises.open(oversizedPath, 'w'); + await handle.truncate(100 * 1024 * 1024 + 1); + await handle.close(); + mockMimeGetType.mockReturnValue('image/png'); + + const result = await processSingleFileContent(oversizedPath, mockConfig); + + expect(result.errorType).toBe(ToolErrorType.FILE_TOO_LARGE); + expect(result.llmContent).toContain('100 MB source limit'); + }); + + it('forwards a corrupt canonical image verbatim instead of failing the read', async () => { + const corruptPath = path.join(tempRootDir, 'corrupt.png'); + const bytes = Buffer.from('not a real png'); + await fsPromises.writeFile(corruptPath, bytes); + mockMimeGetType.mockReturnValue('image/png'); + + const result = await processSingleFileContent(corruptPath, mockConfig); + + expect(result.error).toBeUndefined(); + expect(result.llmContent).toEqual({ + inlineData: { + data: bytes.toString('base64'), + mimeType: 'image/png', + displayName: 'corrupt.png', + }, + }); + expect(result.returnDisplay).toContain('Read image file: corrupt.png'); + }); + + it('forwards an animated canonical image verbatim instead of failing the read', async () => { + const twoFrameGif = Buffer.from( + '47494638396101000100800000000000ffffff21f90400010000002c000000000100010000020244010021f90400010000002c00000000010001000002024c01003b', + 'hex', + ); + const animatedPath = path.join(tempRootDir, 'animated.webp'); + await sharp(twoFrameGif, { animated: true }).webp().toFile(animatedPath); + mockMimeGetType.mockReturnValue('image/webp'); + + const result = await processSingleFileContent(animatedPath, mockConfig); + + const onDisk = await fsPromises.readFile(animatedPath); + expect(result.error).toBeUndefined(); + expect(result.llmContent).toEqual({ + inlineData: { + data: onDisk.toString('base64'), + mimeType: 'image/webp', + displayName: 'animated.webp', + }, + }); + expect(result.returnDisplay).toContain('Read image file: animated.webp'); + }); + + it('forwards non-canonical content behind a canonical extension verbatim', async () => { + const gifBytes = Buffer.from( + '47494638396101000100800000000000ffffff21f90400010000002c000000000100010000020244010021f90400010000002c00000000010001000002024c01003b', + 'hex', + ); + const mismatchPath = path.join(tempRootDir, 'mismatch.png'); + await fsPromises.writeFile(mismatchPath, gifBytes); + mockMimeGetType.mockReturnValue('image/png'); + + const result = await processSingleFileContent(mismatchPath, mockConfig); + + expect(result.error).toBeUndefined(); + expect(result.llmContent).toEqual({ + inlineData: { + data: gifBytes.toString('base64'), + mimeType: 'image/png', + displayName: 'mismatch.png', + }, + }); + }); + + it('applies EXIF orientation before describing and rendering an overview', async () => { + const orientedPath = path.join(tempRootDir, 'oriented.jpg'); + await sharp({ + create: { + width: 60, + height: 40, + channels: 3, + background: '#306090', + }, + }) + .jpeg() + .withMetadata({ orientation: 6 }) + .toFile(orientedPath); + mockMimeGetType.mockReturnValue('image/jpeg'); + + const result = await processSingleFileContent(orientedPath, mockConfig); + const parts = result.llmContent as Part[]; + const metadata = await sharp( + Buffer.from(parts[1]!.inlineData!.data!, 'base64'), + ).metadata(); + + expect(parts[0]?.text).toContain('oriented source: 40x60'); + expect(metadata).toMatchObject({ width: 40, height: 60 }); + }); + + it('flattens transparent overview pixels onto white', async () => { + const transparentPath = path.join(tempRootDir, 'transparent.webp'); + await sharp({ + create: { + width: 20, + height: 20, + channels: 4, + background: { r: 0, g: 0, b: 0, alpha: 0 }, + }, + }) + .webp() + .toFile(transparentPath); + mockMimeGetType.mockReturnValue('image/webp'); + + const result = await processSingleFileContent( + transparentPath, + mockConfig, + ); + const parts = result.llmContent as Part[]; + const { data, info } = await sharp( + Buffer.from(parts[1]!.inlineData!.data!, 'base64'), + ) + .raw() + .toBuffer({ resolveWithObject: true }); + const center = + (Math.floor(info.height / 2) * info.width + + Math.floor(info.width / 2)) * + info.channels; + + expect(Array.from(data.subarray(center, center + 3))).toEqual([ + 255, 255, 255, + ]); + }); + + it.each([ + ['animated GIF', 'animation.gif', 'image/gif'], + ['BMP', 'bitmap.bmp', 'image/bmp'], + ])('keeps %s image bytes unchanged', async (_label, name, mimeType) => { + const filePath = path.join(tempRootDir, name); + const bytes = Buffer.from('unchanged image bytes'); + actualNodeFs.writeFileSync(filePath, bytes); + mockMimeGetType.mockReturnValue(mimeType); + + const result = await processSingleFileContent(filePath, mockConfig); + + expect(result.llmContent).toEqual({ + inlineData: { + data: bytes.toString('base64'), + mimeType, + displayName: name, + }, + }); + }); + it('should reject image files when model does not support image', async () => { const fakePngData = Buffer.from('fake png data'); actualNodeFs.writeFileSync(testImageFilePath, fakePngData); @@ -1224,8 +1429,16 @@ describe('fileUtils', () => { }); it('keeps image inline when preserveUnsupportedImage is true', async () => { - const fakePngData = Buffer.from('fake png data'); - actualNodeFs.writeFileSync(testImageFilePath, fakePngData); + await sharp({ + create: { + width: 8, + height: 8, + channels: 3, + background: '#306090', + }, + }) + .png() + .toFile(testImageFilePath); mockMimeGetType.mockReturnValue('image/png'); const mockConfigNoImage = { @@ -1239,10 +1452,13 @@ describe('fileUtils', () => { { preserveUnsupportedImage: true }, ); expect(typeof result.llmContent).toBe('object'); - expect( - (result.llmContent as { inlineData: { mimeType: string } }).inlineData - .mimeType, - ).toBe('image/png'); + const parts = result.llmContent as Part[]; + expect(parts[0]?.text).toContain('Image overview'); + expect(parts[1]?.inlineData).toMatchObject({ + mimeType: 'image/jpeg', + data: expect.any(String), + displayName: 'image.png', + }); expect(result.returnDisplay).toContain('Read image file'); }); @@ -1285,6 +1501,29 @@ describe('fileUtils', () => { expect(result.llmContent).toContain('does not support audio input'); }); + it('keeps supported audio bytes unchanged', async () => { + const audioPath = path.join(tempRootDir, 'clip.mp3'); + const audioBytes = Buffer.from('fake audio data'); + actualNodeFs.writeFileSync(audioPath, audioBytes); + mockMimeGetType.mockReturnValue('audio/mpeg'); + const audioConfig = { + ...mockConfig, + getContentGeneratorConfig: () => ({ + modalities: { image: true, audio: true, video: true }, + }), + } as unknown as Config; + + const result = await processSingleFileContent(audioPath, audioConfig); + + expect(result.llmContent).toEqual({ + inlineData: { + data: audioBytes.toString('base64'), + mimeType: 'audio/mpeg', + displayName: 'clip.mp3', + }, + }); + }); + it('processes an .m4v video as inline data despite the mime/lite gap', async () => { // Regression guard for the /learn local-video path: mime/lite returns // null for .m4v, so without the detectFileType override the file fell @@ -3088,16 +3327,11 @@ describe('fileUtils', () => { }); it('should still return an error if an inline media file exceeds 10MB', async () => { - mockMimeGetType.mockReturnValue('image/png'); - actualNodeFs.writeFileSync( - testImageFilePath, - Buffer.alloc(11 * 1024 * 1024), - ); + const largeGifPath = path.join(tempRootDir, 'large.gif'); + mockMimeGetType.mockReturnValue('image/gif'); + actualNodeFs.writeFileSync(largeGifPath, Buffer.alloc(11 * 1024 * 1024)); - const result = await processSingleFileContent( - testImageFilePath, - mockConfig, - ); + const result = await processSingleFileContent(largeGifPath, mockConfig); expect(result.error).toContain('File size exceeds the 10MB limit'); expect(result.returnDisplay).toContain( diff --git a/packages/core/src/utils/fileUtils.ts b/packages/core/src/utils/fileUtils.ts index 67bd88b0e8d..2d132e0be3b 100644 --- a/packages/core/src/utils/fileUtils.ts +++ b/packages/core/src/utils/fileUtils.ts @@ -41,8 +41,18 @@ import { DEFAULT_RANGE_READ_BYTES, TEXT_RANGE_FAST_PATH_MAX_SIZE, } from './text-range-constants.js'; +import { + IMAGE_MAX_SOURCE_BYTES, + ImageViewError, + renderImageOverview, +} from './image-view.js'; const debugLogger = createDebugLogger('FILE_UTILS'); +const CANONICAL_IMAGE_MIME_TYPES = new Set([ + 'image/jpeg', + 'image/png', + 'image/webp', +]); // Default values for encoding and separator format export const DEFAULT_ENCODING: BufferEncoding = 'utf-8'; @@ -1093,6 +1103,12 @@ export async function processSingleFileContent( } const fileType = await detectFileType(filePath); + const mediaMimeType = + mime.getType(filePath) ?? + MIME_LITE_MISSING_VIDEO_TYPES.get(path.extname(filePath).toLowerCase()) ?? + 'application/octet-stream'; + const shouldRenderImageOverview = + fileType === 'image' && CANONICAL_IMAGE_MIME_TYPES.has(mediaMimeType); const relativePathForDisplay = path .relative(rootDirectory, filePath) .replace(/\\/g, '/'); @@ -1229,7 +1245,20 @@ export async function processSingleFileContent( }; } } - if (fileSizeInMB > 9.9 && !willExtractPdfText && fileType !== 'text') { + if (shouldRenderImageOverview && stats.size > IMAGE_MAX_SOURCE_BYTES) { + return { + llmContent: 'Image file exceeds the 100 MB source limit.', + returnDisplay: 'Image file exceeds the 100 MB source limit.', + error: `Image file exceeds the 100 MB source limit: ${filePath}`, + errorType: ToolErrorType.FILE_TOO_LARGE, + }; + } + if ( + fileSizeInMB > 9.9 && + !willExtractPdfText && + fileType !== 'text' && + !shouldRenderImageOverview + ) { return { llmContent: 'File size exceeds the 10MB limit.', returnDisplay: 'File size exceeds the 10MB limit.', @@ -1416,7 +1445,79 @@ export async function processSingleFileContent( stats, }; } - case 'image': + case 'image': { + if (shouldRenderImageOverview) { + try { + const view = await renderImageOverview( + filePath, + signal ?? new AbortController().signal, + ); + return { + llmContent: [ + { + text: + `Image overview: ${view.outputWidth}x${view.outputHeight}; ` + + `oriented source: ${view.sourceWidth}x${view.sourceHeight}. ` + + `If details are too small, use tool_search for "zoom image", then ` + + `call zoom_image with coordinates normalized from 0 to 1000.`, + }, + { + inlineData: { + data: view.bytes.toString('base64'), + mimeType: view.mimeType, + displayName, + }, + }, + ], + returnDisplay: `Read image file: ${relativePathForDisplay}`, + }; + } catch (error) { + signal?.throwIfAborted(); + if (error instanceof ImageViewError) { + // Non-size render failures (sharp missing, animated, + // unsupported, or corrupt input) fall through to the legacy + // inline-bytes branch below rather than hard-failing the read, + // matching main's forward-verbatim behaviour. The size codes + // stay a hard error because that branch cannot shrink them. + if ( + error.code === 'source_too_large' || + error.code === 'output_too_large' + ) { + const userMessage = error.message.replace(`: ${filePath}`, ''); + return { + llmContent: userMessage, + returnDisplay: userMessage, + error: error.message, + errorType: ToolErrorType.FILE_TOO_LARGE, + }; + } + } else { + throw error; + } + } + } + const contentBuffer = await fs.promises.readFile(filePath); + const base64Data = contentBuffer.toString('base64'); + const base64SizeInMB = base64Data.length / (1024 * 1024); + if (base64SizeInMB > 9.9) { + return { + llmContent: `File exceeds the 10MB data URI limit after base64 encoding (${base64SizeInMB.toFixed(2)}MB encoded).`, + returnDisplay: `File exceeds the 10MB data URI limit after base64 encoding.`, + error: `File exceeds the 10MB data URI limit after base64 encoding: ${filePath} (${base64SizeInMB.toFixed(2)}MB encoded)`, + errorType: ToolErrorType.FILE_TOO_LARGE, + }; + } + return { + llmContent: { + inlineData: { + data: base64Data, + mimeType: mediaMimeType, + displayName, + }, + }, + returnDisplay: `Read image file: ${relativePathForDisplay}`, + }; + } case 'audio': case 'video': { const contentBuffer = await fs.promises.readFile(filePath); @@ -1435,12 +1536,7 @@ export async function processSingleFileContent( llmContent: { inlineData: { data: base64Data, - mimeType: - mime.getType(filePath) ?? - MIME_LITE_MISSING_VIDEO_TYPES.get( - path.extname(filePath).toLowerCase(), - ) ?? - 'application/octet-stream', + mimeType: mediaMimeType, displayName, }, }, diff --git a/packages/core/src/utils/image-view.test.ts b/packages/core/src/utils/image-view.test.ts new file mode 100644 index 00000000000..fa1aa32ac68 --- /dev/null +++ b/packages/core/src/utils/image-view.test.ts @@ -0,0 +1,122 @@ +/** + * @license + * Copyright 2026 Qwen + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import sharp from 'sharp'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { + renderImageOverview, + renderNormalizedImageCrop, +} from './image-view.js'; + +describe('image views', () => { + let root: string; + const signal = new AbortController().signal; + + beforeEach(async () => { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'image-view-')); + }); + + afterEach(async () => { + await fs.rm(root, { recursive: true, force: true }); + }); + + it('keeps a small overview at its oriented source size', async () => { + const filePath = path.join(root, 'small.png'); + await sharp({ + create: { + width: 20, + height: 10, + channels: 3, + background: '#306090', + }, + }) + .png() + .toFile(filePath); + + const view = await renderImageOverview(filePath, signal); + const metadata = await sharp(view.bytes).metadata(); + + expect(view).toMatchObject({ + mimeType: 'image/jpeg', + sourceWidth: 20, + sourceHeight: 10, + selectedWidth: 20, + selectedHeight: 10, + outputWidth: 20, + outputHeight: 10, + }); + expect(metadata).toMatchObject({ width: 20, height: 10, format: 'jpeg' }); + }); + + it('bounds a large overview by the shared edge and patch budget', async () => { + const filePath = path.join(root, 'large.png'); + await sharp({ + create: { + width: 4000, + height: 2000, + channels: 3, + background: '#804020', + }, + }) + .png() + .toFile(filePath); + + const view = await renderImageOverview(filePath, signal); + + expect(Math.max(view.outputWidth, view.outputHeight)).toBeLessThanOrEqual( + 1568, + ); + expect( + Math.ceil(view.outputWidth / 28) * Math.ceil(view.outputHeight / 28), + ).toBeLessThanOrEqual(1568); + expect(view.bytes.length).toBeLessThanOrEqual(9 * 1024 * 1024); + }); + + it('may magnify a normalized crop while preserving its source dimensions', async () => { + const filePath = path.join(root, 'crop.png'); + await sharp({ + create: { + width: 400, + height: 400, + channels: 3, + background: '#306090', + }, + }) + .png() + .toFile(filePath); + + const view = await renderNormalizedImageCrop( + filePath, + { x1: 0, y1: 0, x2: 25, y2: 25 }, + signal, + ); + const metadata = await sharp(view.bytes).metadata(); + + expect(view).toMatchObject({ + mimeType: 'image/jpeg', + sourceWidth: 400, + sourceHeight: 400, + selectedWidth: 10, + selectedHeight: 10, + outputWidth: 80, + outputHeight: 80, + }); + expect(metadata).toMatchObject({ width: 80, height: 80, format: 'jpeg' }); + expect(view.bytes.length).toBeLessThanOrEqual(9 * 1024 * 1024); + }); + + it('reports decode_failed for a corrupt canonical image', async () => { + const filePath = path.join(root, 'corrupt.png'); + await fs.writeFile(filePath, 'not a real png'); + + await expect(renderImageOverview(filePath, signal)).rejects.toMatchObject({ + code: 'decode_failed', + }); + }); +}); diff --git a/packages/core/src/utils/image-view.ts b/packages/core/src/utils/image-view.ts new file mode 100644 index 00000000000..339339b33c4 --- /dev/null +++ b/packages/core/src/utils/image-view.ts @@ -0,0 +1,314 @@ +/** + * @license + * Copyright 2026 Qwen + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import type { Metadata } from 'sharp'; + +const IMAGE_VIEW_MAX_EDGE = 1568; +const IMAGE_VIEW_MAX_PATCHES = 1568; +const IMAGE_PATCH_SIZE = 28; +const IMAGE_MAX_UPSCALE = 8; +const IMAGE_JPEG_QUALITY = 92; +export const IMAGE_MAX_SOURCE_BYTES = 100 * 1024 * 1024; +const IMAGE_MAX_OUTPUT_BYTES = 9 * 1024 * 1024; +const SUPPORTED_IMAGE_FORMATS = new Set(['jpeg', 'png', 'webp']); + +export interface NormalizedRegion { + x1: number; + y1: number; + x2: number; + y2: number; +} + +export interface ImageView { + bytes: Buffer; + mimeType: 'image/jpeg'; + sourceWidth: number; + sourceHeight: number; + selectedWidth: number; + selectedHeight: number; + outputWidth: number; + outputHeight: number; +} + +interface ImageSize { + width: number; + height: number; +} + +interface PreparedImage { + bytes: Buffer; + metadata: Metadata; + sharp: typeof import('sharp'); +} + +export type ImageViewErrorCode = + | 'renderer_unavailable' + | 'file_not_found' + | 'target_is_directory' + | 'target_not_regular_file' + | 'source_too_large' + | 'unsupported_image' + | 'animated_image' + | 'decode_failed' + | 'output_too_large'; + +export class ImageViewError extends Error { + constructor( + readonly code: ImageViewErrorCode, + message: string, + ) { + super(message); + } +} + +function fitsVisualBudget({ width, height }: ImageSize): boolean { + return ( + width <= IMAGE_VIEW_MAX_EDGE && + height <= IMAGE_VIEW_MAX_EDGE && + Math.ceil(width / IMAGE_PATCH_SIZE) * + Math.ceil(height / IMAGE_PATCH_SIZE) <= + IMAGE_VIEW_MAX_PATCHES + ); +} + +function boundedSize( + width: number, + height: number, + maxUpscale: number, +): ImageSize { + const widthIsLongEdge = width >= height; + const maxLongEdge = Math.min( + IMAGE_VIEW_MAX_EDGE, + Math.max(width, height) * maxUpscale, + ); + let low = 1; + let high = maxLongEdge; + let best: ImageSize = { width: 1, height: 1 }; + + while (low <= high) { + const longEdge = Math.floor((low + high) / 2); + const candidate = widthIsLongEdge + ? { + width: longEdge, + height: Math.max(1, Math.round((height / width) * longEdge)), + } + : { + width: Math.max(1, Math.round((width / height) * longEdge)), + height: longEdge, + }; + if (fitsVisualBudget(candidate)) { + best = candidate; + low = longEdge + 1; + } else { + high = longEdge - 1; + } + } + + return best; +} + +async function prepareImage( + filePath: string, + signal: AbortSignal, +): Promise { + signal.throwIfAborted(); + let sharp: typeof import('sharp'); + try { + // sharp is a CJS `export =` module, so the callable is on `.default` + // at runtime even though NodeNext types collapse that namespace away. + sharp = ( + (await import('sharp')) as unknown as { + default: typeof import('sharp'); + } + ).default; + } catch { + throw new ImageViewError( + 'renderer_unavailable', + 'Image rendering is unavailable because the "sharp" image module could not be loaded.', + ); + } + + let stats: Awaited>; + try { + stats = await fs.stat(filePath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + throw new ImageViewError( + 'file_not_found', + `Image file not found: ${filePath}`, + ); + } + throw error; + } + if (stats.isDirectory()) { + throw new ImageViewError( + 'target_is_directory', + `Image path is a directory: ${filePath}`, + ); + } + if (!stats.isFile()) { + throw new ImageViewError( + 'target_not_regular_file', + `Image path is not a regular file: ${filePath}`, + ); + } + if (stats.size > IMAGE_MAX_SOURCE_BYTES) { + throw new ImageViewError( + 'source_too_large', + `Image file exceeds the 100 MB source limit: ${filePath}`, + ); + } + + const bytes = await fs.readFile(filePath, { signal }); + if (bytes.length > IMAGE_MAX_SOURCE_BYTES) { + throw new ImageViewError( + 'source_too_large', + `Image file exceeds the 100 MB source limit: ${filePath}`, + ); + } + + let metadata: Metadata; + try { + metadata = await sharp(bytes, { + failOn: 'error', + limitInputPixels: true, + }).metadata(); + } catch { + signal.throwIfAborted(); + throw new ImageViewError( + 'decode_failed', + `Failed to decode image (file may be corrupt or not a static PNG, JPEG, or WebP): ${filePath}`, + ); + } + signal.throwIfAborted(); + + if (!SUPPORTED_IMAGE_FORMATS.has(metadata.format)) { + throw new ImageViewError( + 'unsupported_image', + `Unsupported image. Expected a static PNG, JPEG, or WebP file: ${filePath}`, + ); + } + if ((metadata.pages ?? 1) > 1) { + throw new ImageViewError( + 'animated_image', + `Only static images are supported: ${filePath}`, + ); + } + + return { bytes, metadata, sharp }; +} + +async function renderImageView( + filePath: string, + prepared: PreparedImage, + selection: { left: number; top: number; width: number; height: number }, + outputSize: ImageSize, + signal: AbortSignal, +): Promise { + const { bytes, metadata, sharp } = prepared; + let output: Buffer; + try { + output = await sharp(bytes, { + autoOrient: true, + failOn: 'error', + limitInputPixels: true, + }) + .extract(selection) + .resize(outputSize.width, outputSize.height, { + fit: 'fill', + kernel: sharp.kernel.lanczos3, + }) + .flatten({ background: '#ffffff' }) + .jpeg({ + quality: IMAGE_JPEG_QUALITY, + chromaSubsampling: '4:4:4', + }) + .toBuffer(); + } catch { + signal.throwIfAborted(); + throw new ImageViewError( + 'decode_failed', + `Failed to render image overview: ${filePath}`, + ); + } + signal.throwIfAborted(); + if (output.length > IMAGE_MAX_OUTPUT_BYTES) { + throw new ImageViewError( + 'output_too_large', + `Rendered image exceeds the 9 MB output limit: ${filePath}`, + ); + } + + return { + bytes: output, + mimeType: 'image/jpeg', + sourceWidth: metadata.autoOrient.width, + sourceHeight: metadata.autoOrient.height, + selectedWidth: selection.width, + selectedHeight: selection.height, + outputWidth: outputSize.width, + outputHeight: outputSize.height, + }; +} + +export async function renderImageOverview( + filePath: string, + signal: AbortSignal, +): Promise { + const prepared = await prepareImage(filePath, signal); + const sourceWidth = prepared.metadata.autoOrient.width; + const sourceHeight = prepared.metadata.autoOrient.height; + const outputSize = boundedSize(sourceWidth, sourceHeight, 1); + return renderImageView( + filePath, + prepared, + { left: 0, top: 0, width: sourceWidth, height: sourceHeight }, + outputSize, + signal, + ); +} + +export async function renderNormalizedImageCrop( + filePath: string, + region: NormalizedRegion, + signal: AbortSignal, +): Promise { + const prepared = await prepareImage(filePath, signal); + const sourceWidth = prepared.metadata.autoOrient.width; + const sourceHeight = prepared.metadata.autoOrient.height; + const left = Math.min( + sourceWidth - 1, + Math.max(0, Math.floor((region.x1 / 1000) * sourceWidth)), + ); + const top = Math.min( + sourceHeight - 1, + Math.max(0, Math.floor((region.y1 / 1000) * sourceHeight)), + ); + const right = Math.min( + sourceWidth, + Math.max(left + 1, Math.ceil((region.x2 / 1000) * sourceWidth)), + ); + const bottom = Math.min( + sourceHeight, + Math.max(top + 1, Math.ceil((region.y2 / 1000) * sourceHeight)), + ); + const selectedWidth = right - left; + const selectedHeight = bottom - top; + const outputSize = boundedSize( + selectedWidth, + selectedHeight, + IMAGE_MAX_UPSCALE, + ); + + return renderImageView( + filePath, + prepared, + { left, top, width: selectedWidth, height: selectedHeight }, + outputSize, + signal, + ); +} diff --git a/packages/core/src/utils/pathReader.test.ts b/packages/core/src/utils/pathReader.test.ts index 5b5e951acbe..1db43d198a0 100644 --- a/packages/core/src/utils/pathReader.test.ts +++ b/packages/core/src/utils/pathReader.test.ts @@ -7,6 +7,7 @@ import { describe, it, expect, afterEach, vi } from 'vitest'; import mock from 'mock-fs'; import * as path from 'node:path'; +import sharp from 'sharp'; import { WorkspaceContext } from './workspaceContext.js'; import { readPathFromWorkspace } from './pathReader.js'; import type { Config } from '../config/config.js'; @@ -104,11 +105,17 @@ describe('readPathFromWorkspace', () => { expect(result).toEqual(['hello from cwd']); }); - it('should read an image file and return it as inlineData (Part object)', async () => { - // Use a real PNG header for robustness - const imageData = Buffer.from([ - 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, - ]); + it('should read an image file as overview text and inlineData', async () => { + const imageData = await sharp({ + create: { + width: 20, + height: 10, + channels: 3, + background: '#306090', + }, + }) + .png() + .toBuffer(); mock({ [CWD]: { 'image.png': imageData, @@ -119,12 +126,17 @@ describe('readPathFromWorkspace', () => { } as unknown as FileDiscoveryService; const config = createMockConfig(CWD, [], mockFileService); const result = await readPathFromWorkspace('image.png', config); - // Expect [Part] for image content + // Expect overview text immediately followed by the bounded image. expect(result).toEqual([ + { + text: expect.stringContaining( + 'Image overview: 20x10; oriented source: 20x10', + ), + }, { inlineData: { - mimeType: 'image/png', - data: imageData.toString('base64'), + mimeType: 'image/jpeg', + data: expect.any(String), displayName: 'image.png', }, }, @@ -236,9 +248,16 @@ describe('readPathFromWorkspace', () => { }); it('should handle mixed content and include files from subdirectories', async () => { - const imageData = Buffer.from([ - 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, - ]); + const imageData = await sharp({ + create: { + width: 8, + height: 8, + channels: 3, + background: '#306090', + }, + }) + .png() + .toBuffer(); mock({ [CWD]: { 'mixed-dir': { @@ -274,8 +293,8 @@ describe('readPathFromWorkspace', () => { ); expect(imagePart).toEqual({ inlineData: { - mimeType: 'image/png', - data: imageData.toString('base64'), + mimeType: 'image/jpeg', + data: expect.any(String), displayName: 'photo.png', }, }); diff --git a/packages/core/src/utils/readManyFiles.test.ts b/packages/core/src/utils/readManyFiles.test.ts index 4351432c80c..e87d1e89cfa 100644 --- a/packages/core/src/utils/readManyFiles.test.ts +++ b/packages/core/src/utils/readManyFiles.test.ts @@ -9,6 +9,7 @@ import fs from 'node:fs/promises'; import * as nodeFs from 'node:fs'; import path from 'node:path'; import os from 'node:os'; +import sharp from 'sharp'; import type { Part, PartListUnion } from '@google/genai'; import { readManyFiles } from './readManyFiles.js'; import { FileDiscoveryService } from '../services/fileDiscoveryService.js'; @@ -192,11 +193,19 @@ describe('readManyFiles', () => { expect(result.files[0]!.filePath).toBe(absolutePath); }); - it('preserves unsupported images when the bridge handoff flag is set', async () => { + it('renders canonical images through the overview pipeline even when the bridge handoff flag is set', async () => { const relativePath = 'screenshot.png'; const absolutePath = path.join(tempRootDir, relativePath); - const imageBytes = Buffer.from('fake png data'); - await fs.writeFile(absolutePath, imageBytes); + await sharp({ + create: { + width: 20, + height: 10, + channels: 3, + background: '#306090', + }, + }) + .png() + .toFile(absolutePath); const mockConfig = createMockConfig(tempRootDir); const result = await readManyFiles(mockConfig, { @@ -209,13 +218,24 @@ describe('readManyFiles', () => { expect( (imagePart as { inlineData: { mimeType: string; data: string } }) .inlineData, - ).toEqual({ - mimeType: 'image/png', - data: imageBytes.toString('base64'), + ).toMatchObject({ + mimeType: 'image/jpeg', + data: expect.any(String), displayName: 'screenshot.png', }); + const parts = result.contentParts as Part[]; + const imageIndex = parts.indexOf(imagePart!); + expect(parts[imageIndex - 2]?.text).toBe( + `\nContent from ${absolutePath}:\n`, + ); + expect(parts[imageIndex - 1]?.text).toContain( + 'Image overview: 20x10; oriented source: 20x10', + ); expect(result.files).toHaveLength(1); - expect(result.files[0]!.content).toEqual(imagePart); + expect(result.files[0]!.content).toEqual([ + parts[imageIndex - 1], + imagePart, + ]); }); it('skips unsupported images when the bridge handoff flag is absent', async () => {