diff --git a/src/services/api/errors.openaiCompatibility.test.ts b/src/services/api/errors.openaiCompatibility.test.ts index ebb31342c2..8c830d9017 100644 --- a/src/services/api/errors.openaiCompatibility.test.ts +++ b/src/services/api/errors.openaiCompatibility.test.ts @@ -28,7 +28,7 @@ test('maps endpoint_not_found category markers to actionable setup guidance', () expect(text).toContain('/v1') }) -test('vision_not_supported shows image-specific guidance for remote host', () => { +test('vision_not_supported shows image-specific guidance (issue #1421 canonical message)', () => { const error = APIError.generate( 404, undefined, @@ -40,9 +40,29 @@ test('vision_not_supported shows image-specific guidance for remote host', () => const text = getFirstText(message) expect(message.isApiErrorMessage).toBe(true) - expect(text).toContain('images') - expect(text).toContain('mimo-v2.5-pro') - expect(text).toContain('opengateway.gitlawb.com') + expect(text).toContain('image') + expect(text).toContain('does not support') + // The command is `/model` in interactive sessions and `--model` in + // non-interactive (test/SDK) sessions — both forms are intentional. + expect(text).toMatch(/(\/model|--model)/) + expect(text).not.toContain('OPENAI_BASE_URL') +}) + +test('vision_not_supported from Xiaomi Mimo 400 "text is not set" uses the same canonical message (issue #1421)', () => { + const error = APIError.generate( + 400, + undefined, + 'OpenAI API error 400: {"error":{"code":"400","message":"Param Incorrect","param":"`text` is not set"}} [openai_category=vision_not_supported,host=api.xiaomimimo.com] Hint: The provider rejected an image-bearing request because it lacked a text part.', + new Headers(), + ) + + const message = getAssistantMessageFromError(error, 'mimo-v2.5-pro') + const text = getFirstText(message) + + expect(message.isApiErrorMessage).toBe(true) + expect(text).toContain('image') + expect(text).toContain('does not support') + expect(text).toMatch(/(\/model|--model)/) expect(text).not.toContain('OPENAI_BASE_URL') }) diff --git a/src/services/api/errors.ts b/src/services/api/errors.ts index 68d5bf879f..d15448f2c5 100644 --- a/src/services/api/errors.ts +++ b/src/services/api/errors.ts @@ -99,8 +99,9 @@ function mapOpenAICompatibilityFailureToAssistantMessage(options: { case 'vision_not_supported': return createAssistantAPIErrorMessage({ - content: `The provider at ${options.host} returned 404 for a request containing images. The model (${options.model}) may not support image/vision inputs. Try removing images from your message, or ${switchCmd} to a vision-capable model.`, + content: getVisionNotSupportedErrorMessage(), error: 'invalid_request', + errorDetails: stripOpenAICompatibilityMetadata(options.rawMessage), }) case 'model_not_found': @@ -399,6 +400,31 @@ export function getRequestTooLargeErrorMessage(): string { ? `Request too large (${limits}). Try with a smaller file.` : `Request too large (${limits}). Double press esc to go back and try with a smaller file.` } + +const VISION_NOT_SUPPORTED_MESSAGE_PREFIX = + 'The active model does not support image/vision inputs. The provider rejected the request because it contained an image. Remove the image, or' + +export function getVisionNotSupportedErrorMessages(): string[] { + return [ + `${VISION_NOT_SUPPORTED_MESSAGE_PREFIX} switch to a vision-capable model with --model.`, + `${VISION_NOT_SUPPORTED_MESSAGE_PREFIX} run /model to switch to a vision-capable model.`, + ] +} + +/** + * Canonical message for the `vision_not_supported` OpenAI compatibility + * failure (issue #1421). Returned as a stable string so that + * `normalizeMessagesForAPI`'s `errorToBlockTypes` map can self-heal existing + * transcripts by stripping `image` blocks from the preceding user message + * on resume. + */ +export function getVisionNotSupportedErrorMessage(): string { + const [nonInteractiveMessage, interactiveMessage] = + getVisionNotSupportedErrorMessages() + return getIsNonInteractiveSession() + ? nonInteractiveMessage! + : interactiveMessage! +} export const OAUTH_ORG_NOT_ALLOWED_ERROR_MESSAGE = 'Your account does not have access to OpenClaude. Please run /login.' diff --git a/src/services/api/openaiErrorClassification.test.ts b/src/services/api/openaiErrorClassification.test.ts index 75ab07d3fd..b7ef772918 100644 --- a/src/services/api/openaiErrorClassification.test.ts +++ b/src/services/api/openaiErrorClassification.test.ts @@ -73,6 +73,39 @@ test('classifies 404 with images as vision_not_supported', () => { expect(failure.hint).toContain('image') }) +test('classifies 400 with "text is not set" + images as vision_not_supported (issue #1421)', () => { + const failure = classifyOpenAIHttpFailure({ + status: 400, + body: '{"error":{"code":"400","message":"Param Incorrect","param":"`text` is not set","type":""}}', + hasImages: true, + }) + + expect(failure.category).toBe('vision_not_supported') + expect(failure.retryable).toBe(false) + expect(failure.hint).toContain('image') +}) + +test('classifies 400 with "text is required" + images as vision_not_supported (issue #1421)', () => { + const failure = classifyOpenAIHttpFailure({ + status: 400, + body: '{"error":{"message":"text parameter is required"}}', + hasImages: true, + }) + + expect(failure.category).toBe('vision_not_supported') +}) + +test('does not classify 400 with "text is not set" when request has no images', () => { + const failure = classifyOpenAIHttpFailure({ + status: 400, + body: '{"error":{"message":"text is not set"}}', + hasImages: false, + }) + + // Without images, "text is not set" is unrelated to vision capability. + expect(failure.category).not.toBe('vision_not_supported') +}) + test('classifies context-overflow responses', () => { const failure = classifyOpenAIHttpFailure({ status: 500, diff --git a/src/services/api/openaiErrorClassification.ts b/src/services/api/openaiErrorClassification.ts index c8b097fbf5..0ec068a325 100644 --- a/src/services/api/openaiErrorClassification.ts +++ b/src/services/api/openaiErrorClassification.ts @@ -158,6 +158,34 @@ function isMalformedProviderResponse(body: string): boolean { ) } +/** + * Detect provider messages that complain about a missing/required `text` + * field on an otherwise image-bearing payload. Xiaomi Mimo surfaces this as + * `{"error":{"code":"400","message":"Param Incorrect","param":"`text` is not set"}}` + * (with backticks around `text`) when a `role: "tool"` message carries + * images but no text part. Other OpenAI-compatible providers may phrase + * it differently — match liberally. + * + * Only meaningful when `hasImages` is true (we never want this branch to fire + * for text-only requests, which legitimately lack a text field on vision-only + * payloads). + */ +function isMissingTextPartMessage(body: string): boolean { + // Strip backticks so `\`text\` is not set` matches the same patterns as + // `text is not set` — the Xiaomi Mimo 400 body wraps `text` in backticks + // inside the `param` field, which trips naive substring matching. + const lower = body.toLowerCase().replace(/`/g, '') + return ( + lower.includes('text is not set') || + lower.includes('text is required') || + lower.includes('text parameter is required') || + lower.includes('text parameter is missing') || + lower.includes('missing text') || + lower.includes('"param":"text"') || + lower.includes('"param": "text"') + ) +} + function isModelNotFoundMessage(body: string): boolean { const lower = body.toLowerCase() return ( @@ -344,6 +372,26 @@ export function classifyOpenAIHttpFailure(options: { } } + // Xiaomi Mimo and similar OpenAI-compatible providers reject image-bearing + // `role: "tool"` messages with a 400 carrying `text is not set` instead of + // a 404. Classify the same way as the 404 + hasImages branch so the user + // gets actionable guidance rather than the raw API error (issue #1421). + if ( + options.status === 400 && + options.hasImages && + isMissingTextPartMessage(body) + ) { + return { + source: 'http', + category: 'vision_not_supported', + retryable: false, + status: options.status, + message: body, + requestUrl: options.url, + hint: 'The provider rejected a request containing an image (likely a tool result) because it did not include a text part. The model may not support image/vision inputs.', + } + } + if (options.status === 404) { const isRemote = hostname !== null && !isLocalHost return { diff --git a/src/services/api/openaiShim.test.ts b/src/services/api/openaiShim.test.ts index 4ba62de62d..f0cc9db67d 100644 --- a/src/services/api/openaiShim.test.ts +++ b/src/services/api/openaiShim.test.ts @@ -1950,7 +1950,11 @@ test('preserves image tool results as placeholders in follow-up requests', async text?: string image_url?: { url: string } }> + // Issue #1421: image-only tool results now get a placeholder text part + // prepended so OpenAI-compatible providers that require a `text` field on + // `role: "tool"` messages (e.g. Xiaomi Mimo) don't 400 with "text is not set". expect(parts).toEqual([ + { type: 'text', text: 'Image attached.' }, { type: 'image_url', image_url: { url: 'data:image/png;base64,ZmFrZQ==' }, diff --git a/src/services/api/openaiShim.ts b/src/services/api/openaiShim.ts index efb29c23c2..52acecea36 100644 --- a/src/services/api/openaiShim.ts +++ b/src/services/api/openaiShim.ts @@ -408,7 +408,12 @@ function convertToolResultContent( parts.unshift({ type: 'text', text: 'Error:' }) } - return parts + // Defense in depth (issue #1421): some OpenAI-compatible providers (e.g. + // Xiaomi Mimo) reject `role: "tool"` messages whose `content` is image-only + // with a 400 "text is not set". Prepend a placeholder text part so the + // payload always carries a text component alongside any images, mirroring + // the existing behavior for user-role messages. + return ensureTextPartForImageContent(parts) } function convertContentBlocks( diff --git a/src/tools/FileReadTool/FileReadTool.ts b/src/tools/FileReadTool/FileReadTool.ts index e6caad1d94..97d2265495 100644 --- a/src/tools/FileReadTool/FileReadTool.ts +++ b/src/tools/FileReadTool/FileReadTool.ts @@ -64,6 +64,7 @@ import { isPDFSupported, parsePDFPageRange, } from '../../utils/pdfUtils.js' +import { checkVisionCapabilityForFile } from '../../utils/visionUtils.js' import { checkReadPermissionForTool, matchingRuleForInput, @@ -524,14 +525,6 @@ export const FileReadTool = buildTool({ } } - // SECURITY: UNC path check (no I/O) — defer filesystem operations - // until after user grants permission to prevent NTLM credential leaks - const isUncPath = - fullFilePath.startsWith('\\\\') || fullFilePath.startsWith('//') - if (isUncPath) { - return { result: true } - } - // Binary extension check (string check on extension only, no I/O). // PDF, images, and SVG are excluded - this tool renders them natively. const ext = path.extname(fullFilePath).toLowerCase() @@ -547,6 +540,35 @@ export const FileReadTool = buildTool({ } } + // Vision-capability gate: refuse image reads when the active model + // explicitly lacks `supportsVision` (e.g. Xiaomi Mimo V2.5 Pro / Flash, + // Llama, Mistral). Returning early surfaces a clear `` + // to the model so it can pivot to a text-based approach (Bash `file`, + // `identify`, OCR) or `/model` to switch to a vision-capable model — + // instead of producing an image-only tool result that the provider + // rejects with a generic 400 (issue #1421). + const visionCheck = checkVisionCapabilityForFile( + fullFilePath, + toolUseContext.options.mainLoopModel, + { + baseUrl: + toolUseContext.options.providerOverride?.baseURL ?? + process.env.OPENAI_BASE_URL ?? + process.env.OPENAI_API_BASE, + }, + ) + if (visionCheck.result === false) { + return visionCheck + } + + // SECURITY: UNC path check (no I/O) — defer filesystem operations + // until after user grants permission to prevent NTLM credential leaks + const isUncPath = + fullFilePath.startsWith('\\\\') || fullFilePath.startsWith('//') + if (isUncPath) { + return { result: true } + } + // Block specific device files that would hang (infinite output or blocking input). // This is a path-based check with no I/O — safe special files like /dev/null are allowed. if (isBlockedDevicePath(fullFilePath)) { diff --git a/src/tools/FileReadTool/prompt.vision.test.ts b/src/tools/FileReadTool/prompt.vision.test.ts new file mode 100644 index 0000000000..6b4c1213ab --- /dev/null +++ b/src/tools/FileReadTool/prompt.vision.test.ts @@ -0,0 +1,130 @@ +import { afterEach, beforeEach, describe, expect, test } from 'bun:test' + +import { + acquireSharedMutationLock, + releaseSharedMutationLock, +} from '../../test/sharedMutationLock.js' +import type { ToolUseContext } from '../../Tool.js' +import { getEmptyToolPermissionContext } from '../../Tool.js' + +const moduleNonce = `vision-${Date.now()}-${Math.random()}` +const { FileReadTool } = (await import( + `./FileReadTool.js?${moduleNonce}` +)) as typeof import('./FileReadTool.js') +const { renderPromptTemplate } = (await import( + `./prompt.js?${moduleNonce}` +)) as typeof import('./prompt.js') + +const originalOpenAIBaseUrl = process.env.OPENAI_BASE_URL +const originalOpenAIApiBase = process.env.OPENAI_API_BASE + +beforeEach(async () => { + await acquireSharedMutationLock('tools/FileReadTool/prompt.vision.test.ts') + delete process.env.OPENAI_BASE_URL + delete process.env.OPENAI_API_BASE +}) + +afterEach(() => { + try { + if (originalOpenAIBaseUrl === undefined) { + delete process.env.OPENAI_BASE_URL + } else { + process.env.OPENAI_BASE_URL = originalOpenAIBaseUrl + } + if (originalOpenAIApiBase === undefined) { + delete process.env.OPENAI_API_BASE + } else { + process.env.OPENAI_API_BASE = originalOpenAIApiBase + } + } finally { + releaseSharedMutationLock() + } +}) + +function createToolUseContext( + mainLoopModel: string, + providerOverride?: ToolUseContext['options']['providerOverride'], +): ToolUseContext { + return { + options: { + mainLoopModel, + providerOverride, + }, + getAppState: () => ({ + toolPermissionContext: getEmptyToolPermissionContext(), + }), + } as unknown as ToolUseContext +} + +describe('renderPromptTemplate — vision sentence (issue #1421)', () => { + test('includes the image-reading sentence when the active model supports vision (default Claude)', () => { + const rendered = renderPromptTemplate( + '- Results are returned using cat -n format, with line numbers starting at 1', + '', + '', + ) + + expect(rendered).toContain('This tool allows Claude Code to read images') + }) + + test('always includes the Jupyter notebook sentence and the directory-listing hint', () => { + const rendered = renderPromptTemplate( + '- Results are returned using cat -n format, with line numbers starting at 1', + '', + '', + ) + + expect(rendered).toContain('Jupyter notebooks') + expect(rendered).toContain('not directories') + }) +}) + +describe('FileReadTool.validateInput — vision gate (issue #1421)', () => { + test('returns a structured denial for image reads on registered non-vision models', async () => { + const result = await FileReadTool.validateInput( + { file_path: 'fixture.png' }, + createToolUseContext('mimo-v2.5-pro'), + ) + + expect(result).toEqual({ + result: false, + errorCode: 10, + message: expect.stringContaining('does not support image inputs'), + }) + }) + + test('provider override base URL wins over ambient OpenAI-compatible env', async () => { + process.env.OPENAI_BASE_URL = 'https://api.xiaomimimo.com/v1' + + const result = await FileReadTool.validateInput( + { file_path: 'fixture.png' }, + createToolUseContext('mimo-v2.5', { + model: 'mimo-v2.5', + baseURL: 'https://opencode.ai/zen/go/v1', + apiKey: 'test-key', + }), + ) + + expect(result).toMatchObject({ result: false, errorCode: 10 }) + }) + + test('falls back to OPENAI_BASE_URL when no provider override is present', async () => { + process.env.OPENAI_BASE_URL = 'https://opencode.ai/zen/go/v1' + + const result = await FileReadTool.validateInput( + { file_path: 'fixture.png' }, + createToolUseContext('mimo-v2.5'), + ) + + expect(result).toMatchObject({ result: false, errorCode: 10 }) + }) + + test('validates UNC image paths before the UNC no-I/O early return', async () => { + const result = await FileReadTool.validateInput( + { file_path: '\\\\server\\share\\fixture.png' }, + createToolUseContext('mimo-v2.5-pro'), + ) + + expect(result).toMatchObject({ result: false, errorCode: 10 }) + }) +}) diff --git a/src/utils/messages.ts b/src/utils/messages.ts index 07d6620c00..0a684c84d9 100644 --- a/src/utils/messages.ts +++ b/src/utils/messages.ts @@ -36,6 +36,7 @@ import { getPdfPasswordProtectedErrorMessage, getPdfTooLargeErrorMessage, getRequestTooLargeErrorMessage, + getVisionNotSupportedErrorMessages, } from '../services/api/errors.js' import type { AnyObject, Progress } from '../Tool.js' import { isConnectorTextBlock } from '../types/connectorText.js' @@ -2069,6 +2070,17 @@ export function normalizeMessagesForAPI( [getPdfInvalidErrorMessage()]: new Set(['document']), [getImageTooLargeErrorMessage()]: new Set(['image']), [getRequestTooLargeErrorMessage()]: new Set(['document', 'image']), + // Issue #1421: existing transcripts poisoned by a 400 "text is not set" + // (Xiaomi Mimo + non-vision model) carry an image-only tool_result that + // would re-trigger the same 400 on every retry. Match the canonical + // message so `normalizeMessagesForAPI` strips the `image` blocks from + // the preceding tool_result user message on resume / next turn. + ...Object.fromEntries( + getVisionNotSupportedErrorMessages().map(message => [ + message, + new Set(['image']), + ]), + ), } // Walk the reordered messages to build a targeted strip map: diff --git a/src/utils/visionUtils.test.ts b/src/utils/visionUtils.test.ts new file mode 100644 index 0000000000..2282379f3e --- /dev/null +++ b/src/utils/visionUtils.test.ts @@ -0,0 +1,193 @@ +import { describe, expect, test } from 'bun:test' + +import { + checkVisionCapabilityForFile, + findModelDescriptorForApiName, + findModelDescriptorForApiNameWithRoute, + isVisionSupported, + VISION_NOT_SUPPORTED_ERROR_CODE, +} from './visionUtils.js' + +describe('VISION_NOT_SUPPORTED_ERROR_CODE', () => { + test('is a non-conflicting positive integer distinct from other Read error codes', () => { + expect(VISION_NOT_SUPPORTED_ERROR_CODE).toBe(10) + }) +}) + +describe('findModelDescriptorForApiName', () => { + test('returns undefined for empty / whitespace input', () => { + expect(findModelDescriptorForApiName(undefined)).toBeUndefined() + expect(findModelDescriptorForApiName('')).toBeUndefined() + expect(findModelDescriptorForApiName(' ')).toBeUndefined() + }) + + test('returns the registered descriptor for an exact id match', () => { + const descriptor = findModelDescriptorForApiName('mimo-v2.5-pro') + expect(descriptor).toBeDefined() + expect(descriptor?.id).toBe('mimo-v2.5-pro') + expect(descriptor?.capabilities?.supportsVision).toBeFalsy() + }) + + test('returns the registered descriptor for a case-insensitive match', () => { + const descriptor = findModelDescriptorForApiName('MIMO-V2.5-PRO') + expect(descriptor?.id).toBe('mimo-v2.5-pro') + }) + + test('returns the registered descriptor for a prefix match', () => { + const descriptor = findModelDescriptorForApiName('claude-sonnet-4-6') + expect(descriptor).toBeDefined() + expect(descriptor?.id).toContain('claude') + }) + + test('uses catalog mappings before prefix fallback', () => { + const descriptor = findModelDescriptorForApiName('mimo-v2.5-free') + expect(descriptor?.id).toBe('opencode-mimo-v2.5-free') + expect(descriptor?.capabilities?.supportsVision).toBe(false) + }) + + test('uses route-specific catalog mappings before global model ids', () => { + const descriptor = findModelDescriptorForApiNameWithRoute( + 'mimo-v2.5', + 'opencode-go', + ) + expect(descriptor?.id).toBe('opencode-go-mimo-v2.5') + expect(descriptor?.capabilities?.supportsVision).toBe(false) + }) + + test('returns undefined for unknown models so callers fail open', () => { + expect(findModelDescriptorForApiName('definitely-not-a-real-model-xyz')).toBeUndefined() + }) +}) + +describe('isVisionSupported', () => { + test('returns false for a registered non-vision model (Xiaomi Mimo V2.5 Pro)', () => { + expect(isVisionSupported('mimo-v2.5-pro')).toBe(false) + }) + + test('returns false for the Xiaomi Mimo V2 Flash variant', () => { + expect(isVisionSupported('mimo-v2-flash')).toBe(false) + }) + + test('returns true for the registered vision variant (Xiaomi Mimo V2.5)', () => { + expect(isVisionSupported('mimo-v2.5')).toBe(true) + }) + + test('does not let ambient OPENAI_BASE_URL change route-free checks', () => { + const originalBaseUrl = process.env.OPENAI_BASE_URL + process.env.OPENAI_BASE_URL = 'https://opencode.ai/zen/go/v1' + try { + expect(isVisionSupported('mimo-v2.5')).toBe(true) + } finally { + if (originalBaseUrl === undefined) { + delete process.env.OPENAI_BASE_URL + } else { + process.env.OPENAI_BASE_URL = originalBaseUrl + } + } + }) + + test('returns false for non-vision catalog aliases that share a vision-model prefix', () => { + expect(isVisionSupported('mimo-v2.5-free')).toBe(false) + }) + + test('returns false for route-specific catalog aliases that collide with global vision models', () => { + expect( + isVisionSupported('mimo-v2.5', { + baseUrl: 'https://opencode.ai/zen/go/v1', + }), + ).toBe(false) + }) + + test('returns true for Claude models', () => { + expect(isVisionSupported('claude-sonnet-4-6')).toBe(true) + }) + + test('returns true for Gemini models', () => { + expect(isVisionSupported('gemini-2.5-pro')).toBe(true) + }) + + test('falls open for unknown models so custom / non-registered providers keep working', () => { + expect(isVisionSupported('custom-vision-corp/secret-model-v1')).toBe(true) + expect(isVisionSupported('not-a-real-model-xyz')).toBe(true) + }) +}) + +describe('checkVisionCapabilityForFile (issue #1421)', () => { + test('refuses PNG read for a registered non-vision model with an actionable message', () => { + const result = checkVisionCapabilityForFile( + 'C:\\temp\\openclaude\\tests\\fixtures\\screenshot.png', + 'mimo-v2.5-pro', + ) + + expect(result.result).toBe(false) + if (result.result === false) { + expect(result.message).toContain('mimo-v2.5-pro') + expect(result.message).toContain('does not support image') + expect(result.message).toContain('/model') + expect(result.errorCode).toBe(VISION_NOT_SUPPORTED_ERROR_CODE) + } + }) + + test('refuses JPG read for Xiaomi Mimo V2 Flash', () => { + const result = checkVisionCapabilityForFile('C:\\foo\\bar.jpg', 'mimo-v2-flash') + expect(result.result).toBe(false) + if (result.result === false) { + expect(result.errorCode).toBe(VISION_NOT_SUPPORTED_ERROR_CODE) + } + }) + + test('refuses GIF / WebP reads for non-vision models', () => { + expect(checkVisionCapabilityForFile('a.gif', 'mimo-v2.5-pro').result).toBe(false) + expect(checkVisionCapabilityForFile('a.webp', 'mimo-v2.5-pro').result).toBe(false) + }) + + test('allows PNG read for a registered vision-capable model', () => { + const result = checkVisionCapabilityForFile( + 'C:\\temp\\openclaude\\tests\\fixtures\\screenshot.png', + 'claude-sonnet-4-6', + ) + expect(result.result).toBe(true) + }) + + test('refuses PNG read for a route-specific non-vision catalog model that collides with a global vision model', () => { + const result = checkVisionCapabilityForFile( + 'C:\\foo\\bar.png', + 'mimo-v2.5', + { baseUrl: 'https://opencode.ai/zen/go/v1' }, + ) + expect(result.result).toBe(false) + }) + + test('allows PNG read for unknown models (fail-open so non-registered providers keep working)', () => { + const result = checkVisionCapabilityForFile( + 'C:\\foo\\bar.png', + 'my-custom-provider/vision-experimental', + ) + expect(result.result).toBe(true) + }) + + test('does not gate text-file reads on non-vision models', () => { + const result = checkVisionCapabilityForFile('C:\\foo\\bar.txt', 'mimo-v2.5-pro') + expect(result.result).toBe(true) + }) + + test('does not gate PDF reads (PDF has its own capability gate)', () => { + const result = checkVisionCapabilityForFile('C:\\foo\\bar.pdf', 'mimo-v2.5-pro') + expect(result.result).toBe(true) + }) + + test('handles uppercase image extensions', () => { + expect( + checkVisionCapabilityForFile('C:\\FOO\\BAR.PNG', 'mimo-v2.5-pro').result, + ).toBe(false) + expect( + checkVisionCapabilityForFile('C:\\FOO\\BAR.PNG', 'claude-sonnet-4-6').result, + ).toBe(true) + }) + + test('handles files without an extension', () => { + expect( + checkVisionCapabilityForFile('C:\\foo\\bar', 'mimo-v2.5-pro').result, + ).toBe(true) + }) +}) diff --git a/src/utils/visionUtils.ts b/src/utils/visionUtils.ts new file mode 100644 index 0000000000..adb4ddd3cf --- /dev/null +++ b/src/utils/visionUtils.ts @@ -0,0 +1,198 @@ +// src/utils/visionUtils.ts +// Vision capability detection for the active model. +// +// `supportsVision` is declared on every model descriptor but currently has no +// runtime consumer. This module is the single source of truth for "can this +// model see images?" queries. Mirrors the `isPDFSupported()` pattern in +// `pdfUtils.ts` so existing call sites stay uniform. + +import { + ensureIntegrationsLoaded, + getAllGateways, + getAllModels, + getAllVendors, + getCatalogEntriesForRoute, + getModel, + resolveRouteIdFromBaseUrl, +} from '../integrations/index.js' +import { getMainLoopModel } from './model/model.js' + +export const VISION_NOT_SUPPORTED_ERROR_CODE = 10 + +// Common image extensions — kept here (rather than reading from FileReadTool) +// so the gate is testable in isolation, without depending on the FileReadTool +// module getting mocked by other tests (e.g. `compact.test.ts`). +const IMAGE_EXTENSIONS = new Set(['png', 'jpg', 'jpeg', 'gif', 'webp']) + +function normalizedName(value: string): string { + return value.trim().toLowerCase() +} + +function hasModelNamePrefix(modelApiName: string, registeredName: string): boolean { + const trimmedName = registeredName.trim() + if (!trimmedName) return false + if (!modelApiName.startsWith(trimmedName)) return false + + const next = modelApiName[trimmedName.length] + return next === undefined || next === ':' || next === '/' || next === '@' +} + +function findModelDescriptorFromCatalog( + modelApiName: string, + routeId?: string, +) { + const normalized = normalizedName(modelApiName) + const routeIds = + routeId !== undefined + ? [routeId] + : [ + ...new Set([ + ...getAllGateways().map(route => route.id), + ...getAllVendors().map(route => route.id), + ]), + ] + + for (const routeId of routeIds) { + const entry = getCatalogEntriesForRoute(routeId).find(candidate => { + return ( + normalizedName(candidate.apiName) === normalized || + normalizedName(candidate.id) === normalized + ) + }) + if (entry?.modelDescriptorId) { + const descriptor = getModel(entry.modelDescriptorId) + if (descriptor) return descriptor + } + } + + return undefined +} + +/** + * Find the model descriptor for a given API name, matching across + * `id`, `defaultModel`, and any `providerModelMap` entries. + * + * Returns undefined when the model isn't in the registry — callers should + * treat unknown models as vision-capable (fail-open) so custom / non-registered + * providers don't regress. The post-flight error surface + * (`vision_not_supported` in `openaiErrorClassification.ts`) handles the case + * where an unknown model still rejects image-bearing requests. + */ +export function findModelDescriptorForApiName(modelApiName: string | undefined) { + return findModelDescriptorForApiNameWithRoute(modelApiName) +} + +export function findModelDescriptorForApiNameWithRoute( + modelApiName: string | undefined, + routeId?: string, +) { + const trimmed = modelApiName?.trim() + if (!trimmed) return undefined + + ensureIntegrationsLoaded() + + if (routeId !== undefined) { + const routeCatalogDescriptor = findModelDescriptorFromCatalog(trimmed, routeId) + if (routeCatalogDescriptor) return routeCatalogDescriptor + } + + const direct = getModel(trimmed) + if (direct) return direct + + const catalogDescriptor = findModelDescriptorFromCatalog(trimmed) + if (catalogDescriptor) return catalogDescriptor + + const normalized = normalizedName(trimmed) + const models = getAllModels() + + const candidates = models + .map(model => ({ + model, + names: [ + model.id, + model.defaultModel, + ...Object.values(model.providerModelMap ?? {}), + ].filter((value): value is string => Boolean(value?.trim())), + })) + .sort((left, right) => { + const leftLongest = Math.max( + ...left.names.map(name => name.length), + 0, + ) + const rightLongest = Math.max( + ...right.names.map(name => name.length), + 0, + ) + return rightLongest - leftLongest + }) + + for (const candidate of candidates) { + if (candidate.names.some(name => trimmed === name.trim())) { + return candidate.model + } + } + for (const candidate of candidates) { + if (candidate.names.some(name => hasModelNamePrefix(trimmed, name))) { + return candidate.model + } + } + for (const candidate of candidates) { + if ( + candidate.names.some(name => { + const lowered = normalizedName(name) + return normalized === lowered || hasModelNamePrefix(normalized, lowered) + }) + ) { + return candidate.model + } + } + return undefined +} + +/** + * Returns true when the given (or currently-active) model is registered as + * supporting image/vision inputs. Returns true (fail-open) for unknown models + * so providers outside the registry continue to receive images until they + * reject them — at which point the post-flight error classifier surfaces an + * actionable `vision_not_supported` message instead of the raw API error. + */ +export function isVisionSupported( + model?: string, + options?: { routeId?: string; baseUrl?: string }, +): boolean { + const target = model ?? getMainLoopModel() + const routeId = + options?.routeId ?? resolveRouteIdFromBaseUrl(options?.baseUrl) ?? undefined + const descriptor = findModelDescriptorForApiNameWithRoute(target, routeId) + if (!descriptor) return true + return descriptor.capabilities?.supportsVision === true +} + +/** + * Pre-flight vision check for a file read. Returns a refusal when the file + * is an image and the active model is registered as not supporting vision + * inputs (e.g. Xiaomi Mimo V2.5 Pro / Flash, Llama, Mistral). Returns null + * when the file is non-image OR the model supports vision OR the model is + * unknown (fail-open). + * + * Extracted into `visionUtils.ts` (rather than living inside `FileReadTool`) + * so it can be unit-tested without depending on the FileReadTool module — + * which is mocked by other test files (e.g. `compact.test.ts`) and would + * otherwise leave this gate un-tested in the full suite (issue #1421). + */ +export function checkVisionCapabilityForFile( + filePath: string, + mainLoopModel: string, + options?: { routeId?: string; baseUrl?: string }, +): + | { result: false; message: string; errorCode: number } + | { result: true } { + const ext = filePath.toLowerCase().slice(filePath.lastIndexOf('.') + 1) + if (!IMAGE_EXTENSIONS.has(ext)) return { result: true } + if (isVisionSupported(mainLoopModel, options)) return { result: true } + return { + result: false, + message: `The active model (${mainLoopModel}) does not support image inputs. To analyze this file, use a text-based tool (e.g. the Bash tool with \`file\`, \`identify\`, or an OCR command), or run /model to switch to a vision-capable model.`, + errorCode: VISION_NOT_SUPPORTED_ERROR_CODE, + } +}