diff --git a/packages/cli/src/acp-integration/session/Session.test.ts b/packages/cli/src/acp-integration/session/Session.test.ts index e76786e8a57..3a99a3fad0b 100644 --- a/packages/cli/src/acp-integration/session/Session.test.ts +++ b/packages/cli/src/acp-integration/session/Session.test.ts @@ -41,6 +41,7 @@ import { MessageType } from '../../ui/types.js'; const debugLoggerWarnSpy = vi.hoisted(() => vi.fn()); const debugLoggerDebugSpy = vi.hoisted(() => vi.fn()); +const runVisionBridgeSpy = vi.hoisted(() => vi.fn()); // Records every LoopTickResolver construction's deps so a test can assert what // Session computed (e.g. the home confinement root) without a private-field peek. const loopTickResolverDepsSpy = vi.hoisted(() => vi.fn()); @@ -58,6 +59,7 @@ vi.mock('@qwen-code/qwen-code-core', async (importOriginal) => { }), generatePromptSuggestion: vi.fn(), logPromptSuggestion: vi.fn(), + runVisionBridge: runVisionBridgeSpy, // Transparent recording wrapper: records the constructor deps, then behaves // exactly like the real resolver (subclass → instanceof + methods preserved). LoopTickResolver: class extends actual.LoopTickResolver { @@ -344,6 +346,7 @@ describe('Session', () => { } beforeEach(() => { + runVisionBridgeSpy.mockReset(); currentModel = 'qwen3-code-plus'; currentAuthType = AuthType.USE_OPENAI; switchModelSpy = vi @@ -2212,6 +2215,281 @@ describe('Session', () => { } }); + it('routes ACP image prompts through the vision bridge for text-only primary models', async () => { + mockConfig.getEffectiveInputModalities = vi.fn().mockReturnValue({}); + mockConfig.getDefaultVisionBridgeModel = vi.fn().mockReturnValue({ + id: 'qwen3.7-plus', + }); + runVisionBridgeSpy.mockResolvedValue({ + applied: true, + status: 'ok', + parts: [{ text: 'look at this' }, { text: '[transcribed image]' }], + transcript: '[transcribed image]', + convertedCount: 1, + omittedCount: 0, + modelId: 'qwen3.7-plus', + }); + mockChat.sendMessageStream = vi + .fn() + .mockResolvedValue(createEmptyStream()); + + await session.prompt({ + sessionId: 'test-session-id', + prompt: [ + { type: 'text', text: 'look at this' }, + { + type: 'image', + mimeType: 'image/png', + data: 'iVBORw0KGgo=', + }, + ], + }); + + expect(runVisionBridgeSpy).toHaveBeenCalledWith( + expect.objectContaining({ + config: mockConfig, + signal: expect.any(AbortSignal), + }), + ); + const sent = firstSentMessage(); + expect(textParts(sent)).toContain('[transcribed image]'); + expect(sent.some((part) => 'inlineData' in part)).toBe(false); + }); + + it('strips image parts when the vision bridge is cancelled before applying', async () => { + mockConfig.getEffectiveInputModalities = vi.fn().mockReturnValue({}); + mockConfig.getDefaultVisionBridgeModel = vi.fn().mockReturnValue({ + id: 'qwen3.7-plus', + }); + runVisionBridgeSpy.mockResolvedValue({ + applied: false, + status: 'skipped', + convertedCount: 0, + omittedCount: 0, + modelId: 'qwen3.7-plus', + egressOccurred: true, + }); + mockChat.sendMessageStream = vi + .fn() + .mockResolvedValue(createEmptyStream()); + + await session.prompt({ + sessionId: 'test-session-id', + prompt: [ + { type: 'text', text: 'look at this' }, + { + type: 'image', + mimeType: 'image/png', + data: 'iVBORw0KGgo=', + }, + ], + }); + + const sent = firstSentMessage(); + expect(sent.some((part) => 'inlineData' in part)).toBe(false); + expect( + textParts(sent).some((t: string) => t.includes('look at this')), + ).toBe(true); + }); + + it('preserves oversized inline images for the vision bridge', async () => { + const ENV_KEY = 'QWEN_CODE_MAX_INLINE_MEDIA_BYTES'; + const original = process.env[ENV_KEY]; + process.env[ENV_KEY] = '8'; + try { + mockConfig.getEffectiveInputModalities = vi.fn().mockReturnValue({}); + mockConfig.getDefaultVisionBridgeModel = vi.fn().mockReturnValue({ + id: 'qwen3.7-plus', + }); + runVisionBridgeSpy.mockResolvedValue({ + applied: true, + status: 'ok', + parts: [{ text: 'look at this' }, { text: '[large image]' }], + transcript: '[large image]', + convertedCount: 1, + omittedCount: 0, + modelId: 'qwen3.7-plus', + }); + mockChat.sendMessageStream = vi + .fn() + .mockResolvedValue(createEmptyStream()); + + await session.prompt({ + sessionId: 'test-session-id', + prompt: [ + { type: 'text', text: 'look at this' }, + { + type: 'image', + mimeType: 'image/png', + data: 'QUJDREVGR0hJSktMTU5PUFFSU1Q=', + }, + ], + }); + + const bridgeParts = runVisionBridgeSpy.mock.calls[0]?.[0] + ?.parts as Part[]; + expect(bridgeParts.some((part) => 'inlineData' in part)).toBe(true); + expect(textParts(firstSentMessage())).toContain('[large image]'); + } finally { + if (original === undefined) delete process.env[ENV_KEY]; + else process.env[ENV_KEY] = original; + } + }); + + it('falls back to text-only parts when the vision bridge throws', async () => { + mockConfig.getEffectiveInputModalities = vi.fn().mockReturnValue({}); + mockConfig.getDefaultVisionBridgeModel = vi.fn().mockReturnValue({ + id: 'qwen3.7-plus', + }); + runVisionBridgeSpy.mockRejectedValue(new Error('provider unavailable')); + mockChat.sendMessageStream = vi + .fn() + .mockResolvedValue(createEmptyStream()); + + await session.prompt({ + sessionId: 'test-session-id', + prompt: [ + { type: 'text', text: 'look at this' }, + { + type: 'image', + mimeType: 'image/png', + data: 'iVBORw0KGgo=', + }, + ], + }); + + const sent = firstSentMessage(); + expect(sent.some((part) => 'inlineData' in part)).toBe(false); + expect( + textParts(sent).some((t: string) => t.includes('look at this')), + ).toBe(true); + expect(debugLoggerDebugSpy).toHaveBeenCalledWith( + expect.stringContaining('provider unavailable'), + ); + }); + + it('forwards failed bridge replacement parts to the primary model', async () => { + mockConfig.getEffectiveInputModalities = vi.fn().mockReturnValue({}); + mockConfig.getDefaultVisionBridgeModel = vi.fn().mockReturnValue({ + id: 'qwen3.7-plus', + }); + runVisionBridgeSpy.mockResolvedValue({ + applied: true, + status: 'failed', + parts: [{ text: 'look at this' }, { text: '[bridge failed]' }], + convertedCount: 0, + omittedCount: 1, + modelId: 'qwen3.7-plus', + egressOccurred: true, + error: 'quota exceeded', + }); + mockChat.sendMessageStream = vi + .fn() + .mockResolvedValue(createEmptyStream()); + + await session.prompt({ + sessionId: 'test-session-id', + prompt: [ + { type: 'text', text: 'look at this' }, + { + type: 'image', + mimeType: 'image/png', + data: 'iVBORw0KGgo=', + }, + ], + }); + + const sent = firstSentMessage(); + expect(textParts(sent)).toContain('[bridge failed]'); + expect(sent.some((part) => 'inlineData' in part)).toBe(false); + expect(debugLoggerDebugSpy).toHaveBeenCalledWith( + expect.stringContaining('error=quota exceeded'), + ); + }); + + it('does not run the vision bridge when the primary model supports images', async () => { + mockConfig.getEffectiveInputModalities = vi + .fn() + .mockReturnValue({ image: true }); + mockConfig.getDefaultVisionBridgeModel = vi.fn().mockReturnValue({ + id: 'qwen3.7-plus', + }); + mockChat.sendMessageStream = vi + .fn() + .mockResolvedValue(createEmptyStream()); + + await session.prompt({ + sessionId: 'test-session-id', + prompt: [ + { type: 'text', text: 'look at this' }, + { + type: 'image', + mimeType: 'image/png', + data: 'iVBORw0KGgo=', + }, + ], + }); + + expect(runVisionBridgeSpy).not.toHaveBeenCalled(); + expect(firstSentMessage().some((part) => 'inlineData' in part)).toBe( + true, + ); + }); + + it('preserves unsupported image @ files for the vision bridge', async () => { + mockConfig.getEffectiveInputModalities = vi.fn().mockReturnValue({}); + mockConfig.getDefaultVisionBridgeModel = vi.fn().mockReturnValue({ + id: 'qwen3.7-plus', + }); + const readManyFilesSpy = vi + .spyOn(core, 'readManyFiles') + .mockResolvedValue({ + contentParts: { + inlineData: { mimeType: 'image/png', data: 'iVBORw0KGgo=' }, + }, + } as Awaited>); + runVisionBridgeSpy.mockResolvedValue({ + applied: true, + status: 'ok', + parts: [{ text: 'look at this' }, { text: '[file image]' }], + transcript: '[file image]', + convertedCount: 1, + omittedCount: 0, + modelId: 'qwen3.7-plus', + }); + mockChat.sendMessageStream = vi + .fn() + .mockResolvedValue(createEmptyStream()); + + try { + await session.prompt({ + sessionId: 'test-session-id', + prompt: [ + { type: 'text', text: 'look at this' }, + { + type: 'resource_link', + uri: 'file:///tmp/image.png', + mimeType: 'image/png', + name: 'image.png', + }, + ], + }); + + expect(readManyFilesSpy).toHaveBeenCalledWith( + mockConfig, + expect.objectContaining({ + preserveUnsupportedImageForBridge: true, + }), + ); + const bridgeParts = runVisionBridgeSpy.mock.calls[0]?.[0] + ?.parts as Part[]; + expect(bridgeParts.some((part) => 'inlineData' in part)).toBe(true); + expect(textParts(firstSentMessage())).toContain('[file image]'); + } finally { + readManyFilesSpy.mockRestore(); + } + }); + describe('conversation_finished telemetry (#4602 review)', () => { it('emits conversation_finished once when a turn completes normally', async () => { const finishedSpy = vi diff --git a/packages/cli/src/acp-integration/session/Session.ts b/packages/cli/src/acp-integration/session/Session.ts index ebfde682e61..5eafdecfc0e 100644 --- a/packages/cli/src/acp-integration/session/Session.ts +++ b/packages/cli/src/acp-integration/session/Session.ts @@ -31,6 +31,7 @@ import type { ToolCallRequestInfo, ToolCallResponseInfo, LoopTickResult, + VisionBridgeResult, } from '@qwen-code/qwen-code-core'; import { AuthType, @@ -118,6 +119,11 @@ import { getProviderToolCallId, parsePositiveIntegerEnv, DEFAULT_TOKEN_LIMIT, + hasImageParts, + normalizeParts, + runVisionBridge, + shouldRunVisionBridge, + splitImageParts, } from '@qwen-code/qwen-code-core'; import { NOT_CURRENTLY_GENERATING_CANCEL_MESSAGE } from '@qwen-code/acp-bridge/bridgeErrors'; // Single source of truth shared with the daemon-side answerer (BridgeClient), @@ -5360,6 +5366,9 @@ export class Session implements SessionContext { const embeddedContext: EmbeddedResourceResource[] = []; const extensionMentions = new Map(); + const preserveUnsupportedImageForBridge = shouldRunVisionBridge( + this.config, + ); const parts = message.map((part) => { switch (part.type) { @@ -5367,6 +5376,20 @@ export class Session implements SessionContext { collectExtensionMentionRefs(part.text, extensionMentions); return { text: part.text }; case 'image': + if (preserveUnsupportedImageForBridge) { + return { + inlineData: { + mimeType: part.mimeType, + data: part.data, + }, + }; + } + return clampInlineMediaPart({ + inlineData: { + mimeType: part.mimeType, + data: part.data, + }, + }); case 'audio': return clampInlineMediaPart({ inlineData: { @@ -5409,11 +5432,14 @@ export class Session implements SessionContext { embeddedContext.length === 0 && extensionParts.length === 0 ) { - return parts; + return this.#applyVisionBridgeIfNeeded(parts, abortSignal); } if (atPathCommandParts.length === 0 && embeddedContext.length === 0) { - return [...parts, ...extensionParts]; + return this.#applyVisionBridgeIfNeeded( + [...parts, ...extensionParts], + abortSignal, + ); } // Extract paths from @ commands - pass directly to readManyFiles without filtering @@ -5448,6 +5474,9 @@ export class Session implements SessionContext { const readResult = await readManyFiles(this.config, { paths: pathSpecsToRead, signal: abortSignal, + ...(preserveUnsupportedImageForBridge + ? { preserveUnsupportedImageForBridge } + : {}), }); const contentParts = Array.isArray(readResult.contentParts) @@ -5462,6 +5491,8 @@ export class Session implements SessionContext { for (const part of contentParts) { if (typeof part === 'string') { processedQueryParts.push({ text: part }); + } else if (preserveUnsupportedImageForBridge && hasImageParts([part])) { + processedQueryParts.push(part); } else { processedQueryParts.push(clampInlineMediaPart(part)); } @@ -5481,18 +5512,102 @@ export class Session implements SessionContext { } // Type guard for blob resources if ('blob' in contextPart && contextPart.blob) { + const inlinePart = { + inlineData: { + mimeType: contextPart.mimeType ?? 'application/octet-stream', + data: contextPart.blob, + }, + }; processedQueryParts.push( - clampInlineMediaPart({ - inlineData: { - mimeType: contextPart.mimeType ?? 'application/octet-stream', - data: contextPart.blob, - }, - }), + preserveUnsupportedImageForBridge && hasImageParts([inlinePart]) + ? inlinePart + : clampInlineMediaPart(inlinePart), + ); + } + } + + return this.#applyVisionBridgeIfNeeded(processedQueryParts, abortSignal); + } + + async #applyVisionBridgeIfNeeded( + parts: Part[], + abortSignal: AbortSignal, + ): Promise { + if (!hasImageParts(parts) || !shouldRunVisionBridge(this.config)) { + return parts; + } + + let bridgeResult: VisionBridgeResult; + try { + debugLogger.debug('vision bridge: gate matched, running conversion'); + bridgeResult = await runVisionBridge({ + config: this.config, + parts, + signal: abortSignal, + }); + } catch (error) { + debugLogger.debug( + `vision bridge: failed before replacement; falling back to text-only parts error=${String(error instanceof Error ? error.message : error)}`, + ); + return splitImageParts(parts).nonImageParts; + } + debugLogger.debug( + `vision bridge: status=${bridgeResult.status} applied=${bridgeResult.applied} model=${bridgeResult.modelId ?? '(none)'}${bridgeResult.error ? ` error=${bridgeResult.error}` : ''}`, + ); + + if (bridgeResult.status !== 'skipped' || bridgeResult.egressOccurred) { + try { + await this.messageEmitter.emitAgentMessage( + this.#formatVisionBridgeNotice(bridgeResult), + ); + } catch (error) { + debugLogger.debug( + `vision bridge: failed to emit notice; continuing with bridge result error=${String(error instanceof Error ? error.message : error)}`, ); } } - return processedQueryParts; + if (abortSignal.aborted) { + debugLogger.debug('vision bridge: turn aborted after bridge returned'); + return splitImageParts(parts).nonImageParts; + } + + if (bridgeResult.applied && bridgeResult.parts != null) { + return normalizeParts(bridgeResult.parts); + } + + // Bridge did not apply (e.g. skipped after cancel). Strip images before + // forwarding to the text-only primary model — never send raw inlineData to + // a model that cannot interpret it. + return splitImageParts(parts).nonImageParts; + } + + #formatVisionBridgeNotice(result: VisionBridgeResult): string { + const modelName = result.modelId ?? 'vision model'; + const target = result.modelEndpoint + ? `${modelName} (${result.modelEndpoint})` + : modelName; + const egressNote = result.egressOccurred + ? ` Your image and prompt/context were sent to ${target}.` + : ''; + + if (result.status === 'failed') { + const reason = result.egressOccurred + ? 'the vision model request failed' + : 'the vision bridge could not run'; + return `Vision bridge (${modelName}) failed: ${reason}.${egressNote} The image was not interpreted.`; + } + + if (result.status === 'skipped') { + return `Vision bridge cancelled.${egressNote}`; + } + + // On success the image was always sent, so disclose egress unconditionally. + const omitted = + result.omittedCount > 0 + ? ` (${result.omittedCount} image(s) omitted)` + : ''; + return `Converted ${result.convertedCount} image(s)${omitted} to text via ${target}. Your image and prompt/context were sent to that model.`; } async #resolveExtensionMentionParts(