From 7b772c965cfc0e1457a1fcdae425b42237b1a1f8 Mon Sep 17 00:00:00 2001 From: tly Date: Sat, 1 Aug 2026 16:19:34 +0800 Subject: [PATCH 1/7] feat(cli): render inline terminal images --- docs/design/terminal-inline-images.md | 139 ++++ .../src/ui/components/HistoryItemDisplay.tsx | 2 + .../src/ui/components/TerminalImage.test.tsx | 260 +++++++ .../cli/src/ui/components/TerminalImage.tsx | 255 +++++++ .../messages/ConversationMessages.test.tsx | 24 + .../messages/ConversationMessages.tsx | 64 +- .../messages/ToolGroupMessage.test.tsx | 16 + .../components/messages/ToolGroupMessage.tsx | 13 +- .../components/messages/ToolMessage.test.tsx | 18 + .../ui/components/messages/ToolMessage.tsx | 18 + .../cli/src/ui/hooks/useGeminiStream.test.tsx | 429 ++++++++++- packages/cli/src/ui/hooks/useGeminiStream.ts | 232 +++++- .../src/ui/hooks/useHistoryManager.test.ts | 84 +- .../cli/src/ui/hooks/useHistoryManager.ts | 63 +- .../ui/hooks/useReactToolScheduler.test.tsx | 29 +- .../cli/src/ui/hooks/useReactToolScheduler.ts | 22 +- packages/cli/src/ui/types.ts | 10 + .../cli/src/ui/utils/inline-image-parts.ts | 86 +++ .../cli/src/ui/utils/mermaidImageRenderer.ts | 318 +------- .../src/ui/utils/resumeHistoryUtils.test.ts | 122 +++ .../cli/src/ui/utils/resumeHistoryUtils.ts | 53 +- .../cli/src/ui/utils/terminal-image.test.ts | 191 +++++ packages/cli/src/ui/utils/terminal-image.ts | 718 ++++++++++++++++++ packages/core/src/core/turn.test.ts | 91 +++ packages/core/src/core/turn.ts | 58 +- 25 files changed, 2932 insertions(+), 383 deletions(-) create mode 100644 docs/design/terminal-inline-images.md create mode 100644 packages/cli/src/ui/components/TerminalImage.test.tsx create mode 100644 packages/cli/src/ui/components/TerminalImage.tsx create mode 100644 packages/cli/src/ui/utils/inline-image-parts.ts create mode 100644 packages/cli/src/ui/utils/terminal-image.test.ts create mode 100644 packages/cli/src/ui/utils/terminal-image.ts diff --git a/docs/design/terminal-inline-images.md b/docs/design/terminal-inline-images.md new file mode 100644 index 00000000000..5e2d114f602 --- /dev/null +++ b/docs/design/terminal-inline-images.md @@ -0,0 +1,139 @@ +# Terminal Inline Images + +## Problem + +The interactive CLI discards model `inlineData` image parts at the +`Turn`-to-TUI boundary. Tool images survive in nested +`functionResponse.parts`, but the tool display reduces them to text +placeholders. As a result, image-generating models and screenshot-producing +tools cannot show their output in the conversation even when the terminal has +an image protocol. + +Mermaid rendering already contains Kitty and iTerm2 protocol primitives, but +they are coupled to Mermaid process execution and cannot be reused by normal +messages. + +## Scope + +This change provides the first, render-and-forget slice of issue #8090: + +- extract terminal capability detection, protocol encoding, image sizing, and + Kitty placeholder generation into a shared utility; +- render inline model images without changing the existing text stream + contract; +- render images nested in successful, failed, or cancelled tool responses; +- restore both model and tool images from recorded sessions; +- show a deterministic text placeholder when an image cannot be rendered. + +Kitty image deletion, resize-driven replacement, scroll lifecycle management, +and terminal cell pixel queries are intentionally deferred. They require a +separate lifecycle owner above individual history items. + +## Data Flow + +### Model output + +`ServerGeminiContentEvent.value` remains the concatenated text string consumed +by all existing clients. When a response chunk also contains image +`inlineData`, the event gains an optional ordered `parts` field containing only +displayable non-thought text and image parts. + +Only the interactive TUI reads `parts`. It buffers text and image entries in +their original order and represents an image as an otherwise normal `gemini` or +`gemini_content` history item. Runs normally remain in the dynamic region until +a response boundary, allowing a fresh retry or model fallback to discard the +uncommitted failed attempt. Existing size- and height-driven incremental commit +boundaries still apply to very long output. Text-only events keep their exact +runtime shape, so existing consumers are unaffected. + +The unchanged `value` contract continues to serve core client aggregation, loop +detection, non-interactive output, daemon/channel bridges, ACP, SDK, Web UI, +VS Code, and desktop consumers. The optional field is additive and does not add +a new event discriminant that those consumers would need to handle. + +Recorded assistant messages already retain their original parts. Resume logic +reconstructs ordered text/image runs from those parts instead of flattening the +images away. + +Because encoded images are much larger than ordinary history text, UI memory +compaction drops payloads from old assistant image items while retaining the 20 +most recent items. Cleared images leave a visible marker instead of becoming a +blank history row. Tool image payloads participate in the existing tool-result +compaction limit as well. + +### Tool output + +Tool media is stored in `functionResponse.parts`. A CLI-only extractor reads +image `inlineData` from both top-level and nested response parts. Live scheduler +mapping and resume mapping attach the extracted images to the existing +`IndividualToolCallDisplay`. + +Tools carrying images are rendered individually even when their text-only form +would normally be collapsed into a read/search summary. `ToolMessage` then +routes each image through the same `TerminalImage` component used for assistant +messages. + +## Rendering + +The shared renderer: + +1. validates bounded base64 input before decoding; +2. verifies a supported image header and reads its pixel dimensions; +3. calculates a cell bounding box from the available width and a conservative + default cell aspect ratio; +4. selects Kitty or iTerm2 only for a positively identified local TTY; +5. returns either a protocol render result or a text placeholder. + +Kitty-capable terminals use virtual placement plus Unicode placeholders. The +PNG transfer is written through the raw terminal output context, while the +placeholder cells give Ink a stable layout anchor. + +iTerm2 OSC 1337 sequences cannot be embedded in Ink text because Ink strips +terminal control tokens. In the default alternate-screen viewport, +`TerminalImage` therefore reserves the calculated rows, measures its +post-layout cell position, and writes the OSC sequence at that visible screen +position while preserving and restoring the user's cursor. If the measured +position is outside the visible viewport, it leaves the text placeholder in +place instead of writing at an unrelated cursor location. Main-screen +scrollback has no reliable absolute origin, so iTerm2 rendering also uses the +placeholder when terminal-buffer mode is disabled. + +Detection is disabled under tmux, screen, and SSH because protocol forwarding +cannot be assumed. Kitty and Ghostty use Kitty virtual placements; iTerm2, +WezTerm, and Warp use OSC 1337. Each terminal is selected from its documented +environment markers. Tests can force a protocol through the shared detection +options without changing production detection. + +Kitty transfers these images as PNG (`f=100`). Other validated formats render +through iTerm2 where supported and otherwise use the text placeholder. + +## Fallback and Accessibility + +The fallback format is: + +```text +[image: 1024x768 png] +``` + +When dimensions cannot be verified it becomes `[image: png]`. MIME labels are +derived only from validated `image/*` media types; arbitrary response strings +are never emitted as terminal control data. + +Screen-reader mode always uses the text placeholder. Unsupported terminals, +invalid base64, oversized payloads, unsupported formats, and unsafe/off-screen +iTerm2 placement also use the placeholder. + +## Test Plan + +- Unit-test terminal detection, multiplexers, encoders, base64 bounds, image + metadata, sizing, and fallback labels. +- Unit-test Kitty raw transfer and iTerm2 measured placement in the React + component. +- Verify `Turn` preserves mixed text/image/text ordering while keeping `value`. +- Verify the live TUI commits mixed content in order. +- Verify live and resumed tool responses expose nested images. +- Verify resumed assistant history preserves text/image ordering. +- Verify memory compaction clears old assistant and tool image payloads. +- Re-run existing Mermaid renderer and component tests after extraction. +- Manually exercise supported and unsupported terminal paths using a generated + PNG fixture. diff --git a/packages/cli/src/ui/components/HistoryItemDisplay.tsx b/packages/cli/src/ui/components/HistoryItemDisplay.tsx index 1364fd48975..be117b41c9f 100644 --- a/packages/cli/src/ui/components/HistoryItemDisplay.tsx +++ b/packages/cli/src/ui/components/HistoryItemDisplay.tsx @@ -296,6 +296,7 @@ const HistoryItemDisplayComponent: React.FC = ({ )} = ({ {itemForDisplay.type === 'gemini_content' && ( { + const actual = await importOriginal(); + return { + ...actual, + useIsScreenReaderEnabled: vi.fn(() => false), + }; +}); + +const PNG_1X1_BASE64 = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII='; +const originalIsTTY = Object.getOwnPropertyDescriptor(process.stdout, 'isTTY'); +const originalRows = Object.getOwnPropertyDescriptor(process.stdout, 'rows'); +const TERMINAL_ENV_KEYS = [ + 'QWEN_CODE_DISABLE_TERMINAL_IMAGES', + 'QWEN_CODE_TERMINAL_IMAGE_PROTOCOL', + 'TMUX', + 'STY', + 'SSH_TTY', + 'SSH_CLIENT', + 'SSH_CONNECTION', +] as const; +const originalTerminalEnv = new Map( + TERMINAL_ENV_KEYS.map((key) => [key, process.env[key]]), +); + +function setStdoutIsTTY(value: boolean): void { + Object.defineProperty(process.stdout, 'isTTY', { + configurable: true, + value, + }); +} + +function setStdoutRows(value: number): void { + Object.defineProperty(process.stdout, 'rows', { + configurable: true, + value, + }); + process.stdout.emit('resize'); +} + +beforeEach(() => { + for (const key of TERMINAL_ENV_KEYS) { + delete process.env[key]; + } + setStdoutIsTTY(true); + setStdoutRows(24); + vi.mocked(useIsScreenReaderEnabled).mockReturnValue(false); +}); + +afterEach(() => { + if (originalIsTTY) { + Object.defineProperty(process.stdout, 'isTTY', originalIsTTY); + } else { + delete (process.stdout as { isTTY?: boolean }).isTTY; + } + if (originalRows) { + Object.defineProperty(process.stdout, 'rows', originalRows); + } else { + delete (process.stdout as { rows?: number }).rows; + } + for (const key of TERMINAL_ENV_KEYS) { + const originalValue = originalTerminalEnv.get(key); + if (originalValue === undefined) { + delete process.env[key]; + } else { + process.env[key] = originalValue; + } + } +}); + +describe('', () => { + it('writes Kitty image data through raw output and renders its placeholder', async () => { + process.env['QWEN_CODE_TERMINAL_IMAGE_PROTOCOL'] = 'kitty'; + const writeRaw = vi.fn(); + const { lastFrame } = render( + + + , + ); + + await vi.waitFor(() => expect(writeRaw).toHaveBeenCalledOnce()); + expect(writeRaw.mock.calls[0]?.[0]).toContain('\u001b_Ga=T,f=100'); + expect(lastFrame()?.split('\n')).toHaveLength(4); + }); + + it('cancels a delayed Kitty write when the image unmounts', async () => { + process.env['QWEN_CODE_TERMINAL_IMAGE_PROTOCOL'] = 'kitty'; + const writeRaw = vi.fn(); + const view = render( + + + , + ); + + view.unmount(); + await new Promise((resolve) => setImmediate(resolve)); + expect(writeRaw).not.toHaveBeenCalled(); + }); + + it('writes iTerm2 data at the measured cursor location', async () => { + process.env['QWEN_CODE_TERMINAL_IMAGE_PROTOCOL'] = 'iterm2'; + const writeRaw = vi.fn(); + render( + + + + + , + ); + + await vi.waitFor(() => expect(writeRaw).toHaveBeenCalledOnce()); + const sequence = writeRaw.mock.calls[0]?.[0] as string; + expect(sequence.startsWith('\u001b7\u001b[')).toBe(true); + expect(sequence).toContain('\u001b]1337;File=inline=1'); + expect(sequence.endsWith('\u001b8')).toBe(true); + }); + + it('re-emits an iTerm2 image after it leaves and re-enters the viewport', async () => { + process.env['QWEN_CODE_TERMINAL_IMAGE_PROTOCOL'] = 'iterm2'; + const writeRaw = vi.fn(); + const view = render( + + + + + , + ); + + await vi.waitFor(() => expect(writeRaw).toHaveBeenCalledOnce()); + setStdoutRows(2); + await vi.waitFor(() => expect(view.lastFrame()).toContain('1x1 png]')); + setStdoutRows(24); + await vi.waitFor(() => expect(writeRaw).toHaveBeenCalledTimes(2)); + }); + + it('does not use absolute iTerm2 placement in the main-screen buffer', () => { + process.env['QWEN_CODE_TERMINAL_IMAGE_PROTOCOL'] = 'iterm2'; + const writeRaw = vi.fn(); + const { lastFrame } = render( + + + + + , + ); + + expect(lastFrame()).toContain('[image: 1x1 png]'); + expect(writeRaw).not.toHaveBeenCalled(); + }); + + it('uses descriptive text without protocol output for screen readers', async () => { + process.env['QWEN_CODE_TERMINAL_IMAGE_PROTOCOL'] = 'kitty'; + vi.mocked(useIsScreenReaderEnabled).mockReturnValue(true); + const writeRaw = vi.fn(); + const { lastFrame } = render( + + + , + ); + + await vi.waitFor(() => expect(lastFrame()).toContain('[image: 1x1 png]')); + expect(writeRaw).not.toHaveBeenCalled(); + }); + + it('renders a readable placeholder when image protocols are unavailable', () => { + process.env['QWEN_CODE_TERMINAL_IMAGE_PROTOCOL'] = 'off'; + const writeRaw = vi.fn(); + const { lastFrame } = render( + + + , + ); + + expect(lastFrame()).toContain('[image: 1x1 png]'); + expect(writeRaw).not.toHaveBeenCalled(); + }); +}); + +describe('calculateITerm2Placement', () => { + it('rejects image rows that would be scrolled above the viewport', () => { + const root = { + yogaNode: { + getComputedHeight: () => 30, + getComputedLeft: () => 0, + getComputedTop: () => 0, + }, + } as unknown as DOMElement; + const node = { + parentNode: root, + yogaNode: { + getComputedHeight: () => 2, + getComputedWidth: () => 10, + getComputedLeft: () => 0, + getComputedTop: () => 0, + }, + } as unknown as DOMElement; + + expect(calculateITerm2Placement(node, 24, 2)).toBeNull(); + }); + + it('rejects images that would extend past the right viewport edge', () => { + const root = { + yogaNode: { + getComputedHeight: () => 10, + getComputedLeft: () => 0, + getComputedTop: () => 0, + }, + } as unknown as DOMElement; + const node = { + parentNode: root, + yogaNode: { + getComputedHeight: () => 2, + getComputedWidth: () => 10, + getComputedLeft: () => 75, + getComputedTop: () => 0, + }, + } as unknown as DOMElement; + + expect(calculateITerm2Placement(node, 24, 2, 80, 10)).toBeNull(); + }); +}); diff --git a/packages/cli/src/ui/components/TerminalImage.tsx b/packages/cli/src/ui/components/TerminalImage.tsx new file mode 100644 index 00000000000..1d0a94a5808 --- /dev/null +++ b/packages/cli/src/ui/components/TerminalImage.tsx @@ -0,0 +1,255 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import type React from 'react'; +import { + memo, + useEffect, + useLayoutEffect, + useMemo, + useRef, + useState, +} from 'react'; +import { Box, Text, type DOMElement, useIsScreenReaderEnabled } from 'ink'; +import type { InlineImageData } from '../types.js'; +import { theme } from '../semantic-colors.js'; +import { useTerminalOutput } from '../contexts/TerminalOutputContext.js'; +import { useVirtualViewport } from '../contexts/VirtualViewportContext.js'; +import { useTerminalSize } from '../hooks/useTerminalSize.js'; +import { + measureElementPosition, + measureFrameHeight, +} from '../utils/measure-element-position.js'; +import { prepareTerminalImage } from '../utils/terminal-image.js'; + +interface TerminalImageProps { + image: InlineImageData; + contentWidth: number; + availableTerminalHeight?: number; +} + +interface ITerm2Placement { + column: number; + row: number; +} + +interface ITerm2Emission extends ITerm2Placement { + imageSequence: string; +} + +export function calculateITerm2Placement( + node: DOMElement, + terminalHeight: number, + requiredRows?: number, + terminalWidth?: number, + requiredColumns?: number, +): ITerm2Placement | null { + const metrics = measureElementPosition(node); + const imageRows = requiredRows ?? metrics.height; + const frameHeight = measureFrameHeight(node); + const renderedHeight = frameHeight - metrics.height + imageRows; + const frameTop = Math.min(0, terminalHeight - renderedHeight); + const row = frameTop + metrics.y; + const column = metrics.x; + const imageColumns = requiredColumns ?? metrics.width; + + if ( + metrics.width <= 0 || + imageRows <= 0 || + imageColumns <= 0 || + column < 0 || + row < 0 || + row + imageRows > terminalHeight || + (terminalWidth !== undefined && column + imageColumns > terminalWidth) + ) { + return null; + } + + return { + column, + row, + }; +} + +export function buildITerm2PlacementSequence( + imageSequence: string, + placement: ITerm2Placement, +): string { + return `\u001b7\u001b[${placement.row + 1};${placement.column + 1}H${imageSequence}\u001b8`; +} + +const TerminalImageInternal: React.FC = ({ + image, + contentWidth, + availableTerminalHeight, +}) => { + const writeRaw = useTerminalOutput(); + const { columns: terminalWidth, rows: terminalHeight } = useTerminalSize(); + const isScreenReaderEnabled = useIsScreenReaderEnabled(); + const useAbsoluteTerminalCoordinates = useVirtualViewport(); + const containerRef = useRef(null); + const emittedKittySequenceRef = useRef(null); + const emittedITerm2PlacementRef = useRef(null); + const [iterm2PlacementAvailable, setITerm2PlacementAvailable] = + useState(true); + const prepared = useMemo( + () => + prepareTerminalImage({ + data: image.data, + mimeType: image.mimeType, + contentWidth, + availableTerminalHeight, + }), + [availableTerminalHeight, contentWidth, image.data, image.mimeType], + ); + + useEffect(() => { + if ( + isScreenReaderEnabled || + prepared.kind !== 'terminal-image' || + prepared.protocol !== 'kitty' || + !prepared.placeholder || + emittedKittySequenceRef.current === prepared.sequence + ) { + return; + } + emittedKittySequenceRef.current = prepared.sequence; + let cancelled = false; + let written = false; + process.nextTick(() => { + if (cancelled) { + return; + } + written = true; + writeRaw(prepared.sequence); + }); + return () => { + cancelled = true; + if (!written && emittedKittySequenceRef.current === prepared.sequence) { + emittedKittySequenceRef.current = null; + } + }; + }, [isScreenReaderEnabled, prepared, writeRaw]); + + // Parent layout and virtual scrolling can move a history item without + // changing this component's props, so placement must be measured after every + // render rather than from a dependency list. + // eslint-disable-next-line react-hooks/exhaustive-deps + useLayoutEffect(() => { + if ( + isScreenReaderEnabled || + prepared.kind !== 'terminal-image' || + prepared.protocol !== 'iterm2' || + !useAbsoluteTerminalCoordinates || + !containerRef.current + ) { + emittedITerm2PlacementRef.current = null; + return; + } + + const placement = calculateITerm2Placement( + containerRef.current, + terminalHeight, + prepared.rows, + terminalWidth, + prepared.widthCells, + ); + const isAvailable = placement !== null; + if (!placement) { + // Ink rewrites the reserved rows with fallback text while the image is + // outside the viewport. Forget the old placement so returning to the + // same coordinates emits the OSC image again instead of leaving only + // the fallback behind. + emittedITerm2PlacementRef.current = null; + } + if (isAvailable !== iterm2PlacementAvailable) { + setITerm2PlacementAvailable(isAvailable); + return; + } + if (!placement) { + return; + } + + const previousEmission = emittedITerm2PlacementRef.current; + if ( + previousEmission?.imageSequence === prepared.sequence && + previousEmission.column === placement.column && + previousEmission.row === placement.row + ) { + return; + } + const emission: ITerm2Emission = { + imageSequence: prepared.sequence, + ...placement, + }; + emittedITerm2PlacementRef.current = emission; + const sequence = buildITerm2PlacementSequence(prepared.sequence, placement); + let cancelled = false; + let written = false; + process.nextTick(() => { + if (cancelled) { + return; + } + written = true; + writeRaw(sequence); + }); + return () => { + cancelled = true; + if (!written && emittedITerm2PlacementRef.current === emission) { + emittedITerm2PlacementRef.current = null; + } + }; + }); + + const fallbackText = + prepared.kind === 'terminal-image' ? prepared.fallbackText : prepared.text; + if ( + isScreenReaderEnabled || + prepared.kind === 'fallback' || + (prepared.protocol === 'iterm2' && !useAbsoluteTerminalCoordinates) + ) { + return {fallbackText}; + } + + if (prepared.protocol === 'kitty' && prepared.placeholder) { + return ( + + {prepared.placeholder.lines.map((line, index) => ( + + + {line} + + + ))} + + ); + } + + return ( + + {iterm2PlacementAvailable ? ( + Array.from({ length: prepared.rows }, (_, index) => ( + + {' '} + + )) + ) : ( + {fallbackText} + )} + + ); +}; + +export const TerminalImage = memo(TerminalImageInternal); diff --git a/packages/cli/src/ui/components/messages/ConversationMessages.test.tsx b/packages/cli/src/ui/components/messages/ConversationMessages.test.tsx index 9020e8237d1..fb945cd55b6 100644 --- a/packages/cli/src/ui/components/messages/ConversationMessages.test.tsx +++ b/packages/cli/src/ui/components/messages/ConversationMessages.test.tsx @@ -5,12 +5,36 @@ */ import { render } from 'ink-testing-library'; +import { Text } from 'ink'; +import { vi } from 'vitest'; import { + AssistantMessage, ThinkMessage, ThinkMessageContent, toggleKeyHint, } from './ConversationMessages.js'; +vi.mock('../TerminalImage.js', () => ({ + TerminalImage: ({ image }: { image: { mimeType: string } }) => ( + MockTerminalImage:{image.mimeType} + ), +})); + +describe('', () => { + it('routes assistant images through TerminalImage', () => { + const { lastFrame } = render( + , + ); + + expect(lastFrame()).toContain('MockTerminalImage:image/png'); + }); +}); + describe('', () => { const defaultProps = { text: 'Analyzing the code structure', diff --git a/packages/cli/src/ui/components/messages/ConversationMessages.tsx b/packages/cli/src/ui/components/messages/ConversationMessages.tsx index 949a3e4d34b..79de0fef859 100644 --- a/packages/cli/src/ui/components/messages/ConversationMessages.tsx +++ b/packages/cli/src/ui/components/messages/ConversationMessages.tsx @@ -22,6 +22,8 @@ import { ErrorBoundary } from '../shared/ErrorBoundary.js'; import { ICON } from '../../constants.js'; import { sanitizeTerminalText } from '../../utils/textUtils.js'; import { formatDuration } from '../../utils/displayUtils.js'; +import type { InlineImageData } from '../../types.js'; +import { TerminalImage } from '../TerminalImage.js'; const debugLogger = createDebugLogger('THINK_RENDER'); @@ -40,6 +42,7 @@ interface UserShellMessageProps { interface AssistantMessageProps { text: string; + images?: InlineImageData[]; isPending: boolean; availableTerminalHeight?: number; contentWidth: number; @@ -48,6 +51,7 @@ interface AssistantMessageProps { interface AssistantMessageContentProps { text: string; + images?: InlineImageData[]; isPending: boolean; availableTerminalHeight?: number; contentWidth: number; @@ -89,6 +93,7 @@ interface PrefixedTextMessageProps { interface PrefixedMarkdownMessageProps { text: string; + images?: InlineImageData[]; prefix: string; prefixColor: string; isPending: boolean; @@ -101,6 +106,7 @@ interface PrefixedMarkdownMessageProps { interface ContinuationMarkdownMessageProps { text: string; + images?: InlineImageData[]; isPending: boolean; availableTerminalHeight?: number; contentWidth: number; @@ -148,6 +154,7 @@ const PrefixedTextMessage: React.FC = ({ const PrefixedMarkdownMessage: React.FC = ({ text, + images, prefix, prefixColor, isPending, @@ -167,14 +174,24 @@ const PrefixedMarkdownMessage: React.FC = ({ - + {text.length > 0 && ( + + )} + {images?.map((image, index) => ( + + ))} ); @@ -184,6 +201,7 @@ const ContinuationMarkdownMessage: React.FC< ContinuationMarkdownMessageProps > = ({ text, + images, isPending, availableTerminalHeight, contentWidth, @@ -195,14 +213,24 @@ const ContinuationMarkdownMessage: React.FC< return ( - + {text.length > 0 && ( + + )} + {images?.map((image, index) => ( + + ))} ); }; @@ -236,6 +264,7 @@ export const UserShellMessage: React.FC = ({ text }) => { export const AssistantMessage: React.FC = ({ text, + images, isPending, availableTerminalHeight, contentWidth, @@ -243,6 +272,7 @@ export const AssistantMessage: React.FC = ({ }) => ( = ({ text, + images, isPending, availableTerminalHeight, contentWidth, @@ -264,6 +295,7 @@ export const AssistantMessageContent: React.FC< }) => ( ', () => { expect(frame).not.toContain('MockTool'); }); + it('renders image-bearing collapsible tools individually', () => { + const toolCalls = [ + createToolCall({ + callId: 'image-read', + name: 'ReadFile', + description: 'chart.png', + images: [{ data: 'aW1hZ2U=', mimeType: 'image/png' }], + }), + ]; + const { lastFrame } = renderWithProviders( + , + ); + + expect(lastFrame()).toContain('MockTool[image-read]'); + }); + it('renders mixed group with summary + individual tools', () => { const toolCalls = [ createToolCall({ callId: 'r1', name: 'ReadFile', description: 'a.ts' }), diff --git a/packages/cli/src/ui/components/messages/ToolGroupMessage.tsx b/packages/cli/src/ui/components/messages/ToolGroupMessage.tsx index 09488a37622..115a660a0a8 100644 --- a/packages/cli/src/ui/components/messages/ToolGroupMessage.tsx +++ b/packages/cli/src/ui/components/messages/ToolGroupMessage.tsx @@ -388,13 +388,17 @@ export const ToolGroupMessage: React.FC = ({ ? [] : inlineToolCalls.filter( (t) => - isCollapsibleTool(t.name) && t.status !== ToolCallStatus.Canceled, + isCollapsibleTool(t.name) && + t.status !== ToolCallStatus.Canceled && + !t.images?.length, ); const nonCollapsibleTools = forceExpandAll ? inlineToolCalls : inlineToolCalls.filter( (t) => - !isCollapsibleTool(t.name) || t.status === ToolCallStatus.Canceled, + !isCollapsibleTool(t.name) || + t.status === ToolCallStatus.Canceled || + Boolean(t.images?.length), ); // Memory badge — shared between all-collapsible and mixed paths. @@ -448,7 +452,10 @@ export const ToolGroupMessage: React.FC = ({ let countToolCallsWithResults = 0; for (const tool of nonCollapsibleTools) { - if (tool.resultDisplay !== undefined && tool.resultDisplay !== '') { + if ( + (tool.resultDisplay !== undefined && tool.resultDisplay !== '') || + tool.images?.length + ) { countToolCallsWithResults++; } } diff --git a/packages/cli/src/ui/components/messages/ToolMessage.test.tsx b/packages/cli/src/ui/components/messages/ToolMessage.test.tsx index 5ec2dc101ab..a827ac654f8 100644 --- a/packages/cli/src/ui/components/messages/ToolMessage.test.tsx +++ b/packages/cli/src/ui/components/messages/ToolMessage.test.tsx @@ -70,6 +70,12 @@ vi.mock('../AnsiOutput.js', () => ({ }, })); +vi.mock('../TerminalImage.js', () => ({ + TerminalImage: ({ image }: { image: { mimeType: string } }) => ( + MockTerminalImage:{image.mimeType} + ), +})); + // Mock child components or utilities if they are complex or have side effects vi.mock('../GeminiRespondingSpinner.js', () => ({ GeminiRespondingSpinner: ({ @@ -175,6 +181,18 @@ describe('', () => { expect(output).not.toContain('MockMarkdown:Test result'); // collapsed }); + it('renders inline images returned by a tool', () => { + const { lastFrame } = renderWithContext( + , + StreamingState.Idle, + ); + + expect(lastFrame()).toContain('MockTerminalImage:image/png'); + }); + it('always shows the vision bridge disclosure for a completed read', () => { const { lastFrame } = renderWithContext( = ({ name, description, resultDisplay, + images, visionBridgeNotice, detailedDisplay, status, @@ -957,6 +959,22 @@ export const ToolMessage: React.FC = ({ )} + {images && images.length > 0 && ( + + {images.map((image, index) => ( + + ))} + + )} {isThisShellFocused && config && ( { }); }); + it('preserves text and inline image ordering in streamed content', async () => { + vi.useFakeTimers(); + + let releaseStream!: () => void; + const holdStream = new Promise((resolve) => { + releaseStream = resolve; + }); + const image = { + data: 'aW1hZ2U=', + mimeType: 'image/png', + displayName: 'chart.png', + }; + mockSendMessageStream.mockReturnValue( + (async function* () { + yield { + type: ServerGeminiEventType.Content, + value: 'beforeafter', + parts: [ + { text: 'before' }, + { inlineData: image }, + { text: 'after' }, + ], + }; + await holdStream; + })(), + ); + + const { result } = renderTestHook(); + act(() => { + void result.current.submitQuery('show a chart'); + }); + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(60); + }); + + const committedAssistantItems = mockAddItem.mock.calls + .map(([item]) => item as HistoryItem) + .filter( + (item) => item.type === 'gemini' || item.type === 'gemini_content', + ); + expect(committedAssistantItems).toEqual([]); + expect(result.current.pendingHistoryItems).toEqual([ + expect.objectContaining({ type: 'gemini', text: 'before' }), + { type: 'gemini_content', text: '', images: [image] }, + { type: 'gemini_content', text: 'after' }, + ]); + + act(() => result.current.cancelOngoingRequest()); + expect( + mockAddItem.mock.calls + .map(([item]) => item as HistoryItem) + .filter( + (item) => item.type === 'gemini' || item.type === 'gemini_content', + ), + ).toEqual([ + expect.objectContaining({ type: 'gemini', text: 'before' }), + { type: 'gemini_content', text: '', images: [image] }, + { type: 'gemini_content', text: 'after' }, + ]); + await act(async () => { + releaseStream(); + }); + }); + + it('does not overwrite an image with whitespace before the next image', async () => { + vi.useFakeTimers(); + + let releaseStream!: () => void; + const holdStream = new Promise((resolve) => { + releaseStream = resolve; + }); + const firstImage = { + data: 'Zmlyc3Q=', + mimeType: 'image/png', + displayName: 'first.png', + }; + const secondImage = { + data: 'c2Vjb25k', + mimeType: 'image/png', + displayName: 'second.png', + }; + mockSendMessageStream.mockReturnValue( + (async function* () { + yield { + type: ServerGeminiEventType.Content, + value: '\n', + parts: [ + { inlineData: firstImage }, + { text: '\n' }, + { inlineData: secondImage }, + ], + }; + await holdStream; + })(), + ); + + const { result } = renderTestHook(); + act(() => { + void result.current.submitQuery('show two charts'); + }); + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(60); + }); + + expect(result.current.pendingHistoryItems).toEqual([ + expect.objectContaining({ + type: 'gemini', + text: '', + images: [firstImage], + }), + { type: 'gemini_content', text: '', images: [secondImage] }, + ]); + + act(() => result.current.cancelOngoingRequest()); + await act(async () => { + releaseStream(); + }); + }); + + it('keeps an image committable when the stream pauses after whitespace', async () => { + vi.useFakeTimers(); + + let releaseStream!: () => void; + const holdStream = new Promise((resolve) => { + releaseStream = resolve; + }); + const image = { + data: 'aW1hZ2U=', + mimeType: 'image/png', + displayName: 'trailing-space.png', + }; + mockSendMessageStream.mockReturnValue( + (async function* () { + yield { + type: ServerGeminiEventType.Content, + value: '\n', + parts: [{ inlineData: image }, { text: '\n' }], + }; + await holdStream; + })(), + ); + + const { result } = renderTestHook(); + act(() => { + void result.current.submitQuery('show a chart'); + }); + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(60); + }); + + expect(result.current.pendingHistoryItems).toEqual([ + expect.objectContaining({ + type: 'gemini', + text: '', + images: [image], + }), + ]); + + act(() => result.current.cancelOngoingRequest()); + expect( + mockAddItem.mock.calls + .map(([item]) => item as HistoryItem) + .filter( + (item) => item.type === 'gemini' || item.type === 'gemini_content', + ), + ).toEqual([ + expect.objectContaining({ + type: 'gemini', + text: '', + images: [image], + }), + ]); + await act(async () => { + releaseStream(); + }); + }); + + it('discards every staged mixed-content run on a fresh retry', async () => { + vi.useFakeTimers(); + + let emitRetry!: () => void; + const waitForRetry = new Promise((resolve) => { + emitRetry = resolve; + }); + let releaseStream!: () => void; + const holdStream = new Promise((resolve) => { + releaseStream = resolve; + }); + const image = { + data: 'aW1hZ2U=', + mimeType: 'image/png', + displayName: 'failed.png', + }; + mockSendMessageStream.mockReturnValue( + (async function* () { + yield { + type: ServerGeminiEventType.Content, + value: 'beforeafter', + parts: [ + { text: 'before' }, + { inlineData: image }, + { text: 'after' }, + ], + }; + await waitForRetry; + yield { + type: ServerGeminiEventType.Retry, + isContinuation: false, + }; + yield { + type: ServerGeminiEventType.Content, + value: 'replacement', + }; + await holdStream; + })(), + ); + + const { result } = renderTestHook(); + act(() => { + void result.current.submitQuery('show a chart'); + }); + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(60); + }); + + expect(result.current.pendingHistoryItems).toEqual([ + expect.objectContaining({ type: 'gemini', text: 'before' }), + { type: 'gemini_content', text: '', images: [image] }, + { type: 'gemini_content', text: 'after' }, + ]); + + await act(async () => { + emitRetry(); + await Promise.resolve(); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(60); + }); + + const committedAssistantItems = mockAddItem.mock.calls + .map(([item]) => item as HistoryItem) + .filter( + (item) => item.type === 'gemini' || item.type === 'gemini_content', + ); + expect(committedAssistantItems).toEqual([]); + expect(result.current.pendingHistoryItems).toEqual([ + expect.objectContaining({ type: 'gemini', text: 'replacement' }), + ]); + + act(() => result.current.cancelOngoingRequest()); + await act(async () => { + releaseStream(); + }); + }); + + it('discards staged mixed content before an explicit retry after a thrown stream', async () => { + const failedImage = { + data: 'aW1hZ2U=', + mimeType: 'image/png', + displayName: 'failed.png', + }; + mockSendMessageStream + .mockReturnValueOnce( + (async function* () { + yield { + type: ServerGeminiEventType.Content, + value: 'beforeafter', + parts: [ + { text: 'before' }, + { inlineData: failedImage }, + { text: 'after' }, + ], + }; + throw new Error('stream failed'); + })(), + ) + .mockReturnValueOnce( + (async function* () { + yield { + type: ServerGeminiEventType.Content, + value: 'replacement', + }; + yield { + type: ServerGeminiEventType.Finished, + value: { reason: 'STOP', usageMetadata: undefined }, + }; + })(), + ); + + const { result } = renderTestHook(); + await act(async () => { + await result.current.submitQuery('show a chart'); + }); + + expect(result.current.pendingHistoryItems).toEqual( + expect.arrayContaining([ + expect.objectContaining({ type: 'gemini', text: 'before' }), + { type: 'gemini_content', text: '', images: [failedImage] }, + { type: 'gemini_content', text: 'after' }, + ]), + ); + + await act(async () => { + await result.current.submitQuery('show a chart', SendMessageType.Retry); + }); + + const committedAssistantItems = mockAddItem.mock.calls + .map(([item]) => item as HistoryItem) + .filter( + (item) => item.type === 'gemini' || item.type === 'gemini_content', + ); + expect(committedAssistantItems).toEqual([ + expect.objectContaining({ type: 'gemini', text: 'replacement' }), + ]); + expect(result.current.pendingHistoryItems).not.toEqual( + expect.arrayContaining([ + expect.objectContaining({ images: [failedImage] }), + ]), + ); + }); + + it('commits staged mixed content before a new user turn after a thrown stream', async () => { + const failedImage = { + data: 'aW1hZ2U=', + mimeType: 'image/png', + displayName: 'partial.png', + }; + mockSendMessageStream + .mockReturnValueOnce( + (async function* () { + yield { + type: ServerGeminiEventType.Content, + value: 'beforeafter', + parts: [ + { text: 'before' }, + { inlineData: failedImage }, + { text: 'after' }, + ], + }; + throw new Error('stream failed'); + })(), + ) + .mockReturnValueOnce( + (async function* () { + yield { + type: ServerGeminiEventType.Content, + value: 'next answer', + }; + yield { + type: ServerGeminiEventType.Finished, + value: { reason: 'STOP', usageMetadata: undefined }, + }; + })(), + ); + + const { result } = renderTestHook(); + await act(async () => { + await result.current.submitQuery('first question'); + }); + await act(async () => { + await result.current.submitQuery('second question'); + }); + + const relevantItems = mockAddItem.mock.calls + .map(([item]) => item as HistoryItem) + .filter( + (item) => + item.type === 'user' || + item.type === 'gemini' || + item.type === 'gemini_content' || + item.type === 'error', + ); + expect(relevantItems).toEqual([ + expect.objectContaining({ type: 'user', text: 'first question' }), + expect.objectContaining({ type: 'gemini', text: 'before' }), + { type: 'gemini_content', text: '', images: [failedImage] }, + { type: 'gemini_content', text: 'after' }, + expect.objectContaining({ type: 'error' }), + expect.objectContaining({ type: 'user', text: 'second question' }), + expect.objectContaining({ type: 'gemini', text: 'next answer' }), + ]); + }); + it('does not render leading blank content chunks as an empty assistant item', async () => { vi.useFakeTimers(); @@ -11878,12 +12272,22 @@ describe('useGeminiStream', () => { }); describe('HookSystemMessage Event', () => { - it('commits buffered content before a displayed Goal state', async () => { + it('commits staged inline content before a displayed Goal state', async () => { + const image = { + data: 'aW1hZ2U=', + mimeType: 'image/png', + displayName: 'goal.png', + }; mockSendMessageStream.mockReturnValue( (async function* () { yield { type: ServerGeminiEventType.Content, value: 'Final Goal output', + parts: [ + { text: 'Final ' }, + { inlineData: image }, + { text: 'Goal output' }, + ], }; yield { type: ServerGeminiEventType.GoalState, @@ -11910,14 +12314,21 @@ describe('useGeminiStream', () => { await act(async () => { await result.current.submitQuery('finish the Goal'); }); - const contentIndex = mockAddItem.mock.calls.findIndex( - ([item]) => item.type === 'gemini' && item.text === 'Final Goal output', - ); - const goalIndex = mockAddItem.mock.calls.findIndex( - ([item]) => item.type === 'goal_state' && item.cause === 'complete', - ); - expect(contentIndex).toBeGreaterThanOrEqual(0); - expect(goalIndex).toBeGreaterThan(contentIndex); + expect( + mockAddItem.mock.calls + .map(([item]) => item as HistoryItem) + .filter( + (item) => + item.type === 'gemini' || + item.type === 'gemini_content' || + item.type === 'goal_state', + ), + ).toEqual([ + expect.objectContaining({ type: 'gemini', text: 'Final ' }), + { type: 'gemini_content', text: '', images: [image] }, + { type: 'gemini_content', text: 'Goal output' }, + expect.objectContaining({ type: 'goal_state', cause: 'complete' }), + ]); }); it('should handle HookSystemMessage event and add stop_hook_system_message history item', async () => { diff --git a/packages/cli/src/ui/hooks/useGeminiStream.ts b/packages/cli/src/ui/hooks/useGeminiStream.ts index 0837efb7518..6ee331ca7a2 100644 --- a/packages/cli/src/ui/hooks/useGeminiStream.ts +++ b/packages/cli/src/ui/hooks/useGeminiStream.ts @@ -75,6 +75,7 @@ import type { HistoryItemWithoutId, HistoryItemToolGroup, HistoryItemGemini, + InlineImageData, SlashCommandProcessorResult, } from '../types.js'; import { StreamingState, MessageType, ToolCallStatus } from '../types.js'; @@ -367,6 +368,7 @@ const LOADING_THOUGHT_DESCRIPTION_MAX_CHARS = 4_096; type BufferedStreamEvent = | { kind: 'content'; value: string } + | { kind: 'image'; value: InlineImageData } | { kind: 'thought'; value: ThoughtSummary }; function showCitations(settings: LoadedSettings): boolean { @@ -693,6 +695,44 @@ export const useGeminiStream = ( const auxiliaryAbortRefsRef = useRef>(new Set()); const [pendingHistoryItem, pendingHistoryItemRef, setPendingHistoryItem] = useStateAndRef(null); + // Mixed assistant output needs multiple live rows to preserve + // text/image/text ordering. Keep completed runs in the dynamic region until + // the response reaches a normal commit boundary so a fresh retry or model + // fallback can still discard the entire failed attempt. + const [ + pendingAssistantItems, + pendingAssistantItemsRef, + setPendingAssistantItems, + ] = useStateAndRef([]); + const commitPendingAssistantItems = useCallback( + (userMessageTimestamp: number) => { + const items = pendingAssistantItemsRef.current; + if (items.length === 0) { + return; + } + for (const item of items) { + commitItem(item, userMessageTimestamp); + } + setPendingAssistantItems([]); + }, + [commitItem, pendingAssistantItemsRef, setPendingAssistantItems], + ); + const commitItemInOrder = useCallback( + (item: HistoryItemWithoutId, userMessageTimestamp: number): number => { + commitPendingAssistantItems(userMessageTimestamp); + return commitItem(item, userMessageTimestamp); + }, + [commitItem, commitPendingAssistantItems], + ); + const stagePendingAssistantItem = useCallback((): boolean => { + const item = pendingHistoryItemRef.current; + if (item?.type !== 'gemini' && item?.type !== 'gemini_content') { + return false; + } + setPendingAssistantItems((items) => [...items, item]); + setPendingHistoryItem(null); + return true; + }, [pendingHistoryItemRef, setPendingAssistantItems, setPendingHistoryItem]); // Streamed model reasoning for the current turn. Rendered (height-limited) // above the answer while thinking, then committed to history as a // collapsible `gemini_thought` block when the answer/tool/turn begins. @@ -1027,7 +1067,7 @@ export const useGeminiStream = ( logApiCancel(config, cancellationEvent); if (pendingHistoryItemRef.current) { - commitItem(pendingHistoryItemRef.current, Date.now()); + commitItemInOrder(pendingHistoryItemRef.current, Date.now()); } addItem( { @@ -1078,7 +1118,7 @@ export const useGeminiStream = ( }, [ streamingState, addItem, - commitItem, + commitItemInOrder, setPendingHistoryItem, onCancelSubmit, pendingHistoryItemRef, @@ -1421,6 +1461,7 @@ export const useGeminiStream = ( eventValue: ContentEvent['value'], currentGeminiMessageBuffer: string, userMessageTimestamp: number, + startAsContinuation = false, ): string => { if (turnCancelledRef.current) { // Prevents additional output after a user initiated cancel. @@ -1435,6 +1476,17 @@ export const useGeminiStream = ( // React history by the time AppContainer's guard runs). turnSawContentEventRef.current = true; let newGeminiMessageBuffer = currentGeminiMessageBuffer + eventValue; + const pendingItem = pendingHistoryItemRef.current; + if ( + (pendingItem?.type === 'gemini' || + pendingItem?.type === 'gemini_content') && + pendingItem.images?.length + ) { + if (newGeminiMessageBuffer.trim().length === 0) { + return newGeminiMessageBuffer; + } + stagePendingAssistantItem(); + } if ( pendingHistoryItemRef.current?.type !== 'gemini' && pendingHistoryItemRef.current?.type !== 'gemini_content' @@ -1443,20 +1495,28 @@ export const useGeminiStream = ( return newGeminiMessageBuffer; } if (pendingHistoryItemRef.current) { - commitItem(pendingHistoryItemRef.current, userMessageTimestamp); + commitItemInOrder( + pendingHistoryItemRef.current, + userMessageTimestamp, + ); } - setPendingHistoryItem({ - type: 'gemini', - text: '', - timestamp: Date.now(), - }); + setPendingHistoryItem( + startAsContinuation + ? { type: 'gemini_content', text: '' } + : { type: 'gemini', text: '', timestamp: Date.now() }, + ); newGeminiMessageBuffer = stripLeadingBlankLines(newGeminiMessageBuffer); } // Split large messages for better rendering performance. Ideally, // we should maximize the amount of output sent to . - let nextPendingType = pendingHistoryItemRef.current?.type as - | 'gemini' - | 'gemini_content'; + let nextPendingType: 'gemini' | 'gemini_content' = + pendingHistoryItemRef.current?.type === 'gemini_content' + ? 'gemini_content' + : pendingHistoryItemRef.current?.type === 'gemini' + ? 'gemini' + : startAsContinuation + ? 'gemini_content' + : 'gemini'; while (newGeminiMessageBuffer.length > STREAM_PENDING_ITEM_MAX_CHARS) { const splitPoint = findLastSafeSplitPoint( newGeminiMessageBuffer, @@ -1481,7 +1541,7 @@ export const useGeminiStream = ( newGeminiMessageBuffer, safeSplitPoint, ); - commitItem( + commitItemInOrder( { type: nextPendingType, text: beforeText, @@ -1595,7 +1655,7 @@ export const useGeminiStream = ( newGeminiMessageBuffer, splitPoint, ); - commitItem( + commitItemInOrder( { type: nextPendingType, text: beforeText, @@ -1621,9 +1681,10 @@ export const useGeminiStream = ( return newGeminiMessageBuffer; }, [ - commitItem, + commitItemInOrder, pendingHistoryItemRef, setPendingHistoryItem, + stagePendingAssistantItem, terminalWidth, terminalHeight, availableTerminalHeightRef, @@ -1811,7 +1872,10 @@ export const useGeminiStream = ( }; addItem(pendingItem, userMessageTimestamp); } else { - commitItem(pendingHistoryItemRef.current, userMessageTimestamp); + commitItemInOrder( + pendingHistoryItemRef.current, + userMessageTimestamp, + ); } setPendingHistoryItem(null); } @@ -1826,7 +1890,7 @@ export const useGeminiStream = ( [ addItem, commitPendingThought, - commitItem, + commitItemInOrder, pendingHistoryItemRef, setPendingHistoryItem, setThought, @@ -1848,7 +1912,7 @@ export const useGeminiStream = ( // Persist any streamed reasoning (collapsed) above the error. commitPendingThought(userMessageTimestamp); if (pendingHistoryItemRef.current) { - commitItem(pendingHistoryItemRef.current, userMessageTimestamp); + commitItemInOrder(pendingHistoryItemRef.current, userMessageTimestamp); setPendingHistoryItem(null); } // Only show Ctrl+Y hint if not already showing an auto-retry countdown @@ -1892,7 +1956,7 @@ export const useGeminiStream = ( }, [ commitPendingThought, - commitItem, + commitItemInOrder, pendingHistoryItemRef, setPendingHistoryItem, setPendingRetryErrorItem, @@ -1909,14 +1973,14 @@ export const useGeminiStream = ( } if (pendingHistoryItemRef.current) { - commitItem(pendingHistoryItemRef.current, userMessageTimestamp); + commitItemInOrder(pendingHistoryItemRef.current, userMessageTimestamp); setPendingHistoryItem(null); } addItem({ type: MessageType.INFO, text }, userMessageTimestamp); }, [ addItem, - commitItem, + commitItemInOrder, pendingHistoryItemRef, setPendingHistoryItem, settings, @@ -1987,7 +2051,7 @@ export const useGeminiStream = ( ) => { autonomousLoopTickResolverRef.current?.resetCache(); if (pendingHistoryItemRef.current) { - commitItem(pendingHistoryItemRef.current, userMessageTimestamp); + commitItemInOrder(pendingHistoryItemRef.current, userMessageTimestamp); setPendingHistoryItem(null); } const activeModel = modelOverrideRef.current ?? config.getModel(); @@ -2011,7 +2075,13 @@ export const useGeminiStream = ( Date.now(), ); }, - [addItem, commitItem, config, pendingHistoryItemRef, setPendingHistoryItem], + [ + addItem, + commitItemInOrder, + config, + pendingHistoryItemRef, + setPendingHistoryItem, + ], ); const handleMaxSessionTurnsEvent = useCallback( @@ -2084,7 +2154,7 @@ export const useGeminiStream = ( userMessageTimestamp: number, ) => { if (pendingHistoryItemRef.current) { - commitItem(pendingHistoryItemRef.current, userMessageTimestamp); + commitItemInOrder(pendingHistoryItemRef.current, userMessageTimestamp); setPendingHistoryItem(null); } addItem( @@ -2096,7 +2166,7 @@ export const useGeminiStream = ( userMessageTimestamp, ); }, - [addItem, commitItem, pendingHistoryItemRef, setPendingHistoryItem], + [addItem, commitItemInOrder, pendingHistoryItemRef, setPendingHistoryItem], ); const handleStopHookLoopEvent = useCallback( @@ -2109,7 +2179,7 @@ export const useGeminiStream = ( userMessageTimestamp: number, ) => { if (pendingHistoryItemRef.current) { - commitItem(pendingHistoryItemRef.current, userMessageTimestamp); + commitItemInOrder(pendingHistoryItemRef.current, userMessageTimestamp); setPendingHistoryItem(null); } addItem( @@ -2122,7 +2192,7 @@ export const useGeminiStream = ( userMessageTimestamp, ); }, - [addItem, commitItem, pendingHistoryItemRef, setPendingHistoryItem], + [addItem, commitItemInOrder, pendingHistoryItemRef, setPendingHistoryItem], ); const processGeminiStreamEvents = useCallback( @@ -2136,6 +2206,9 @@ export const useGeminiStream = ( let geminiMessageBuffer = ''; let thoughtBuffer = ''; let scheduledToolContinuation = false; + let assistantOutputStarted = + pendingHistoryItemRef.current?.type === 'gemini' || + pendingHistoryItemRef.current?.type === 'gemini_content'; const toolCallRequests: ToolCallRequestInfo[] = []; const bufferedEvents: BufferedStreamEvent[] = []; let flushTimer: ReturnType | null = null; @@ -2176,7 +2249,37 @@ export const useGeminiStream = ( contentParts.join(''), geminiMessageBuffer, userMessageTimestamp, + assistantOutputStarted, ); + if (contentParts.some((part) => part.trim().length > 0)) { + assistantOutputStarted = true; + } + continue; + } + + if (nextEvent.kind === 'image') { + if (turnCancelledRef.current) { + continue; + } + setIsReceivingContent(true); + turnSawContentEventRef.current = true; + if (pendingHistoryItemRef.current) { + if (!stagePendingAssistantItem()) { + commitItemInOrder( + pendingHistoryItemRef.current, + userMessageTimestamp, + ); + setPendingHistoryItem(null); + } + } + geminiMessageBuffer = ''; + setPendingHistoryItem({ + type: assistantOutputStarted ? 'gemini_content' : 'gemini', + text: '', + images: [nextEvent.value], + ...(!assistantOutputStarted ? { timestamp: Date.now() } : {}), + }); + assistantOutputStarted = true; continue; } @@ -2238,7 +2341,7 @@ export const useGeminiStream = ( scheduleBufferedStreamFlush(); } break; - case ServerGeminiEventType.Content: + case ServerGeminiEventType.Content: { // Thinking is done once the answer starts streaming; reset the // title status. On the thinking→answer transition, flush any // buffered reasoning so the full thought is captured, then commit @@ -2253,9 +2356,22 @@ export const useGeminiStream = ( thoughtBuffer = ''; } setThought((prev) => (prev ? null : prev)); - bufferedEvents.push({ kind: 'content', value: event.value }); + const displayParts = event.parts ?? [{ text: event.value }]; + for (const part of displayParts) { + if ('text' in part) { + if (part.text.length > 0) { + bufferedEvents.push({ kind: 'content', value: part.text }); + } + } else { + bufferedEvents.push({ + kind: 'image', + value: part.inlineData, + }); + } + } scheduleBufferedStreamFlush(); break; + } case ServerGeminiEventType.ToolCallRequest: // Thinking is done once a tool call is issued; flush buffered // reasoning then commit it to history (collapsed) above the tool @@ -2327,11 +2443,15 @@ export const useGeminiStream = ( // as "t" → "te" → "tes" cumulative rendering even though each // turn is persisted as a clean, separate assistant message. if (pendingHistoryItemRef.current) { - commitItem(pendingHistoryItemRef.current, userMessageTimestamp); + commitItemInOrder( + pendingHistoryItemRef.current, + userMessageTimestamp, + ); setPendingHistoryItem(null); } geminiMessageBuffer = ''; thoughtBuffer = ''; + assistantOutputStarted = false; setThought(null); break; case ServerGeminiEventType.Citation: @@ -2354,6 +2474,7 @@ export const useGeminiStream = ( // losing the partial text we meant to preserve. if (!event.isContinuation) { discardBufferedStreamEvents(); + setPendingAssistantItems([]); if (pendingHistoryItemRef.current) { setPendingHistoryItem(null); } @@ -2361,6 +2482,7 @@ export const useGeminiStream = ( thoughtBuffer = ''; setThought(null); geminiMessageBuffer = ''; + assistantOutputStarted = false; } else { flushBufferedStreamEvents(); } @@ -2384,6 +2506,7 @@ export const useGeminiStream = ( // switching to the next fallback model. Discard partial content // from the failed attempt and show a notification. discardBufferedStreamEvents(); + setPendingAssistantItems([]); if (pendingHistoryItemRef.current) { setPendingHistoryItem(null); } @@ -2391,6 +2514,7 @@ export const useGeminiStream = ( thoughtBuffer = ''; setThought(null); geminiMessageBuffer = ''; + assistantOutputStarted = false; toolCallRequests.length = 0; clearRetryCountdown(); const fromModel = @@ -2410,7 +2534,10 @@ export const useGeminiStream = ( // Display system message from Stop hooks with "Stop says:" prefix // First commit any pending AI response to ensure correct ordering if (pendingHistoryItemRef.current) { - commitItem(pendingHistoryItemRef.current, userMessageTimestamp); + commitItemInOrder( + pendingHistoryItemRef.current, + userMessageTimestamp, + ); setPendingHistoryItem(null); } addItem( @@ -2438,7 +2565,7 @@ export const useGeminiStream = ( if (event.cause && shouldDisplayGoalStateCause(event.cause)) { flushBufferedStreamEvents(); if (pendingHistoryItemRef.current) { - commitItem( + commitItemInOrder( pendingHistoryItemRef.current, userMessageTimestamp, ); @@ -2596,7 +2723,9 @@ export const useGeminiStream = ( handleStopHookLoopEvent, bindGoalTurn, addItem, - commitItem, + commitItemInOrder, + stagePendingAssistantItem, + setPendingAssistantItems, dualOutput, ], ); @@ -2954,6 +3083,34 @@ export const useGeminiStream = ( const userMessageTimestamp = Date.now(); + // A thrown stream can leave partial assistant runs in the dynamic + // region. An explicit Ctrl+Y retry is a fresh attempt, matching a core + // non-continuation Retry event, so discard every run from the failed + // attempt before the replacement stream starts. A different top-level + // turn preserves what the user already saw, but must commit it before + // prepareQueryForGemini appends the next user item. + if (submitType === SendMessageType.Retry) { + setPendingAssistantItems([]); + const pendingItem = pendingHistoryItemRef.current; + if ( + pendingItem?.type === 'gemini' || + pendingItem?.type === 'gemini_content' + ) { + setPendingHistoryItem(null); + } + } else if (!isTurnContinuation && !allowConcurrentBtwDuringResponse) { + const pendingItem = pendingHistoryItemRef.current; + if ( + pendingItem?.type === 'gemini' || + pendingItem?.type === 'gemini_content' + ) { + commitItemInOrder(pendingItem, userMessageTimestamp); + setPendingHistoryItem(null); + } else { + commitPendingAssistantItems(userMessageTimestamp); + } + } + // Reset quota error flag when starting a new query (not a continuation). // Notifications (background agent/shell/monitor completions) are system // events, not new user turns: they must not clear the user's model @@ -3303,7 +3460,10 @@ export const useGeminiStream = ( } if (pendingHistoryItemRef.current) { - commitItem(pendingHistoryItemRef.current, userMessageTimestamp); + commitItemInOrder( + pendingHistoryItemRef.current, + userMessageTimestamp, + ); setPendingHistoryItem(null); } @@ -3508,7 +3668,9 @@ export const useGeminiStream = ( processGeminiStreamEvents, pendingHistoryItemRef, addItem, - commitItem, + commitPendingAssistantItems, + commitItemInOrder, + setPendingAssistantItems, setPendingHistoryItem, setInitError, geminiClient, @@ -4356,6 +4518,7 @@ export const useGeminiStream = ( [ // Reasoning renders above the streaming answer. pendingThoughtItem, + ...pendingAssistantItems, pendingHistoryItem, pendingRetryErrorItem, pendingRetryCountdownItem, @@ -4363,6 +4526,7 @@ export const useGeminiStream = ( ].filter((i) => i !== undefined && i !== null), [ pendingThoughtItem, + pendingAssistantItems, pendingHistoryItem, pendingRetryErrorItem, pendingRetryCountdownItem, diff --git a/packages/cli/src/ui/hooks/useHistoryManager.test.ts b/packages/cli/src/ui/hooks/useHistoryManager.test.ts index 382115a50bd..05ccabb4fa1 100644 --- a/packages/cli/src/ui/hooks/useHistoryManager.test.ts +++ b/packages/cli/src/ui/hooks/useHistoryManager.test.ts @@ -6,7 +6,11 @@ import { describe, it, expect, vi, beforeEach } from 'vitest'; import { renderHook, act } from '@testing-library/react'; -import { useHistory, UI_COMPACT_CLEARED_MESSAGE } from './useHistoryManager.js'; +import { + useHistory, + UI_COMPACT_CLEARED_MESSAGE, + UI_COMPACT_CLEARED_IMAGE_MESSAGE, +} from './useHistoryManager.js'; import type { UseHistoryManagerReturn } from './useHistoryManager.js'; import type { HistoryItemWithoutId, HistoryItemToolGroup } from '../types.js'; import { ToolCallStatus } from '../types.js'; @@ -370,6 +374,84 @@ describe('useHistoryManager', () => { expect(recentTool.detailedDisplay).toBe('full secret file content here'); }); + it('clears image payloads from old tool results', () => { + const { result } = renderHook(() => useHistory()); + const ts = Date.now(); + + for (let i = 0; i < 25; i++) { + act(() => { + result.current.addItem( + { + type: 'tool_group', + tools: [ + { + callId: String(i), + name: 'screenshot', + description: '', + resultDisplay: undefined, + images: [{ data: 'aW1hZ2U=', mimeType: 'image/png' }], + status: ToolCallStatus.Success, + confirmationDetails: undefined, + }, + ], + } as unknown as HistoryItemWithoutId, + ts + i, + ); + }); + } + + act(() => { + result.current.compactOldItems(); + }); + + const oldestTool = ( + result.current.history[0] as unknown as HistoryItemToolGroup + ).tools[0]; + expect(oldestTool.resultDisplay).toBe(UI_COMPACT_CLEARED_MESSAGE); + expect(oldestTool.images).toBeUndefined(); + + const recentTool = ( + result.current.history[24] as unknown as HistoryItemToolGroup + ).tools[0]; + expect(recentTool.images).toHaveLength(1); + }); + + it('clears old assistant image payloads while keeping recent images', () => { + const { result } = renderHook(() => useHistory()); + const ts = Date.now(); + + for (let i = 0; i < 25; i++) { + act(() => { + result.current.addItem( + { + type: i === 0 ? 'gemini' : 'gemini_content', + text: i === 0 ? 'Generated chart' : '', + images: [{ data: `aW1hZ2Ut${i}`, mimeType: 'image/png' }], + }, + ts + i, + ); + }); + } + + act(() => { + result.current.compactOldItems(); + }); + + const oldestItem = result.current.history[0]; + expect(oldestItem).toMatchObject({ + type: 'gemini', + text: `Generated chart\n\n${UI_COMPACT_CLEARED_IMAGE_MESSAGE}`, + }); + expect( + oldestItem.type === 'gemini' ? oldestItem.images : undefined, + ).toBeUndefined(); + + const recentItem = result.current.history[24]; + expect( + recentItem.type === 'gemini_content' ? recentItem.images : undefined, + ).toHaveLength(1); + }); + it('clears a tool that carries detailedDisplay but no resultDisplay (defensive)', () => { const { result } = renderHook(() => useHistory()); const ts = Date.now(); diff --git a/packages/cli/src/ui/hooks/useHistoryManager.ts b/packages/cli/src/ui/hooks/useHistoryManager.ts index fc5b41df276..c0402555259 100644 --- a/packages/cli/src/ui/hooks/useHistoryManager.ts +++ b/packages/cli/src/ui/hooks/useHistoryManager.ts @@ -17,6 +17,8 @@ type HistoryItemUpdater = ( ) => Partial; export const UI_COMPACT_CLEARED_MESSAGE = '[Old tool result content cleared]'; +export const UI_COMPACT_CLEARED_IMAGE_MESSAGE = + '[Old assistant image content cleared]'; const UI_COMPACT_KEEP_RECENT = 20; export interface UseHistoryManagerReturn { @@ -148,22 +150,30 @@ export function useHistory(): UseHistoryManagerReturn { let thoughtRemoved = 0; let toolGroupsCompacted = 0; + let assistantImageItemsCompacted = 0; let totalThoughts = 0; let totalToolGroupsWithOutput = 0; + let totalAssistantItemsWithImages = 0; for (const item of prev) { if ( item.type === 'gemini_thought' || item.type === 'gemini_thought_content' ) { totalThoughts++; + } else if ( + (item.type === 'gemini' || item.type === 'gemini_content') && + item.images?.length + ) { + totalAssistantItemsWithImages++; } else if ( item.type === 'tool_group' && item.tools.some( (t) => (t.resultDisplay != null && t.resultDisplay !== UI_COMPACT_CLEARED_MESSAGE) || - t.detailedDisplay != null, + t.detailedDisplay != null || + Boolean(t.images?.length), ) ) { totalToolGroupsWithOutput++; @@ -177,8 +187,13 @@ export function useHistory(): UseHistoryManagerReturn { 0, totalToolGroupsWithOutput - UI_COMPACT_KEEP_RECENT, ); + const assistantImageItemsToCompact = Math.max( + 0, + totalAssistantItemsWithImages - UI_COMPACT_KEEP_RECENT, + ); let thoughtsDropped = 0; let toolGroupsSeen = 0; + let assistantImageItemsSeen = 0; const next = prev .filter((item) => { @@ -195,16 +210,33 @@ export function useHistory(): UseHistoryManagerReturn { return true; }) .map((item) => { + if ( + (item.type === 'gemini' || item.type === 'gemini_content') && + item.images?.length + ) { + assistantImageItemsSeen++; + if (assistantImageItemsSeen <= assistantImageItemsToCompact) { + assistantImageItemsCompacted++; + return { + ...item, + text: item.text + ? `${item.text}\n\n${UI_COMPACT_CLEARED_IMAGE_MESSAGE}` + : UI_COMPACT_CLEARED_IMAGE_MESSAGE, + images: undefined, + }; + } + } if (item.type !== 'tool_group') return item; // Check for any non-null resultDisplay (covers string, FileDiff, - // AnsiOutputDisplay, AgentResultDisplay, etc.) OR a lingering - // `detailedDisplay` — so a tool that somehow carries only the raw - // transcript detail still triggers compaction and gets cleared. + // AnsiOutputDisplay, AgentResultDisplay, etc.), a lingering + // `detailedDisplay`, or image payloads. Every retained output form + // must participate in the same keep-recent limit. const hasOldOutput = item.tools.some( (t) => (t.resultDisplay != null && t.resultDisplay !== UI_COMPACT_CLEARED_MESSAGE) || - t.detailedDisplay != null, + t.detailedDisplay != null || + Boolean(t.images?.length), ); if (!hasOldOutput) return item; toolGroupsSeen++; @@ -216,19 +248,21 @@ export function useHistory(): UseHistoryManagerReturn { if ( (t.resultDisplay != null && t.resultDisplay !== UI_COMPACT_CLEARED_MESSAGE) || - t.detailedDisplay != null + t.detailedDisplay != null || + t.images?.length ) { // Also drop `detailedDisplay` (the raw functionResponse text // kept for the Ctrl+O full-detail transcript): clearing only // `resultDisplay` would let a post-compaction transcript reopen // re-surface the supposedly cleared read/search/list output, // defeating the memory/privacy compaction. The `detailedDisplay` - // arm keeps the guard robust even if a tool ever carries the - // raw detail without a `resultDisplay`. + // and `images` arms keep the guard robust when a tool carries + // raw detail or media without a `resultDisplay`. return { ...t, resultDisplay: UI_COMPACT_CLEARED_MESSAGE, detailedDisplay: undefined, + images: undefined, }; } return t; @@ -236,17 +270,26 @@ export function useHistory(): UseHistoryManagerReturn { }; }); - if (thoughtRemoved > 0 || toolGroupsCompacted > 0) { + if ( + thoughtRemoved > 0 || + toolGroupsCompacted > 0 || + assistantImageItemsCompacted > 0 + ) { if (debugLogger.isEnabled()) { debugLogger.debug( `[COMPACT_UI_HISTORY] removed ${thoughtRemoved} thought item(s), ` + + `compacted ${assistantImageItemsCompacted} assistant image item(s), ` + `compacted ${toolGroupsCompacted} tool group(s), ` + `historyLength ${prev.length} -> ${next.length}, ` + `memory=${(process.memoryUsage().heapUsed / 1024 / 1024).toFixed(1)}MB`, ); } } - return thoughtRemoved > 0 || toolGroupsCompacted > 0 ? next : prev; + return thoughtRemoved > 0 || + toolGroupsCompacted > 0 || + assistantImageItemsCompacted > 0 + ? next + : prev; }); }, []); diff --git a/packages/cli/src/ui/hooks/useReactToolScheduler.test.tsx b/packages/cli/src/ui/hooks/useReactToolScheduler.test.tsx index f68ee1b1148..2b0414c34e0 100644 --- a/packages/cli/src/ui/hooks/useReactToolScheduler.test.tsx +++ b/packages/cli/src/ui/hooks/useReactToolScheduler.test.tsx @@ -5,11 +5,15 @@ */ import { describe, it, expect } from 'vitest'; +import type { Part } from '@google/genai'; import { mapToDisplay, type TrackedToolCall } from './useReactToolScheduler.js'; // Build a minimal successful tracked tool call with the fields mapToDisplay's // success branch reads. `displayName` drives the collapsible gate. -const makeSuccess = (displayName: string): TrackedToolCall => +const makeSuccess = ( + displayName: string, + responseMedia: Part[] = [], +): TrackedToolCall => ({ status: 'success', request: { callId: 'call-1', name: 'read_file', args: {} }, @@ -23,6 +27,7 @@ const makeSuccess = (displayName: string): TrackedToolCall => id: 'call-1', name: 'read_file', response: { output: 'FULL FILE CONTENT' }, + ...(responseMedia.length > 0 ? { parts: responseMedia } : {}), }, }, ], @@ -45,4 +50,26 @@ describe('mapToDisplay — detailedDisplay (§4.9 live path)', () => { const group = mapToDisplay(makeSuccess('Edit')); expect(group.tools[0].detailedDisplay).toBeUndefined(); }); + + it('extracts nested inline images from tool response parts', () => { + const group = mapToDisplay( + makeSuccess('Read File', [ + { + inlineData: { + data: 'dG9vbC1pbWFnZQ==', + mimeType: 'image/png', + displayName: 'result.png', + }, + }, + ]), + ); + + expect(group.tools[0].images).toEqual([ + { + data: 'dG9vbC1pbWFnZQ==', + mimeType: 'image/png', + displayName: 'result.png', + }, + ]); + }); }); diff --git a/packages/cli/src/ui/hooks/useReactToolScheduler.ts b/packages/cli/src/ui/hooks/useReactToolScheduler.ts index 9ac33108558..5872fdf7250 100644 --- a/packages/cli/src/ui/hooks/useReactToolScheduler.ts +++ b/packages/cli/src/ui/hooks/useReactToolScheduler.ts @@ -38,6 +38,7 @@ import type { } from '../types.js'; import { ToolCallStatus } from '../types.js'; import { isCollapsibleTool } from '../components/messages/CompactToolGroupDisplay.js'; +import { extractInlineImages } from '../utils/inline-image-parts.js'; const debugLogger = createDebugLogger('REACT_TOOL_SCHEDULER'); @@ -386,7 +387,10 @@ export function mapToDisplay( }; switch (trackedCall.status) { - case 'success': + case 'success': { + const images = extractInlineImages( + trackedCall.response.responseParts, + ); return { ...baseDisplayProperties, status: mapCoreStatusToDisplayStatus(trackedCall.status), @@ -410,9 +414,14 @@ export function mapToDisplay( detailedDisplay: isCollapsibleTool(displayName) ? getToolResponseDisplayText(trackedCall.response.responseParts) : undefined, + ...(images.length > 0 ? { images } : {}), confirmationDetails: undefined, }; - case 'error': + } + case 'error': { + const images = extractInlineImages( + trackedCall.response.responseParts, + ); return { ...baseDisplayProperties, status: mapCoreStatusToDisplayStatus(trackedCall.status), @@ -424,9 +433,14 @@ export function mapToDisplay( visionBridgeNotice: trackedCall.response.visionBridgeNotice, } : {}), + ...(images.length > 0 ? { images } : {}), confirmationDetails: undefined, }; - case 'cancelled': + } + case 'cancelled': { + const images = extractInlineImages( + trackedCall.response.responseParts, + ); return { ...baseDisplayProperties, status: mapCoreStatusToDisplayStatus(trackedCall.status), @@ -438,8 +452,10 @@ export function mapToDisplay( visionBridgeNotice: trackedCall.response.visionBridgeNotice, } : {}), + ...(images.length > 0 ? { images } : {}), confirmationDetails: undefined, }; + } case 'awaiting_approval': return { ...baseDisplayProperties, diff --git a/packages/cli/src/ui/types.ts b/packages/cli/src/ui/types.ts index 29baace5db0..09a9896e11c 100644 --- a/packages/cli/src/ui/types.ts +++ b/packages/cli/src/ui/types.ts @@ -54,6 +54,12 @@ export enum ToolCallStatus { Error = 'Error', } +export interface InlineImageData { + data: string; + mimeType: string; + displayName?: string; +} + export interface ToolCallEvent { type: 'tool_call'; status: ToolCallStatus; @@ -79,6 +85,8 @@ export interface IndividualToolCallDisplay { * is only a count. Undefined → fall back to the summary. */ detailedDisplay?: string; + /** Inline images carried by this tool's persisted response parts. */ + images?: InlineImageData[]; status: ToolCallStatus; confirmationDetails: ToolCallConfirmationDetails | undefined; renderOutputAsMarkdown?: boolean; @@ -138,12 +146,14 @@ export type HistoryItemUser = HistoryItemBase & { export type HistoryItemGemini = HistoryItemBase & { type: 'gemini'; text: string; + images?: InlineImageData[]; timestamp?: number; }; export type HistoryItemGeminiContent = HistoryItemBase & { type: 'gemini_content'; text: string; + images?: InlineImageData[]; }; export type HistoryItemGeminiThought = HistoryItemBase & { diff --git a/packages/cli/src/ui/utils/inline-image-parts.ts b/packages/cli/src/ui/utils/inline-image-parts.ts new file mode 100644 index 00000000000..545a0f6d9f0 --- /dev/null +++ b/packages/cli/src/ui/utils/inline-image-parts.ts @@ -0,0 +1,86 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import type { Part } from '@google/genai'; +import type { InlineImageData } from '../types.js'; + +export type InlineContentRun = + | { kind: 'text'; text: string } + | { kind: 'image'; image: InlineImageData }; + +export function getInlineImageData(part: Part): InlineImageData | null { + const inlineData = part.inlineData; + if ( + !inlineData?.mimeType?.trim().toLowerCase().startsWith('image/') || + typeof inlineData.data !== 'string' || + inlineData.data.length === 0 + ) { + return null; + } + + return { + data: inlineData.data, + mimeType: inlineData.mimeType, + ...(typeof inlineData.displayName === 'string' + ? { displayName: inlineData.displayName } + : {}), + }; +} + +export function extractInlineImages( + parts: Part[] | undefined, +): InlineImageData[] { + if (!parts) { + return []; + } + + const images: InlineImageData[] = []; + for (const part of parts) { + const topLevelImage = getInlineImageData(part); + if (topLevelImage) { + images.push(topLevelImage); + } + + for (const nested of part.functionResponse?.parts ?? []) { + const nestedImage = getInlineImageData(nested as Part); + if (nestedImage) { + images.push(nestedImage); + } + } + } + return images; +} + +export function extractInlineContentRuns( + parts: Part[] | undefined, + textSeparator = '', +): InlineContentRun[] { + if (!parts) { + return []; + } + + const runs: InlineContentRun[] = []; + let textParts: string[] = []; + const flushText = () => { + if (textParts.length === 0) return; + runs.push({ kind: 'text', text: textParts.join(textSeparator) }); + textParts = []; + }; + + for (const part of parts) { + if (part.thought) continue; + if (part.text) { + textParts.push(part.text); + } + const image = getInlineImageData(part); + if (image) { + flushText(); + runs.push({ kind: 'image', image }); + } + } + flushText(); + return runs; +} diff --git a/packages/cli/src/ui/utils/mermaidImageRenderer.ts b/packages/cli/src/ui/utils/mermaidImageRenderer.ts index 594073a936e..75d885c4b57 100644 --- a/packages/cli/src/ui/utils/mermaidImageRenderer.ts +++ b/packages/cli/src/ui/utils/mermaidImageRenderer.ts @@ -4,13 +4,34 @@ * SPDX-License-Identifier: Apache-2.0 */ -import crypto from 'node:crypto'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; +import { createHash } from 'node:crypto'; import { spawn, spawnSync } from 'node:child_process'; - -export type TerminalImageProtocol = 'kitty' | 'iterm2'; +import { + buildKittyPlaceholder, + createKittyImageId, + detectTerminalImageProtocol as detectSharedTerminalImageProtocol, + encodeITerm2InlineImage, + encodeKittyVirtualImage, + readPngSize, + type ImageDimensions as PngSize, + type KittyImagePlaceholder, + type TerminalImageProtocol, +} from './terminal-image.js'; + +export { + buildKittyPlaceholder, + encodeITerm2InlineImage, + encodeKittyImage, + encodeKittyVirtualImage, + readPngSize, +} from './terminal-image.js'; +export type { + KittyImagePlaceholder, + TerminalImageProtocol, +} from './terminal-image.js'; export interface MermaidImageRenderOptions { source: string; @@ -46,17 +67,6 @@ export type MermaidImageRenderResult = | MermaidAnsiImageResult | MermaidImageUnavailableResult; -interface PngSize { - width: number; - height: number; -} - -export interface KittyImagePlaceholder { - color: string; - imageId: number; - lines: string[]; -} - const CACHE_LIMIT = 40; const PNG_CACHE_LIMIT = 20; const CACHE_BYTE_LIMIT = 32 * 1024 * 1024; @@ -68,8 +78,6 @@ const MAX_RENDERER_OUTPUT_CHARS = 16 * 1024; const MAX_RENDER_TIMEOUT_MS = 60_000; const OUTPUT_TRUNCATION_MARKER = '\n... renderer output truncated ...'; const NPX_MERMAID_CLI = 'npx:@mermaid-js/mermaid-cli@11.12.0'; -const PNG_SIGNATURE = '89504e470d0a1a0a'; -const KITTY_PLACEHOLDER = '\u{10EEEE}'; const RENDERER_ENV_ALLOWLIST = [ 'PATH', 'PATHEXT', @@ -88,136 +96,6 @@ const RENDERER_ENV_ALLOWLIST = [ 'PUPPETEER_CACHE_DIR', 'PLAYWRIGHT_BROWSERS_PATH', ] as const; -const KITTY_PLACEHOLDER_DIACRITICS = [ - '\u{305}', - '\u{30D}', - '\u{30E}', - '\u{310}', - '\u{312}', - '\u{33D}', - '\u{33E}', - '\u{33F}', - '\u{346}', - '\u{34A}', - '\u{34B}', - '\u{34C}', - '\u{350}', - '\u{351}', - '\u{352}', - '\u{357}', - '\u{35B}', - '\u{363}', - '\u{364}', - '\u{365}', - '\u{366}', - '\u{367}', - '\u{368}', - '\u{369}', - '\u{36A}', - '\u{36B}', - '\u{36C}', - '\u{36D}', - '\u{36E}', - '\u{36F}', - '\u{483}', - '\u{484}', - '\u{485}', - '\u{486}', - '\u{487}', - '\u{592}', - '\u{593}', - '\u{594}', - '\u{595}', - '\u{597}', - '\u{598}', - '\u{599}', - '\u{59C}', - '\u{59D}', - '\u{59E}', - '\u{59F}', - '\u{5A0}', - '\u{5A1}', - '\u{5A8}', - '\u{5A9}', - '\u{5AB}', - '\u{5AC}', - '\u{5AF}', - '\u{5C4}', - '\u{610}', - '\u{611}', - '\u{612}', - '\u{613}', - '\u{614}', - '\u{615}', - '\u{616}', - '\u{617}', - '\u{657}', - '\u{658}', - '\u{659}', - '\u{65A}', - '\u{65B}', - '\u{65D}', - '\u{65E}', - '\u{6D6}', - '\u{6D7}', - '\u{6D8}', - '\u{6D9}', - '\u{6DA}', - '\u{6DB}', - '\u{6DC}', - '\u{6DF}', - '\u{6E0}', - '\u{6E1}', - '\u{6E2}', - '\u{6E4}', - '\u{6E7}', - '\u{6E8}', - '\u{6EB}', - '\u{6EC}', - '\u{730}', - '\u{732}', - '\u{733}', - '\u{735}', - '\u{736}', - '\u{73A}', - '\u{73D}', - '\u{73F}', - '\u{740}', - '\u{741}', - '\u{743}', - '\u{745}', - '\u{747}', - '\u{749}', - '\u{74A}', - '\u{7EB}', - '\u{7EC}', - '\u{7ED}', - '\u{7EE}', - '\u{7EF}', - '\u{7F0}', - '\u{7F1}', - '\u{7F3}', - '\u{816}', - '\u{817}', - '\u{818}', - '\u{819}', - '\u{81B}', - '\u{81C}', - '\u{81D}', - '\u{81E}', - '\u{81F}', - '\u{820}', - '\u{821}', - '\u{822}', - '\u{823}', - '\u{825}', - '\u{826}', - '\u{827}', - '\u{829}', - '\u{82A}', - '\u{82B}', - '\u{82C}', -]; const cachedResults = new Map(); const cachedPngResults = new Map< string, @@ -229,130 +107,10 @@ let cachedPngResultsBytes = 0; export function detectTerminalImageProtocol( env: NodeJS.ProcessEnv = process.env, ): TerminalImageProtocol | null { - if (env['QWEN_CODE_DISABLE_MERMAID_IMAGES'] === '1') { - return null; - } - - const forced = env['QWEN_CODE_MERMAID_IMAGE_PROTOCOL']?.toLowerCase(); - if (forced === 'off' || forced === 'none' || forced === '0') { - return null; - } - - if ( - !process.stdout.isTTY || - env['TMUX'] || - env['SSH_TTY'] || - env['SSH_CLIENT'] - ) { - return null; - } - - if (forced) { - if (forced === 'kitty') return 'kitty'; - if (forced === 'iterm' || forced === 'iterm2') return 'iterm2'; - } - - const term = env['TERM']?.toLowerCase() ?? ''; - const termProgram = env['TERM_PROGRAM']?.toLowerCase() ?? ''; - - if ( - env['KITTY_WINDOW_ID'] || - term.includes('kitty') || - termProgram.includes('ghostty') - ) { - return 'kitty'; - } - - if (termProgram === 'iterm.app' || termProgram.includes('wezterm')) { - return 'iterm2'; - } - - return null; -} - -export function encodeITerm2InlineImage( - png: Buffer, - widthCells: number, - rows: number, -): string { - return `\u001b]1337;File=inline=1;width=${widthCells};height=${rows};preserveAspectRatio=1:${png.toString( - 'base64', - )}\u0007`; -} - -export function encodeKittyImage( - png: Buffer, - widthCells: number, - rows: number, -): string { - return encodeKittyImageCommand(png, `a=T,f=100,c=${widthCells},r=${rows}`); -} - -export function encodeKittyVirtualImage( - png: Buffer, - imageId: number, - widthCells: number, - rows: number, -): string { - return encodeKittyImageCommand( - png, - `a=T,f=100,i=${imageId},q=2,U=1,c=${widthCells},r=${rows}`, - ); -} - -function encodeKittyImageCommand(png: Buffer, firstControl: string): string { - const encoded = png.toString('base64'); - const chunkSize = 4096; - const chunks: string[] = []; - - for (let offset = 0; offset < encoded.length; offset += chunkSize) { - const chunk = encoded.slice(offset, offset + chunkSize); - const hasMore = offset + chunkSize < encoded.length; - const control = - offset === 0 - ? `${firstControl},m=${hasMore ? 1 : 0}` - : `m=${hasMore ? 1 : 0}`; - chunks.push(`\u001b_G${control};${chunk}\u001b\\`); - } - - return chunks.join(''); -} - -export function buildKittyPlaceholder( - imageId: number, - widthCells: number, - rows: number, -): KittyImagePlaceholder { - const clampedRows = Math.min(rows, KITTY_PLACEHOLDER_DIACRITICS.length); - const clampedWidth = Math.min( - widthCells, - KITTY_PLACEHOLDER_DIACRITICS.length, - ); - const lines = Array.from({ length: clampedRows }, (_, row) => { - const rowDiacritic = KITTY_PLACEHOLDER_DIACRITICS[row]; - const cells = Array.from({ length: clampedWidth }, (_, column) => { - const columnDiacritic = KITTY_PLACEHOLDER_DIACRITICS[column]; - return `${KITTY_PLACEHOLDER}${rowDiacritic}${columnDiacritic}`; - }); - return cells.join(''); + return detectSharedTerminalImageProtocol(env, { + disabled: env['QWEN_CODE_DISABLE_MERMAID_IMAGES'] === '1', + forceProtocol: env['QWEN_CODE_MERMAID_IMAGE_PROTOCOL'], }); - - return { - color: `#${imageId.toString(16).padStart(6, '0')}`, - imageId, - lines, - }; -} - -export function readPngSize(png: Buffer): PngSize | null { - if (png.length < 24 || png.subarray(0, 8).toString('hex') !== PNG_SIGNATURE) { - return null; - } - - return { - width: png.readUInt32BE(16), - height: png.readUInt32BE(20), - }; } function isMermaidImageRenderingDisabled(env: NodeJS.ProcessEnv): boolean { @@ -676,22 +434,6 @@ export async function renderMermaidImageAsync({ }); } -function createKittyImageId( - png: Buffer, - imageShape: { widthCells: number; rows: number }, -): number { - const hash = crypto - .createHash('sha256') - .update(png) - .update('\0') - .update(String(imageShape.widthCells)) - .update('\0') - .update(String(imageShape.rows)) - .digest(); - const id = hash.readUIntBE(0, 3); - return id === 0 ? 1 : id; -} - function getResultCache(key: string): MermaidImageRenderResult | undefined { const cached = cachedResults.get(key); if (cached) { @@ -717,8 +459,7 @@ function createPngCacheKey( mmdc: string, env: NodeJS.ProcessEnv, ): string { - return crypto - .createHash('sha256') + return createHash('sha256') .update(source) .update('\0') .update(mmdc) @@ -735,8 +476,7 @@ function createCacheKey( mmdc: string, env: NodeJS.ProcessEnv, ): string { - return crypto - .createHash('sha256') + return createHash('sha256') .update(source) .update('\0') .update(String(contentWidth)) diff --git a/packages/cli/src/ui/utils/resumeHistoryUtils.test.ts b/packages/cli/src/ui/utils/resumeHistoryUtils.test.ts index 0a6bbdcd993..7f30e885687 100644 --- a/packages/cli/src/ui/utils/resumeHistoryUtils.test.ts +++ b/packages/cli/src/ui/utils/resumeHistoryUtils.test.ts @@ -805,6 +805,128 @@ describe('resumeHistoryUtils', () => { expect(items[0]).not.toHaveProperty('sentToModel'); }); + it('restores assistant text and images in their original order', () => { + const conversation = { + messages: [ + { + type: 'assistant', + timestamp: '2026-01-15T19:00:00.000Z', + message: { + parts: [ + { text: 'before' } as Part, + { + inlineData: { + data: 'aW1hZ2U=', + mimeType: 'image/png', + displayName: 'chart.png', + }, + } as Part, + { text: 'after' } as Part, + ], + }, + }, + ], + } as unknown as ConversationRecord; + + const items = buildResumedHistoryItems( + { conversation } as ResumedSessionData, + makeConfig({}), + 100, + ); + + expect(items).toEqual([ + { + id: 101, + type: 'gemini', + text: 'before', + timestamp: new Date('2026-01-15T19:00:00.000Z').getTime(), + }, + { + id: 102, + type: 'gemini_content', + text: '', + images: [ + { + data: 'aW1hZ2U=', + mimeType: 'image/png', + displayName: 'chart.png', + }, + ], + }, + { id: 103, type: 'gemini_content', text: 'after' }, + ]); + }); + + it('restores images nested in persisted tool response parts', () => { + const conversation = { + messages: [ + { + type: 'assistant', + message: { + parts: [ + { + functionCall: { + id: 'call-image', + name: 'replace', + args: {}, + }, + } as unknown as Part, + ], + }, + }, + { + type: 'tool_result', + toolCallResult: { + callId: 'call-image', + resultDisplay: 'Generated chart', + status: 'success', + responseParts: [ + { + functionResponse: { + id: 'call-image', + name: 'replace', + response: { output: 'Generated chart' }, + parts: [ + { + inlineData: { + data: 'dG9vbC1pbWFnZQ==', + mimeType: 'image/webp', + }, + }, + ], + }, + }, + ], + }, + }, + ], + } as unknown as ConversationRecord; + + const items = buildResumedHistoryItems( + { conversation } as ResumedSessionData, + makeConfig({ replace: mockTool }), + 200, + ); + + expect(items).toEqual([ + { + id: 201, + type: 'tool_group', + tools: [ + expect.objectContaining({ + callId: 'call-image', + images: [ + { + data: 'dG9vbC1pbWFnZQ==', + mimeType: 'image/webp', + }, + ], + }), + ], + }, + ]); + }); + describe('detailedDisplay (§4.9 Ctrl+O full detail on resume)', () => { type ToolGroupItem = Extract; const firstTool = (items: HistoryItem[]) => diff --git a/packages/cli/src/ui/utils/resumeHistoryUtils.ts b/packages/cli/src/ui/utils/resumeHistoryUtils.ts index 385898bf9de..e104101c594 100644 --- a/packages/cli/src/ui/utils/resumeHistoryUtils.ts +++ b/packages/cli/src/ui/utils/resumeHistoryUtils.ts @@ -27,6 +27,7 @@ import type { HistoryItemInfo, HistoryItemWithoutId, IndividualToolCallDisplay, + InlineImageData, } from '../types.js'; import { ToolCallStatus, MessageType } from '../types.js'; import { t } from '../../i18n/index.js'; @@ -36,6 +37,10 @@ import { indexGapsByChild, } from './history-gap-notice.js'; import { shouldDisplayGoalStateCause } from './goal-runtime.js'; +import { + extractInlineContentRuns, + extractInlineImages, +} from './inline-image-parts.js'; /** * Projects a plain user record to its display text. @@ -217,6 +222,7 @@ function convertToHistoryItems( resultDisplay: ToolResultDisplay | undefined; visionBridgeNotice?: string; detailedDisplay?: string; + images?: InlineImageData[]; status: ToolCallStatus; confirmationDetails: undefined; }> = []; @@ -439,8 +445,7 @@ function convertToHistoryItems( // verbatim because there is no live loading area in that view. const thoughtText = !config ? extractThoughtTextFromParts(parts) : ''; - // Extract text content (non-function-call, non-thought) - const text = extractTextFromParts(parts); + const displayRuns = extractInlineContentRuns(parts, '\n'); // Extract function calls const functionCalls = extractFunctionCalls(parts); @@ -458,9 +463,8 @@ function convertToHistoryItems( items.push({ type: 'gemini_thought', text: thoughtText }); } - // If there's text content, add it as a gemini message - if (text) { - // Flush any pending tool group before text + if (displayRuns.length > 0) { + // Flush any pending tool group before assistant output. if (currentToolGroup.length > 0) { items.push({ type: 'tool_group', @@ -468,11 +472,27 @@ function convertToHistoryItems( }); currentToolGroup = []; } - items.push({ - type: 'gemini', - text, - timestamp: new Date(record.timestamp).getTime(), - }); + for (const [index, run] of displayRuns.entries()) { + const type = index === 0 ? 'gemini' : 'gemini_content'; + items.push( + run.kind === 'text' + ? { + type, + text: run.text, + ...(index === 0 + ? { timestamp: new Date(record.timestamp).getTime() } + : {}), + } + : { + type, + text: '', + images: [run.image], + ...(index === 0 + ? { timestamp: new Date(record.timestamp).getTime() } + : {}), + }, + ); + } } // Track function calls for pairing with results @@ -500,6 +520,9 @@ function convertToHistoryItems( const callId = record.toolCallResult.callId; const toolCall = currentToolGroup.find((t) => t.callId === callId); if (toolCall) { + const responseParts = + (record.toolCallResult.responseParts as Part[] | undefined) ?? + (record.message?.parts as Part[] | undefined); // Preserve the resultDisplay as-is - it can be a string or structured object const rawDisplay = record.toolCallResult.resultDisplay; toolCall.resultDisplay = rawDisplay; @@ -515,6 +538,10 @@ function convertToHistoryItems( rawStatus === 'error' ? ToolCallStatus.Error : ToolCallStatus.Success; + const images = extractInlineImages(responseParts); + if (images.length > 0) { + toolCall.images = images; + } // Full detail for the Ctrl+O transcript (§4.9): the complete // functionResponse parts are persisted on the tool_result record // (only resultDisplay is sanitized), so resume yields full detail @@ -529,10 +556,8 @@ function convertToHistoryItems( toolCall.status === ToolCallStatus.Success && isCollapsibleTool(toolCall.name) ) { - toolCall.detailedDisplay = getToolResponseDisplayText( - (record.toolCallResult.responseParts as Part[] | undefined) ?? - (record.message?.parts as Part[] | undefined), - ); + toolCall.detailedDisplay = + getToolResponseDisplayText(responseParts); } } pendingToolCalls.delete(callId || ''); diff --git a/packages/cli/src/ui/utils/terminal-image.test.ts b/packages/cli/src/ui/utils/terminal-image.test.ts new file mode 100644 index 00000000000..2c299f03b16 --- /dev/null +++ b/packages/cli/src/ui/utils/terminal-image.test.ts @@ -0,0 +1,191 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from 'vitest'; +import { + detectTerminalImageProtocol, + fitTerminalImage, + formatImageFallback, + prepareTerminalImage, + readImageSize, +} from './terminal-image.js'; + +const PNG_1X1 = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=', + 'base64', +); +const JPEG_2X1 = Buffer.from([ + 0xff, 0xd8, 0xff, 0xc0, 0x00, 0x11, 0x08, 0x00, 0x01, 0x00, 0x02, 0x03, 0x01, + 0x11, 0x00, 0x02, 0x11, 0x00, 0x03, 0x11, 0x00, 0xff, 0xd9, +]); +const WEBP_VP8X_3X2 = Buffer.from([ + 0x52, 0x49, 0x46, 0x46, 0x16, 0x00, 0x00, 0x00, 0x57, 0x45, 0x42, 0x50, 0x56, + 0x50, 0x38, 0x58, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, + 0x00, 0x01, 0x00, 0x00, +]); + +describe('terminal image protocol detection', () => { + it.each([ + [{ TERM: 'xterm-kitty' }, 'kitty'], + [{ TERM_PROGRAM: 'Ghostty' }, 'kitty'], + [{ TERM_PROGRAM: 'WezTerm' }, 'iterm2'], + [{ WARP_SESSION_ID: 'session' }, 'iterm2'], + [{ TERM_PROGRAM: 'iTerm.app' }, 'iterm2'], + ] as const)('detects supported terminal environments', (env, protocol) => { + expect(detectTerminalImageProtocol(env, { isTTY: true })).toBe(protocol); + }); + + it('disables images for multiplexed, remote, and non-TTY sessions', () => { + expect( + detectTerminalImageProtocol( + { TERM: 'xterm-kitty', TMUX: 'session' }, + { isTTY: true }, + ), + ).toBeNull(); + expect( + detectTerminalImageProtocol( + { TERM: 'xterm-kitty', SSH_CONNECTION: 'remote' }, + { isTTY: true }, + ), + ).toBeNull(); + expect( + detectTerminalImageProtocol({ TERM: 'xterm-kitty' }, { isTTY: false }), + ).toBeNull(); + }); + + it('supports explicit protocol selection and opt-out', () => { + expect( + detectTerminalImageProtocol({}, { isTTY: true, forceProtocol: 'kitty' }), + ).toBe('kitty'); + expect( + detectTerminalImageProtocol({}, { isTTY: true, forceProtocol: 'off' }), + ).toBeNull(); + expect( + detectTerminalImageProtocol( + { QWEN_CODE_DISABLE_TERMINAL_IMAGES: '1' }, + { isTTY: true, forceProtocol: 'kitty' }, + ), + ).toBeNull(); + }); +}); + +describe('terminal image preparation', () => { + it('prepares PNG data for Kitty virtual placement', () => { + const result = prepareTerminalImage({ + data: PNG_1X1.toString('base64'), + mimeType: 'image/png', + contentWidth: 40, + availableTerminalHeight: 12, + env: {}, + detection: { isTTY: true, forceProtocol: 'kitty' }, + }); + + expect(result).toMatchObject({ + kind: 'terminal-image', + protocol: 'kitty', + dimensions: { width: 1, height: 1 }, + fallbackText: '[image: 1x1 png]', + }); + expect(result.kind === 'terminal-image' && result.sequence).toContain( + '\u001b_Ga=T,f=100', + ); + expect( + result.kind === 'terminal-image' && result.placeholder?.lines, + ).toHaveLength(12); + }); + + it('prepares JPEG data for iTerm2 and rejects it for Kitty', () => { + const common = { + data: JPEG_2X1.toString('base64'), + mimeType: 'image/jpeg', + contentWidth: 40, + env: {}, + }; + + const iterm = prepareTerminalImage({ + ...common, + detection: { isTTY: true, forceProtocol: 'iterm2' }, + }); + expect(iterm).toMatchObject({ + kind: 'terminal-image', + protocol: 'iterm2', + dimensions: { width: 2, height: 1 }, + }); + expect(iterm.kind === 'terminal-image' && iterm.sequence).toContain( + '\u001b]1337;File=inline=1', + ); + + expect( + prepareTerminalImage({ + ...common, + detection: { isTTY: true, forceProtocol: 'kitty' }, + }), + ).toMatchObject({ + kind: 'fallback', + text: '[image: 2x1 jpeg]', + reason: 'unsupported-protocol-format', + }); + }); + + it('returns descriptive fallbacks for invalid data and unsupported terminals', () => { + expect( + prepareTerminalImage({ + data: 'not base64!', + mimeType: 'image/png', + contentWidth: 40, + }), + ).toEqual({ + kind: 'fallback', + text: '[image: png]', + reason: 'invalid-data', + }); + expect( + prepareTerminalImage({ + data: 'A'.repeat(Math.ceil((8 * 1024 * 1024 * 4) / 3) + 5), + mimeType: 'image/png', + contentWidth: 40, + }), + ).toMatchObject({ kind: 'fallback', reason: 'invalid-data' }); + + expect( + prepareTerminalImage({ + data: PNG_1X1.toString('base64'), + mimeType: 'image/png', + contentWidth: 40, + env: {}, + detection: { isTTY: true }, + }), + ).toMatchObject({ + kind: 'fallback', + text: '[image: 1x1 png]', + reason: 'unsupported-terminal', + }); + }); + + it('reads supported image headers and fits within terminal bounds', () => { + expect(readImageSize(JPEG_2X1, ' IMAGE/JPEG ')).toEqual({ + width: 2, + height: 1, + }); + expect(readImageSize(WEBP_VP8X_3X2, 'image/webp')).toEqual({ + width: 3, + height: 2, + }); + const invalidPng = Buffer.from(PNG_1X1); + invalidPng.write('NOPE', 12, 'ascii'); + expect(readImageSize(invalidPng, 'image/png')).toBeNull(); + expect(fitTerminalImage({ width: 100, height: 200 }, 200, 10)).toEqual({ + widthCells: 10, + rows: 10, + }); + expect( + fitTerminalImage({ width: 100, height: 200 }, Number.NaN, Number.NaN), + ).toEqual({ widthCells: 1, rows: 1 }); + expect(formatImageFallback('image/webp', { width: 20, height: 10 })).toBe( + '[image: 20x10 webp]', + ); + }); +}); diff --git a/packages/cli/src/ui/utils/terminal-image.ts b/packages/cli/src/ui/utils/terminal-image.ts new file mode 100644 index 00000000000..247e36ad3c3 --- /dev/null +++ b/packages/cli/src/ui/utils/terminal-image.ts @@ -0,0 +1,718 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import crypto from 'node:crypto'; + +export type TerminalImageProtocol = 'kitty' | 'iterm2'; + +export interface ImageDimensions { + width: number; + height: number; +} + +export interface TerminalImageDetectionOptions { + disabled?: boolean; + forceProtocol?: string; + isTTY?: boolean; +} + +export interface KittyImagePlaceholder { + color: string; + imageId: number; + lines: string[]; +} + +export interface PrepareTerminalImageOptions { + data: string; + mimeType: string; + contentWidth: number; + availableTerminalHeight?: number; + env?: NodeJS.ProcessEnv; + detection?: TerminalImageDetectionOptions; +} + +export interface PreparedTerminalImage { + kind: 'terminal-image'; + sequence: string; + rows: number; + widthCells: number; + protocol: TerminalImageProtocol; + dimensions: ImageDimensions; + fallbackText: string; + placeholder?: KittyImagePlaceholder; +} + +export interface TerminalImageFallback { + kind: 'fallback'; + text: string; + dimensions?: ImageDimensions; + reason: + | 'invalid-mime-type' + | 'invalid-data' + | 'unsupported-format' + | 'unsupported-terminal' + | 'unsupported-protocol-format'; +} + +export type TerminalImageRenderResult = + | PreparedTerminalImage + | TerminalImageFallback; + +const MAX_INLINE_IMAGE_BYTES = 8 * 1024 * 1024; +const MAX_IMAGE_DIMENSION = 1_000_000; +const DEFAULT_MAX_IMAGE_ROWS = 24; +const MAX_IMAGE_ROWS = 32; +const MAX_IMAGE_COLUMNS = 80; +const DEFAULT_CELL_ASPECT_RATIO = 0.5; +const PNG_SIGNATURE = '89504e470d0a1a0a'; +const KITTY_PLACEHOLDER = '\u{10EEEE}'; +const KITTY_PLACEHOLDER_DIACRITICS = [ + '\u{305}', + '\u{30D}', + '\u{30E}', + '\u{310}', + '\u{312}', + '\u{33D}', + '\u{33E}', + '\u{33F}', + '\u{346}', + '\u{34A}', + '\u{34B}', + '\u{34C}', + '\u{350}', + '\u{351}', + '\u{352}', + '\u{357}', + '\u{35B}', + '\u{363}', + '\u{364}', + '\u{365}', + '\u{366}', + '\u{367}', + '\u{368}', + '\u{369}', + '\u{36A}', + '\u{36B}', + '\u{36C}', + '\u{36D}', + '\u{36E}', + '\u{36F}', + '\u{483}', + '\u{484}', + '\u{485}', + '\u{486}', + '\u{487}', + '\u{592}', + '\u{593}', + '\u{594}', + '\u{595}', + '\u{597}', + '\u{598}', + '\u{599}', + '\u{59C}', + '\u{59D}', + '\u{59E}', + '\u{59F}', + '\u{5A0}', + '\u{5A1}', + '\u{5A8}', + '\u{5A9}', + '\u{5AB}', + '\u{5AC}', + '\u{5AF}', + '\u{5C4}', + '\u{610}', + '\u{611}', + '\u{612}', + '\u{613}', + '\u{614}', + '\u{615}', + '\u{616}', + '\u{617}', + '\u{657}', + '\u{658}', + '\u{659}', + '\u{65A}', + '\u{65B}', + '\u{65D}', + '\u{65E}', + '\u{6D6}', + '\u{6D7}', + '\u{6D8}', + '\u{6D9}', + '\u{6DA}', + '\u{6DB}', + '\u{6DC}', + '\u{6DF}', + '\u{6E0}', + '\u{6E1}', + '\u{6E2}', + '\u{6E4}', + '\u{6E7}', + '\u{6E8}', + '\u{6EB}', + '\u{6EC}', + '\u{730}', + '\u{732}', + '\u{733}', + '\u{735}', + '\u{736}', + '\u{73A}', + '\u{73D}', + '\u{73F}', + '\u{740}', + '\u{741}', + '\u{743}', + '\u{745}', + '\u{747}', + '\u{749}', + '\u{74A}', + '\u{7EB}', + '\u{7EC}', + '\u{7ED}', + '\u{7EE}', + '\u{7EF}', + '\u{7F0}', + '\u{7F1}', + '\u{7F3}', + '\u{816}', + '\u{817}', + '\u{818}', + '\u{819}', + '\u{81B}', + '\u{81C}', + '\u{81D}', + '\u{81E}', + '\u{81F}', + '\u{820}', + '\u{821}', + '\u{822}', + '\u{823}', + '\u{825}', + '\u{826}', + '\u{827}', + '\u{829}', + '\u{82A}', + '\u{82B}', + '\u{82C}', +] as const; + +function normalizeForcedProtocol( + value: string | undefined, +): TerminalImageProtocol | null | undefined { + const normalized = value?.toLowerCase(); + if (!normalized) { + return undefined; + } + if (normalized === 'off' || normalized === 'none' || normalized === '0') { + return null; + } + if (normalized === 'kitty') { + return 'kitty'; + } + if (normalized === 'iterm' || normalized === 'iterm2') { + return 'iterm2'; + } + return undefined; +} + +export function detectTerminalImageProtocol( + env: NodeJS.ProcessEnv = process.env, + options: TerminalImageDetectionOptions = {}, +): TerminalImageProtocol | null { + if (options.disabled || env['QWEN_CODE_DISABLE_TERMINAL_IMAGES'] === '1') { + return null; + } + + const term = env['TERM']?.toLowerCase() ?? ''; + const isMultiplexed = + Boolean(env['TMUX']) || + Boolean(env['STY']) || + term.startsWith('tmux') || + term.startsWith('screen'); + const isRemote = + Boolean(env['SSH_TTY']) || + Boolean(env['SSH_CLIENT']) || + Boolean(env['SSH_CONNECTION']); + if ((options.isTTY ?? process.stdout.isTTY === true) !== true) { + return null; + } + if (isMultiplexed || isRemote) { + return null; + } + + const forced = normalizeForcedProtocol( + options.forceProtocol ?? env['QWEN_CODE_TERMINAL_IMAGE_PROTOCOL'], + ); + if (forced !== undefined) { + return forced; + } + + const termProgram = env['TERM_PROGRAM']?.toLowerCase() ?? ''; + if ( + env['KITTY_WINDOW_ID'] || + env['GHOSTTY_RESOURCES_DIR'] || + term.includes('kitty') || + term.includes('ghostty') || + termProgram === 'kitty' || + termProgram.includes('ghostty') + ) { + return 'kitty'; + } + + if (env['WEZTERM_PANE'] || termProgram.includes('wezterm')) { + return 'iterm2'; + } + + if ( + env['WARP_SESSION_ID'] || + env['WARP_TERMINAL_SESSION_UUID'] || + termProgram === 'warpterminal' + ) { + return 'iterm2'; + } + + if (env['ITERM_SESSION_ID'] || termProgram === 'iterm.app') { + return 'iterm2'; + } + + return null; +} + +export function encodeITerm2InlineImage( + image: Buffer, + widthCells: number, + rows: number, +): string { + return `\u001b]1337;File=inline=1;width=${widthCells};height=${rows};preserveAspectRatio=1:${image.toString( + 'base64', + )}\u0007`; +} + +export function encodeKittyImage( + png: Buffer, + widthCells: number, + rows: number, +): string { + return encodeKittyImageCommand(png, `a=T,f=100,c=${widthCells},r=${rows}`); +} + +export function encodeKittyVirtualImage( + png: Buffer, + imageId: number, + widthCells: number, + rows: number, +): string { + return encodeKittyImageCommand( + png, + `a=T,f=100,i=${imageId},q=2,U=1,c=${widthCells},r=${rows}`, + ); +} + +function encodeKittyImageCommand(image: Buffer, firstControl: string): string { + const encoded = image.toString('base64'); + const chunkSize = 4096; + const chunks: string[] = []; + + for (let offset = 0; offset < encoded.length; offset += chunkSize) { + const chunk = encoded.slice(offset, offset + chunkSize); + const hasMore = offset + chunkSize < encoded.length; + const control = + offset === 0 + ? `${firstControl},m=${hasMore ? 1 : 0}` + : `m=${hasMore ? 1 : 0}`; + chunks.push(`\u001b_G${control};${chunk}\u001b\\`); + } + + return chunks.join(''); +} + +export function buildKittyPlaceholder( + imageId: number, + widthCells: number, + rows: number, +): KittyImagePlaceholder { + const clampedRows = Math.min(rows, KITTY_PLACEHOLDER_DIACRITICS.length); + const clampedWidth = Math.min( + widthCells, + KITTY_PLACEHOLDER_DIACRITICS.length, + ); + const lines = Array.from({ length: clampedRows }, (_, row) => { + const rowDiacritic = KITTY_PLACEHOLDER_DIACRITICS[row]; + const cells = Array.from({ length: clampedWidth }, (_, column) => { + const columnDiacritic = KITTY_PLACEHOLDER_DIACRITICS[column]; + return `${KITTY_PLACEHOLDER}${rowDiacritic}${columnDiacritic}`; + }); + return cells.join(''); + }); + + return { + color: `#${imageId.toString(16).padStart(6, '0')}`, + imageId, + lines, + }; +} + +function validDimensions( + width: number, + height: number, +): ImageDimensions | null { + if ( + !Number.isInteger(width) || + !Number.isInteger(height) || + width <= 0 || + height <= 0 || + width > MAX_IMAGE_DIMENSION || + height > MAX_IMAGE_DIMENSION + ) { + return null; + } + return { width, height }; +} + +export function readPngSize(png: Buffer): ImageDimensions | null { + if ( + png.length < 24 || + png.subarray(0, 8).toString('hex') !== PNG_SIGNATURE || + png.readUInt32BE(8) !== 13 || + png.subarray(12, 16).toString('ascii') !== 'IHDR' + ) { + return null; + } + + return validDimensions(png.readUInt32BE(16), png.readUInt32BE(20)); +} + +function readJpegSize(jpeg: Buffer): ImageDimensions | null { + if (jpeg.length < 4 || jpeg[0] !== 0xff || jpeg[1] !== 0xd8) { + return null; + } + + let offset = 2; + while (offset + 8 < jpeg.length) { + if (jpeg[offset] !== 0xff) { + offset += 1; + continue; + } + + while (offset < jpeg.length && jpeg[offset] === 0xff) { + offset += 1; + } + const marker = jpeg[offset]; + if (marker === undefined || marker === 0xd9 || marker === 0xda) { + return null; + } + if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) { + offset += 1; + continue; + } + if (offset + 2 >= jpeg.length) { + return null; + } + + const segmentLength = jpeg.readUInt16BE(offset + 1); + if (segmentLength < 2 || offset + 1 + segmentLength > jpeg.length) { + return null; + } + const isStartOfFrame = + (marker >= 0xc0 && marker <= 0xc3) || + (marker >= 0xc5 && marker <= 0xc7) || + (marker >= 0xc9 && marker <= 0xcb) || + (marker >= 0xcd && marker <= 0xcf); + if (isStartOfFrame) { + if (segmentLength < 7) { + return null; + } + return validDimensions( + jpeg.readUInt16BE(offset + 6), + jpeg.readUInt16BE(offset + 4), + ); + } + offset += segmentLength + 1; + } + + return null; +} + +function readGifSize(gif: Buffer): ImageDimensions | null { + if (gif.length < 10) { + return null; + } + const signature = gif.subarray(0, 6).toString('ascii'); + if (signature !== 'GIF87a' && signature !== 'GIF89a') { + return null; + } + return validDimensions(gif.readUInt16LE(6), gif.readUInt16LE(8)); +} + +function readWebpSize(webp: Buffer): ImageDimensions | null { + if ( + webp.length < 21 || + webp.subarray(0, 4).toString('ascii') !== 'RIFF' || + webp.subarray(8, 12).toString('ascii') !== 'WEBP' || + webp.readUInt32LE(4) + 8 > webp.length + ) { + return null; + } + + const chunkType = webp.subarray(12, 16).toString('ascii'); + const chunkLength = webp.readUInt32LE(16); + if (chunkLength > webp.length - 20) { + return null; + } + if (chunkType === 'VP8 ') { + if ( + chunkLength < 10 || + webp.length < 30 || + webp[23] !== 0x9d || + webp[24] !== 0x01 || + webp[25] !== 0x2a + ) { + return null; + } + return validDimensions( + webp.readUInt16LE(26) & 0x3fff, + webp.readUInt16LE(28) & 0x3fff, + ); + } + if (chunkType === 'VP8L') { + if (chunkLength < 5 || webp.length < 25 || webp[20] !== 0x2f) { + return null; + } + const bits = webp.readUInt32LE(21); + return validDimensions((bits & 0x3fff) + 1, ((bits >>> 14) & 0x3fff) + 1); + } + if (chunkType === 'VP8X') { + if (chunkLength < 10 || webp.length < 30) { + return null; + } + const width = + (webp[24] ?? 0) | ((webp[25] ?? 0) << 8) | ((webp[26] ?? 0) << 16); + const height = + (webp[27] ?? 0) | ((webp[28] ?? 0) << 8) | ((webp[29] ?? 0) << 16); + return validDimensions(width + 1, height + 1); + } + + return null; +} + +export function readImageSize( + image: Buffer, + mimeType: string, +): ImageDimensions | null { + switch (mimeType.trim().toLowerCase()) { + case 'image/png': + return readPngSize(image); + case 'image/jpeg': + case 'image/jpg': + return readJpegSize(image); + case 'image/gif': + return readGifSize(image); + case 'image/webp': + return readWebpSize(image); + default: + return null; + } +} + +function getImageFormat(mimeType: string): string | null { + const normalized = mimeType.trim().toLowerCase(); + const match = /^image\/([a-z0-9][a-z0-9.+-]*)$/.exec(normalized); + return match?.[1] ?? null; +} + +export function formatImageFallback( + mimeType: string, + dimensions?: ImageDimensions, +): string { + const format = getImageFormat(mimeType); + if (!format) { + return '[image]'; + } + const size = dimensions ? `${dimensions.width}x${dimensions.height} ` : ''; + return `[image: ${size}${format}]`; +} + +function decodeBase64Image(data: string): Buffer | null { + const maxEncodedLength = Math.ceil((MAX_INLINE_IMAGE_BYTES * 4) / 3) + 4; + if (data.length === 0 || data.length > maxEncodedLength) { + return null; + } + + const normalized = data.replace(/\s/g, ''); + if ( + normalized.length === 0 || + normalized.length % 4 === 1 || + !/^[A-Za-z0-9+/]*={0,2}$/.test(normalized) + ) { + return null; + } + + const decoded = Buffer.from(normalized, 'base64'); + if (decoded.length === 0 || decoded.length > MAX_INLINE_IMAGE_BYTES) { + return null; + } + const canonicalInput = normalized.replace(/=+$/, ''); + const canonicalDecoded = decoded.toString('base64').replace(/=+$/, ''); + return canonicalInput === canonicalDecoded ? decoded : null; +} + +export function fitTerminalImage( + dimensions: ImageDimensions, + contentWidth: number, + availableTerminalHeight?: number, +): { widthCells: number; rows: number } { + const requestedWidth = Number.isFinite(contentWidth) + ? Math.floor(contentWidth) + : 1; + const maxWidth = Math.max(1, Math.min(requestedWidth, MAX_IMAGE_COLUMNS)); + const requestedRows = + availableTerminalHeight === undefined + ? DEFAULT_MAX_IMAGE_ROWS + : Number.isFinite(availableTerminalHeight) + ? Math.floor(availableTerminalHeight) + : 1; + const maxRows = Math.max(1, Math.min(requestedRows, MAX_IMAGE_ROWS)); + const naturalRows = Math.max( + 1, + Math.ceil( + (dimensions.height / dimensions.width) * + maxWidth * + DEFAULT_CELL_ASPECT_RATIO, + ), + ); + if (naturalRows <= maxRows) { + return { widthCells: maxWidth, rows: naturalRows }; + } + + const widthCells = Math.max( + 1, + Math.floor((maxWidth * maxRows) / naturalRows), + ); + const rows = Math.max( + 1, + Math.min( + maxRows, + Math.ceil( + (dimensions.height / dimensions.width) * + widthCells * + DEFAULT_CELL_ASPECT_RATIO, + ), + ), + ); + return { widthCells, rows }; +} + +export function createKittyImageId( + image: Buffer, + imageShape: { widthCells: number; rows: number }, +): number { + const hash = crypto + .createHash('sha256') + .update(image) + .update('\0') + .update(String(imageShape.widthCells)) + .update('\0') + .update(String(imageShape.rows)) + .digest(); + const id = hash.readUIntBE(0, 3); + return id === 0 ? 1 : id; +} + +export function prepareTerminalImage({ + data, + mimeType, + contentWidth, + availableTerminalHeight, + env = process.env, + detection, +}: PrepareTerminalImageOptions): TerminalImageRenderResult { + const format = getImageFormat(mimeType); + if (!format) { + return { + kind: 'fallback', + text: '[image]', + reason: 'invalid-mime-type', + }; + } + + const image = decodeBase64Image(data); + if (!image) { + return { + kind: 'fallback', + text: formatImageFallback(mimeType), + reason: 'invalid-data', + }; + } + + const dimensions = readImageSize(image, mimeType); + if (!dimensions) { + return { + kind: 'fallback', + text: formatImageFallback(mimeType), + reason: 'unsupported-format', + }; + } + + const fallbackText = formatImageFallback(mimeType, dimensions); + const protocol = detectTerminalImageProtocol(env, detection); + if (!protocol) { + return { + kind: 'fallback', + text: fallbackText, + dimensions, + reason: 'unsupported-terminal', + }; + } + if (protocol === 'kitty' && mimeType.trim().toLowerCase() !== 'image/png') { + return { + kind: 'fallback', + text: fallbackText, + dimensions, + reason: 'unsupported-protocol-format', + }; + } + + const imageShape = fitTerminalImage( + dimensions, + contentWidth, + availableTerminalHeight, + ); + if (protocol === 'kitty') { + const imageId = createKittyImageId(image, imageShape); + return { + kind: 'terminal-image', + sequence: encodeKittyVirtualImage( + image, + imageId, + imageShape.widthCells, + imageShape.rows, + ), + ...imageShape, + protocol, + dimensions, + fallbackText, + placeholder: buildKittyPlaceholder( + imageId, + imageShape.widthCells, + imageShape.rows, + ), + }; + } + + return { + kind: 'terminal-image', + sequence: encodeITerm2InlineImage( + image, + imageShape.widthCells, + imageShape.rows, + ), + ...imageShape, + protocol, + dimensions, + fallbackText, + }; +} diff --git a/packages/core/src/core/turn.test.ts b/packages/core/src/core/turn.test.ts index a049b4b0d0c..be37bf336e5 100644 --- a/packages/core/src/core/turn.test.ts +++ b/packages/core/src/core/turn.test.ts @@ -183,6 +183,97 @@ describe('Turn', () => { ]); }); + it('should preserve ordered image parts in content events', async () => { + const mockResponseStream = (async function* () { + yield { + type: StreamEventType.CHUNK, + value: { + candidates: [ + { + content: { + parts: [ + { text: 'before' }, + { + inlineData: { + data: 'aW1hZ2U=', + mimeType: 'image/png', + displayName: 'chart.png', + }, + }, + { thought: true, text: 'hidden' }, + { text: 'after' }, + ], + }, + }, + ], + } as GenerateContentResponse, + }; + yield { + type: StreamEventType.CHUNK, + value: { + candidates: [ + { + content: { + parts: [ + { + inlineData: { + data: 'c2Vjb25k', + mimeType: 'image/webp', + }, + }, + ], + }, + }, + ], + } as GenerateContentResponse, + }; + })(); + mockSendMessageStream.mockResolvedValue(mockResponseStream); + + const events = []; + for await (const event of turn.run( + 'test-model', + [{ text: 'Hi' }], + new AbortController().signal, + )) { + events.push(event); + } + + expect(events).toEqual([ + { + type: GeminiEventType.Thought, + value: { subject: '', description: 'hidden' }, + }, + { + type: GeminiEventType.Content, + value: 'beforeafter', + parts: [ + { text: 'before' }, + { + inlineData: { + data: 'aW1hZ2U=', + mimeType: 'image/png', + displayName: 'chart.png', + }, + }, + { text: 'after' }, + ], + }, + { + type: GeminiEventType.Content, + value: '', + parts: [ + { + inlineData: { + data: 'c2Vjb25k', + mimeType: 'image/webp', + }, + }, + ], + }, + ]); + }); + it('should emit Thought events when a thought part is present', async () => { const mockResponseStream = (async function* () { yield { diff --git a/packages/core/src/core/turn.ts b/packages/core/src/core/turn.ts index 9c8838a4dd6..25bec267b14 100644 --- a/packages/core/src/core/turn.ts +++ b/packages/core/src/core/turn.ts @@ -277,9 +277,21 @@ export interface ServerToolCallConfirmationDetails { details: ToolCallConfirmationDetails; } +export type ServerGeminiContentPart = + | { text: string } + | { + inlineData: { + data: string; + mimeType: string; + displayName?: string; + }; + }; + export type ServerGeminiContentEvent = { type: GeminiEventType.Content; value: string; + /** Ordered display parts, present only when the chunk contains an image. */ + parts?: ServerGeminiContentPart[]; }; export type ServerGeminiThoughtEvent = { @@ -450,6 +462,40 @@ export type ServerGeminiStreamEvent = | ServerGeminiSessionTokenLimitExceededEvent | ServerGeminiRetryEvent; +function getDisplayContentParts( + response: GenerateContentResponse, +): ServerGeminiContentPart[] { + const parts = response.candidates?.[0]?.content?.parts ?? []; + const displayParts: ServerGeminiContentPart[] = []; + + for (const part of parts) { + if (part.thought) { + continue; + } + if (typeof part.text === 'string' && part.text.length > 0) { + displayParts.push({ text: part.text }); + } + const inlineData = part.inlineData; + if ( + inlineData?.mimeType?.trim().toLowerCase().startsWith('image/') && + typeof inlineData.data === 'string' && + inlineData.data.length > 0 + ) { + displayParts.push({ + inlineData: { + data: inlineData.data, + mimeType: inlineData.mimeType, + ...(typeof inlineData.displayName === 'string' + ? { displayName: inlineData.displayName } + : {}), + }, + }); + } + } + + return displayParts; +} + // A turn manages the agentic loop turn within the server context. export class Turn { readonly pendingToolCalls: ToolCallRequestInfo[] = []; @@ -555,9 +601,15 @@ export class Turn { }; } - const text = getResponseText(resp); - if (text) { - yield { type: GeminiEventType.Content, value: text }; + const text = getResponseText(resp) ?? ''; + const displayParts = getDisplayContentParts(resp); + const hasImage = displayParts.some((part) => 'inlineData' in part); + if (text || hasImage) { + yield { + type: GeminiEventType.Content, + value: text, + ...(hasImage ? { parts: displayParts } : {}), + }; } // Handle function calls (requesting tool execution) From f74d9548f49cc8b03661972e5f8b037678709fcb Mon Sep 17 00:00:00 2001 From: tly Date: Sun, 2 Aug 2026 00:19:33 +0800 Subject: [PATCH 2/7] fix(cli): preserve text around hidden citations --- .../messages/ConversationMessages.test.tsx | 16 ++ .../cli/src/ui/hooks/useGeminiStream.test.tsx | 159 ++++++++++++++++++ packages/cli/src/ui/hooks/useGeminiStream.ts | 5 +- .../src/ui/utils/inline-image-parts.test.ts | 20 +++ 4 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 packages/cli/src/ui/utils/inline-image-parts.test.ts diff --git a/packages/cli/src/ui/components/messages/ConversationMessages.test.tsx b/packages/cli/src/ui/components/messages/ConversationMessages.test.tsx index fb945cd55b6..49fec49abd0 100644 --- a/packages/cli/src/ui/components/messages/ConversationMessages.test.tsx +++ b/packages/cli/src/ui/components/messages/ConversationMessages.test.tsx @@ -9,6 +9,7 @@ import { Text } from 'ink'; import { vi } from 'vitest'; import { AssistantMessage, + AssistantMessageContent, ThinkMessage, ThinkMessageContent, toggleKeyHint, @@ -35,6 +36,21 @@ describe('', () => { }); }); +describe('', () => { + it('routes continuation images through TerminalImage', () => { + const { lastFrame } = render( + , + ); + + expect(lastFrame()).toContain('MockTerminalImage:image/png'); + }); +}); + describe('', () => { const defaultProps = { text: 'Analyzing the code structure', diff --git a/packages/cli/src/ui/hooks/useGeminiStream.test.tsx b/packages/cli/src/ui/hooks/useGeminiStream.test.tsx index db77fb7b4e8..5d002106428 100644 --- a/packages/cli/src/ui/hooks/useGeminiStream.test.tsx +++ b/packages/cli/src/ui/hooks/useGeminiStream.test.tsx @@ -6359,6 +6359,90 @@ describe('useGeminiStream', () => { }); }); + it('discards every staged mixed-content run on model fallback', async () => { + vi.useFakeTimers(); + + let emitFallback!: () => void; + const waitForFallback = new Promise((resolve) => { + emitFallback = resolve; + }); + let releaseStream!: () => void; + const holdStream = new Promise((resolve) => { + releaseStream = resolve; + }); + const image = { + data: 'aW1hZ2U=', + mimeType: 'image/png', + displayName: 'failed.png', + }; + mockSendMessageStream.mockReturnValue( + (async function* () { + yield { + type: ServerGeminiEventType.Content, + value: 'beforeafter', + parts: [ + { text: 'before' }, + { inlineData: image }, + { text: 'after' }, + ], + }; + await waitForFallback; + yield { + type: ServerGeminiEventType.ModelFallback, + fromModel: 'primary-model', + toModel: 'fallback-model', + fallbackIndex: 1, + }; + await holdStream; + })(), + ); + + const { result } = renderTestHook(); + act(() => { + void result.current.submitQuery('show a chart'); + }); + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(60); + }); + + expect(result.current.pendingHistoryItems).toEqual([ + expect.objectContaining({ type: 'gemini', text: 'before' }), + { type: 'gemini_content', text: '', images: [image] }, + { type: 'gemini_content', text: 'after' }, + ]); + + await act(async () => { + emitFallback(); + await Promise.resolve(); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(60); + }); + + expect( + mockAddItem.mock.calls + .map(([item]) => item as HistoryItem) + .filter( + (item) => item.type === 'gemini' || item.type === 'gemini_content', + ), + ).toEqual([]); + expect(result.current.pendingHistoryItems).toEqual([]); + expect(mockAddItem).toHaveBeenCalledWith( + { + type: 'notification', + text: 'Model primary-model unavailable, falling back to fallback-model', + }, + expect.any(Number), + ); + + act(() => result.current.cancelOngoingRequest()); + await act(async () => { + releaseStream(); + }); + }); + it('discards staged mixed content before an explicit retry after a thrown stream', async () => { const failedImage = { data: 'aW1hZ2U=', @@ -9485,6 +9569,81 @@ describe('useGeminiStream', () => { }); }); + describe('Citation event', () => { + it('preserves streamed text across hidden citation events', async () => { + const settingsWithCitationsHidden = { + ...mockLoadedSettings, + merged: { + ...mockLoadedSettings.merged, + ui: { + ...mockLoadedSettings.merged.ui, + showCitations: false, + }, + }, + } as LoadedSettings; + mockSendMessageStream.mockReturnValue( + (async function* () { + yield { + type: ServerGeminiEventType.Content, + value: 'Hello world', + }; + yield { + type: ServerGeminiEventType.Citation, + value: 'Citation text', + }; + yield { + type: ServerGeminiEventType.Content, + value: ' more', + }; + yield { + type: ServerGeminiEventType.Finished, + value: { reason: 'STOP', usageMetadata: undefined }, + }; + })(), + ); + + const { result } = renderHook(() => + useGeminiStream( + new MockedGeminiClientClass(mockConfig), + [], + mockAddItem, + mockConfig, + true, + settingsWithCitationsHidden, + mockOnDebugMessage, + mockHandleSlashCommand, + false, + () => 'vscode' as EditorType, + () => {}, + () => Promise.resolve(), + false, + () => {}, + () => {}, + () => {}, + () => {}, + 80, + 24, + ), + ); + + await act(async () => { + await result.current.submitQuery('test hidden citation'); + }); + + expect( + mockAddItem.mock.calls + .map(([item]) => item as HistoryItem) + .filter((item) => item.type === 'gemini'), + ).toEqual([ + expect.objectContaining({ type: 'gemini', text: 'Hello world more' }), + ]); + expect(mockAddItem).not.toHaveBeenCalledWith( + expect.objectContaining({ type: MessageType.INFO }), + expect.any(Number), + ); + }); + }); + describe('handleFinishedEvent', () => { it('commits mixed assistant output before a MAX_TOKENS warning', async () => { const image = { diff --git a/packages/cli/src/ui/hooks/useGeminiStream.ts b/packages/cli/src/ui/hooks/useGeminiStream.ts index edda0c94c99..f7eda0a0b2b 100644 --- a/packages/cli/src/ui/hooks/useGeminiStream.ts +++ b/packages/cli/src/ui/hooks/useGeminiStream.ts @@ -2474,7 +2474,9 @@ export const useGeminiStream = ( case ServerGeminiEventType.Citation: flushBufferedStreamEvents(); handleCitationEvent(event.value, userMessageTimestamp); - geminiMessageBuffer = ''; + if (showCitations(settings)) { + geminiMessageBuffer = ''; + } break; case ServerGeminiEventType.LoopDetected: flushBufferedStreamEvents(); @@ -2734,6 +2736,7 @@ export const useGeminiStream = ( handleMaxSessionTurnsEvent, handleSessionTokenLimitExceededEvent, handleCitationEvent, + settings, startRetryCountdown, clearRetryCountdown, setThought, diff --git a/packages/cli/src/ui/utils/inline-image-parts.test.ts b/packages/cli/src/ui/utils/inline-image-parts.test.ts new file mode 100644 index 00000000000..8b47ccf8b26 --- /dev/null +++ b/packages/cli/src/ui/utils/inline-image-parts.test.ts @@ -0,0 +1,20 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { describe, expect, it } from 'vitest'; +import { extractInlineImages } from './inline-image-parts.js'; + +describe('extractInlineImages', () => { + it('extracts an image from a top-level tool response part', () => { + const image = { + data: 'aW1hZ2U=', + mimeType: 'image/png', + displayName: 'chart.png', + }; + + expect(extractInlineImages([{ inlineData: image }])).toEqual([image]); + }); +}); From 447bccb9496057fa7b77d71477ed551269db7c89 Mon Sep 17 00:00:00 2001 From: tly Date: Sun, 2 Aug 2026 09:12:27 +0800 Subject: [PATCH 3/7] fix(cli): clear pending image state on reset --- packages/cli/src/ui/AppContainer.tsx | 4 + .../src/ui/components/TerminalImage.test.tsx | 17 +++ .../cli/src/ui/components/TerminalImage.tsx | 6 +- .../ui/hooks/slashCommandProcessor.test.ts | 14 ++ .../cli/src/ui/hooks/slashCommandProcessor.ts | 2 + .../cli/src/ui/hooks/useGeminiStream.test.tsx | 142 ++++++++++++++++++ packages/cli/src/ui/hooks/useGeminiStream.ts | 10 ++ .../src/ui/utils/inline-image-parts.test.ts | 60 +++++++- .../ui/utils/terminal-image-renderer.test.ts | 12 ++ 9 files changed, 265 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/ui/AppContainer.tsx b/packages/cli/src/ui/AppContainer.tsx index fe870e0e9c8..66fc4a3bbf5 100644 --- a/packages/cli/src/ui/AppContainer.tsx +++ b/packages/cli/src/ui/AppContainer.tsx @@ -1578,6 +1578,7 @@ export const AppContainer = (props: AppContainerProps) => { // whose implementations are swapped in once the real callbacks exist. const openRewindSelectorRef = useRef<() => void>(() => {}); const cancelOngoingRequestRef = useRef<() => void>(() => {}); + const clearPendingStateRef = useRef<() => void>(() => {}); // /diff opens a per-turn diff dialog. Unlike rewind, no double-press or // history-bound guard is needed, so the open/close handlers can live here @@ -1732,6 +1733,7 @@ export const AppContainer = (props: AppContainerProps) => { handleBranch, openDeleteDialog, openHelpDialog, + clearPendingState: () => clearPendingStateRef.current(), }), [ openAuthDialog, @@ -2041,6 +2043,7 @@ export const AppContainer = (props: AppContainerProps) => { submitQuery, initError, pendingHistoryItems: pendingGeminiHistoryItems, + clearPendingState, thought, cancelOngoingRequest, preemptGoalTurn, @@ -2079,6 +2082,7 @@ export const AppContainer = (props: AppContainerProps) => { goalQueueRef, ); cancelOngoingRequestRef.current = cancelOngoingRequest; + clearPendingStateRef.current = clearPendingState; // Now that streamingState is available, keep isIdleRef in sync and // flush any deferred update notifications when the model finishes responding. diff --git a/packages/cli/src/ui/components/TerminalImage.test.tsx b/packages/cli/src/ui/components/TerminalImage.test.tsx index 33afeaa47a9..52e7b26461d 100644 --- a/packages/cli/src/ui/components/TerminalImage.test.tsx +++ b/packages/cli/src/ui/components/TerminalImage.test.tsx @@ -227,6 +227,23 @@ describe('TerminalImage', () => { ); }); + it('explains why an inline image renderer is unavailable', () => { + mockedPrepareInlineTerminalImage.mockReturnValue({ + fallbackText: '[image: 1x1 png]', + result: { + kind: 'unavailable', + reason: 'chafa is not installed', + }, + }); + + const { lastFrame } = render( + , + ); + + expect(lastFrame()).toContain('[image: 1x1 png]'); + expect(lastFrame()).toContain('chafa is not installed'); + }); + it('uses the deterministic inline placeholder for screen readers', () => { vi.mocked(useIsScreenReaderEnabled).mockReturnValue(true); mockedPrepareInlineTerminalImage.mockReturnValue({ diff --git a/packages/cli/src/ui/components/TerminalImage.tsx b/packages/cli/src/ui/components/TerminalImage.tsx index 0d84aa12210..0bc33650f8d 100644 --- a/packages/cli/src/ui/components/TerminalImage.tsx +++ b/packages/cli/src/ui/components/TerminalImage.tsx @@ -172,7 +172,11 @@ const InlineTerminalImage: React.FC = ({ return ( diff --git a/packages/cli/src/ui/hooks/slashCommandProcessor.test.ts b/packages/cli/src/ui/hooks/slashCommandProcessor.test.ts index 1a4dcc2185e..e79f55e8bf2 100644 --- a/packages/cli/src/ui/hooks/slashCommandProcessor.test.ts +++ b/packages/cli/src/ui/hooks/slashCommandProcessor.test.ts @@ -145,6 +145,7 @@ describe('useSlashCommandProcessor', () => { const mockOpenMemoryDialog = vi.fn(); const mockOpenModelDialog = vi.fn(); const mockSetQuittingMessages = vi.fn(); + const mockClearPendingState = vi.fn(); const mockConfig = makeFakeConfig({}); mockConfig.getChatRecordingService = vi.fn().mockReturnValue({ @@ -181,6 +182,7 @@ describe('useSlashCommandProcessor', () => { openMcpDialog: vi.fn(), openHooksDialog: vi.fn(), openRewindSelector: vi.fn(), + clearPendingState: mockClearPendingState, }); beforeEach(() => { @@ -2632,6 +2634,18 @@ describe('useSlashCommandProcessor', () => { }); describe('ui.clear and /btw dialog', () => { + it('should discard pending Gemini state when ui.clear is called', async () => { + const result = setupProcessorHook(); + await waitFor(() => expect(result.current.commandContext).toBeDefined()); + + act(() => { + result.current.commandContext.ui.clear(); + }); + + expect(mockClearPendingState).toHaveBeenCalledTimes(1); + expect(mockClearItems).toHaveBeenCalledTimes(1); + }); + it('should dismiss an active btw dialog when ui.clear is called', async () => { const result = setupProcessorHook(); await waitFor(() => expect(result.current.commandContext).toBeDefined()); diff --git a/packages/cli/src/ui/hooks/slashCommandProcessor.ts b/packages/cli/src/ui/hooks/slashCommandProcessor.ts index 68922cba960..4e635d41e00 100644 --- a/packages/cli/src/ui/hooks/slashCommandProcessor.ts +++ b/packages/cli/src/ui/hooks/slashCommandProcessor.ts @@ -163,6 +163,7 @@ export interface SlashCommandProcessorActions { openRewindSelector: () => void; openDiffDialog: () => void; openHelpDialog: () => void; + clearPendingState: () => void; } /** @@ -479,6 +480,7 @@ export const useSlashCommandProcessor = ( addItem, clear: () => { cancelBtw(); + actions.clearPendingState(); clearItems(); clearScreen(); refreshStatic(); diff --git a/packages/cli/src/ui/hooks/useGeminiStream.test.tsx b/packages/cli/src/ui/hooks/useGeminiStream.test.tsx index 5d002106428..76f25055ebf 100644 --- a/packages/cli/src/ui/hooks/useGeminiStream.test.tsx +++ b/packages/cli/src/ui/hooks/useGeminiStream.test.tsx @@ -6359,6 +6359,82 @@ describe('useGeminiStream', () => { }); }); + it('preserves staged mixed-content runs on a continuation retry', async () => { + vi.useFakeTimers(); + + let emitRetry!: () => void; + const waitForRetry = new Promise((resolve) => { + emitRetry = resolve; + }); + let releaseStream!: () => void; + const holdStream = new Promise((resolve) => { + releaseStream = resolve; + }); + const image = { + data: 'aW1hZ2U=', + mimeType: 'image/png', + displayName: 'partial.png', + }; + mockSendMessageStream.mockReturnValue( + (async function* () { + yield { + type: ServerGeminiEventType.Content, + value: 'beforeafter', + parts: [ + { text: 'before' }, + { inlineData: image }, + { text: 'after' }, + ], + }; + await waitForRetry; + yield { + type: ServerGeminiEventType.Retry, + isContinuation: true, + }; + yield { + type: ServerGeminiEventType.Content, + value: ' continued', + }; + await holdStream; + })(), + ); + + const { result } = renderTestHook(); + act(() => { + void result.current.submitQuery('show a chart'); + }); + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(60); + }); + + expect(result.current.pendingHistoryItems).toEqual([ + expect.objectContaining({ type: 'gemini', text: 'before' }), + { type: 'gemini_content', text: '', images: [image] }, + { type: 'gemini_content', text: 'after' }, + ]); + + await act(async () => { + emitRetry(); + await Promise.resolve(); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(60); + }); + + expect(result.current.pendingHistoryItems).toEqual([ + expect.objectContaining({ type: 'gemini', text: 'before' }), + { type: 'gemini_content', text: '', images: [image] }, + { type: 'gemini_content', text: 'after continued' }, + ]); + + act(() => result.current.cancelOngoingRequest()); + await act(async () => { + releaseStream(); + }); + }); + it('discards every staged mixed-content run on model fallback', async () => { vi.useFakeTimers(); @@ -6571,6 +6647,72 @@ describe('useGeminiStream', () => { ]); }); + it('does not restore staged mixed content after pending state is cleared', async () => { + const failedImage = { + data: 'aW1hZ2U=', + mimeType: 'image/png', + displayName: 'cleared.png', + }; + mockSendMessageStream + .mockReturnValueOnce( + (async function* () { + yield { + type: ServerGeminiEventType.Content, + value: 'beforeafter', + parts: [ + { text: 'before' }, + { inlineData: failedImage }, + { text: 'after' }, + ], + }; + throw new Error('stream failed'); + })(), + ) + .mockReturnValueOnce( + (async function* () { + yield { + type: ServerGeminiEventType.Content, + value: 'new answer', + }; + yield { + type: ServerGeminiEventType.Finished, + value: { reason: 'STOP', usageMetadata: undefined }, + }; + })(), + ); + + const { result } = renderTestHook(); + await act(async () => { + await result.current.submitQuery('first question'); + }); + expect(result.current.pendingHistoryItems).toEqual( + expect.arrayContaining([ + expect.objectContaining({ type: 'gemini', text: 'before' }), + { type: 'gemini_content', text: '', images: [failedImage] }, + { type: 'gemini_content', text: 'after' }, + expect.objectContaining({ type: 'error' }), + ]), + ); + + act(() => { + result.current.clearPendingState(); + }); + expect(result.current.pendingHistoryItems).toEqual([]); + + await act(async () => { + await result.current.submitQuery('second question'); + }); + + const assistantItems = mockAddItem.mock.calls + .map(([item]) => item as HistoryItem) + .filter( + (item) => item.type === 'gemini' || item.type === 'gemini_content', + ); + expect(assistantItems).toEqual([ + expect.objectContaining({ type: 'gemini', text: 'new answer' }), + ]); + }); + it('does not render leading blank content chunks as an empty assistant item', async () => { vi.useFakeTimers(); diff --git a/packages/cli/src/ui/hooks/useGeminiStream.ts b/packages/cli/src/ui/hooks/useGeminiStream.ts index f7eda0a0b2b..7f3b496ded3 100644 --- a/packages/cli/src/ui/hooks/useGeminiStream.ts +++ b/packages/cli/src/ui/hooks/useGeminiStream.ts @@ -749,6 +749,15 @@ export const useGeminiStream = ( pendingRetryCountdownItemRef, setPendingRetryCountdownItem, ] = useStateAndRef(null); + const clearPendingState = useCallback(() => { + setPendingAssistantItems([]); + setPendingHistoryItem(null); + setPendingRetryErrorItem(null); + }, [ + setPendingAssistantItems, + setPendingHistoryItem, + setPendingRetryErrorItem, + ]); const retryCountdownTimerRef = useRef | null>( null, ); @@ -5115,6 +5124,7 @@ export const useGeminiStream = ( submitQuery, initError, pendingHistoryItems, + clearPendingState, thought, cancelOngoingRequest, preemptGoalTurn, diff --git a/packages/cli/src/ui/utils/inline-image-parts.test.ts b/packages/cli/src/ui/utils/inline-image-parts.test.ts index 8b47ccf8b26..a59fcbe1fa8 100644 --- a/packages/cli/src/ui/utils/inline-image-parts.test.ts +++ b/packages/cli/src/ui/utils/inline-image-parts.test.ts @@ -5,7 +5,10 @@ */ import { describe, expect, it } from 'vitest'; -import { extractInlineImages } from './inline-image-parts.js'; +import { + extractInlineContentRuns, + extractInlineImages, +} from './inline-image-parts.js'; describe('extractInlineImages', () => { it('extracts an image from a top-level tool response part', () => { @@ -17,4 +20,59 @@ describe('extractInlineImages', () => { expect(extractInlineImages([{ inlineData: image }])).toEqual([image]); }); + + it('extracts an image from nested function response parts', () => { + const image = { + data: 'bmVzdGVkLWltYWdl', + mimeType: 'image/webp', + }; + + expect( + extractInlineImages([ + { + functionResponse: { + id: 'call-1', + name: 'generate_image', + response: { output: 'done' }, + parts: [{ inlineData: image }], + }, + }, + ]), + ).toEqual([image]); + }); + + it('ignores non-image inline data', () => { + expect( + extractInlineImages([ + { + inlineData: { + data: 'bm90LWFuLWltYWdl', + mimeType: 'text/plain', + }, + }, + ]), + ).toEqual([]); + }); +}); + +describe('extractInlineContentRuns', () => { + it('preserves text-image-text order and skips thought parts', () => { + const image = { + data: 'aW1hZ2U=', + mimeType: 'image/png', + }; + + expect( + extractInlineContentRuns([ + { text: 'before' }, + { text: 'hidden reasoning', thought: true }, + { inlineData: image }, + { text: 'after' }, + ]), + ).toEqual([ + { kind: 'text', text: 'before' }, + { kind: 'image', image }, + { kind: 'text', text: 'after' }, + ]); + }); }); diff --git a/packages/cli/src/ui/utils/terminal-image-renderer.test.ts b/packages/cli/src/ui/utils/terminal-image-renderer.test.ts index 959a9f5f803..93dc65d7ac8 100644 --- a/packages/cli/src/ui/utils/terminal-image-renderer.test.ts +++ b/packages/cli/src/ui/utils/terminal-image-renderer.test.ts @@ -146,6 +146,18 @@ describe('terminalImageRenderer', () => { ).toEqual({ fallbackText: '[image: png]', result: null }); }); + it('rejects inline PNG dimensions above the shared image limit', () => { + expect( + prepareInlineTerminalImage({ + data: pngWithSize(1_000_001, 1).toString('base64'), + mimeType: 'image/png', + contentWidth: 24, + env: { TERM: 'xterm-kitty' }, + stdoutIsTTY: true, + }), + ).toEqual({ fallbackText: '[image: png]', result: null }); + }); + it('does not render inline image data when output is disabled', () => { expect( prepareInlineTerminalImage({ From 0edfa6ce1c4bbe1bc518e39d56b4d0903b3c5ce8 Mon Sep 17 00:00:00 2001 From: tly Date: Mon, 3 Aug 2026 01:42:50 +0800 Subject: [PATCH 4/7] fix(cli): bound inline image rendering --- .qwen/e2e-tests/terminal-inline-images.md | 11 +- docs/design/terminal-inline-images.md | 17 +- .../ui/components/HistoryItemDisplay.test.tsx | 30 +++ .../src/ui/components/HistoryItemDisplay.tsx | 2 + .../messages/ConversationMessages.test.tsx | 13 + .../messages/ConversationMessages.tsx | 17 ++ .../messages/ToolGroupMessage.test.tsx | 16 ++ .../components/messages/ToolGroupMessage.tsx | 10 +- .../components/messages/ToolMessage.test.tsx | 9 + .../ui/components/messages/ToolMessage.tsx | 10 +- .../cli/src/ui/hooks/useGeminiStream.test.tsx | 226 +++++++++++++++++- packages/cli/src/ui/hooks/useGeminiStream.ts | 78 +++++- .../ui/hooks/useReactToolScheduler.test.tsx | 23 ++ .../cli/src/ui/hooks/useReactToolScheduler.ts | 47 ++-- packages/cli/src/ui/types.ts | 4 + .../src/ui/utils/inline-image-parts.test.ts | 75 +++++- .../cli/src/ui/utils/inline-image-parts.ts | 66 +++-- .../src/ui/utils/resumeHistoryUtils.test.ts | 38 +++ .../cli/src/ui/utils/resumeHistoryUtils.ts | 48 ++-- .../src/ui/utils/terminal-image-renderer.ts | 4 +- 20 files changed, 653 insertions(+), 91 deletions(-) diff --git a/.qwen/e2e-tests/terminal-inline-images.md b/.qwen/e2e-tests/terminal-inline-images.md index 3d8444c9b6e..697617850a3 100644 --- a/.qwen/e2e-tests/terminal-inline-images.md +++ b/.qwen/e2e-tests/terminal-inline-images.md @@ -33,6 +33,8 @@ baseline is grounded in issue #8090 and the unchanged `main` event mapping. 4. Confirm both successful and failed/cancelled tool rows retain their images. 5. Open Ctrl+O and resume the session; confirm assistant and tool image order is reconstructed from persisted parts. +6. Return six images in one assistant output and one tool response; confirm the + first four render and the row ends with `[+2 more images]`. ### Kitty/Ghostty @@ -57,10 +59,11 @@ baseline is grounded in issue #8090 and the unchanged `main` event mapping. 1. Repeat without `chafa` and outside direct Kitty/Ghostty. 2. Confirm a valid PNG displays `[image: x png]`. -3. Repeat with malformed base64, a payload above 8 MiB, invalid - IHDR dimensions, and a non-PNG MIME type. -4. Confirm no raw image sequence is written and a deterministic placeholder - remains visible. +3. Repeat with malformed base64, a payload above 8 MiB, invalid IHDR + dimensions, and a non-PNG MIME type. +4. Confirm no raw image sequence is written. Confirm the oversized payload is + excluded from UI history, while admitted malformed/non-PNG data uses a + deterministic placeholder. 5. Repeat with `INK_SCREEN_READER=true`; confirm only the placeholder is emitted. diff --git a/docs/design/terminal-inline-images.md b/docs/design/terminal-inline-images.md index 086206da642..65b4275120f 100644 --- a/docs/design/terminal-inline-images.md +++ b/docs/design/terminal-inline-images.md @@ -26,6 +26,8 @@ This is the render-and-forget slice requested by issue #8090: - keep text/image ordering across retry, model fallback, cancellation, stream boundaries, and goal-state events; - bound retained image payloads during UI history compaction; +- render at most four images per assistant output or tool row and collapse the + remainder into a `[+K more images]` marker; - show a deterministic text placeholder when an image cannot be rendered. Kitty deletion, resize-driven replacement, terminal cell pixel queries, and @@ -43,6 +45,10 @@ non-thought text and image parts. Only the interactive TUI reads `parts`. It stages text and image history items in their original order. A fresh retry or model fallback discards the failed attempt's staged output, while a normal response boundary commits it. +The TUI admits at most four images for one assistant output and represents any +remaining image parts with a small overflow marker. Visible status rows end the +current assistant display block, so later text starts with the normal assistant +prefix instead of being attached to the status row as a continuation. Text-only events keep their existing runtime shape, so non-interactive output, SDK, ACP, daemon, channel, Web UI, and VS Code consumers continue using `value` unchanged. @@ -55,7 +61,8 @@ reconstructs ordered text/image runs instead of flattening images away. Tool media is stored in `functionResponse.parts`. A CLI extractor reads image `inlineData` from top-level and nested response parts. Live scheduler mapping and resume mapping attach the images to the existing -`IndividualToolCallDisplay`. +`IndividualToolCallDisplay`. Each tool row keeps the first four images and an +overflow count for the rest. Tools carrying images render individually even when their text-only form would normally collapse into a read/search summary. `ToolMessage` routes the images @@ -83,6 +90,10 @@ becomes `[image: png]`; unsupported image MIME types retain their sanitized format label, such as `[image: jpeg]`. Screen-reader mode always uses the text placeholder and emits no raw image sequence. +The same encoded-length limit is applied before inline data enters CLI history +or tool-display state. Payloads that exceed the renderer's 8 MiB decoded-image +budget are therefore not retained by the UI. + The first slice renders validated PNG data only. Other image MIME types remain visible as deterministic placeholders rather than entering a second protocol or decoding path. @@ -93,6 +104,8 @@ Encoded images are much larger than ordinary history text. UI compaction drops payloads from old assistant image items while retaining the 20 most recent items. Cleared images leave a visible marker instead of becoming blank rows. Tool image payloads participate in the existing tool-result compaction limit. +The four-image admission cap also bounds synchronous Kitty/chafa rendering for +each assistant output and tool row. ## Test Plan @@ -105,4 +118,6 @@ Tool image payloads participate in the existing tool-result compaction limit. boundaries, and goal-state events. - Verify live and restored tool responses expose nested images. - Verify restored assistant history preserves text/image ordering. +- Verify live, restored, and tool output enforce the image cap and expose the + overflow count without retaining oversized payloads. - Verify memory compaction clears old assistant and tool image payloads. diff --git a/packages/cli/src/ui/components/HistoryItemDisplay.test.tsx b/packages/cli/src/ui/components/HistoryItemDisplay.test.tsx index 9e7ff78b5b3..fce85d88a77 100644 --- a/packages/cli/src/ui/components/HistoryItemDisplay.test.tsx +++ b/packages/cli/src/ui/components/HistoryItemDisplay.test.tsx @@ -5,6 +5,7 @@ */ import { describe, it, expect, vi } from 'vitest'; +import { Text } from 'ink'; import { HistoryItemDisplay } from './HistoryItemDisplay.js'; import { type HistoryItem, ToolCallStatus } from '../types.js'; import { MessageType } from '../types.js'; @@ -30,6 +31,12 @@ vi.mock('./messages/ToolGroupMessage.js', () => ({ ToolGroupMessage: vi.fn(() =>
), })); +vi.mock('./TerminalImage.js', () => ({ + TerminalImage: ({ image }: { image: { mimeType: string } }) => ( + MockTerminalImage:{image.mimeType} + ), +})); + vi.mock('../hooks/useMouseEvents.js', () => ({ useMouseEvents: vi.fn(), })); @@ -94,6 +101,29 @@ describe('', () => { expect(output).toContain('◆\uFE0E Hello'); }); + it.each(['gemini', 'gemini_content'] as const)( + 'passes images from a %s history item to the assistant renderer', + (type) => { + const item: HistoryItem = { + id: 1, + type, + text: '', + images: [{ data: 'aW1hZ2U=', mimeType: 'image/png' }], + omittedImageCount: 2, + }; + const { lastFrame } = renderWithProviders( + , + ); + + expect(lastFrame()).toContain('MockTerminalImage:image/png'); + expect(lastFrame()).toContain('[+2 more images]'); + }, + ); + it('renders tool summaries without a leading spacer row', () => { const item: HistoryItem = { id: 1, diff --git a/packages/cli/src/ui/components/HistoryItemDisplay.tsx b/packages/cli/src/ui/components/HistoryItemDisplay.tsx index fd8b2ddea8c..7ff5a9e54da 100644 --- a/packages/cli/src/ui/components/HistoryItemDisplay.tsx +++ b/packages/cli/src/ui/components/HistoryItemDisplay.tsx @@ -301,6 +301,7 @@ const HistoryItemDisplayComponent: React.FC = ({ = ({ ', () => { expect(lastFrame()).toContain('MockTerminalImage:image/png'); }); + + it('renders the number of omitted images', () => { + const { lastFrame } = render( + , + ); + + expect(lastFrame()).toContain('[+2 more images]'); + }); }); describe('', () => { diff --git a/packages/cli/src/ui/components/messages/ConversationMessages.tsx b/packages/cli/src/ui/components/messages/ConversationMessages.tsx index 79de0fef859..4c0eee6b686 100644 --- a/packages/cli/src/ui/components/messages/ConversationMessages.tsx +++ b/packages/cli/src/ui/components/messages/ConversationMessages.tsx @@ -24,6 +24,7 @@ import { sanitizeTerminalText } from '../../utils/textUtils.js'; import { formatDuration } from '../../utils/displayUtils.js'; import type { InlineImageData } from '../../types.js'; import { TerminalImage } from '../TerminalImage.js'; +import { formatInlineImageOverflow } from '../../utils/inline-image-parts.js'; const debugLogger = createDebugLogger('THINK_RENDER'); @@ -43,6 +44,7 @@ interface UserShellMessageProps { interface AssistantMessageProps { text: string; images?: InlineImageData[]; + omittedImageCount?: number; isPending: boolean; availableTerminalHeight?: number; contentWidth: number; @@ -52,6 +54,7 @@ interface AssistantMessageProps { interface AssistantMessageContentProps { text: string; images?: InlineImageData[]; + omittedImageCount?: number; isPending: boolean; availableTerminalHeight?: number; contentWidth: number; @@ -94,6 +97,7 @@ interface PrefixedTextMessageProps { interface PrefixedMarkdownMessageProps { text: string; images?: InlineImageData[]; + omittedImageCount?: number; prefix: string; prefixColor: string; isPending: boolean; @@ -107,6 +111,7 @@ interface PrefixedMarkdownMessageProps { interface ContinuationMarkdownMessageProps { text: string; images?: InlineImageData[]; + omittedImageCount?: number; isPending: boolean; availableTerminalHeight?: number; contentWidth: number; @@ -155,6 +160,7 @@ const PrefixedTextMessage: React.FC = ({ const PrefixedMarkdownMessage: React.FC = ({ text, images, + omittedImageCount, prefix, prefixColor, isPending, @@ -192,6 +198,9 @@ const PrefixedMarkdownMessage: React.FC = ({ availableTerminalHeight={availableTerminalHeight} /> ))} + {omittedImageCount !== undefined && omittedImageCount > 0 && ( + {formatInlineImageOverflow(omittedImageCount)} + )} ); @@ -202,6 +211,7 @@ const ContinuationMarkdownMessage: React.FC< > = ({ text, images, + omittedImageCount, isPending, availableTerminalHeight, contentWidth, @@ -231,6 +241,9 @@ const ContinuationMarkdownMessage: React.FC< availableTerminalHeight={availableTerminalHeight} /> ))} + {omittedImageCount !== undefined && omittedImageCount > 0 && ( + {formatInlineImageOverflow(omittedImageCount)} + )} ); }; @@ -265,6 +278,7 @@ export const UserShellMessage: React.FC = ({ text }) => { export const AssistantMessage: React.FC = ({ text, images, + omittedImageCount, isPending, availableTerminalHeight, contentWidth, @@ -273,6 +287,7 @@ export const AssistantMessage: React.FC = ({ = ({ text, images, + omittedImageCount, isPending, availableTerminalHeight, contentWidth, @@ -296,6 +312,7 @@ export const AssistantMessageContent: React.FC< ', () => { expect(lastFrame()).toContain('MockTool[image-read]'); }); + it('renders an overflow-only collapsible tool individually', () => { + const toolCalls = [ + createToolCall({ + callId: 'overflow-read', + name: 'ReadFile', + description: 'many charts', + omittedImageCount: 2, + }), + ]; + const { lastFrame } = renderWithProviders( + , + ); + + expect(lastFrame()).toContain('MockTool[overflow-read]'); + }); + it('renders mixed group with summary + individual tools', () => { const toolCalls = [ createToolCall({ callId: 'r1', name: 'ReadFile', description: 'a.ts' }), diff --git a/packages/cli/src/ui/components/messages/ToolGroupMessage.tsx b/packages/cli/src/ui/components/messages/ToolGroupMessage.tsx index 115a660a0a8..f3f4fceb26c 100644 --- a/packages/cli/src/ui/components/messages/ToolGroupMessage.tsx +++ b/packages/cli/src/ui/components/messages/ToolGroupMessage.tsx @@ -45,6 +45,10 @@ function isRunningAgent( ); } +function hasInlineImageOutput(tool: IndividualToolCallDisplay): boolean { + return Boolean(tool.images?.length || tool.omittedImageCount); +} + /** * Predicate: tool entry whose `resultDisplay` is an `AgentResultDisplay` * (i.e. a `task_execution` subagent invocation), regardless of status. @@ -390,7 +394,7 @@ export const ToolGroupMessage: React.FC = ({ (t) => isCollapsibleTool(t.name) && t.status !== ToolCallStatus.Canceled && - !t.images?.length, + !hasInlineImageOutput(t), ); const nonCollapsibleTools = forceExpandAll ? inlineToolCalls @@ -398,7 +402,7 @@ export const ToolGroupMessage: React.FC = ({ (t) => !isCollapsibleTool(t.name) || t.status === ToolCallStatus.Canceled || - Boolean(t.images?.length), + hasInlineImageOutput(t), ); // Memory badge — shared between all-collapsible and mixed paths. @@ -454,7 +458,7 @@ export const ToolGroupMessage: React.FC = ({ for (const tool of nonCollapsibleTools) { if ( (tool.resultDisplay !== undefined && tool.resultDisplay !== '') || - tool.images?.length + hasInlineImageOutput(tool) ) { countToolCallsWithResults++; } diff --git a/packages/cli/src/ui/components/messages/ToolMessage.test.tsx b/packages/cli/src/ui/components/messages/ToolMessage.test.tsx index 4c3315ca2bf..edab4b8cbb7 100644 --- a/packages/cli/src/ui/components/messages/ToolMessage.test.tsx +++ b/packages/cli/src/ui/components/messages/ToolMessage.test.tsx @@ -203,6 +203,15 @@ describe('', () => { expect(lastFrame()).toContain('MockTerminalImage:image/png'); }); + it('renders the number of omitted inline images', () => { + const { lastFrame } = renderWithContext( + , + StreamingState.Idle, + ); + + expect(lastFrame()).toContain('[+2 more images]'); + }); + it('always shows the vision bridge disclosure for a completed read', () => { const { lastFrame } = renderWithContext( = ({ description, resultDisplay, images, + omittedImageCount, visionBridgeNotice, detailedDisplay, status, @@ -974,13 +976,14 @@ export const ToolMessage: React.FC = ({ )} - {images && images.length > 0 && ( + {((images?.length ?? 0) > 0 || + (omittedImageCount !== undefined && omittedImageCount > 0)) && ( - {images.map((image, index) => ( + {images?.map((image, index) => ( = ({ availableTerminalHeight={availableHeight} /> ))} + {omittedImageCount !== undefined && omittedImageCount > 0 && ( + {formatInlineImageOverflow(omittedImageCount)} + )} )} {isThisShellFocused && config && ( diff --git a/packages/cli/src/ui/hooks/useGeminiStream.test.tsx b/packages/cli/src/ui/hooks/useGeminiStream.test.tsx index 76f25055ebf..0c1a1cff35b 100644 --- a/packages/cli/src/ui/hooks/useGeminiStream.test.tsx +++ b/packages/cli/src/ui/hooks/useGeminiStream.test.tsx @@ -44,6 +44,10 @@ import type { HistoryItem, SlashCommandProcessorResult } from '../types.js'; import { MessageType, StreamingState, ToolCallStatus } from '../types.js'; import type { LoadedSettings } from '../../config/settings.js'; import { findLastSafeSplitPoint } from '../utils/markdownUtilities.js'; +import { + MAX_INLINE_IMAGE_ENCODED_LENGTH, + MAX_INLINE_IMAGES_PER_ITEM, +} from '../utils/inline-image-parts.js'; import type { DirectUserAdmission, QueuedGoalTurn } from './useMessageQueue.js'; // --- MOCKS --- @@ -6218,6 +6222,167 @@ describe('useGeminiStream', () => { }); }); + it('caps inline images for one assistant output and reports the overflow', async () => { + vi.useFakeTimers(); + + let releaseStream!: () => void; + const holdStream = new Promise((resolve) => { + releaseStream = resolve; + }); + const images = Array.from( + { length: MAX_INLINE_IMAGES_PER_ITEM + 2 }, + (_, index) => ({ + data: Buffer.from(`assistant-image-${index}`).toString('base64'), + mimeType: 'image/png', + }), + ); + mockSendMessageStream.mockReturnValue( + (async function* () { + yield { + type: ServerGeminiEventType.Content, + value: '', + parts: images.map((inlineData) => ({ inlineData })), + }; + await holdStream; + })(), + ); + + const { result } = renderTestHook(); + act(() => { + void result.current.submitQuery('show many charts'); + }); + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(60); + }); + + const assistantItems = result.current.pendingHistoryItems.filter( + (item) => item.type === 'gemini' || item.type === 'gemini_content', + ); + expect(assistantItems.flatMap((item) => item.images ?? [])).toEqual( + images.slice(0, MAX_INLINE_IMAGES_PER_ITEM), + ); + expect(assistantItems).toHaveLength(MAX_INLINE_IMAGES_PER_ITEM + 1); + expect(assistantItems.at(-1)).toMatchObject({ + text: '', + omittedImageCount: 2, + }); + + act(() => result.current.cancelOngoingRequest()); + await act(async () => { + releaseStream(); + }); + }); + + it('resets the inline image cap at a finished response boundary', async () => { + const firstOutputImages = Array.from( + { length: MAX_INLINE_IMAGES_PER_ITEM }, + (_, index) => ({ + data: Buffer.from(`first-output-${index}`).toString('base64'), + mimeType: 'image/png', + }), + ); + const nextOutputImage = { + data: Buffer.from('next-output').toString('base64'), + mimeType: 'image/png', + }; + mockSendMessageStream.mockReturnValue( + (async function* () { + yield { + type: ServerGeminiEventType.Content, + value: '', + parts: firstOutputImages.map((inlineData) => ({ inlineData })), + }; + yield { + type: ServerGeminiEventType.Finished, + value: { reason: 'STOP', usageMetadata: undefined }, + }; + yield { + type: ServerGeminiEventType.Content, + value: '', + parts: [{ inlineData: nextOutputImage }], + }; + yield { + type: ServerGeminiEventType.Finished, + value: { reason: 'STOP', usageMetadata: undefined }, + }; + })(), + ); + + const { result } = renderTestHook(); + await act(async () => { + await result.current.submitQuery('show charts across responses'); + }); + + const assistantItems = mockAddItem.mock.calls + .map(([item]) => item as HistoryItem) + .filter( + (item) => item.type === 'gemini' || item.type === 'gemini_content', + ); + expect(assistantItems.flatMap((item) => item.images ?? [])).toEqual([ + ...firstOutputImages, + nextOutputImage, + ]); + expect(assistantItems.some((item) => item.omittedImageCount)).toBe(false); + expect(assistantItems.at(-1)).toMatchObject({ + type: 'gemini', + images: [nextOutputImage], + }); + }); + + it('does not retain an oversized inline image payload in UI state', async () => { + vi.useFakeTimers(); + + let releaseStream!: () => void; + const holdStream = new Promise((resolve) => { + releaseStream = resolve; + }); + const oversizedData = 'A'.repeat(MAX_INLINE_IMAGE_ENCODED_LENGTH + 1); + mockSendMessageStream.mockReturnValue( + (async function* () { + yield { + type: ServerGeminiEventType.Content, + value: '', + parts: [ + { + inlineData: { + data: oversizedData, + mimeType: 'image/png', + }, + }, + ], + }; + await holdStream; + })(), + ); + + const { result } = renderTestHook(); + act(() => { + void result.current.submitQuery('show oversized chart'); + }); + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(60); + }); + + const assistantItems = result.current.pendingHistoryItems.filter( + (item) => item.type === 'gemini' || item.type === 'gemini_content', + ); + expect(assistantItems).toEqual([]); + expect(JSON.stringify(result.current.pendingHistoryItems)).not.toContain( + oversizedData, + ); + + act(() => result.current.cancelOngoingRequest()); + await act(async () => { + releaseStream(); + }); + }); + it('keeps an image committable when the stream pauses after whitespace', async () => { vi.useFakeTimers(); @@ -9786,6 +9951,63 @@ describe('useGeminiStream', () => { }); }); + describe('ChatCompressed event', () => { + it('starts a fresh prefixed text item after the status row', async () => { + mockSendMessageStream.mockReturnValue( + (async function* () { + yield { + type: ServerGeminiEventType.Content, + value: 'before compression', + }; + yield { + type: ServerGeminiEventType.ChatCompressed, + value: { + originalTokenCount: 100, + newTokenCount: 50, + }, + }; + yield { + type: ServerGeminiEventType.Content, + value: 'after compression', + }; + yield { + type: ServerGeminiEventType.Finished, + value: { reason: 'STOP', usageMetadata: undefined }, + }; + })(), + ); + + const { result } = renderTestHook(); + await act(async () => { + await result.current.submitQuery('test compression boundary'); + }); + + expect( + mockAddItem.mock.calls + .map(([item]) => item as HistoryItem) + .filter( + (item) => + item.type === 'gemini' || + item.type === 'gemini_content' || + item.type === 'info', + ), + ).toEqual([ + expect.objectContaining({ + type: 'gemini', + text: 'before compression', + }), + expect.objectContaining({ + type: 'info', + text: expect.stringContaining('compressed context'), + }), + expect.objectContaining({ + type: 'gemini', + text: 'after compression', + }), + ]); + }); + }); + describe('handleFinishedEvent', () => { it('commits mixed assistant output before a MAX_TOKENS warning', async () => { const image = { @@ -12659,7 +12881,7 @@ describe('useGeminiStream', () => { }); describe('HookSystemMessage Event', () => { - it('commits staged inline content before a displayed Goal state', async () => { + it('commits staged inline content and restarts after a displayed Goal state', async () => { const image = { data: 'aW1hZ2U=', mimeType: 'image/png', @@ -12723,7 +12945,7 @@ describe('useGeminiStream', () => { { type: 'gemini_content', text: '', images: [image] }, { type: 'gemini_content', text: 'Goal output' }, expect.objectContaining({ type: 'goal_state', cause: 'complete' }), - { type: 'gemini_content', text: ' continued' }, + expect.objectContaining({ type: 'gemini', text: ' continued' }), ]); }); diff --git a/packages/cli/src/ui/hooks/useGeminiStream.ts b/packages/cli/src/ui/hooks/useGeminiStream.ts index 7f3b496ded3..3e21567fa86 100644 --- a/packages/cli/src/ui/hooks/useGeminiStream.ts +++ b/packages/cli/src/ui/hooks/useGeminiStream.ts @@ -125,6 +125,10 @@ import { } from './useMessageQueue.js'; import { classifyApiError } from '../../utils/classify-api-error.js'; import { cleanupReviewWorktreeLeases } from '../../services/review-worktree-lease.js'; +import { + getInlineImageData, + MAX_INLINE_IMAGES_PER_ITEM, +} from '../utils/inline-image-parts.js'; const debugLogger = createDebugLogger('GEMINI_STREAM'); @@ -1489,7 +1493,7 @@ export const useGeminiStream = ( if ( (pendingItem?.type === 'gemini' || pendingItem?.type === 'gemini_content') && - pendingItem.images?.length + (pendingItem.images?.length || pendingItem.omittedImageCount) ) { if (newGeminiMessageBuffer.trim().length === 0) { return newGeminiMessageBuffer; @@ -2218,6 +2222,16 @@ export const useGeminiStream = ( let assistantOutputStarted = pendingHistoryItemRef.current?.type === 'gemini' || pendingHistoryItemRef.current?.type === 'gemini_content'; + let assistantInlineImageCount = [ + ...pendingAssistantItemsRef.current, + pendingHistoryItemRef.current, + ].reduce( + (count, item) => + item?.type === 'gemini' || item?.type === 'gemini_content' + ? count + (item.images?.length ?? 0) + : count, + 0, + ); const toolCallRequests: ToolCallRequestInfo[] = []; const bufferedEvents: BufferedStreamEvent[] = []; let flushTimer: ReturnType | null = null; @@ -2272,6 +2286,26 @@ export const useGeminiStream = ( } setIsReceivingContent(true); turnSawContentEventRef.current = true; + const pendingItem = pendingHistoryItemRef.current; + const isOverflowOnlyItem = + (pendingItem?.type === 'gemini' || + pendingItem?.type === 'gemini_content') && + pendingItem.text.length === 0 && + !pendingItem.images?.length && + Boolean(pendingItem.omittedImageCount); + const shouldDisplayImage = + assistantInlineImageCount < MAX_INLINE_IMAGES_PER_ITEM; + + if (!shouldDisplayImage && isOverflowOnlyItem) { + setPendingHistoryItem({ + ...pendingItem, + omittedImageCount: (pendingItem.omittedImageCount ?? 0) + 1, + }); + geminiMessageBuffer = ''; + assistantOutputStarted = true; + continue; + } + if (pendingHistoryItemRef.current) { if (!stagePendingAssistantItem()) { commitItemInOrder( @@ -2282,12 +2316,22 @@ export const useGeminiStream = ( } } geminiMessageBuffer = ''; - setPendingHistoryItem({ - type: assistantOutputStarted ? 'gemini_content' : 'gemini', - text: '', - images: [nextEvent.value], - ...(!assistantOutputStarted ? { timestamp: Date.now() } : {}), - }); + if (shouldDisplayImage) { + setPendingHistoryItem({ + type: assistantOutputStarted ? 'gemini_content' : 'gemini', + text: '', + images: [nextEvent.value], + ...(!assistantOutputStarted ? { timestamp: Date.now() } : {}), + }); + assistantInlineImageCount++; + } else { + setPendingHistoryItem({ + type: assistantOutputStarted ? 'gemini_content' : 'gemini', + text: '', + omittedImageCount: 1, + ...(!assistantOutputStarted ? { timestamp: Date.now() } : {}), + }); + } assistantOutputStarted = true; continue; } @@ -2372,10 +2416,12 @@ export const useGeminiStream = ( bufferedEvents.push({ kind: 'content', value: part.text }); } } else { - bufferedEvents.push({ - kind: 'image', - value: part.inlineData, + const image = getInlineImageData({ + inlineData: part.inlineData, }); + if (image) { + bufferedEvents.push({ kind: 'image', value: image }); + } } } scheduleBufferedStreamFlush(); @@ -2422,6 +2468,7 @@ export const useGeminiStream = ( flushBufferedStreamEvents(); handleChatCompressionEvent(event.value, userMessageTimestamp); geminiMessageBuffer = ''; + assistantOutputStarted = false; break; case ServerGeminiEventType.ToolCallConfirmation: case ServerGeminiEventType.ToolCallResponse: @@ -2438,6 +2485,7 @@ export const useGeminiStream = ( } handleMaxSessionTurnsEvent(); geminiMessageBuffer = ''; + assistantOutputStarted = false; break; case ServerGeminiEventType.SessionTokenLimitExceeded: flushBufferedStreamEvents(); @@ -2450,6 +2498,7 @@ export const useGeminiStream = ( } handleSessionTokenLimitExceededEvent(event.value); geminiMessageBuffer = ''; + assistantOutputStarted = false; break; case ServerGeminiEventType.Finished: flushBufferedStreamEvents(); @@ -2474,6 +2523,7 @@ export const useGeminiStream = ( geminiMessageBuffer = ''; thoughtBuffer = ''; assistantOutputStarted = false; + assistantInlineImageCount = 0; setThought(null); handleFinishedEvent( event as ServerGeminiFinishedEvent, @@ -2485,6 +2535,7 @@ export const useGeminiStream = ( handleCitationEvent(event.value, userMessageTimestamp); if (showCitations(settings)) { geminiMessageBuffer = ''; + assistantOutputStarted = false; } break; case ServerGeminiEventType.LoopDetected: @@ -2512,6 +2563,7 @@ export const useGeminiStream = ( setThought(null); geminiMessageBuffer = ''; assistantOutputStarted = false; + assistantInlineImageCount = 0; } else { flushBufferedStreamEvents(); } @@ -2544,6 +2596,7 @@ export const useGeminiStream = ( setThought(null); geminiMessageBuffer = ''; assistantOutputStarted = false; + assistantInlineImageCount = 0; toolCallRequests.length = 0; clearRetryCountdown(); const fromModel = @@ -2577,6 +2630,7 @@ export const useGeminiStream = ( userMessageTimestamp, ); geminiMessageBuffer = ''; + assistantOutputStarted = false; break; case ServerGeminiEventType.UserPromptSubmitBlocked: flushBufferedStreamEvents(); @@ -2585,11 +2639,13 @@ export const useGeminiStream = ( userMessageTimestamp, ); geminiMessageBuffer = ''; + assistantOutputStarted = false; break; case ServerGeminiEventType.StopHookLoop: flushBufferedStreamEvents(); handleStopHookLoopEvent(event.value, userMessageTimestamp); geminiMessageBuffer = ''; + assistantOutputStarted = false; break; case ServerGeminiEventType.ActiveGoal: break; @@ -2612,6 +2668,7 @@ export const useGeminiStream = ( userMessageTimestamp, ); geminiMessageBuffer = ''; + assistantOutputStarted = false; } break; default: { @@ -2751,6 +2808,7 @@ export const useGeminiStream = ( setThought, commitPendingThought, pendingHistoryItemRef, + pendingAssistantItemsRef, pendingThoughtItemRef, setPendingHistoryItem, handleUserPromptSubmitBlockedEvent, diff --git a/packages/cli/src/ui/hooks/useReactToolScheduler.test.tsx b/packages/cli/src/ui/hooks/useReactToolScheduler.test.tsx index b3700369674..6a7c324ec78 100644 --- a/packages/cli/src/ui/hooks/useReactToolScheduler.test.tsx +++ b/packages/cli/src/ui/hooks/useReactToolScheduler.test.tsx @@ -7,6 +7,7 @@ import { describe, it, expect } from 'vitest'; import type { Part } from '@google/genai'; import { mapToDisplay, type TrackedToolCall } from './useReactToolScheduler.js'; +import { MAX_INLINE_IMAGES_PER_ITEM } from '../utils/inline-image-parts.js'; // Build a minimal successful tracked tool call with the fields mapToDisplay's // success branch reads. `displayName` drives the collapsible gate. @@ -81,4 +82,26 @@ describe('mapToDisplay — detailedDisplay (§4.9 live path)', () => { ]); }, ); + + it('caps tool images and reports the overflow count', () => { + const images = Array.from( + { length: MAX_INLINE_IMAGES_PER_ITEM + 2 }, + (_, index) => ({ + inlineData: { + data: Buffer.from(`tool-image-${index}`).toString('base64'), + mimeType: 'image/png', + }, + }), + ); + + const tool = mapToDisplay(makeCompleted('success', 'Read File', images)) + .tools[0]; + + expect(tool.images).toEqual( + images + .slice(0, MAX_INLINE_IMAGES_PER_ITEM) + .map((part) => part.inlineData), + ); + expect(tool.omittedImageCount).toBe(2); + }); }); diff --git a/packages/cli/src/ui/hooks/useReactToolScheduler.ts b/packages/cli/src/ui/hooks/useReactToolScheduler.ts index 5872fdf7250..7e2e57e814e 100644 --- a/packages/cli/src/ui/hooks/useReactToolScheduler.ts +++ b/packages/cli/src/ui/hooks/useReactToolScheduler.ts @@ -38,7 +38,7 @@ import type { } from '../types.js'; import { ToolCallStatus } from '../types.js'; import { isCollapsibleTool } from '../components/messages/CompactToolGroupDisplay.js'; -import { extractInlineImages } from '../utils/inline-image-parts.js'; +import { collectInlineImages } from '../utils/inline-image-parts.js'; const debugLogger = createDebugLogger('REACT_TOOL_SCHEDULER'); @@ -386,11 +386,15 @@ export function mapToDisplay( : undefined, }; + const inlineImageCollection = + trackedCall.status === 'success' || + trackedCall.status === 'error' || + trackedCall.status === 'cancelled' + ? collectInlineImages(trackedCall.response.responseParts) + : null; + switch (trackedCall.status) { case 'success': { - const images = extractInlineImages( - trackedCall.response.responseParts, - ); return { ...baseDisplayProperties, status: mapCoreStatusToDisplayStatus(trackedCall.status), @@ -414,33 +418,19 @@ export function mapToDisplay( detailedDisplay: isCollapsibleTool(displayName) ? getToolResponseDisplayText(trackedCall.response.responseParts) : undefined, - ...(images.length > 0 ? { images } : {}), - confirmationDetails: undefined, - }; - } - case 'error': { - const images = extractInlineImages( - trackedCall.response.responseParts, - ); - return { - ...baseDisplayProperties, - status: mapCoreStatusToDisplayStatus(trackedCall.status), - resultDisplay: compactToolResultDisplayForHistory( - trackedCall.response.resultDisplay, - ), - ...(trackedCall.response.visionBridgeNotice !== undefined + ...(inlineImageCollection?.images.length + ? { images: inlineImageCollection.images } + : {}), + ...(inlineImageCollection?.omittedImageCount ? { - visionBridgeNotice: trackedCall.response.visionBridgeNotice, + omittedImageCount: inlineImageCollection.omittedImageCount, } : {}), - ...(images.length > 0 ? { images } : {}), confirmationDetails: undefined, }; } + case 'error': case 'cancelled': { - const images = extractInlineImages( - trackedCall.response.responseParts, - ); return { ...baseDisplayProperties, status: mapCoreStatusToDisplayStatus(trackedCall.status), @@ -452,7 +442,14 @@ export function mapToDisplay( visionBridgeNotice: trackedCall.response.visionBridgeNotice, } : {}), - ...(images.length > 0 ? { images } : {}), + ...(inlineImageCollection?.images.length + ? { images: inlineImageCollection.images } + : {}), + ...(inlineImageCollection?.omittedImageCount + ? { + omittedImageCount: inlineImageCollection.omittedImageCount, + } + : {}), confirmationDetails: undefined, }; } diff --git a/packages/cli/src/ui/types.ts b/packages/cli/src/ui/types.ts index 09a9896e11c..8cf69cb8d99 100644 --- a/packages/cli/src/ui/types.ts +++ b/packages/cli/src/ui/types.ts @@ -87,6 +87,8 @@ export interface IndividualToolCallDisplay { detailedDisplay?: string; /** Inline images carried by this tool's persisted response parts. */ images?: InlineImageData[]; + /** Images hidden after the per-row rendering limit. */ + omittedImageCount?: number; status: ToolCallStatus; confirmationDetails: ToolCallConfirmationDetails | undefined; renderOutputAsMarkdown?: boolean; @@ -147,6 +149,7 @@ export type HistoryItemGemini = HistoryItemBase & { type: 'gemini'; text: string; images?: InlineImageData[]; + omittedImageCount?: number; timestamp?: number; }; @@ -154,6 +157,7 @@ export type HistoryItemGeminiContent = HistoryItemBase & { type: 'gemini_content'; text: string; images?: InlineImageData[]; + omittedImageCount?: number; }; export type HistoryItemGeminiThought = HistoryItemBase & { diff --git a/packages/cli/src/ui/utils/inline-image-parts.test.ts b/packages/cli/src/ui/utils/inline-image-parts.test.ts index a59fcbe1fa8..7d383041902 100644 --- a/packages/cli/src/ui/utils/inline-image-parts.test.ts +++ b/packages/cli/src/ui/utils/inline-image-parts.test.ts @@ -6,11 +6,14 @@ import { describe, expect, it } from 'vitest'; import { + collectInlineImages, extractInlineContentRuns, - extractInlineImages, + getInlineImageData, + MAX_INLINE_IMAGE_ENCODED_LENGTH, + MAX_INLINE_IMAGES_PER_ITEM, } from './inline-image-parts.js'; -describe('extractInlineImages', () => { +describe('collectInlineImages', () => { it('extracts an image from a top-level tool response part', () => { const image = { data: 'aW1hZ2U=', @@ -18,7 +21,10 @@ describe('extractInlineImages', () => { displayName: 'chart.png', }; - expect(extractInlineImages([{ inlineData: image }])).toEqual([image]); + expect(collectInlineImages([{ inlineData: image }])).toEqual({ + images: [image], + omittedImageCount: 0, + }); }); it('extracts an image from nested function response parts', () => { @@ -28,7 +34,7 @@ describe('extractInlineImages', () => { }; expect( - extractInlineImages([ + collectInlineImages([ { functionResponse: { id: 'call-1', @@ -38,12 +44,12 @@ describe('extractInlineImages', () => { }, }, ]), - ).toEqual([image]); + ).toEqual({ images: [image], omittedImageCount: 0 }); }); it('ignores non-image inline data', () => { expect( - extractInlineImages([ + collectInlineImages([ { inlineData: { data: 'bm90LWFuLWltYWdl', @@ -51,7 +57,37 @@ describe('extractInlineImages', () => { }, }, ]), - ).toEqual([]); + ).toEqual({ images: [], omittedImageCount: 0 }); + }); + + it('caps images and reports how many were omitted', () => { + const images = Array.from( + { length: MAX_INLINE_IMAGES_PER_ITEM + 2 }, + (_, index) => ({ + data: Buffer.from(`image-${index}`).toString('base64'), + mimeType: 'image/png', + }), + ); + + expect( + collectInlineImages(images.map((inlineData) => ({ inlineData }))), + ).toEqual({ + images: images.slice(0, MAX_INLINE_IMAGES_PER_ITEM), + omittedImageCount: 2, + }); + }); +}); + +describe('getInlineImageData', () => { + it('rejects payloads above the renderer encoded-length limit', () => { + expect( + getInlineImageData({ + inlineData: { + data: 'A'.repeat(MAX_INLINE_IMAGE_ENCODED_LENGTH + 1), + mimeType: 'image/png', + }, + }), + ).toBeNull(); }); }); @@ -75,4 +111,29 @@ describe('extractInlineContentRuns', () => { { kind: 'text', text: 'after' }, ]); }); + + it('caps images and preserves an overflow marker at the first omission', () => { + const images = Array.from( + { length: MAX_INLINE_IMAGES_PER_ITEM + 2 }, + (_, index) => ({ + data: Buffer.from(`image-${index}`).toString('base64'), + mimeType: 'image/png', + }), + ); + + expect( + extractInlineContentRuns([ + { text: 'before' }, + ...images.map((inlineData) => ({ inlineData })), + { text: 'after' }, + ]), + ).toEqual([ + { kind: 'text', text: 'before' }, + ...images + .slice(0, MAX_INLINE_IMAGES_PER_ITEM) + .map((image) => ({ kind: 'image' as const, image })), + { kind: 'omitted_images', count: 2 }, + { kind: 'text', text: 'after' }, + ]); + }); }); diff --git a/packages/cli/src/ui/utils/inline-image-parts.ts b/packages/cli/src/ui/utils/inline-image-parts.ts index 545a0f6d9f0..2bbb363affc 100644 --- a/packages/cli/src/ui/utils/inline-image-parts.ts +++ b/packages/cli/src/ui/utils/inline-image-parts.ts @@ -5,18 +5,34 @@ */ import type { Part } from '@google/genai'; +import { MAX_TERMINAL_IMAGE_BYTES } from '@qwen-code/qwen-code-core'; import type { InlineImageData } from '../types.js'; +export const MAX_INLINE_IMAGES_PER_ITEM = 4; +export const MAX_INLINE_IMAGE_ENCODED_LENGTH = + Math.ceil((MAX_TERMINAL_IMAGE_BYTES * 4) / 3) + 4; + export type InlineContentRun = | { kind: 'text'; text: string } - | { kind: 'image'; image: InlineImageData }; + | { kind: 'image'; image: InlineImageData } + | { kind: 'omitted_images'; count: number }; + +export interface InlineImageCollection { + images: InlineImageData[]; + omittedImageCount: number; +} + +export function formatInlineImageOverflow(count: number): string { + return `[+${count} more ${count === 1 ? 'image' : 'images'}]`; +} export function getInlineImageData(part: Part): InlineImageData | null { const inlineData = part.inlineData; if ( !inlineData?.mimeType?.trim().toLowerCase().startsWith('image/') || typeof inlineData.data !== 'string' || - inlineData.data.length === 0 + inlineData.data.length === 0 || + inlineData.data.length > MAX_INLINE_IMAGE_ENCODED_LENGTH ) { return null; } @@ -30,28 +46,35 @@ export function getInlineImageData(part: Part): InlineImageData | null { }; } -export function extractInlineImages( +export function collectInlineImages( parts: Part[] | undefined, -): InlineImageData[] { +): InlineImageCollection { if (!parts) { - return []; + return { images: [], omittedImageCount: 0 }; } const images: InlineImageData[] = []; - for (const part of parts) { - const topLevelImage = getInlineImageData(part); - if (topLevelImage) { - images.push(topLevelImage); + let omittedImageCount = 0; + const collectImage = (part: Part) => { + const image = getInlineImageData(part); + if (!image) { + return; + } + if (images.length < MAX_INLINE_IMAGES_PER_ITEM) { + images.push(image); + } else { + omittedImageCount++; } + }; + + for (const part of parts) { + collectImage(part); for (const nested of part.functionResponse?.parts ?? []) { - const nestedImage = getInlineImageData(nested as Part); - if (nestedImage) { - images.push(nestedImage); - } + collectImage(nested as Part); } } - return images; + return { images, omittedImageCount }; } export function extractInlineContentRuns( @@ -64,6 +87,11 @@ export function extractInlineContentRuns( const runs: InlineContentRun[] = []; let textParts: string[] = []; + let displayedImageCount = 0; + let overflowRun: Extract< + InlineContentRun, + { kind: 'omitted_images' } + > | null = null; const flushText = () => { if (textParts.length === 0) return; runs.push({ kind: 'text', text: textParts.join(textSeparator) }); @@ -78,7 +106,15 @@ export function extractInlineContentRuns( const image = getInlineImageData(part); if (image) { flushText(); - runs.push({ kind: 'image', image }); + if (displayedImageCount < MAX_INLINE_IMAGES_PER_ITEM) { + runs.push({ kind: 'image', image }); + displayedImageCount++; + } else if (overflowRun) { + overflowRun.count++; + } else { + overflowRun = { kind: 'omitted_images', count: 1 }; + runs.push(overflowRun); + } } } flushText(); diff --git a/packages/cli/src/ui/utils/resumeHistoryUtils.test.ts b/packages/cli/src/ui/utils/resumeHistoryUtils.test.ts index 7f30e885687..86770516208 100644 --- a/packages/cli/src/ui/utils/resumeHistoryUtils.test.ts +++ b/packages/cli/src/ui/utils/resumeHistoryUtils.test.ts @@ -21,6 +21,7 @@ import type { } from '@qwen-code/qwen-code-core'; import type { Part } from '@google/genai'; import type { HistoryItem } from '../types.js'; +import { MAX_INLINE_IMAGES_PER_ITEM } from './inline-image-parts.js'; const makeConfig = (tools: Record) => ({ @@ -857,6 +858,43 @@ describe('resumeHistoryUtils', () => { ]); }); + it('caps restored assistant images and retains the overflow count', () => { + const images = Array.from( + { length: MAX_INLINE_IMAGES_PER_ITEM + 2 }, + (_, index) => ({ + data: Buffer.from(`restored-image-${index}`).toString('base64'), + mimeType: 'image/png', + }), + ); + const conversation = { + messages: [ + { + type: 'assistant', + timestamp: '2026-01-15T19:00:00.000Z', + message: { + parts: images.map((inlineData) => ({ inlineData })), + }, + }, + ], + } as unknown as ConversationRecord; + + const items = buildResumedHistoryItems( + { conversation } as ResumedSessionData, + makeConfig({}), + ).filter( + (item) => item.type === 'gemini' || item.type === 'gemini_content', + ); + + expect(items.flatMap((item) => item.images ?? [])).toEqual( + images.slice(0, MAX_INLINE_IMAGES_PER_ITEM), + ); + expect(items.at(-1)).toMatchObject({ + type: 'gemini_content', + text: '', + omittedImageCount: 2, + }); + }); + it('restores images nested in persisted tool response parts', () => { const conversation = { messages: [ diff --git a/packages/cli/src/ui/utils/resumeHistoryUtils.ts b/packages/cli/src/ui/utils/resumeHistoryUtils.ts index e104101c594..6ed5d934d48 100644 --- a/packages/cli/src/ui/utils/resumeHistoryUtils.ts +++ b/packages/cli/src/ui/utils/resumeHistoryUtils.ts @@ -38,8 +38,8 @@ import { } from './history-gap-notice.js'; import { shouldDisplayGoalStateCause } from './goal-runtime.js'; import { + collectInlineImages, extractInlineContentRuns, - extractInlineImages, } from './inline-image-parts.js'; /** @@ -223,6 +223,7 @@ function convertToHistoryItems( visionBridgeNotice?: string; detailedDisplay?: string; images?: InlineImageData[]; + omittedImageCount?: number; status: ToolCallStatus; confirmationDetails: undefined; }> = []; @@ -474,24 +475,27 @@ function convertToHistoryItems( } for (const [index, run] of displayRuns.entries()) { const type = index === 0 ? 'gemini' : 'gemini_content'; - items.push( - run.kind === 'text' - ? { - type, - text: run.text, - ...(index === 0 - ? { timestamp: new Date(record.timestamp).getTime() } - : {}), - } - : { - type, - text: '', - images: [run.image], - ...(index === 0 - ? { timestamp: new Date(record.timestamp).getTime() } - : {}), - }, - ); + const timestamp = + index === 0 + ? { timestamp: new Date(record.timestamp).getTime() } + : {}; + if (run.kind === 'text') { + items.push({ type, text: run.text, ...timestamp }); + } else if (run.kind === 'image') { + items.push({ + type, + text: '', + images: [run.image], + ...timestamp, + }); + } else { + items.push({ + type, + text: '', + omittedImageCount: run.count, + ...timestamp, + }); + } } } @@ -538,10 +542,14 @@ function convertToHistoryItems( rawStatus === 'error' ? ToolCallStatus.Error : ToolCallStatus.Success; - const images = extractInlineImages(responseParts); + const { images, omittedImageCount } = + collectInlineImages(responseParts); if (images.length > 0) { toolCall.images = images; } + if (omittedImageCount > 0) { + toolCall.omittedImageCount = omittedImageCount; + } // Full detail for the Ctrl+O transcript (§4.9): the complete // functionResponse parts are persisted on the tool_result record // (only resultDisplay is sanitized), so resume yields full detail diff --git a/packages/cli/src/ui/utils/terminal-image-renderer.ts b/packages/cli/src/ui/utils/terminal-image-renderer.ts index e2b3695e58c..843b6793f17 100644 --- a/packages/cli/src/ui/utils/terminal-image-renderer.ts +++ b/packages/cli/src/ui/utils/terminal-image-renderer.ts @@ -21,6 +21,7 @@ import { shouldRunThroughShell, type KittyImagePlaceholder, } from './mermaidImageRenderer.js'; +import { MAX_INLINE_IMAGE_ENCODED_LENGTH } from './inline-image-parts.js'; const CHAFA_TIMEOUT_MS = 8000; const CHAFA_MAX_OUTPUT_BYTES = 2 * 1024 * 1024; @@ -340,8 +341,7 @@ function getImageFormat(mimeType: string): string | null { } function decodeInlineImage(data: string): Buffer | null { - const maxEncodedLength = Math.ceil((MAX_TERMINAL_IMAGE_BYTES * 4) / 3) + 4; - if (data.length === 0 || data.length > maxEncodedLength) { + if (data.length === 0 || data.length > MAX_INLINE_IMAGE_ENCODED_LENGTH) { return null; } From cf7f9e76af950becf3271d1335b44c1e0a443716 Mon Sep 17 00:00:00 2001 From: tly Date: Mon, 3 Aug 2026 09:31:00 +0800 Subject: [PATCH 5/7] fix(cli): clear compacted image overflow markers --- .../src/ui/hooks/useHistoryManager.test.ts | 52 +++++++++++++++++++ .../cli/src/ui/hooks/useHistoryManager.ts | 13 +++-- .../ui/utils/terminal-image-renderer.test.ts | 5 +- 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/packages/cli/src/ui/hooks/useHistoryManager.test.ts b/packages/cli/src/ui/hooks/useHistoryManager.test.ts index 05ccabb4fa1..b88e5738ee4 100644 --- a/packages/cli/src/ui/hooks/useHistoryManager.test.ts +++ b/packages/cli/src/ui/hooks/useHistoryManager.test.ts @@ -390,6 +390,7 @@ describe('useHistoryManager', () => { description: '', resultDisplay: undefined, images: [{ data: 'aW1hZ2U=', mimeType: 'image/png' }], + omittedImageCount: 2, status: ToolCallStatus.Success, confirmationDetails: undefined, }, @@ -409,11 +410,13 @@ describe('useHistoryManager', () => { ).tools[0]; expect(oldestTool.resultDisplay).toBe(UI_COMPACT_CLEARED_MESSAGE); expect(oldestTool.images).toBeUndefined(); + expect(oldestTool.omittedImageCount).toBeUndefined(); const recentTool = ( result.current.history[24] as unknown as HistoryItemToolGroup ).tools[0]; expect(recentTool.images).toHaveLength(1); + expect(recentTool.omittedImageCount).toBe(2); }); it('clears old assistant image payloads while keeping recent images', () => { @@ -427,6 +430,7 @@ describe('useHistoryManager', () => { type: i === 0 ? 'gemini' : 'gemini_content', text: i === 0 ? 'Generated chart' : '', images: [{ data: `aW1hZ2Ut${i}`, mimeType: 'image/png' }], + omittedImageCount: 2, }, ts + i, ); @@ -445,11 +449,59 @@ describe('useHistoryManager', () => { expect( oldestItem.type === 'gemini' ? oldestItem.images : undefined, ).toBeUndefined(); + expect( + oldestItem.type === 'gemini' ? oldestItem.omittedImageCount : undefined, + ).toBeUndefined(); const recentItem = result.current.history[24]; expect( recentItem.type === 'gemini_content' ? recentItem.images : undefined, ).toHaveLength(1); + expect( + recentItem.type === 'gemini_content' + ? recentItem.omittedImageCount + : undefined, + ).toBe(2); + }); + + it('compacts old assistant image overflow markers without payloads', () => { + const { result } = renderHook(() => useHistory()); + const ts = Date.now(); + + for (let i = 0; i < 25; i++) { + act(() => { + result.current.addItem( + { + type: 'gemini_content', + text: '', + omittedImageCount: 2, + }, + ts + i, + ); + }); + } + + act(() => { + result.current.compactOldItems(); + }); + + const oldestItem = result.current.history[0]; + expect(oldestItem).toMatchObject({ + type: 'gemini_content', + text: UI_COMPACT_CLEARED_IMAGE_MESSAGE, + }); + expect( + oldestItem.type === 'gemini_content' + ? oldestItem.omittedImageCount + : undefined, + ).toBeUndefined(); + + const recentItem = result.current.history[24]; + expect( + recentItem.type === 'gemini_content' + ? recentItem.omittedImageCount + : undefined, + ).toBe(2); }); it('clears a tool that carries detailedDisplay but no resultDisplay (defensive)', () => { diff --git a/packages/cli/src/ui/hooks/useHistoryManager.ts b/packages/cli/src/ui/hooks/useHistoryManager.ts index c0402555259..9aed5ba379d 100644 --- a/packages/cli/src/ui/hooks/useHistoryManager.ts +++ b/packages/cli/src/ui/hooks/useHistoryManager.ts @@ -163,7 +163,7 @@ export function useHistory(): UseHistoryManagerReturn { totalThoughts++; } else if ( (item.type === 'gemini' || item.type === 'gemini_content') && - item.images?.length + (item.images?.length || item.omittedImageCount) ) { totalAssistantItemsWithImages++; } else if ( @@ -173,7 +173,7 @@ export function useHistory(): UseHistoryManagerReturn { (t.resultDisplay != null && t.resultDisplay !== UI_COMPACT_CLEARED_MESSAGE) || t.detailedDisplay != null || - Boolean(t.images?.length), + Boolean(t.images?.length || t.omittedImageCount), ) ) { totalToolGroupsWithOutput++; @@ -212,7 +212,7 @@ export function useHistory(): UseHistoryManagerReturn { .map((item) => { if ( (item.type === 'gemini' || item.type === 'gemini_content') && - item.images?.length + (item.images?.length || item.omittedImageCount) ) { assistantImageItemsSeen++; if (assistantImageItemsSeen <= assistantImageItemsToCompact) { @@ -223,6 +223,7 @@ export function useHistory(): UseHistoryManagerReturn { ? `${item.text}\n\n${UI_COMPACT_CLEARED_IMAGE_MESSAGE}` : UI_COMPACT_CLEARED_IMAGE_MESSAGE, images: undefined, + omittedImageCount: undefined, }; } } @@ -236,7 +237,7 @@ export function useHistory(): UseHistoryManagerReturn { (t.resultDisplay != null && t.resultDisplay !== UI_COMPACT_CLEARED_MESSAGE) || t.detailedDisplay != null || - Boolean(t.images?.length), + Boolean(t.images?.length || t.omittedImageCount), ); if (!hasOldOutput) return item; toolGroupsSeen++; @@ -249,7 +250,8 @@ export function useHistory(): UseHistoryManagerReturn { (t.resultDisplay != null && t.resultDisplay !== UI_COMPACT_CLEARED_MESSAGE) || t.detailedDisplay != null || - t.images?.length + t.images?.length || + t.omittedImageCount ) { // Also drop `detailedDisplay` (the raw functionResponse text // kept for the Ctrl+O full-detail transcript): clearing only @@ -263,6 +265,7 @@ export function useHistory(): UseHistoryManagerReturn { resultDisplay: UI_COMPACT_CLEARED_MESSAGE, detailedDisplay: undefined, images: undefined, + omittedImageCount: undefined, }; } return t; diff --git a/packages/cli/src/ui/utils/terminal-image-renderer.test.ts b/packages/cli/src/ui/utils/terminal-image-renderer.test.ts index 93dc65d7ac8..d6f33d75277 100644 --- a/packages/cli/src/ui/utils/terminal-image-renderer.test.ts +++ b/packages/cli/src/ui/utils/terminal-image-renderer.test.ts @@ -18,6 +18,7 @@ import { TRANSMITTED_KEY_LIMIT, wasKittyImageWritten, } from './terminal-image-renderer.js'; +import { MAX_INLINE_IMAGE_ENCODED_LENGTH } from './inline-image-parts.js'; const PNG_1X1 = Buffer.from( 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=', @@ -131,9 +132,7 @@ describe('terminalImageRenderer', () => { }); it('rejects inline payloads above the shared image limit before decoding', () => { - const oversizedBase64 = 'A'.repeat( - Math.ceil(((8 * 1024 * 1024 + 1) * 4) / 3), - ); + const oversizedBase64 = 'A'.repeat(MAX_INLINE_IMAGE_ENCODED_LENGTH + 1); expect( prepareInlineTerminalImage({ From f97d233d0f50337bc67340c522f849f3e60f6734 Mon Sep 17 00:00:00 2001 From: tly Date: Mon, 3 Aug 2026 20:34:35 +0800 Subject: [PATCH 6/7] docs(cli): clarify inline image resume scope --- .qwen/e2e-tests/terminal-inline-images.md | 12 ++++--- docs/design/terminal-inline-images.md | 31 ++++++++++++------- .../src/ui/utils/resumeHistoryUtils.test.ts | 6 ++-- 3 files changed, 30 insertions(+), 19 deletions(-) diff --git a/.qwen/e2e-tests/terminal-inline-images.md b/.qwen/e2e-tests/terminal-inline-images.md index 697617850a3..bcf3506ea5b 100644 --- a/.qwen/e2e-tests/terminal-inline-images.md +++ b/.qwen/e2e-tests/terminal-inline-images.md @@ -29,10 +29,12 @@ baseline is grounded in issue #8090 and the unchanged `main` event mapping. 1. Return a 1x1 PNG between two assistant text parts. 2. Confirm the transcript order is `text -> image -> text`. -3. Return a PNG in a tool's top-level and nested `functionResponse.parts`. -4. Confirm both successful and failed/cancelled tool rows retain their images. -5. Open Ctrl+O and resume the session; confirm assistant and tool image order is - reconstructed from persisted parts. +3. Return a PNG in a successful tool's top-level and nested + `functionResponse.parts`. +4. Confirm the successful tool row retains its images. +5. Open Ctrl+O and resume the session; confirm successful tool image order is + reconstructed from persisted parts. Assistant output resumes its persisted + text; assistant inline images are not persisted by the current Core recorder. 6. Return six images in one assistant output and one tool response; confirm the first four render and the row ends with `[+2 more images]`. @@ -62,7 +64,7 @@ baseline is grounded in issue #8090 and the unchanged `main` event mapping. 3. Repeat with malformed base64, a payload above 8 MiB, invalid IHDR dimensions, and a non-PNG MIME type. 4. Confirm no raw image sequence is written. Confirm the oversized payload is - excluded from UI history, while admitted malformed/non-PNG data uses a + dropped before UI history, while admitted malformed/non-PNG data uses a deterministic placeholder. 5. Repeat with `INK_SCREEN_READER=true`; confirm only the placeholder is emitted. diff --git a/docs/design/terminal-inline-images.md b/docs/design/terminal-inline-images.md index 65b4275120f..7eed41287df 100644 --- a/docs/design/terminal-inline-images.md +++ b/docs/design/terminal-inline-images.md @@ -20,9 +20,9 @@ This is the render-and-forget slice requested by issue #8090: - preserve ordered text and image parts on content events without changing the existing concatenated `value` contract; -- render live and restored assistant PNGs through the #8217 component and - renderer; -- render PNGs nested in successful, failed, or cancelled tool responses; +- render live assistant PNGs and restored successful tool PNGs through the + #8217 component and renderer; +- render PNGs nested in successful tool responses; - keep text/image ordering across retry, model fallback, cancellation, stream boundaries, and goal-state events; - bound retained image payloads during UI history compaction; @@ -53,16 +53,21 @@ Text-only events keep their existing runtime shape, so non-interactive output, SDK, ACP, daemon, channel, Web UI, and VS Code consumers continue using `value` unchanged. -Recorded assistant messages already retain their original parts. Resume logic -reconstructs ordered text/image runs instead of flattening images away. +Resume logic reconstructs ordered text/image runs when persisted parts contain +them. Tool responses retain nested image parts in the session record. The +current Core recorder flattens assistant output to text, so live assistant +images are not restored by `--continue`; assistant-image persistence is outside +this slice. ### Tool output Tool media is stored in `functionResponse.parts`. A CLI extractor reads image `inlineData` from top-level and nested response parts. Live scheduler mapping and resume mapping attach the images to the existing -`IndividualToolCallDisplay`. Each tool row keeps the first four images and an -overflow count for the rest. +`IndividualToolCallDisplay`. Each successful tool row keeps the first four +images and an overflow count for the rest. Failed and cancelled tool records +currently do not carry inline image parts from Core, so their image handling is +defensive rather than a supported output path in this slice. Tools carrying images render individually even when their text-only form would normally collapse into a read/search summary. `ToolMessage` routes the images @@ -92,7 +97,7 @@ placeholder and emits no raw image sequence. The same encoded-length limit is applied before inline data enters CLI history or tool-display state. Payloads that exceed the renderer's 8 MiB decoded-image -budget are therefore not retained by the UI. +budget are dropped before rendering and do not produce a placeholder. The first slice renders validated PNG data only. Other image MIME types remain visible as deterministic placeholders rather than entering a second protocol @@ -116,8 +121,10 @@ each assistant output and tool row. retaining the old `value` and text-only event shape. - Verify live TUI ordering across retry, fallback, cancellation, stream boundaries, and goal-state events. -- Verify live and restored tool responses expose nested images. -- Verify restored assistant history preserves text/image ordering. -- Verify live, restored, and tool output enforce the image cap and expose the - overflow count without retaining oversized payloads. +- Verify live and restored successful tool responses expose nested images. +- Verify the resume parser preserves assistant text/image ordering for records + that already contain persisted parts; the current recorder's assistant-image + persistence gap remains outside this slice. +- Verify live assistant and successful tool output enforce the image cap and + expose the overflow count; oversized payloads are dropped before UI history. - Verify memory compaction clears old assistant and tool image payloads. diff --git a/packages/cli/src/ui/utils/resumeHistoryUtils.test.ts b/packages/cli/src/ui/utils/resumeHistoryUtils.test.ts index 86770516208..815039582fa 100644 --- a/packages/cli/src/ui/utils/resumeHistoryUtils.test.ts +++ b/packages/cli/src/ui/utils/resumeHistoryUtils.test.ts @@ -806,7 +806,9 @@ describe('resumeHistoryUtils', () => { expect(items[0]).not.toHaveProperty('sentToModel'); }); - it('restores assistant text and images in their original order', () => { + // The current Core recorder flattens assistant output before persistence. + // This fixture covers the parser for records written by a compatible writer. + it('parses persisted assistant text and images in their original order', () => { const conversation = { messages: [ { @@ -858,7 +860,7 @@ describe('resumeHistoryUtils', () => { ]); }); - it('caps restored assistant images and retains the overflow count', () => { + it('caps persisted assistant images and retains the overflow count', () => { const images = Array.from( { length: MAX_INLINE_IMAGES_PER_ITEM + 2 }, (_, index) => ({ From ac7d564f97693fd69f73729680cd41ca095ad4d3 Mon Sep 17 00:00:00 2001 From: tly Date: Tue, 4 Aug 2026 17:27:53 +0800 Subject: [PATCH 7/7] fix(cli): address inline image review feedback --- docs/design/terminal-inline-images.md | 14 +- packages/cli/src/ui/AppContainer.test.tsx | 8 + packages/cli/src/ui/AppContainer.tsx | 11 +- .../src/ui/commands/restoreCommand.test.ts | 2 + .../cli/src/ui/commands/restoreCommand.ts | 1 + packages/cli/src/ui/commands/types.ts | 2 + .../messages/ConversationMessages.test.tsx | 30 ++- .../messages/ConversationMessages.tsx | 12 +- .../components/messages/ToolMessage.test.tsx | 22 +- .../ui/components/messages/ToolMessage.tsx | 6 +- .../cli/src/ui/hooks/slashCommandProcessor.ts | 1 + .../cli/src/ui/hooks/useBranchCommand.test.ts | 7 + packages/cli/src/ui/hooks/useBranchCommand.ts | 13 +- .../cli/src/ui/hooks/useGeminiStream.test.tsx | 254 +++++++++++++++++- .../ui/hooks/useReactToolScheduler.test.tsx | 1 - .../cli/src/ui/hooks/useResumeCommand.test.ts | 6 + packages/cli/src/ui/hooks/useResumeCommand.ts | 4 + packages/cli/src/ui/types.ts | 1 - .../src/ui/utils/inline-image-parts.test.ts | 2 +- .../cli/src/ui/utils/inline-image-parts.ts | 3 - .../src/ui/utils/resumeHistoryUtils.test.ts | 1 - .../ui/utils/terminal-image-renderer.test.ts | 15 ++ .../src/ui/utils/terminal-image-renderer.ts | 43 ++- 23 files changed, 419 insertions(+), 40 deletions(-) diff --git a/docs/design/terminal-inline-images.md b/docs/design/terminal-inline-images.md index 7eed41287df..3e646902030 100644 --- a/docs/design/terminal-inline-images.md +++ b/docs/design/terminal-inline-images.md @@ -31,7 +31,8 @@ This is the render-and-forget slice requested by issue #8090: - show a deterministic text placeholder when an image cannot be rendered. Kitty deletion, resize-driven replacement, terminal cell pixel queries, and -global scroll lifecycle ownership remain out of scope. +global scroll lifecycle ownership remain out of scope and are tracked in +#8520. ## Data Flow @@ -53,11 +54,17 @@ Text-only events keep their existing runtime shape, so non-interactive output, SDK, ACP, daemon, channel, Web UI, and VS Code consumers continue using `value` unchanged. +After a thrown stream, staged output remains transient so an explicit retry can +discard the failed attempt. If an out-of-band shell or slash-command item is +added before the next model submit, that item can enter committed history before +the staged output. The next model submit commits the staged output; history +clear, resume, branch, restore, rewind, and Ctrl+L paths discard it instead. + Resume logic reconstructs ordered text/image runs when persisted parts contain them. Tool responses retain nested image parts in the session record. The current Core recorder flattens assistant output to text, so live assistant images are not restored by `--continue`; assistant-image persistence is outside -this slice. +this slice and tracked in #8521. ### Tool output @@ -80,7 +87,8 @@ adds an in-memory PNG entry point that: 1. validates bounded base64 before decoding; 2. verifies the PNG signature and IHDR dimensions; -3. rejects payloads above 8 MiB or dimensions above 1,000,000 pixels; +3. rejects payloads above 8 MiB, dimensions above 1,000,000 pixels, or images + above 64 million total pixels; 4. reuses the existing terminal sizing and bounded render cache; 5. uses native Kitty placement in direct Kitty/Ghostty sessions; 6. passes PNG bytes to `chafa` over stdin in other supported environments; diff --git a/packages/cli/src/ui/AppContainer.test.tsx b/packages/cli/src/ui/AppContainer.test.tsx index 6a66052271a..0d9988e84e9 100644 --- a/packages/cli/src/ui/AppContainer.test.tsx +++ b/packages/cli/src/ui/AppContainer.test.tsx @@ -230,6 +230,7 @@ describe('AppContainer State Management', () => { const mockedUseKeypress = useKeypress as Mock; let originalStdoutIsTTY: boolean | undefined; let restoreCiEnv = () => {}; + let mockClearPendingState: Mock; const mockedRestorePromptStash = vi.mocked(restorePromptStash); beforeEach(() => { @@ -249,6 +250,7 @@ describe('AppContainer State Management', () => { capturedUIActions = null!; capturedRenderMode = 'render'; capturedThoughtExpanded = null!; + mockClearPendingState = vi.fn(); // **Provide a default return value for EVERY mocked hook.** mockedUseHistory.mockReturnValue({ @@ -338,6 +340,7 @@ describe('AppContainer State Management', () => { retryLastPrompt: vi.fn(), streamingResponseLengthRef: { current: 0 }, isReceivingContent: false, + clearPendingState: mockClearPendingState, }); mockedUseVim.mockReturnValue({ handleInput: vi.fn() }); mockedUseFolderTrust.mockReturnValue({ @@ -1259,6 +1262,7 @@ describe('AppContainer State Management', () => { capturedUIActions.handleClearScreen(); expect(clearSpy).toHaveBeenCalledTimes(1); + expect(mockClearPendingState).toHaveBeenCalledTimes(1); expect(mockStdout.write).not.toHaveBeenCalledWith( ansiEscapes.clearTerminal, ); @@ -5774,6 +5778,10 @@ describe('AppContainer State Management', () => { rewindUserItem(1, 'first prompt', 'prompt-1'), { id: 2, type: 'gemini', text: 'first response' }, ]); + expect(mockClearPendingState).toHaveBeenCalledTimes(1); + expect(mockClearPendingState.mock.invocationCallOrder[0]).toBeLessThan( + harness.loadHistory.mock.invocationCallOrder[0]!, + ); expect(harness.setText).toHaveBeenCalledWith('second prompt'); expect(harness.addItem).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/packages/cli/src/ui/AppContainer.tsx b/packages/cli/src/ui/AppContainer.tsx index 66fc4a3bbf5..eb8211f2706 100644 --- a/packages/cli/src/ui/AppContainer.tsx +++ b/packages/cli/src/ui/AppContainer.tsx @@ -1494,6 +1494,12 @@ export const AppContainer = (props: AppContainerProps) => { }); }, [addHistoryItem, config]); + const clearPendingStateRef = useRef<() => void>(() => {}); + const clearPendingStateFromRef = useCallback( + () => clearPendingStateRef.current(), + [], + ); + const { isResumeDialogOpen, resumeMatchedSessions, @@ -1505,6 +1511,7 @@ export const AppContainer = (props: AppContainerProps) => { settings, historyManager, startNewSession, + clearPendingState: clearPendingStateFromRef, setSessionName, remount: refreshStatic, }); @@ -1514,6 +1521,7 @@ export const AppContainer = (props: AppContainerProps) => { settings, historyManager, startNewSession, + clearPendingState: clearPendingStateFromRef, setSessionName, remount: refreshStatic, }); @@ -1578,7 +1586,6 @@ export const AppContainer = (props: AppContainerProps) => { // whose implementations are swapped in once the real callbacks exist. const openRewindSelectorRef = useRef<() => void>(() => {}); const cancelOngoingRequestRef = useRef<() => void>(() => {}); - const clearPendingStateRef = useRef<() => void>(() => {}); // /diff opens a per-turn diff dialog. Unlike rewind, no double-press or // history-bound guard is needed, so the open/close handlers can live here @@ -2931,6 +2938,7 @@ export const AppContainer = (props: AppContainerProps) => { ); const handleClearScreen = useCallback(() => { + clearPendingStateRef.current(); historyManager.clearItems(); clearScreen(); remountStaticHistory(); @@ -3578,6 +3586,7 @@ export const AppContainer = (props: AppContainerProps) => { const truncatedUi = expandCollapsedHistory( originalHistory.filter((h) => h.id < userItem.id), ); + clearPendingStateRef.current(); historyManager.loadHistory(truncatedUi); refreshStatic(); diff --git a/packages/cli/src/ui/commands/restoreCommand.test.ts b/packages/cli/src/ui/commands/restoreCommand.test.ts index 3be2274748e..aa10b631783 100644 --- a/packages/cli/src/ui/commands/restoreCommand.test.ts +++ b/packages/cli/src/ui/commands/restoreCommand.test.ts @@ -54,6 +54,7 @@ describe('restoreCommand', () => { config: mockConfig, }, }); + mockContext.ui.clearPendingState = vi.fn(); }); afterEach(async () => { @@ -170,6 +171,7 @@ describe('restoreCommand', () => { expect(mockContext.ui.loadHistory).toHaveBeenCalledWith( toolCallData.history, ); + expect(mockContext.ui.clearPendingState).toHaveBeenCalledTimes(1); expect(mockSetHistory).toHaveBeenCalledWith(toolCallData.clientHistory); expect(mockRewind).toHaveBeenCalledWith(toolCallData.promptId, true); expect(mockContext.ui.addItem).toHaveBeenCalledWith( diff --git a/packages/cli/src/ui/commands/restoreCommand.ts b/packages/cli/src/ui/commands/restoreCommand.ts index 8c335c8dec9..8fc3c36f1a6 100644 --- a/packages/cli/src/ui/commands/restoreCommand.ts +++ b/packages/cli/src/ui/commands/restoreCommand.ts @@ -144,6 +144,7 @@ async function restoreAction( content: 'loadHistory function is not available.', }; } + context.ui.clearPendingState?.(); loadHistory(toolCallData.history); } diff --git a/packages/cli/src/ui/commands/types.ts b/packages/cli/src/ui/commands/types.ts index d82680e7bd4..f2444722e31 100644 --- a/packages/cli/src/ui/commands/types.ts +++ b/packages/cli/src/ui/commands/types.ts @@ -63,6 +63,8 @@ export interface CommandContext { addItem: UseHistoryManagerReturn['addItem']; /** Clears all history items and the console screen. */ clear: () => void; + /** Clears transient assistant output before replacing conversation history. */ + clearPendingState?: () => void; /** * Sets the transient debug message displayed in the application footer in debug mode. */ diff --git a/packages/cli/src/ui/components/messages/ConversationMessages.test.tsx b/packages/cli/src/ui/components/messages/ConversationMessages.test.tsx index 2223e406b99..8f24edb90ea 100644 --- a/packages/cli/src/ui/components/messages/ConversationMessages.test.tsx +++ b/packages/cli/src/ui/components/messages/ConversationMessages.test.tsx @@ -16,8 +16,17 @@ import { } from './ConversationMessages.js'; vi.mock('../TerminalImage.js', () => ({ - TerminalImage: ({ image }: { image: { mimeType: string } }) => ( - MockTerminalImage:{image.mimeType} + TerminalImage: ({ + image, + availableTerminalHeight, + }: { + image: { mimeType: string }; + availableTerminalHeight?: number; + }) => ( + + MockTerminalImage:{image.mimeType}:height= + {availableTerminalHeight ?? 'undef'} + ), })); @@ -47,6 +56,23 @@ describe('', () => { expect(lastFrame()).toContain('[+2 more images]'); }); + + it('shares the pending height budget across assistant images', () => { + const { lastFrame } = render( + , + ); + + expect(lastFrame()).toContain('MockTerminalImage:image/png:height=6'); + }); }); describe('', () => { diff --git a/packages/cli/src/ui/components/messages/ConversationMessages.tsx b/packages/cli/src/ui/components/messages/ConversationMessages.tsx index 4c0eee6b686..fa81f473d24 100644 --- a/packages/cli/src/ui/components/messages/ConversationMessages.tsx +++ b/packages/cli/src/ui/components/messages/ConversationMessages.tsx @@ -171,6 +171,10 @@ const PrefixedMarkdownMessage: React.FC = ({ sourceCopyIndexOffsets, }) => { const prefixWidth = getPrefixWidth(prefix); + const imageHeightBudget = + availableTerminalHeight !== undefined && images?.length + ? Math.max(1, Math.floor(availableTerminalHeight / (images.length + 1))) + : availableTerminalHeight; return ( @@ -195,7 +199,7 @@ const PrefixedMarkdownMessage: React.FC = ({ key={index} image={image} contentWidth={contentWidth - prefixWidth} - availableTerminalHeight={availableTerminalHeight} + availableTerminalHeight={imageHeightBudget} /> ))} {omittedImageCount !== undefined && omittedImageCount > 0 && ( @@ -220,6 +224,10 @@ const ContinuationMarkdownMessage: React.FC< sourceCopyIndexOffsets, }) => { const prefixWidth = getPrefixWidth(basePrefix); + const imageHeightBudget = + availableTerminalHeight !== undefined && images?.length + ? Math.max(1, Math.floor(availableTerminalHeight / (images.length + 1))) + : availableTerminalHeight; return ( @@ -238,7 +246,7 @@ const ContinuationMarkdownMessage: React.FC< key={index} image={image} contentWidth={contentWidth - prefixWidth} - availableTerminalHeight={availableTerminalHeight} + availableTerminalHeight={imageHeightBudget} /> ))} {omittedImageCount !== undefined && omittedImageCount > 0 && ( diff --git a/packages/cli/src/ui/components/messages/ToolMessage.test.tsx b/packages/cli/src/ui/components/messages/ToolMessage.test.tsx index edab4b8cbb7..3c5bbac5a73 100644 --- a/packages/cli/src/ui/components/messages/ToolMessage.test.tsx +++ b/packages/cli/src/ui/components/messages/ToolMessage.test.tsx @@ -74,14 +74,16 @@ vi.mock('../TerminalImage.js', () => ({ TerminalImage: ({ data, image, + availableTerminalHeight, }: { data?: { filePath: string; mimeType: string }; image?: { mimeType: string }; + availableTerminalHeight?: number; }) => ( {image - ? `MockTerminalImage:${image.mimeType}` - : `MockTerminalImage:${data?.filePath}:${data?.mimeType}`} + ? `MockTerminalImage:${image.mimeType}:height=${availableTerminalHeight ?? 'undef'}` + : `MockTerminalImage:${data?.filePath}:${data?.mimeType}:height=${availableTerminalHeight ?? 'undef'}`} ), })); @@ -212,6 +214,22 @@ describe('', () => { expect(lastFrame()).toContain('[+2 more images]'); }); + it('shares the tool height budget across inline images', () => { + const { lastFrame } = renderWithContext( + , + StreamingState.Responding, + ); + + expect(lastFrame()).toContain('MockTerminalImage:image/png:height=4'); + }); + it('always shows the vision bridge disclosure for a completed read', () => { const { lastFrame } = renderWithContext( = ({ MIN_LINES_SHOWN + 1, // enforce minimum lines shown ) : undefined; + const inlineImageHeight = + availableHeight !== undefined && images?.length + ? Math.max(1, Math.floor(availableHeight / (images.length + 1))) + : availableHeight; // Cap inline shell output. Applies to both the streaming ANSI display and // the completed string display (shell.ts emits the final result as a plain // string via `returnDisplayMessage = result.output`). ShellStatsBar surfaces @@ -988,7 +992,7 @@ export const ToolMessage: React.FC = ({ key={index} image={image} contentWidth={innerWidth} - availableTerminalHeight={availableHeight} + availableTerminalHeight={inlineImageHeight} /> ))} {omittedImageCount !== undefined && omittedImageCount > 0 && ( diff --git a/packages/cli/src/ui/hooks/slashCommandProcessor.ts b/packages/cli/src/ui/hooks/slashCommandProcessor.ts index 4e635d41e00..73eb734893f 100644 --- a/packages/cli/src/ui/hooks/slashCommandProcessor.ts +++ b/packages/cli/src/ui/hooks/slashCommandProcessor.ts @@ -486,6 +486,7 @@ export const useSlashCommandProcessor = ( refreshStatic(); setSessionName?.(null); }, + clearPendingState: actions.clearPendingState, loadHistory, refreshStatic, setDebugMessage: actions.setDebugMessage, diff --git a/packages/cli/src/ui/hooks/useBranchCommand.test.ts b/packages/cli/src/ui/hooks/useBranchCommand.test.ts index e53797ceb91..cee5f7d3683 100644 --- a/packages/cli/src/ui/hooks/useBranchCommand.test.ts +++ b/packages/cli/src/ui/hooks/useBranchCommand.test.ts @@ -23,6 +23,7 @@ describe('useBranchCommand', () => { let startNewSessionConfig: ReturnType; let getGoalRuntimeReady: ReturnType; let startNewSessionUI: ReturnType; + let clearPendingState: ReturnType; let findSessionTitlesByPrefix: ReturnType; let clearItems: ReturnType; let loadHistory: ReturnType; @@ -54,6 +55,7 @@ describe('useBranchCommand', () => { settings: mockSettings, historyManager: { clearItems, loadHistory, addItem }, startNewSession: startNewSessionUI, + clearPendingState, setSessionName, remount, }); @@ -92,6 +94,7 @@ describe('useBranchCommand', () => { startNewSessionConfig = vi.fn(); getGoalRuntimeReady = vi.fn().mockResolvedValue({}); startNewSessionUI = vi.fn(); + clearPendingState = vi.fn(); clearItems = vi.fn(); loadHistory = vi.fn(); setSessionName = vi.fn(); @@ -164,6 +167,10 @@ describe('useBranchCommand', () => { expect(monitorRegistry.reset).toHaveBeenCalledOnce(); expect(backgroundShellRegistry.reset).toHaveBeenCalledOnce(); expect(workflowRunRegistry.reset).toHaveBeenCalledOnce(); + expect(clearPendingState).toHaveBeenCalledOnce(); + expect(clearPendingState.mock.invocationCallOrder[0]).toBeLessThan( + loadHistory.mock.invocationCallOrder[0]!, + ); expect(startNewSessionUI.mock.invocationCallOrder[0]).toBeLessThan( backgroundTaskRegistry.reset.mock.invocationCallOrder[0]!, ); diff --git a/packages/cli/src/ui/hooks/useBranchCommand.ts b/packages/cli/src/ui/hooks/useBranchCommand.ts index 7fd09d608d2..2ac2d6343e7 100644 --- a/packages/cli/src/ui/hooks/useBranchCommand.ts +++ b/packages/cli/src/ui/hooks/useBranchCommand.ts @@ -68,6 +68,7 @@ export interface UseBranchCommandOptions { 'clearItems' | 'loadHistory' | 'addItem' >; startNewSession: (sessionId: string) => void; + clearPendingState?: () => void; setSessionName?: (name: string | null) => void; remount?: () => void; } @@ -93,8 +94,14 @@ export interface UseBranchCommandResult { export function useBranchCommand( options: UseBranchCommandOptions, ): UseBranchCommandResult { - const { config, historyManager, startNewSession, setSessionName, remount } = - options; + const { + config, + historyManager, + startNewSession, + clearPendingState, + setSessionName, + remount, + } = options; const handleBranch = useCallback( async (name?: string) => { @@ -208,6 +215,7 @@ export function useBranchCommand( collapsePreviewCount, ); startNewSession(newSessionId); + clearPendingState?.(); historyManager.clearItems(); historyManager.loadHistory(uiHistoryItems); uiSwapped = true; @@ -294,6 +302,7 @@ export function useBranchCommand( config, historyManager, startNewSession, + clearPendingState, setSessionName, remount, options.settings.merged.ui?.history?.collapseOnResume, diff --git a/packages/cli/src/ui/hooks/useGeminiStream.test.tsx b/packages/cli/src/ui/hooks/useGeminiStream.test.tsx index 2562e13e8c3..2a30df2d486 100644 --- a/packages/cli/src/ui/hooks/useGeminiStream.test.tsx +++ b/packages/cli/src/ui/hooks/useGeminiStream.test.tsx @@ -6111,7 +6111,6 @@ describe('useGeminiStream', () => { const image = { data: 'aW1hZ2U=', mimeType: 'image/png', - displayName: 'chart.png', }; mockSendMessageStream.mockReturnValue( (async function* () { @@ -6168,6 +6167,55 @@ describe('useGeminiStream', () => { }); }); + it('commits mixed content in order before scheduling a tool call', async () => { + const image = { + data: 'dG9vbC1ib3VuZGFyeQ==', + mimeType: 'image/png', + }; + const toolCall = { + callId: 'tool-after-image', + name: 'read_file', + args: { path: '/tmp/example.ts' }, + isClientInitiated: false, + prompt_id: 'prompt-tool-boundary', + }; + mockSendMessageStream.mockReturnValue( + (async function* () { + yield { + type: ServerGeminiEventType.Content, + value: 'beforeafter', + parts: [ + { text: 'before' }, + { inlineData: image }, + { text: 'after' }, + ], + }; + yield { + type: ServerGeminiEventType.ToolCallRequest, + value: toolCall, + }; + })(), + ); + + const { result } = renderTestHook(); + await act(async () => { + await result.current.submitQuery('read after showing a chart'); + }); + + const assistantItems = mockAddItem.mock.calls + .map(([item]) => item as HistoryItem) + .filter( + (item) => item.type === 'gemini' || item.type === 'gemini_content', + ); + expect(assistantItems).toEqual([ + expect.objectContaining({ type: 'gemini', text: 'before' }), + { type: 'gemini_content', text: '', images: [image] }, + { type: 'gemini_content', text: 'after' }, + ]); + expect(mockScheduleToolCalls).toHaveBeenCalledTimes(1); + expect(mockScheduleToolCalls.mock.calls[0][0]).toEqual([toolCall]); + }); + it('does not overwrite an image with whitespace before the next image', async () => { vi.useFakeTimers(); @@ -6178,12 +6226,10 @@ describe('useGeminiStream', () => { const firstImage = { data: 'Zmlyc3Q=', mimeType: 'image/png', - displayName: 'first.png', }; const secondImage = { data: 'c2Vjb25k', mimeType: 'image/png', - displayName: 'second.png', }; mockSendMessageStream.mockReturnValue( (async function* () { @@ -6280,6 +6326,153 @@ describe('useGeminiStream', () => { }); }); + it('applies the inline image cap across multiple content events', async () => { + const images = Array.from( + { length: MAX_INLINE_IMAGES_PER_ITEM + 2 }, + (_, index) => ({ + data: Buffer.from(`multi-event-image-${index}`).toString('base64'), + mimeType: 'image/png', + }), + ); + mockSendMessageStream.mockReturnValue( + (async function* () { + for (const inlineData of images) { + yield { + type: ServerGeminiEventType.Content, + value: '', + parts: [{ inlineData }], + }; + } + yield { + type: ServerGeminiEventType.Finished, + value: { reason: 'STOP', usageMetadata: undefined }, + }; + })(), + ); + + const { result } = renderTestHook(); + await act(async () => { + await result.current.submitQuery('show many streamed charts'); + }); + + const assistantItems = mockAddItem.mock.calls + .map(([item]) => item as HistoryItem) + .filter( + (item) => item.type === 'gemini' || item.type === 'gemini_content', + ); + expect(assistantItems.flatMap((item) => item.images ?? [])).toEqual( + images.slice(0, MAX_INLINE_IMAGES_PER_ITEM), + ); + expect(assistantItems.at(-1)).toMatchObject({ + text: '', + omittedImageCount: 2, + }); + }); + + it('resets the inline image cap after a fresh retry', async () => { + const failedImages = Array.from( + { length: MAX_INLINE_IMAGES_PER_ITEM }, + (_, index) => ({ + data: Buffer.from(`retry-image-${index}`).toString('base64'), + mimeType: 'image/png', + }), + ); + const replacementImage = { + data: Buffer.from('retry-replacement').toString('base64'), + mimeType: 'image/png', + }; + mockSendMessageStream.mockReturnValue( + (async function* () { + yield { + type: ServerGeminiEventType.Content, + value: '', + parts: failedImages.map((inlineData) => ({ inlineData })), + }; + yield { + type: ServerGeminiEventType.Retry, + isContinuation: false, + }; + yield { + type: ServerGeminiEventType.Content, + value: '', + parts: [{ inlineData: replacementImage }], + }; + yield { + type: ServerGeminiEventType.Finished, + value: { reason: 'STOP', usageMetadata: undefined }, + }; + })(), + ); + + const { result } = renderTestHook(); + await act(async () => { + await result.current.submitQuery('retry the charts'); + }); + + const assistantItems = mockAddItem.mock.calls + .map(([item]) => item as HistoryItem) + .filter( + (item) => item.type === 'gemini' || item.type === 'gemini_content', + ); + expect(assistantItems.flatMap((item) => item.images ?? [])).toEqual([ + replacementImage, + ]); + expect(assistantItems.some((item) => item.omittedImageCount)).toBe(false); + }); + + it('resets the inline image cap after model fallback', async () => { + const failedImages = Array.from( + { length: MAX_INLINE_IMAGES_PER_ITEM }, + (_, index) => ({ + data: Buffer.from(`fallback-image-${index}`).toString('base64'), + mimeType: 'image/png', + }), + ); + const replacementImage = { + data: Buffer.from('fallback-replacement').toString('base64'), + mimeType: 'image/png', + }; + mockSendMessageStream.mockReturnValue( + (async function* () { + yield { + type: ServerGeminiEventType.Content, + value: '', + parts: failedImages.map((inlineData) => ({ inlineData })), + }; + yield { + type: ServerGeminiEventType.ModelFallback, + fromModel: 'primary-model', + toModel: 'fallback-model', + fallbackIndex: 1, + }; + yield { + type: ServerGeminiEventType.Content, + value: '', + parts: [{ inlineData: replacementImage }], + }; + yield { + type: ServerGeminiEventType.Finished, + value: { reason: 'STOP', usageMetadata: undefined }, + }; + })(), + ); + + const { result } = renderTestHook(); + await act(async () => { + await result.current.submitQuery('fallback the charts'); + }); + + const assistantItems = mockAddItem.mock.calls + .map(([item]) => item as HistoryItem) + .filter( + (item) => item.type === 'gemini' || item.type === 'gemini_content', + ); + expect(assistantItems.flatMap((item) => item.images ?? [])).toEqual([ + replacementImage, + ]); + expect(assistantItems.some((item) => item.omittedImageCount)).toBe(false); + }); + it('resets the inline image cap at a finished response boundary', async () => { const firstOutputImages = Array.from( { length: MAX_INLINE_IMAGES_PER_ITEM }, @@ -6397,7 +6590,6 @@ describe('useGeminiStream', () => { const image = { data: 'aW1hZ2U=', mimeType: 'image/png', - displayName: 'trailing-space.png', }; mockSendMessageStream.mockReturnValue( (async function* () { @@ -6462,7 +6654,6 @@ describe('useGeminiStream', () => { const image = { data: 'aW1hZ2U=', mimeType: 'image/png', - displayName: 'failed.png', }; mockSendMessageStream.mockReturnValue( (async function* () { @@ -6542,7 +6733,6 @@ describe('useGeminiStream', () => { const image = { data: 'aW1hZ2U=', mimeType: 'image/png', - displayName: 'partial.png', }; mockSendMessageStream.mockReturnValue( (async function* () { @@ -6618,7 +6808,6 @@ describe('useGeminiStream', () => { const image = { data: 'aW1hZ2U=', mimeType: 'image/png', - displayName: 'failed.png', }; mockSendMessageStream.mockReturnValue( (async function* () { @@ -6692,7 +6881,6 @@ describe('useGeminiStream', () => { const failedImage = { data: 'aW1hZ2U=', mimeType: 'image/png', - displayName: 'failed.png', }; mockSendMessageStream .mockReturnValueOnce( @@ -6758,7 +6946,6 @@ describe('useGeminiStream', () => { const failedImage = { data: 'aW1hZ2U=', mimeType: 'image/png', - displayName: 'partial.png', }; mockSendMessageStream .mockReturnValueOnce( @@ -6820,7 +7007,6 @@ describe('useGeminiStream', () => { const failedImage = { data: 'aW1hZ2U=', mimeType: 'image/png', - displayName: 'cleared.png', }; mockSendMessageStream .mockReturnValueOnce( @@ -9881,6 +10067,51 @@ describe('useGeminiStream', () => { }); describe('Citation event', () => { + it('starts a fresh assistant item after a shown citation', async () => { + mockSendMessageStream.mockReturnValue( + (async function* () { + yield { + type: ServerGeminiEventType.Content, + value: 'Hello world', + }; + yield { + type: ServerGeminiEventType.Citation, + value: 'Citation text', + }; + yield { + type: ServerGeminiEventType.Content, + value: ' more', + }; + yield { + type: ServerGeminiEventType.Finished, + value: { reason: 'STOP', usageMetadata: undefined }, + }; + })(), + ); + + const { result } = renderTestHook(); + await act(async () => { + await result.current.submitQuery('test shown citation'); + }); + + const outputItems = mockAddItem.mock.calls + .map(([item]) => item as HistoryItem) + .filter( + (item) => + item.type === 'gemini' || + item.type === 'gemini_content' || + item.type === MessageType.INFO, + ); + expect(outputItems).toEqual([ + expect.objectContaining({ type: 'gemini', text: 'Hello world' }), + expect.objectContaining({ + type: MessageType.INFO, + text: 'Citation text', + }), + expect.objectContaining({ type: 'gemini', text: ' more' }), + ]); + }); + it('preserves streamed text across hidden citation events', async () => { const settingsWithCitationsHidden = { ...mockLoadedSettings, @@ -10017,7 +10248,6 @@ describe('useGeminiStream', () => { const image = { data: 'aW1hZ2U=', mimeType: 'image/png', - displayName: 'truncated.png', }; // Setup mock to return a stream with MAX_TOKENS finish reason mockSendMessageStream.mockReturnValue( @@ -10115,7 +10345,6 @@ describe('useGeminiStream', () => { const image = { data: 'aW1hZ2U=', mimeType: 'image/png', - displayName: 'boundary.png', }; mockSendMessageStream.mockReturnValue( (async function* () { @@ -12889,7 +13118,6 @@ describe('useGeminiStream', () => { const image = { data: 'aW1hZ2U=', mimeType: 'image/png', - displayName: 'goal.png', }; mockSendMessageStream.mockReturnValue( (async function* () { diff --git a/packages/cli/src/ui/hooks/useReactToolScheduler.test.tsx b/packages/cli/src/ui/hooks/useReactToolScheduler.test.tsx index 6a7c324ec78..0c947c2762e 100644 --- a/packages/cli/src/ui/hooks/useReactToolScheduler.test.tsx +++ b/packages/cli/src/ui/hooks/useReactToolScheduler.test.tsx @@ -77,7 +77,6 @@ describe('mapToDisplay — detailedDisplay (§4.9 live path)', () => { { data: 'dG9vbC1pbWFnZQ==', mimeType: 'image/png', - displayName: 'result.png', }, ]); }, diff --git a/packages/cli/src/ui/hooks/useResumeCommand.test.ts b/packages/cli/src/ui/hooks/useResumeCommand.test.ts index 1ad5dfbca1d..6d3555c776e 100644 --- a/packages/cli/src/ui/hooks/useResumeCommand.test.ts +++ b/packages/cli/src/ui/hooks/useResumeCommand.test.ts @@ -240,6 +240,7 @@ describe('useResumeCommand', () => { loadHistory: vi.fn(), }; const startNewSession = vi.fn(); + const clearPendingState = vi.fn(); const geminiClient = { initialize: vi.fn().mockResolvedValue(undefined), }; @@ -287,6 +288,7 @@ describe('useResumeCommand', () => { settings: mockSettings, historyManager, startNewSession, + clearPendingState, }), ); @@ -325,6 +327,10 @@ describe('useResumeCommand', () => { expect(geminiClient.initialize).toHaveBeenCalledWith(); expect(historyManager.clearItems).toHaveBeenCalledTimes(1); expect(historyManager.loadHistory).toHaveBeenCalledTimes(1); + expect(clearPendingState).toHaveBeenCalledTimes(1); + expect(clearPendingState.mock.invocationCallOrder[0]).toBeLessThan( + historyManager.loadHistory.mock.invocationCallOrder[0]!, + ); expect(resetMonitorRegistry).toHaveBeenCalledTimes(1); expect(config.getGoalRuntimeReady).toHaveBeenCalledTimes(1); }); diff --git a/packages/cli/src/ui/hooks/useResumeCommand.ts b/packages/cli/src/ui/hooks/useResumeCommand.ts index 646da4a2dca..276c2f141b0 100644 --- a/packages/cli/src/ui/hooks/useResumeCommand.ts +++ b/packages/cli/src/ui/hooks/useResumeCommand.ts @@ -32,6 +32,7 @@ export interface UseResumeCommandOptions { 'addItem' | 'clearItems' | 'loadHistory' >; startNewSession: (sessionId: string) => void; + clearPendingState?: () => void; setSessionName?: (name: string | null) => void; remount?: () => void; } @@ -80,6 +81,7 @@ export function useResumeCommand( settings, historyManager, startNewSession, + clearPendingState, setSessionName, remount, } = options; @@ -177,6 +179,7 @@ export function useResumeCommand( // into the old JSONL (split-brain). startNewSession(sessionId); setSessionName?.(customTitle ?? null); + clearPendingState?.(); clearItems(); loadHistory(uiHistoryItems); if (recoveredBackgroundAgentsNotice) { @@ -242,6 +245,7 @@ export function useResumeCommand( clearItems, loadHistory, startNewSession, + clearPendingState, setSessionName, remount, settings.merged.ui?.history?.collapseOnResume, diff --git a/packages/cli/src/ui/types.ts b/packages/cli/src/ui/types.ts index 8cf69cb8d99..469efda1485 100644 --- a/packages/cli/src/ui/types.ts +++ b/packages/cli/src/ui/types.ts @@ -57,7 +57,6 @@ export enum ToolCallStatus { export interface InlineImageData { data: string; mimeType: string; - displayName?: string; } export interface ToolCallEvent { diff --git a/packages/cli/src/ui/utils/inline-image-parts.test.ts b/packages/cli/src/ui/utils/inline-image-parts.test.ts index 7d383041902..b92b3d570c1 100644 --- a/packages/cli/src/ui/utils/inline-image-parts.test.ts +++ b/packages/cli/src/ui/utils/inline-image-parts.test.ts @@ -22,7 +22,7 @@ describe('collectInlineImages', () => { }; expect(collectInlineImages([{ inlineData: image }])).toEqual({ - images: [image], + images: [{ data: image.data, mimeType: image.mimeType }], omittedImageCount: 0, }); }); diff --git a/packages/cli/src/ui/utils/inline-image-parts.ts b/packages/cli/src/ui/utils/inline-image-parts.ts index 2bbb363affc..45c045b3962 100644 --- a/packages/cli/src/ui/utils/inline-image-parts.ts +++ b/packages/cli/src/ui/utils/inline-image-parts.ts @@ -40,9 +40,6 @@ export function getInlineImageData(part: Part): InlineImageData | null { return { data: inlineData.data, mimeType: inlineData.mimeType, - ...(typeof inlineData.displayName === 'string' - ? { displayName: inlineData.displayName } - : {}), }; } diff --git a/packages/cli/src/ui/utils/resumeHistoryUtils.test.ts b/packages/cli/src/ui/utils/resumeHistoryUtils.test.ts index 815039582fa..90557deb34c 100644 --- a/packages/cli/src/ui/utils/resumeHistoryUtils.test.ts +++ b/packages/cli/src/ui/utils/resumeHistoryUtils.test.ts @@ -852,7 +852,6 @@ describe('resumeHistoryUtils', () => { { data: 'aW1hZ2U=', mimeType: 'image/png', - displayName: 'chart.png', }, ], }, diff --git a/packages/cli/src/ui/utils/terminal-image-renderer.test.ts b/packages/cli/src/ui/utils/terminal-image-renderer.test.ts index d6f33d75277..fc993c35ee2 100644 --- a/packages/cli/src/ui/utils/terminal-image-renderer.test.ts +++ b/packages/cli/src/ui/utils/terminal-image-renderer.test.ts @@ -11,6 +11,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { containsCmdShellMetacharacters, getTerminalImageRenderSupport, + MAX_INLINE_IMAGE_PIXELS, markKittyImageWritten, prepareInlineTerminalImage, renderTerminalImage, @@ -157,6 +158,20 @@ describe('terminalImageRenderer', () => { ).toEqual({ fallbackText: '[image: png]', result: null }); }); + it('rejects inline PNGs above the total pixel limit', () => { + const width = 8_001; + const height = Math.floor(MAX_INLINE_IMAGE_PIXELS / width) + 1; + expect( + prepareInlineTerminalImage({ + data: pngWithSize(width, height).toString('base64'), + mimeType: 'image/png', + contentWidth: 24, + env: { TERM: 'xterm-kitty' }, + stdoutIsTTY: true, + }), + ).toEqual({ fallbackText: '[image: png]', result: null }); + }); + it('does not render inline image data when output is disabled', () => { expect( prepareInlineTerminalImage({ diff --git a/packages/cli/src/ui/utils/terminal-image-renderer.ts b/packages/cli/src/ui/utils/terminal-image-renderer.ts index 843b6793f17..f6ff1962dff 100644 --- a/packages/cli/src/ui/utils/terminal-image-renderer.ts +++ b/packages/cli/src/ui/utils/terminal-image-renderer.ts @@ -31,6 +31,7 @@ const ESTIMATED_CELL_WIDTH_PX = 8; const ESTIMATED_CELL_HEIGHT_PX = 16; const MAX_REASON_CHARS = 200; const MAX_INLINE_IMAGE_DIMENSION = 1_000_000; +export const MAX_INLINE_IMAGE_PIXELS = 64_000_000; // cmd.exe treats these as command separators / metacharacters. With shell:true // Node forwards arguments unquoted, so a model-chosen path containing any of // them is a command-injection vector through a .cmd/.bat chafa shim. @@ -45,6 +46,11 @@ const RENDER_CACHE_LIMIT = 40; const RENDER_CACHE_BYTE_LIMIT = 32 * 1024 * 1024; const renderCache = new Map(); let renderCacheBytes = 0; +const INLINE_DECODE_CACHE_LIMIT = 4; +const inlineDecodeCache = new Map< + string, + { png: Buffer; size: { width: number; height: number } } +>(); // A Kitty terminal keeps a transmitted image and redraws it from the placeholder // cells alone. The live-row -> Static-row move and every resize remount @@ -165,14 +171,11 @@ export function prepareInlineTerminalImage({ return { fallbackText: emptyFallback, result: null }; } - const png = decodeInlineImage(data); - if (!png) { - return { fallbackText: emptyFallback, result: null }; - } - const size = readValidatedInlinePngSize(png); - if (!size) { + const decoded = getDecodedInlinePng(data); + if (!decoded) { return { fallbackText: emptyFallback, result: null }; } + const { png, size } = decoded; const fallbackText = `[image: ${size.width}x${size.height} png]`; if (disabled) { @@ -366,6 +369,31 @@ function decodeInlineImage(data: string): Buffer | null { return decoded; } +function getDecodedInlinePng( + data: string, +): { png: Buffer; size: { width: number; height: number } } | null { + const cached = inlineDecodeCache.get(data); + if (cached) { + inlineDecodeCache.delete(data); + inlineDecodeCache.set(data, cached); + return cached; + } + + const png = decodeInlineImage(data); + if (!png) return null; + const size = readValidatedInlinePngSize(png); + if (!size) return null; + + const decoded = { png, size }; + inlineDecodeCache.set(data, decoded); + while (inlineDecodeCache.size > INLINE_DECODE_CACHE_LIMIT) { + const oldest = inlineDecodeCache.keys().next().value; + if (oldest === undefined) break; + inlineDecodeCache.delete(oldest); + } + return decoded; +} + function readValidatedInlinePngSize( png: Buffer, ): { width: number; height: number } | null { @@ -384,7 +412,8 @@ function readValidatedInlinePngSize( size.width <= 0 || size.height <= 0 || size.width > MAX_INLINE_IMAGE_DIMENSION || - size.height > MAX_INLINE_IMAGE_DIMENSION + size.height > MAX_INLINE_IMAGE_DIMENSION || + size.width * size.height > MAX_INLINE_IMAGE_PIXELS ) { return null; }