diff --git a/docs/design/tui-display-image-tool.md b/docs/design/tui-display-image-tool.md new file mode 100644 index 00000000000..4b976b1e5e5 --- /dev/null +++ b/docs/design/tui-display-image-tool.md @@ -0,0 +1,79 @@ +# TUI Display Image Tool + +## Goal + +Add a model-invocable `display_image` tool that previews an existing PNG in +the interactive terminal UI. The tool is display-only: it does not provide +image pixels to the model and is not available in headless, SDK, screen-reader, +or subagent registries. + +## Design + +The core tool validates an absolute workspace path, regular-file status, PNG +signature, and an 8 MiB size limit. On success it returns a small structured +display value: + +```ts +{ + type: 'terminal_image', + filePath: '/absolute/workspace/path/image.png', + mimeType: 'image/png', +} +``` + +Before returning success, the tool asks a CLI-injected provider whether the +current terminal has an available renderer. Native protocols pass immediately; +the fallback checks that `chafa` is available on `PATH` without rendering the +selected file. If no renderer is available, the tool returns an execution error +so both the TUI and the model know that the image was not displayed. +`llmContent` tells the model to use `read_file` if it needs to inspect the image. +Image bytes and terminal escape sequences never enter model history or the +persisted display value. + +`ToolMessage` recognizes the structured value and delegates it to a TUI +component. The component revalidates the path against the current workspace +before reading it. Rendering is prepared during the component's first render +because completed tool rows enter Ink's append-only `Static` region +immediately; an asynchronous state update would be dropped there. +Kitty-compatible terminals use the existing virtual-image and +Unicode-placeholder approach already used for Mermaid diagrams. Other +terminals try `chafa` with symbol output. `chafa` receives only the same +allowlisted environment used by the Mermaid renderer, so API credentials are +not forwarded. If neither path is available, the TUI shows a bounded text +fallback naming the image. + +Previews are capped at 72 columns and 24 rows, then reduced further to fit the +available terminal space. An 8-by-16-pixel terminal cell estimate provides a +conservative natural size so small images are not deliberately enlarged. Both +native and `chafa` rendering use the same aspect-preserving dimensions. + +The tool is registered only when all of these are true: + +- the main config is interactive; +- SDK mode is disabled; +- screen-reader mode is disabled; +- the registry is not being created for a subagent. + +Bare mode keeps its existing minimal tool set. + +## Compatibility + +The first version supports PNG only. Kitty and Ghostty use native image +placement. Warp is intentionally excluded because its Kitty support does not +include the Unicode-placeholder extension used by this renderer, while direct +placement drifts away from Ink content during scrollback and terminal reflow. +Direct iTerm2 placement is also excluded because its inline image protocol is +cursor-positioned and the existing asynchronous Ink path already treats it as +unsafe. Warp, iTerm2, tmux, SSH, and terminals without a native path can still +render through `chafa` when it is installed. + +## Security and persistence + +Both the tool and renderer use `WorkspaceContext.isPathWithinWorkspace`, which +resolves symlinks and prevents traversal outside registered workspace roots. +The renderer treats restored `resultDisplay` data as untrusted and repeats this +check. Persisted sessions contain only the path and MIME type; if the file no +longer exists, restore degrades to a text notice. + +Raw terminal output is generated only by the trusted renderer. Model-controlled +strings and file contents are never written directly to the terminal. diff --git a/packages/cli/src/config/config.ts b/packages/cli/src/config/config.ts index 6cc6a7be953..0cb0c5dc93b 100755 --- a/packages/cli/src/config/config.ts +++ b/packages/cli/src/config/config.ts @@ -2106,6 +2106,14 @@ export async function loadCliConfig( disabledSlashCommands.length > 0 ? disabledSlashCommands : undefined, disabledSkillNamesProvider: bareMode || safeMode ? undefined : disabledSkillNamesProvider, + terminalImageRenderSupportProvider: interactive + ? async () => { + const { getTerminalImageRenderSupport } = await import( + '../ui/utils/terminal-image-renderer.js' + ); + return getTerminalImageRenderSupport(); + } + : undefined, disabledSkillLevels: bareMode || safeMode || !Array.isArray(settings.skills?.disabledLevels) ? undefined diff --git a/packages/cli/src/i18n/locales/en.js b/packages/cli/src/i18n/locales/en.js index d646be5ff00..9e3333e6ab3 100644 --- a/packages/cli/src/i18n/locales/en.js +++ b/packages/cli/src/i18n/locales/en.js @@ -192,6 +192,7 @@ export default { 'toolDisplayName.Agent': 'toolDisplayName.Agent', 'toolDisplayName.Artifact': 'toolDisplayName.Artifact', 'toolDisplayName.RecordArtifact': 'toolDisplayName.RecordArtifact', + 'toolDisplayName.DisplayImage': 'toolDisplayName.DisplayImage', 'toolDisplayName.Skill': 'toolDisplayName.Skill', 'toolDisplayName.EnterPlanMode': 'toolDisplayName.EnterPlanMode', 'toolDisplayName.ExitPlanMode': 'toolDisplayName.ExitPlanMode', diff --git a/packages/cli/src/i18n/locales/zh-TW.js b/packages/cli/src/i18n/locales/zh-TW.js index f2bd5c22dcb..eee98d9e4d3 100644 --- a/packages/cli/src/i18n/locales/zh-TW.js +++ b/packages/cli/src/i18n/locales/zh-TW.js @@ -183,6 +183,7 @@ export default { 'toolDisplayName.Agent': 'Agent', 'toolDisplayName.Artifact': '製品', 'toolDisplayName.RecordArtifact': '記錄製品', + 'toolDisplayName.DisplayImage': '顯示圖片', 'toolDisplayName.Skill': '技能', 'toolDisplayName.EnterPlanMode': '進入計畫模式', 'toolDisplayName.ExitPlanMode': '退出計畫模式', diff --git a/packages/cli/src/i18n/locales/zh.js b/packages/cli/src/i18n/locales/zh.js index 91138e25dc6..d12a7d88058 100644 --- a/packages/cli/src/i18n/locales/zh.js +++ b/packages/cli/src/i18n/locales/zh.js @@ -184,6 +184,7 @@ export default { 'toolDisplayName.Agent': 'Agent', 'toolDisplayName.Artifact': '制品', 'toolDisplayName.RecordArtifact': '记录制品', + 'toolDisplayName.DisplayImage': '显示图片', 'toolDisplayName.Skill': '技能', 'toolDisplayName.EnterPlanMode': '进入计划模式', 'toolDisplayName.ExitPlanMode': '退出计划模式', diff --git a/packages/cli/src/ui/components/TerminalImage.test.tsx b/packages/cli/src/ui/components/TerminalImage.test.tsx new file mode 100644 index 00000000000..3d321b5280b --- /dev/null +++ b/packages/cli/src/ui/components/TerminalImage.test.tsx @@ -0,0 +1,186 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { render } from 'ink-testing-library'; +import { Static } from 'ink'; +import type { Config } from '@qwen-code/qwen-code-core'; +import { TerminalOutputProvider } from '../contexts/TerminalOutputContext.js'; +import { + renderTerminalImage, + type TerminalImageRenderResult, +} from '../utils/terminal-image-renderer.js'; +import { TerminalImage } from './TerminalImage.js'; + +const { writtenKeys } = vi.hoisted(() => ({ writtenKeys: new Set() })); + +vi.mock('../utils/terminal-image-renderer.js', () => ({ + renderTerminalImage: vi.fn(), + wasKittyImageWritten: vi.fn((key: string) => writtenKeys.has(key)), + markKittyImageWritten: vi.fn((key: string) => { + writtenKeys.add(key); + }), +})); + +const mockedRenderTerminalImage = vi.mocked(renderTerminalImage); + +function configWithWorkspaceResult(isWithinWorkspace: boolean): Config { + return { + getWorkspaceContext: () => ({ + isPathWithinWorkspace: () => isWithinWorkspace, + }), + } as unknown as Config; +} + +const IMAGE = { + type: 'terminal_image' as const, + filePath: '/workspace/chart.png', + mimeType: 'image/png' as const, +}; + +const KITTY_RESULT: TerminalImageRenderResult = { + kind: 'kitty', + key: 'kitty-payload', + sequence: '\x1b_Gpayload\x1b\\', + placeholder: { + color: '#00002a', + imageId: 42, + lines: ['placeholder'], + }, +}; + +function renderImage( + result: TerminalImageRenderResult, + writeRaw: (...args: unknown[]) => void = vi.fn(), +) { + mockedRenderTerminalImage.mockReturnValueOnce(result); + return render( + + + {(item) => ( + + )} + + , + ); +} + +describe('TerminalImage', () => { + beforeEach(() => { + writtenKeys.clear(); + vi.clearAllMocks(); + }); + + it('writes trusted Kitty data and renders its placeholder', async () => { + const writeRaw = vi.fn(); + const { lastFrame } = renderImage(KITTY_RESULT, writeRaw); + + await vi.waitFor(() => { + expect(writeRaw).toHaveBeenCalledWith('\x1b_Gpayload\x1b\\'); + }); + expect(lastFrame()).toContain('placeholder'); + }); + + it('renders chafa ansi output', () => { + const { lastFrame } = renderImage({ kind: 'ansi', lines: ['▀▀', '▄▄'] }); + + expect(lastFrame()).toContain('▀▀'); + expect(lastFrame()).toContain('▄▄'); + }); + + it('shows a readable fallback when no renderer is available', () => { + const { lastFrame } = renderImage({ + kind: 'unavailable', + reason: 'chafa is not installed', + }); + + expect(lastFrame()).toContain('chafa is not installed'); + expect(lastFrame()).toContain('chart.png'); + }); + + it('refuses restored paths outside the current workspace', () => { + const { lastFrame } = render( + , + ); + + expect(lastFrame()).toContain('outside the current workspace'); + expect(mockedRenderTerminalImage).not.toHaveBeenCalled(); + }); + + it('does not re-emit the Kitty sequence when the emit effect re-runs', async () => { + mockedRenderTerminalImage.mockReturnValue(KITTY_RESULT); + + const renderWith = (writer: (...args: unknown[]) => void) => ( + + + + ); + + const firstWriteRaw = vi.fn(); + const { rerender } = render(renderWith(firstWriteRaw)); + + await vi.waitFor(() => { + expect(firstWriteRaw).toHaveBeenCalledWith('\x1b_Gpayload\x1b\\'); + }); + expect(firstWriteRaw).toHaveBeenCalledTimes(1); + + const secondWriteRaw = vi.fn(); + rerender(renderWith(secondWriteRaw)); + await new Promise((resolve) => setTimeout(resolve, 10)); + + expect(secondWriteRaw).not.toHaveBeenCalled(); + expect(firstWriteRaw).toHaveBeenCalledTimes(1); + }); + + it('does not re-transmit the Kitty payload when the image remounts', async () => { + mockedRenderTerminalImage.mockReturnValue(KITTY_RESULT); + + const renderWith = (writer: (...args: unknown[]) => void) => ( + + + + ); + + const firstWriteRaw = vi.fn(); + const first = render(renderWith(firstWriteRaw)); + await vi.waitFor(() => { + expect(firstWriteRaw).toHaveBeenCalledWith('\x1b_Gpayload\x1b\\'); + }); + first.unmount(); + + // A fresh mount (live row -> Static row, or a resize) previously + // re-transmitted the whole payload even though the terminal still holds it. + const secondWriteRaw = vi.fn(); + const second = render(renderWith(secondWriteRaw)); + await new Promise((resolve) => setTimeout(resolve, 10)); + + expect(secondWriteRaw).not.toHaveBeenCalled(); + second.unmount(); + }); +}); diff --git a/packages/cli/src/ui/components/TerminalImage.tsx b/packages/cli/src/ui/components/TerminalImage.tsx new file mode 100644 index 00000000000..36169dfcfcc --- /dev/null +++ b/packages/cli/src/ui/components/TerminalImage.tsx @@ -0,0 +1,114 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import path from 'node:path'; +import React from 'react'; +import { Box, Text } from 'ink'; +import type { Config, TerminalImageDisplay } from '@qwen-code/qwen-code-core'; +import { MaxSizedBox } from './shared/MaxSizedBox.js'; +import { useTerminalOutput } from '../contexts/TerminalOutputContext.js'; +import { + markKittyImageWritten, + renderTerminalImage, + wasKittyImageWritten, + type TerminalImageRenderResult, +} from '../utils/terminal-image-renderer.js'; +import { theme } from '../semantic-colors.js'; +import { + sanitizeMultilineForDisplay, + sanitizeTerminalText, +} from '../utils/textUtils.js'; + +interface TerminalImageProps { + data: TerminalImageDisplay; + config: Config; + contentWidth: number; + availableTerminalHeight?: number; +} + +export const TerminalImage: React.FC = ({ + data, + config, + contentWidth, + availableTerminalHeight, +}) => { + const writeRaw = useTerminalOutput(); + const filePath = path.resolve(data.filePath); + const safePath = config.getWorkspaceContext().isPathWithinWorkspace(filePath); + const result = React.useMemo( + () => + safePath + ? renderTerminalImage({ + display: { + type: 'terminal_image', + filePath, + mimeType: data.mimeType, + }, + contentWidth, + availableTerminalHeight, + }) + : null, + [availableTerminalHeight, contentWidth, data.mimeType, filePath, safePath], + ); + + // The Kitty payload is written once per terminal session per render key; the + // terminal keeps the image and redraws it from the placeholder cells, so a + // remount (live row -> Static row, or a resize) must not re-transmit it. + React.useEffect(() => { + if (!result || result.kind !== 'kitty') return; + if (wasKittyImageWritten(result.key)) return; + markKittyImageWritten(result.key); + const sequence = result.sequence; + process.nextTick(() => writeRaw(sequence)); + }, [result, writeRaw]); + + if (!safePath) { + return ( + + Refusing to display an image outside the current workspace. + + ); + } + if (!result) return null; + if (result.kind === 'unavailable') { + const fileName = sanitizeMultilineForDisplay(path.basename(filePath)); + return ( + + {fileName}: {sanitizeTerminalText(result.reason)} + + ); + } + if (result.kind === 'ansi') { + return ( + + {result.lines.map((line, index) => ( + + {line || ' '} + + ))} + + ); + } + return ( + + {result.placeholder.lines.map((line, index) => ( + + + {line} + + + ))} + + ); +}; diff --git a/packages/cli/src/ui/components/messages/ToolMessage.test.tsx b/packages/cli/src/ui/components/messages/ToolMessage.test.tsx index 5ec2dc101ab..7b13aa36adf 100644 --- a/packages/cli/src/ui/components/messages/ToolMessage.test.tsx +++ b/packages/cli/src/ui/components/messages/ToolMessage.test.tsx @@ -70,6 +70,18 @@ vi.mock('../AnsiOutput.js', () => ({ }, })); +vi.mock('../TerminalImage.js', () => ({ + TerminalImage: ({ + data, + }: { + data: { filePath: string; mimeType: string }; + }) => ( + + MockTerminalImage:{data.filePath}:{data.mimeType} + + ), +})); + // Mock child components or utilities if they are complex or have side effects vi.mock('../GeminiRespondingSpinner.js', () => ({ GeminiRespondingSpinner: ({ @@ -327,6 +339,25 @@ describe('', () => { expect(output).toContain('MockMarkdown:Test result'); // not collapsed }); + it('renders structured terminal image results through TerminalImage', () => { + const { lastFrame } = renderWithContext( + , + StreamingState.Idle, + ); + + expect(lastFrame()).toContain( + 'MockTerminalImage:/workspace/chart.png:image/png', + ); + }); + it('renders tool results directly below the header row when forced', () => { const { lastFrame } = renderWithContext( , diff --git a/packages/cli/src/ui/components/messages/ToolMessage.tsx b/packages/cli/src/ui/components/messages/ToolMessage.tsx index d0c2ce052d3..3017d27f313 100644 --- a/packages/cli/src/ui/components/messages/ToolMessage.tsx +++ b/packages/cli/src/ui/components/messages/ToolMessage.tsx @@ -23,9 +23,11 @@ import type { Config, McpToolProgressData, FileDiff, + TerminalImageDisplay, } from '@qwen-code/qwen-code-core'; import { formatVisionBridgeNoticeDisplay, + isTerminalImageDisplay, isVisionBridgeNoticeDisplay, ToolNames, ToolNamesMigration, @@ -55,6 +57,7 @@ import { STATUS_INDICATOR_WIDTH, } from '../shared/ToolStatusIndicator.js'; import { ToolElapsedTime } from '../shared/ToolElapsedTime.js'; +import { TerminalImage } from '../TerminalImage.js'; // Names that resolve to the agent tool: the canonical name plus whatever // legacy request aliases core's migration map declares (e.g. 'task'). @@ -172,6 +175,7 @@ type DisplayRendererResult = | { type: 'string'; data: string } | { type: 'diff'; data: { fileDiff: string; fileName: string } } | { type: 'task'; data: AgentResultDisplay } + | { type: 'image'; data: TerminalImageDisplay } | { type: 'ansi'; data: AnsiOutput; stats?: ShellStatsBarProps }; /** @@ -185,6 +189,10 @@ const useResultDisplayRenderer = ( return { type: 'none' }; } + if (isTerminalImageDisplay(resultDisplay)) { + return { type: 'image', data: resultDisplay }; + } + // Check for TodoResultDisplay if ( typeof resultDisplay === 'object' && @@ -946,6 +954,14 @@ export const ToolMessage: React.FC = ({ )} )} + {effectiveDisplayRenderer.type === 'image' && config && ( + + )} {effectiveDisplayRenderer.type === 'string' && ( = [ + ['keeps a small landscape at its natural size', 320, 160, 200, 100, 40, 10], + ['does not round a small image up', 12, 24, 200, 100, 1, 1], + ['caps a large landscape by width', 1600, 800, 200, 100, 72, 18], + ['caps a large square by default height', 1600, 1600, 200, undefined, 48, 24], + ['fits a large portrait without distortion', 800, 1600, 200, 100, 24, 24], + ['respects a narrower terminal', 1600, 1600, 30, 100, 30, 15], + ['respects a shorter terminal', 1600, 1600, 200, 10, 20, 10], +]; + +describe('terminalImageRenderer', () => { + let tempDir: string; + let imagePath: string; + + beforeEach(async () => { + tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'terminal-image-test-')); + imagePath = path.join(tempDir, 'pixel.png'); + await fs.writeFile(imagePath, PNG_1X1); + }); + + afterEach(async () => { + await fs.rm(tempDir, { recursive: true, force: true }); + }); + + it('renders a Kitty virtual image with placeholder rows', () => { + const result = renderTerminalImage({ + display: { + type: 'terminal_image', + filePath: imagePath, + mimeType: 'image/png', + }, + contentWidth: 24, + availableTerminalHeight: 12, + env: { TERM: 'xterm-kitty' }, + stdoutIsTTY: true, + }); + + expect(result.kind).toBe('kitty'); + if (result.kind !== 'kitty') return; + expect(result.sequence).toContain('\u001b_Ga=T,f=100'); + expect(result.sequence).toContain('q=2,U=1'); + expect(result.sequence).toContain('c=1,r=1'); + expect(result.placeholder.lines).toHaveLength(1); + expect(result.placeholder.lines[0]).toContain('\u{10EEEE}'); + }); + + it('exposes a stable render key on Kitty results so remounts can skip re-transmission', async () => { + await fs.writeFile(imagePath, pngWithSize(1600, 800)); + const options = { + display: { + type: 'terminal_image' as const, + filePath: imagePath, + mimeType: 'image/png' as const, + }, + contentWidth: 200, + availableTerminalHeight: 100, + env: { TERM: 'xterm-kitty' }, + stdoutIsTTY: true, + }; + + const first = renderTerminalImage(options); + const second = renderTerminalImage(options); + expect(first.kind).toBe('kitty'); + expect(second.kind).toBe('kitty'); + if (first.kind !== 'kitty' || second.kind !== 'kitty') return; + expect(first.key).toBeTruthy(); + expect(second.key).toBe(first.key); + + // A different placement shape is a different image on the terminal grid and + // therefore gets its own transmission key. + const resized = renderTerminalImage({ ...options, contentWidth: 30 }); + expect(resized.kind).toBe('kitty'); + if (resized.kind === 'kitty') { + expect(resized.key).not.toBe(first.key); + } + }); + + it('tracks transmitted Kitty images for the session and bounds the set', () => { + const unique = `transmit-${Date.now()}-`; + expect(wasKittyImageWritten(`${unique}0`)).toBe(false); + markKittyImageWritten(`${unique}0`); + expect(wasKittyImageWritten(`${unique}0`)).toBe(true); + + // Exceeding the limit evicts the oldest key, so the set stays bounded. + for (let i = 1; i <= TRANSMITTED_KEY_LIMIT; i++) { + markKittyImageWritten(`${unique}${i}`); + } + expect(wasKittyImageWritten(`${unique}0`)).toBe(false); + expect(wasKittyImageWritten(`${unique}${TRANSMITTED_KEY_LIMIT}`)).toBe( + true, + ); + }); + + it.each(IMAGE_SIZE_CASES)( + '%s', + async ( + _name, + imageWidth, + imageHeight, + contentWidth, + availableTerminalHeight, + expectedWidth, + expectedRows, + ) => { + await fs.writeFile(imagePath, pngWithSize(imageWidth, imageHeight)); + + const result = renderTerminalImage({ + display: { + type: 'terminal_image', + filePath: imagePath, + mimeType: 'image/png', + }, + contentWidth, + availableTerminalHeight, + env: { TERM: 'xterm-kitty' }, + stdoutIsTTY: true, + }); + + expect(result.kind).toBe('kitty'); + if (result.kind !== 'kitty') return; + expect(result.sequence).toContain(`c=${expectedWidth},r=${expectedRows}`); + expect(result.placeholder.lines).toHaveLength(expectedRows); + expect(result.placeholder.lines[0].split('\u{10EEEE}').length - 1).toBe( + expectedWidth, + ); + }, + ); + + it('disables native placement in tmux, SSH, and non-TTY output', () => { + expect(supportsKittyImageProtocol({ TERM: 'xterm-kitty' }, true)).toBe( + true, + ); + expect( + supportsKittyImageProtocol( + { TERM: 'xterm-kitty', TMUX: '/tmp/tmux' }, + true, + ), + ).toBe(false); + expect( + supportsKittyImageProtocol( + { TERM: 'xterm-kitty', SSH_TTY: '/dev/pts/1' }, + true, + ), + ).toBe(false); + expect( + supportsKittyImageProtocol( + { TERM: 'xterm-kitty', SSH_CLIENT: '10.0.0.1 51234 22' }, + true, + ), + ).toBe(false); + expect(supportsKittyImageProtocol({ TERM: 'xterm-kitty' }, false)).toBe( + false, + ); + }); + + it('detects Kitty and Ghostty native placement', () => { + expect( + supportsKittyImageProtocol( + { KITTY_WINDOW_ID: '1', TERM: 'xterm-256color' }, + true, + ), + ).toBe(true); + expect(supportsKittyImageProtocol({ TERM_PROGRAM: 'ghostty' }, true)).toBe( + true, + ); + expect( + supportsKittyImageProtocol( + { TERM_PROGRAM: 'ghostty', TMUX: '/tmp/tmux' }, + true, + ), + ).toBe(false); + }); + + it('does not use Kitty Unicode placeholders in Warp', () => { + expect( + supportsKittyImageProtocol( + { TERM: 'xterm-256color', TERM_PROGRAM: 'WarpTerminal' }, + true, + ), + ).toBe(false); + expect( + supportsKittyImageProtocol( + { TERM: 'xterm-kitty', TERM_PROGRAM: 'WarpTerminal' }, + true, + ), + ).toBe(false); + expect( + getTerminalImageRenderSupport( + { + PATH: tempDir, + TERM: 'xterm-256color', + TERM_PROGRAM: 'WarpTerminal', + }, + true, + ), + ).toEqual({ + available: false, + reason: + 'No compatible native image protocol was detected, and chafa is not installed.', + }); + }); + + it.runIf(process.platform !== 'win32')( + 'falls back to chafa symbol output', + async () => { + const binDir = path.join(tempDir, 'bin'); + await fs.mkdir(binDir); + const chafaPath = path.join(binDir, 'chafa'); + await fs.writeFile( + chafaPath, + '#!/usr/bin/env node\nprocess.stdout.write(process.env.TEST_RENDERER_SECRET ? "LEAKED\\n" : `${process.argv.find((arg) => arg.startsWith("--colors="))} ${process.argv.find((arg) => arg.startsWith("--size="))}\\n`);\n', + ); + await fs.chmod(chafaPath, 0o755); + await fs.writeFile(imagePath, pngWithSize(1600, 800)); + + const result = renderTerminalImage({ + display: { + type: 'terminal_image', + filePath: imagePath, + mimeType: 'image/png', + }, + contentWidth: 20, + env: { + PATH: `${binDir}${path.delimiter}${process.env['PATH'] ?? ''}`, + TERM: 'xterm-256color', + TERM_PROGRAM: 'WarpTerminal', + TEST_RENDERER_SECRET: 'must-not-reach-chafa', + }, + stdoutIsTTY: true, + }); + + expect(result).toEqual({ + kind: 'ansi', + lines: ['--colors=256 --size=20x5'], + }); + expect( + getTerminalImageRenderSupport( + { + PATH: `${binDir}${path.delimiter}${process.env['PATH'] ?? ''}`, + TERM: 'xterm-256color', + TERM_PROGRAM: 'WarpTerminal', + }, + true, + ), + ).toEqual({ available: true }); + }, + ); + + it('rejects cmd.exe metacharacters before invoking a shell shim', async () => { + const dangerousImagePath = path.join(tempDir, 'chart & whoami.png'); + await fs.writeFile(dangerousImagePath, PNG_1X1); + const chafaPath = path.join(tempDir, 'chafa.CMD'); + await fs.writeFile(chafaPath, '@echo off\r\n'); + await fs.chmod(chafaPath, 0o755); + + const platformDescriptor = Object.getOwnPropertyDescriptor( + process, + 'platform', + ); + Object.defineProperty(process, 'platform', { + ...platformDescriptor, + value: 'win32', + }); + try { + const result = renderTerminalImage({ + display: { + type: 'terminal_image', + filePath: dangerousImagePath, + mimeType: 'image/png', + }, + contentWidth: 20, + env: { + PATH: tempDir, + PATHEXT: '.CMD', + TERM_PROGRAM: 'WarpTerminal', + }, + stdoutIsTTY: true, + }); + + expect(result).toEqual({ + kind: 'unavailable', + reason: + 'Image path contains characters that cannot be safely passed to the renderer on this platform.', + }); + } finally { + if (platformDescriptor) { + Object.defineProperty(process, 'platform', platformDescriptor); + } + } + }); + + it.runIf(process.platform !== 'win32')( + 'does not trust a project-local node_modules/.bin/chafa', + async () => { + const localBin = path.join(tempDir, 'node_modules', '.bin'); + await fs.mkdir(localBin, { recursive: true }); + const chafaPath = path.join(localBin, 'chafa'); + await fs.writeFile( + chafaPath, + '#!/usr/bin/env node\nprocess.stdout.write("HACKED\\n");\n', + ); + await fs.chmod(chafaPath, 0o755); + + expect( + getTerminalImageRenderSupport( + { + PATH: localBin, + TERM: 'xterm-256color', + TERM_PROGRAM: 'WarpTerminal', + }, + true, + ), + ).toEqual({ + available: false, + reason: + 'No compatible native image protocol was detected, and chafa is not installed.', + }); + + const result = renderTerminalImage({ + display: { + type: 'terminal_image', + filePath: imagePath, + mimeType: 'image/png', + }, + contentWidth: 20, + env: { + PATH: localBin, + TERM: 'xterm-256color', + TERM_PROGRAM: 'WarpTerminal', + }, + stdoutIsTTY: true, + }); + expect(result).toEqual({ + kind: 'unavailable', + reason: + 'No compatible native image protocol was detected, and chafa is not installed.', + }); + }, + ); + + it.runIf(process.platform !== 'win32')( + 'surfaces chafa stderr when rendering fails', + async () => { + const binDir = path.join(tempDir, 'bin'); + await fs.mkdir(binDir); + const chafaPath = path.join(binDir, 'chafa'); + await fs.writeFile( + chafaPath, + '#!/usr/bin/env node\nprocess.stderr.write("libpng: invalid IHDR data\\n");\nprocess.exit(1);\n', + ); + await fs.chmod(chafaPath, 0o755); + + const result = renderTerminalImage({ + display: { + type: 'terminal_image', + filePath: imagePath, + mimeType: 'image/png', + }, + contentWidth: 20, + env: { + PATH: `${binDir}${path.delimiter}${process.env['PATH'] ?? ''}`, + TERM: 'xterm-256color', + TERM_PROGRAM: 'WarpTerminal', + }, + stdoutIsTTY: true, + }); + + expect(result.kind).toBe('unavailable'); + expect(result.kind === 'unavailable' && result.reason).toContain( + 'libpng: invalid IHDR data', + ); + }, + ); + + it.runIf(process.platform !== 'win32')( + 'bounds a verbose chafa failure to a capped first line', + async () => { + const binDir = path.join(tempDir, 'bin'); + await fs.mkdir(binDir); + const chafaPath = path.join(binDir, 'chafa'); + await fs.writeFile( + chafaPath, + '#!/usr/bin/env node\nprocess.stderr.write("E".repeat(300) + "\\nsecond line\\n");\nprocess.exit(1);\n', + ); + await fs.chmod(chafaPath, 0o755); + + const result = renderTerminalImage({ + display: { + type: 'terminal_image', + filePath: imagePath, + mimeType: 'image/png', + }, + contentWidth: 20, + env: { + PATH: `${binDir}${path.delimiter}${process.env['PATH'] ?? ''}`, + TERM: 'xterm-256color', + TERM_PROGRAM: 'WarpTerminal', + }, + stdoutIsTTY: true, + }); + + expect(result.kind).toBe('unavailable'); + if (result.kind !== 'unavailable') return; + expect(result.reason).toBe(`${'E'.repeat(200)}…`); + expect(result.reason).not.toContain('\n'); + expect(result.reason).not.toContain('second line'); + }, + ); + + it.runIf(process.platform !== 'win32')( + 'caches a render so resize and restore do not re-spawn chafa', + async () => { + const binDir = path.join(tempDir, 'bin'); + await fs.mkdir(binDir); + const chafaPath = path.join(binDir, 'chafa'); + await fs.writeFile( + chafaPath, + '#!/usr/bin/env node\nconst fs = require("fs");\nconst path = require("path");\nfs.appendFileSync(path.join(__dirname, "count.txt"), "x\\n");\nprocess.stdout.write(`${process.argv.find((arg) => arg.startsWith("--size="))}\\n`);\n', + ); + await fs.chmod(chafaPath, 0o755); + await fs.writeFile(imagePath, pngWithSize(1600, 800)); + + const options = { + display: { + type: 'terminal_image' as const, + filePath: imagePath, + mimeType: 'image/png' as const, + }, + contentWidth: 20, + env: { + PATH: `${binDir}${path.delimiter}${process.env['PATH'] ?? ''}`, + TERM: 'xterm-256color', + TERM_PROGRAM: 'WarpTerminal', + }, + stdoutIsTTY: true, + }; + const spawnCount = async () => { + try { + const raw = await fs.readFile(path.join(binDir, 'count.txt'), 'utf8'); + return raw.split('x').length - 1; + } catch { + return 0; + } + }; + + const first = renderTerminalImage(options); + expect(first.kind).toBe('ansi'); + const second = renderTerminalImage(options); + expect(second).toEqual(first); + expect(await spawnCount()).toBe(1); + + // A different shape misses the cache and re-spawns the renderer. + await fs.writeFile(imagePath, pngWithSize(800, 1600)); + const third = renderTerminalImage(options); + expect(third.kind).toBe('ansi'); + expect(await spawnCount()).toBe(2); + + // The same shape with a newer mtime invalidates the cached entry. + const future = new Date(Date.now() + 10_000); + await fs.utimes(imagePath, future, future); + const fourth = renderTerminalImage(options); + expect(fourth.kind).toBe('ansi'); + expect(await spawnCount()).toBe(3); + }, + ); + + it('returns a readable fallback when chafa is unavailable', () => { + const result = renderTerminalImage({ + display: { + type: 'terminal_image', + filePath: imagePath, + mimeType: 'image/png', + }, + contentWidth: 20, + env: { PATH: tempDir }, + stdoutIsTTY: false, + }); + + expect(result.kind).toBe('unavailable'); + expect(result.kind === 'unavailable' && result.reason).toContain( + 'chafa is not installed', + ); + expect(getTerminalImageRenderSupport({ PATH: tempDir }, true)).toEqual({ + available: false, + reason: + 'No compatible native image protocol was detected, and chafa is not installed.', + }); + }); + + it('rejects missing and invalid PNG files during restored rendering', async () => { + const missing = renderTerminalImage({ + display: { + type: 'terminal_image', + filePath: path.join(tempDir, 'missing.png'), + mimeType: 'image/png', + }, + contentWidth: 20, + env: { TERM: 'xterm-kitty' }, + stdoutIsTTY: true, + }); + expect(missing.kind).toBe('unavailable'); + + await fs.writeFile(imagePath, 'not a png'); + const invalid = renderTerminalImage({ + display: { + type: 'terminal_image', + filePath: imagePath, + mimeType: 'image/png', + }, + contentWidth: 20, + env: { TERM: 'xterm-kitty' }, + stdoutIsTTY: true, + }); + expect(invalid).toEqual({ + kind: 'unavailable', + reason: 'Image is not a valid PNG.', + }); + + const oversizedPath = path.join(tempDir, 'oversized.png'); + const handle = await fs.open(oversizedPath, 'w'); + await handle.truncate(8 * 1024 * 1024 + 1); + await handle.close(); + const oversized = renderTerminalImage({ + display: { + type: 'terminal_image', + filePath: oversizedPath, + mimeType: 'image/png', + }, + contentWidth: 20, + env: { TERM: 'xterm-kitty' }, + stdoutIsTTY: true, + }); + expect(oversized.kind).toBe('unavailable'); + expect(oversized.kind === 'unavailable' && oversized.reason).toContain( + 'display limit', + ); + }); +}); + +describe('containsCmdShellMetacharacters', () => { + it('flags cmd.exe metacharacters in a model-supplied path', () => { + for (const dangerous of [ + 'chart & whoami.png', + 'a|b.png', + 'ab.png', + 'a^b.png', + 'a%PATH%.png', + 'a"b.png', + 'a!b.png', + 'a(b).png', + 'a\nb.png', + 'a\rb.png', + ]) { + expect(containsCmdShellMetacharacters(dangerous)).toBe(true); + } + }); + + it('accepts ordinary image paths', () => { + for (const safe of [ + '/workspace/chart.png', + '/workspace/reports/q3-summary.png', + '/workspace/img_2026-08-01.png', + '/workspace/my image.png', + 'C:\\Users\\me\\Pictures\\vacation.png', + ]) { + expect(containsCmdShellMetacharacters(safe)).toBe(false); + } + }); +}); diff --git a/packages/cli/src/ui/utils/terminal-image-renderer.ts b/packages/cli/src/ui/utils/terminal-image-renderer.ts new file mode 100644 index 00000000000..b46fb9517ea --- /dev/null +++ b/packages/cli/src/ui/utils/terminal-image-renderer.ts @@ -0,0 +1,453 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import { execFileSync } from 'node:child_process'; +import { + MAX_TERMINAL_IMAGE_BYTES, + type TerminalImageRenderSupport, + type TerminalImageDisplay, +} from '@qwen-code/qwen-code-core'; +import { + buildKittyPlaceholder, + createRendererChildEnv, + encodeKittyVirtualImage, + findExecutable, + readPngSize, + shouldRunThroughShell, + type KittyImagePlaceholder, +} from './mermaidImageRenderer.js'; + +const CHAFA_TIMEOUT_MS = 8000; +const CHAFA_MAX_OUTPUT_BYTES = 2 * 1024 * 1024; +const MAX_PREVIEW_WIDTH_CELLS = 72; +const MAX_PREVIEW_ROWS = 24; +const ESTIMATED_CELL_WIDTH_PX = 8; +const ESTIMATED_CELL_HEIGHT_PX = 16; +const MAX_REASON_CHARS = 200; +// cmd.exe treats these as command separators / metacharacters. With shell:true +// Node forwards arguments unquoted, so a model-chosen path containing any of +// them is a command-injection vector through a .cmd/.bat chafa shim. +const CMD_SHELL_METACHARACTERS = /[&|<>^%"!()\n\r]/; + +// Completed tool rows enter Ink's append-only Static region immediately, so +// rendering must stay synchronous. A terminal resize or a restored session +// re-renders every visible image, which would otherwise re-read the file and +// re-spawn chafa for each one. This cache makes those repeats cheap; it mirrors +// the bounded LRU the Mermaid renderer keeps for the same reason. +const RENDER_CACHE_LIMIT = 40; +const RENDER_CACHE_BYTE_LIMIT = 32 * 1024 * 1024; +const renderCache = new Map(); +let renderCacheBytes = 0; + +// A Kitty terminal keeps a transmitted image and redraws it from the placeholder +// cells alone. The live-row -> Static-row move and every resize remount +// TerminalImage, which would re-transmit the same base64 payload (megabytes at +// the size cap) even though the terminal already holds that image id. This +// session-scoped set remembers which render keys were written so a remount only +// re-emits the cheap placeholders. It is bounded so a long session cannot grow +// it without limit; evicting an old key costs at most a re-transmit, never a +// blank image, because the payload is rebuilt from the still-cached render. +export const TRANSMITTED_KEY_LIMIT = 256; +const transmittedKeys = new Set(); + +export function wasKittyImageWritten(key: string): boolean { + return transmittedKeys.has(key); +} + +export function markKittyImageWritten(key: string): void { + transmittedKeys.add(key); + if (transmittedKeys.size > TRANSMITTED_KEY_LIMIT) { + const oldest = transmittedKeys.values().next().value; + if (oldest !== undefined) { + transmittedKeys.delete(oldest); + } + } +} + +export type TerminalImageRenderResult = + | { + kind: 'kitty'; + key: string; + sequence: string; + placeholder: KittyImagePlaceholder; + } + | { kind: 'ansi'; lines: string[] } + | { kind: 'unavailable'; reason: string }; + +export interface TerminalImageRenderOptions { + display: TerminalImageDisplay; + contentWidth: number; + availableTerminalHeight?: number; + env?: NodeJS.ProcessEnv; + stdoutIsTTY?: boolean; +} + +export function supportsKittyImageProtocol( + env: NodeJS.ProcessEnv = process.env, + stdoutIsTTY = process.stdout.isTTY, +): boolean { + if (!stdoutIsTTY || env['TMUX'] || env['SSH_TTY'] || env['SSH_CLIENT']) { + return false; + } + + const term = env['TERM']?.toLowerCase() ?? ''; + const termProgram = env['TERM_PROGRAM']?.toLowerCase() ?? ''; + if (termProgram === 'warpterminal') { + return false; + } + + return Boolean( + env['KITTY_WINDOW_ID'] || + term.includes('kitty') || + termProgram.includes('ghostty'), + ); +} + +export function getTerminalImageRenderSupport( + env: NodeJS.ProcessEnv = process.env, + stdoutIsTTY = process.stdout.isTTY, +): TerminalImageRenderSupport { + if (supportsKittyImageProtocol(env, stdoutIsTTY)) { + return { available: true }; + } + + // Detect chafa with a PATH lookup instead of rendering the user's image: + // this runs during display_image execution and must never spawn a + // synchronous subprocess. A render failure still surfaces later, as a + // fallback notice, when renderTerminalImage actually runs chafa. + return findExecutable('chafa', env) + ? { available: true } + : { + available: false, + reason: + 'No compatible native image protocol was detected, and chafa is not installed.', + }; +} + +export function containsCmdShellMetacharacters(filePath: string): boolean { + return CMD_SHELL_METACHARACTERS.test(filePath); +} + +export function renderTerminalImage({ + display, + contentWidth, + availableTerminalHeight, + env = process.env, + stdoutIsTTY = process.stdout.isTTY, +}: TerminalImageRenderOptions): TerminalImageRenderResult { + let stat: fs.Stats; + try { + stat = fs.statSync(display.filePath); + } catch (error) { + return { + kind: 'unavailable', + reason: `Image file is unavailable: ${ + error instanceof Error ? error.message : String(error) + }`, + }; + } + if (!stat.isFile()) { + return { + kind: 'unavailable', + reason: 'Image path is not a regular file.', + }; + } + if (stat.size > MAX_TERMINAL_IMAGE_BYTES) { + return { + kind: 'unavailable', + reason: `Image exceeds the ${MAX_TERMINAL_IMAGE_BYTES} byte display limit.`, + }; + } + + // Read only the 24-byte PNG header to size the preview and build the cache + // key, so a cache hit never re-reads the full file or re-spawns chafa. + const headerSize = readPngHeaderSize(display.filePath); + if (!headerSize) { + return { kind: 'unavailable', reason: 'Image is not a valid PNG.' }; + } + + const shape = fitImageToTerminal( + headerSize, + contentWidth, + availableTerminalHeight, + ); + const useKitty = supportsKittyImageProtocol(env, stdoutIsTTY); + const chafaPath = useKitty ? null : findExecutable('chafa', env); + const cacheKey = createRenderCacheKey( + display.filePath, + stat, + shape, + useKitty, + chafaPath, + ); + const cached = getCachedRenderResult(cacheKey); + if (cached) return cached; + + let png: Buffer; + try { + png = fs.readFileSync(display.filePath); + } catch (error) { + return { + kind: 'unavailable', + reason: `Unable to read image: ${ + error instanceof Error ? error.message : String(error) + }`, + }; + } + if (png.length > MAX_TERMINAL_IMAGE_BYTES) { + return { + kind: 'unavailable', + reason: `Image exceeds the ${MAX_TERMINAL_IMAGE_BYTES} byte display limit.`, + }; + } + + const result: TerminalImageRenderResult = useKitty + ? { ...renderKitty(png, shape), key: cacheKey } + : renderWithChafa(display.filePath, shape, env, chafaPath); + if (result.kind !== 'unavailable') { + rememberRenderResult(cacheKey, result); + } + return result; +} + +function readPngHeaderSize( + filePath: string, +): { width: number; height: number } | null { + let fd: number | undefined; + try { + fd = fs.openSync(filePath, 'r'); + const header = Buffer.alloc(24); + const bytesRead = fs.readSync(fd, header, 0, header.length, 0); + return bytesRead === header.length ? readPngSize(header) : null; + } catch { + return null; + } finally { + if (fd !== undefined) { + try { + fs.closeSync(fd); + } catch { + // Closing a read-only handle should not fail the render. + } + } + } +} + +function createRenderCacheKey( + filePath: string, + stat: fs.Stats, + shape: { widthCells: number; rows: number }, + useKitty: boolean, + chafaPath: string | null, +): string { + return [ + filePath, + stat.mtimeMs, + stat.size, + shape.widthCells, + shape.rows, + useKitty ? 'kitty' : (chafaPath ?? 'none'), + ].join('\0'); +} + +function getCachedRenderResult( + key: string, +): TerminalImageRenderResult | undefined { + const cached = renderCache.get(key); + if (cached) { + renderCache.delete(key); + renderCache.set(key, cached); + } + return cached; +} + +function rememberRenderResult( + key: string, + result: TerminalImageRenderResult, +): void { + const bytes = estimateRenderResultBytes(result); + if (bytes > RENDER_CACHE_BYTE_LIMIT) { + renderCache.delete(key); + return; + } + + const existing = renderCache.get(key); + if (existing) { + renderCacheBytes -= estimateRenderResultBytes(existing); + } + renderCache.set(key, result); + renderCacheBytes += bytes; + while ( + renderCache.size > RENDER_CACHE_LIMIT || + renderCacheBytes > RENDER_CACHE_BYTE_LIMIT + ) { + const oldest = renderCache.keys().next().value; + if (!oldest) break; + const oldestResult = renderCache.get(oldest); + if (oldestResult) { + renderCacheBytes -= estimateRenderResultBytes(oldestResult); + } + renderCache.delete(oldest); + } +} + +function estimateRenderResultBytes(result: TerminalImageRenderResult): number { + switch (result.kind) { + case 'kitty': + return ( + Buffer.byteLength(result.sequence, 'utf8') + + result.placeholder.lines.reduce( + (total, line) => total + Buffer.byteLength(line, 'utf8'), + 0, + ) + ); + case 'ansi': + return result.lines.reduce( + (total, line) => total + Buffer.byteLength(line, 'utf8'), + 0, + ); + case 'unavailable': + return Buffer.byteLength(result.reason, 'utf8'); + default: { + const exhaustive: never = result; + return exhaustive; + } + } +} + +function fitImageToTerminal( + size: { width: number; height: number }, + contentWidth: number, + availableTerminalHeight?: number, +): { widthCells: number; rows: number } { + const maxWidthCells = Math.max( + 1, + Math.min(Math.floor(contentWidth), MAX_PREVIEW_WIDTH_CELLS), + ); + const maxRows = Math.max( + 1, + Math.min( + Math.floor(availableTerminalHeight ?? MAX_PREVIEW_ROWS), + MAX_PREVIEW_ROWS, + ), + ); + const naturalWidthCells = Math.max(1, size.width / ESTIMATED_CELL_WIDTH_PX); + const naturalRows = Math.max(1, size.height / ESTIMATED_CELL_HEIGHT_PX); + const scale = Math.min( + 1, + maxWidthCells / naturalWidthCells, + maxRows / naturalRows, + ); + + return { + widthCells: Math.max( + 1, + Math.min(maxWidthCells, Math.floor(naturalWidthCells * scale)), + ), + rows: Math.max(1, Math.min(maxRows, Math.floor(naturalRows * scale))), + }; +} + +function createImageId( + png: Buffer, + shape: { widthCells: number; rows: number }, +): number { + const hash = crypto + .createHash('sha256') + .update(png) + .update('\0') + .update(String(shape.widthCells)) + .update('\0') + .update(String(shape.rows)) + .digest(); + const id = hash.readUIntBE(0, 3); + return id === 0 ? 1 : id; +} + +function renderKitty( + png: Buffer, + shape: { widthCells: number; rows: number }, +): Omit, 'key'> { + const imageId = createImageId(png, shape); + return { + kind: 'kitty', + sequence: encodeKittyVirtualImage( + png, + imageId, + shape.widthCells, + shape.rows, + ), + placeholder: buildKittyPlaceholder(imageId, shape.widthCells, shape.rows), + }; +} + +function renderWithChafa( + filePath: string, + shape: { widthCells: number; rows: number }, + env: NodeJS.ProcessEnv, + chafaPath: string | null, +): Extract { + // Resolve chafa through the hardened lookup so a project-local + // node_modules/.bin/chafa is never executed unless the user opted in, then + // spawn the resolved path rather than a bare name resolved off PATH. + if (!chafaPath) { + return { + kind: 'unavailable', + reason: + 'No compatible native image protocol was detected, and chafa is not installed.', + }; + } + const useShell = shouldRunThroughShell(chafaPath); + if (useShell && containsCmdShellMetacharacters(filePath)) { + return { + kind: 'unavailable', + reason: + 'Image path contains characters that cannot be safely passed to the renderer on this platform.', + }; + } + try { + const stdout = execFileSync( + chafaPath, + [ + '--animate=off', + '--colors=256', + '--format=symbols', + '--symbols=block', + `--size=${shape.widthCells}x${shape.rows}`, + filePath, + ], + { + encoding: 'utf8', + env: createRendererChildEnv(env), + shell: useShell, + maxBuffer: CHAFA_MAX_OUTPUT_BYTES, + timeout: CHAFA_TIMEOUT_MS, + }, + ); + const lines = stdout.split(/\r?\n/).filter((line) => line.length > 0); + return lines.length > 0 + ? { kind: 'ansi', lines } + : { kind: 'unavailable', reason: 'chafa produced no output.' }; + } catch (error) { + const execError = error as Error & { + stderr?: Buffer | string; + }; + const stderr = firstLineBounded(String(execError.stderr ?? '').trim()); + return { + kind: 'unavailable', + reason: + stderr || execError.message || 'chafa could not render the image.', + }; + } +} + +function firstLineBounded(text: string): string { + const firstLine = text.split(/\r?\n/, 1)[0]?.trim() ?? ''; + return firstLine.length > MAX_REASON_CHARS + ? `${firstLine.slice(0, MAX_REASON_CHARS)}…` + : firstLine; +} diff --git a/packages/core/src/agents/background-agent-resume.test.ts b/packages/core/src/agents/background-agent-resume.test.ts index 55e85f23abf..8a9bea32aee 100644 --- a/packages/core/src/agents/background-agent-resume.test.ts +++ b/packages/core/src/agents/background-agent-resume.test.ts @@ -108,8 +108,19 @@ describe('BackgroundAgentResumeService', () => { // mocks the override helper throws and every resume test fails. const stubToolRegistry = { copyDiscoveredToolsFrom: vi.fn(), + registerFactory: vi.fn(), getAllTools: vi.fn().mockReturnValue([]), - getAllToolNames: vi.fn().mockReturnValue([]), + getAllToolNames: vi + .fn() + .mockReturnValue( + ( + options.currentForkRuntime?.registeredTools ?? + options.currentForkRuntime?.advertisedTools ?? + [] + ) + .map((declaration) => declaration.name) + .filter((name): name is string => Boolean(name)), + ), getTool: vi.fn(), stop: vi.fn().mockResolvedValue(undefined), warmAll: vi.fn().mockResolvedValue(undefined), @@ -2032,6 +2043,7 @@ describe('BackgroundAgentResumeService', () => { executionAllowedTools: ['Read', ToolNames.ASK_USER_QUESTION] as | string[] | undefined, + includeDisplayImage: false, deniedTool: 'Edit', expectedExecutionAllowedTools: ['Read'], }, @@ -2039,14 +2051,24 @@ describe('BackgroundAgentResumeService', () => { format: 'history-only bootstrap', legacyCapabilities: {}, executionAllowedTools: undefined as string[] | undefined, + includeDisplayImage: false, deniedTool: ToolNames.ASK_USER_QUESTION, expectedExecutionAllowedTools: ['Read', 'Edit'], }, + { + format: 'legacy fork without a persisted display policy', + legacyCapabilities: {}, + executionAllowedTools: undefined as string[] | undefined, + includeDisplayImage: true, + deniedTool: ToolNames.DISPLAY_IMAGE, + expectedExecutionAllowedTools: ['Read', 'Edit'], + }, ])( 'resumes fork agents with the current parent prompt and live tool registry ($format)', async ({ legacyCapabilities, executionAllowedTools, + includeDisplayImage, deniedTool, expectedExecutionAllowedTools, }) => { @@ -2151,6 +2173,9 @@ describe('BackgroundAgentResumeService', () => { 1, [ { name: 'Read' }, + ...(includeDisplayImage + ? [{ name: ToolNames.DISPLAY_IMAGE }] + : []), { name: 'Edit' }, { name: ToolNames.ASK_USER_QUESTION }, ], @@ -2175,6 +2200,14 @@ describe('BackgroundAgentResumeService', () => { systemInstruction: currentSystemInstruction, advertisedTools: [ { name: 'Read', description: 'advertised current schema' }, + ...(includeDisplayImage + ? [ + { + name: ToolNames.DISPLAY_IMAGE, + description: 'advertised display schema', + }, + ] + : []), { name: 'Edit', description: 'advertised edit schema' }, { name: ToolNames.ASK_USER_QUESTION, @@ -2184,6 +2217,14 @@ describe('BackgroundAgentResumeService', () => { ], registeredTools: [ { name: 'Read', description: 'registered current schema' }, + ...(includeDisplayImage + ? [ + { + name: ToolNames.DISPLAY_IMAGE, + description: 'registered display schema', + }, + ] + : []), { name: 'Edit', description: 'registered edit schema' }, { name: ToolNames.ASK_USER_QUESTION, @@ -2217,7 +2258,12 @@ describe('BackgroundAgentResumeService', () => { max_turns: FORK_DEFAULT_MAX_TURNS, }); expect(createArgs?.[5]).toEqual({ - tools: ['Read', 'Edit', ToolNames.ASK_USER_QUESTION], + tools: [ + 'Read', + ...(includeDisplayImage ? [ToolNames.DISPLAY_IMAGE] : []), + 'Edit', + ToolNames.ASK_USER_QUESTION, + ], executionAllowedTools: expectedExecutionAllowedTools, }); expect(executeContext).toBeDefined(); @@ -2236,6 +2282,12 @@ describe('BackgroundAgentResumeService', () => { expect(deniedError).not.toContain('not found'); expect(stubToolRegistry.getTool).not.toHaveBeenCalled(); expect(deniedBuild).not.toHaveBeenCalled(); + if (includeDisplayImage) { + expect(stubToolRegistry.registerFactory).toHaveBeenCalledWith( + ToolNames.DISPLAY_IMAGE, + expect.any(Function), + ); + } createSpy.mockRestore(); }, ); diff --git a/packages/core/src/agents/background-agent-resume.ts b/packages/core/src/agents/background-agent-resume.ts index d5b412c6539..da29bcdd98e 100644 --- a/packages/core/src/agents/background-agent-resume.ts +++ b/packages/core/src/agents/background-agent-resume.ts @@ -51,6 +51,8 @@ import { FORK_DEFAULT_MAX_TURNS, FORK_SUBAGENT_TYPE, buildForkExecutionAllowlist, + registerForkDisplayImageForCache, + resolveForkExecutionAllowedTools, runInForkContext, } from '../tools/agent/fork-subagent.js'; import { @@ -905,6 +907,12 @@ export class BackgroundAgentResumeService { const activeRestoreParentPM = approvalOverride.cleanup; agentConfig = activeAgentConfig; restoreParentPM = activeRestoreParentPM; + if (target.isFork) { + registerForkDisplayImageForCache( + activeAgentConfig, + currentForkRuntime!.toolNames, + ); + } // Mirror the launch path's permission-bubbling gate (agent.ts): an // agent whose definition uses `approvalMode: bubble` surfaces // confirmations to the parent UI instead of auto-denying, in @@ -1669,9 +1677,13 @@ export class BackgroundAgentResumeService { }; const toolConfig: ToolConfig = { tools: [...runtime.toolNames], - executionAllowedTools: buildForkExecutionAllowlist( - executionAllowedTools, + // Combine the ask_user_question restriction (buildForkExecutionAllowlist) + // with the display_image restriction (resolveForkExecutionAllowedTools): + // a resumed fork keeps the parent's display_image declaration for cache + // parity but must not execute it. + executionAllowedTools: resolveForkExecutionAllowedTools( runtime.toolNames, + buildForkExecutionAllowlist(executionAllowedTools, runtime.toolNames), ), }; diff --git a/packages/core/src/config/config.test.ts b/packages/core/src/config/config.test.ts index c4a6003fe5d..6ee65c1eed7 100644 --- a/packages/core/src/config/config.test.ts +++ b/packages/core/src/config/config.test.ts @@ -3825,6 +3825,75 @@ describe('Server Config (config.ts)', () => { expect(registeredNames).toContain(ToolNames.RECORD_ARTIFACT); }); + it('registers display_image only for the main interactive TUI', async () => { + const interactive = new Config({ + ...baseParams, + interactive: true, + sdkMode: false, + }); + await interactive.initialize(); + + const registerToolMock = ToolRegistry.prototype.registerFactory as Mock; + expect(registerToolMock.mock.calls.map((call) => call[0])).toContain( + ToolNames.DISPLAY_IMAGE, + ); + + for (const params of [ + { interactive: false, sdkMode: false }, + { interactive: true, sdkMode: true }, + { + interactive: true, + sdkMode: false, + inputFormat: InputFormat.STREAM_JSON, + }, + { + interactive: true, + sdkMode: false, + accessibility: { screenReader: true }, + }, + ]) { + registerToolMock.mockClear(); + const config = new Config({ ...baseParams, ...params }); + await config.initialize(); + expect( + registerToolMock.mock.calls.map((call) => call[0]), + ).not.toContain(ToolNames.DISPLAY_IMAGE); + } + + registerToolMock.mockClear(); + await interactive.createToolRegistry(undefined, { + skipDiscovery: true, + forSubAgent: true, + }); + expect(registerToolMock.mock.calls.map((call) => call[0])).not.toContain( + ToolNames.DISPLAY_IMAGE, + ); + }); + + it('forwards terminal image renderer support to the display tool', async () => { + const provider = vi.fn().mockResolvedValue({ + available: false, + reason: 'renderer unavailable', + }); + const config = new Config({ + ...baseParams, + terminalImageRenderSupportProvider: provider, + }); + + await expect(config.getTerminalImageRenderSupport()).resolves.toEqual({ + available: false, + reason: 'renderer unavailable', + }); + expect(provider).toHaveBeenCalledWith(); + + await expect( + new Config(baseParams).getTerminalImageRenderSupport(), + ).resolves.toEqual({ + available: false, + reason: 'No terminal image renderer is configured.', + }); + }); + it('registers only record_artifact by default for daemon artifact metadata', async () => { const config = new Config({ ...baseParams, diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts index 86274d7644d..815a7eed96a 100644 --- a/packages/core/src/config/config.ts +++ b/packages/core/src/config/config.ts @@ -963,6 +963,7 @@ export interface ConfigParameters { * Names returned must be lower-cased; consumers compare case-insensitively. */ disabledSkillNamesProvider?: () => ReadonlySet; + terminalImageRenderSupportProvider?: () => Promise; /** * Skill discovery levels that should not be loaded. Sourced from * `settings.skills.disabledLevels`. @@ -1363,6 +1364,10 @@ export interface ConfigParameters { settingsWatcher?: { stopWatching(): void }; } +export type TerminalImageRenderSupport = + | { available: true } + | { available: false; reason: string }; + export interface ImageGenerationConfig { model: string; baseUrl: string; @@ -1791,6 +1796,9 @@ export class Config { private readonly disabledSkillNamesProvider: | (() => ReadonlySet) | null; + private readonly terminalImageRenderSupportProvider: + | (() => Promise) + | null; private readonly disabledSkillLevels: ReadonlySet; private readonly customSkillDirs: readonly string[]; // `disabledTools` is set at construction @@ -2105,6 +2113,8 @@ export class Config { ...(params.disabledSlashCommands ?? []), ]); this.disabledSkillNamesProvider = params.disabledSkillNamesProvider ?? null; + this.terminalImageRenderSupportProvider = + params.terminalImageRenderSupportProvider ?? null; this.disabledSkillLevels = new Set(params.disabledSkillLevels ?? []); this.customSkillDirs = Object.freeze([...(params.customSkillDirs ?? [])]); this.disabledTools = new Set(params.disabledTools ?? []); @@ -7049,6 +7059,15 @@ export class Config { return this.interactive; } + async getTerminalImageRenderSupport(): Promise { + return this.terminalImageRenderSupportProvider + ? this.terminalImageRenderSupportProvider() + : { + available: false, + reason: 'No terminal image renderer is configured.', + }; + } + getUseRipgrep(): boolean { return this.useRipgrep; } @@ -7960,6 +7979,17 @@ export class Config { const { WebFetchTool } = await import('../tools/web-fetch.js'); return new WebFetchTool(this); }); + if ( + resolveInteractionMode(this) === 'interactive' && + !this.sdkMode && + !this.getScreenReader() && + !options?.forSubAgent + ) { + await registerLazy(ToolNames.DISPLAY_IMAGE, async () => { + const { DisplayImageTool } = await import('../tools/display-image.js'); + return new DisplayImageTool(this); + }); + } // WebSearch is opt-in: it registers only when explicitly enabled AND the // configured search model resolves to a usable DashScope entry. A failed // gate surfaces a one-time startup notice instead of a silently missing diff --git a/packages/core/src/core/coreToolScheduler.test.ts b/packages/core/src/core/coreToolScheduler.test.ts index 16ccb2eba10..f931e67f0a0 100644 --- a/packages/core/src/core/coreToolScheduler.test.ts +++ b/packages/core/src/core/coreToolScheduler.test.ts @@ -15017,6 +15017,14 @@ describe('extractToolFilePaths', () => { ).toEqual(['/proj/chart.png']); }); + it('extracts file_path for display_image', () => { + expect( + extractToolFilePaths(ToolNames.DISPLAY_IMAGE, { + file_path: '/proj/chart.png', + }), + ).toEqual(['/proj/chart.png']); + }); + it('extracts notebook_path for notebook_edit', () => { expect( extractToolFilePaths('notebook_edit', { diff --git a/packages/core/src/core/coreToolScheduler.ts b/packages/core/src/core/coreToolScheduler.ts index b9ab542d704..d75e7567e19 100644 --- a/packages/core/src/core/coreToolScheduler.ts +++ b/packages/core/src/core/coreToolScheduler.ts @@ -523,6 +523,7 @@ const FS_PATH_TOOL_NAMES: ReadonlySet = new Set([ ToolNames.LS, ToolNames.LSP, ToolNames.NOTEBOOK_EDIT, + ToolNames.DISPLAY_IMAGE, ]); /** @@ -720,6 +721,7 @@ export function extractToolFilePaths( case ToolNames.ZOOM_IMAGE: case ToolNames.EDIT: case ToolNames.WRITE_FILE: + case ToolNames.DISPLAY_IMAGE: push(obj['file_path']); return out; diff --git a/packages/core/src/tools/agent/agent.test.ts b/packages/core/src/tools/agent/agent.test.ts index 7e71bbb2d4b..28bb3d0da99 100644 --- a/packages/core/src/tools/agent/agent.test.ts +++ b/packages/core/src/tools/agent/agent.test.ts @@ -192,6 +192,7 @@ describe('AgentTool', () => { // exercises foreground execution fails. const stubToolRegistry = { copyDiscoveredToolsFrom: vi.fn(), + registerFactory: vi.fn(), getAllTools: vi.fn().mockReturnValue([]), getAllToolNames: vi.fn().mockReturnValue([]), stop: vi.fn().mockResolvedValue(undefined), @@ -3833,6 +3834,62 @@ describe('AgentTool', () => { ); }); + it('preserves display_image in the fork declarations but denies its execution', async () => { + const parentToolDecls = [ + { + name: ToolNames.READ_FILE, + description: 'Read a file', + parameters: { type: 'object', properties: {} }, + }, + { + name: ToolNames.DISPLAY_IMAGE, + description: 'Display an image', + parameters: { type: 'object', properties: {} }, + }, + { + name: ToolNames.EDIT, + description: 'Edit a file', + parameters: { type: 'object', properties: {} }, + }, + ]; + vi.mocked(config.getGeminiClient).mockReturnValue({ + getHistory: vi.fn().mockReturnValue([]), + getChat: vi.fn().mockReturnValue({ + getGenerationConfig: vi.fn().mockReturnValue({ + systemInstruction: 'parent system', + tools: [{ functionDeclarations: parentToolDecls }], + }), + }), + } as unknown as ReturnType); + + const invocation = ( + agentTool as AgentToolWithProtectedMethods + ).createInvocation({ + description: 'inspect images', + prompt: 'inspect the implementation', + subagent_type: 'fork', + fork_tools: [ToolNames.READ_FILE, ToolNames.DISPLAY_IMAGE], + }); + await invocation.execute(); + + const createArgs = vi.mocked(AgentHeadless.create).mock.calls[0]; + const forkConfig = createArgs?.[1]; + const toolConfig = createArgs?.[5]; + expect(toolConfig?.tools).toStrictEqual([ + ToolNames.READ_FILE, + ToolNames.DISPLAY_IMAGE, + ToolNames.EDIT, + ]); + expect(toolConfig?.executionAllowedTools).toEqual([ToolNames.READ_FILE]); + expect( + forkConfig?.getToolRegistry().registerFactory, + ).toHaveBeenCalledWith(ToolNames.DISPLAY_IMAGE, expect.any(Function)); + expect(mockContextState.set).toHaveBeenCalledWith( + 'task_prompt', + expect.stringContaining(JSON.stringify([ToolNames.READ_FILE])), + ); + }); + it('omitting subagent_type uses general-purpose, not fork', async () => { // Omission resolves to the regular general-purpose subagent, never a // context-inheriting fork — even in interactive mode. diff --git a/packages/core/src/tools/agent/agent.ts b/packages/core/src/tools/agent/agent.ts index def18466258..a75745e1a95 100644 --- a/packages/core/src/tools/agent/agent.ts +++ b/packages/core/src/tools/agent/agent.ts @@ -49,6 +49,8 @@ import { buildPinnedWorktreeNotice, buildWorktreeNotice, normalizeForkTurns, + registerForkDisplayImageForCache, + resolveForkExecutionAllowedTools, runInForkContext, selectForkHistory, validateForkToolList, @@ -1684,12 +1686,20 @@ class AgentToolInvocation extends BaseToolInvocation { toolConfig: ToolConfig; }> { const geminiClient = this.config.getGeminiClient(); + const generationConfig = geminiClient?.getChat().getGenerationConfig(); + const parentToolNames = generationConfig?.systemInstruction + ? extractParentToolNames(generationConfig) + : []; + registerForkDisplayImageForCache(agentConfig, parentToolNames); const forkTurns = normalizeForkTurns(this.params.fork_turns); const requestedTools = this.forkProfile?.tools ?? this.params.fork_tools; const requestedExecutionAllowedTools = requestedTools === undefined ? undefined - : buildForkExecutionAllowlist(requestedTools, []); + : resolveForkExecutionAllowedTools( + parentToolNames, + buildForkExecutionAllowlist(requestedTools, []), + ); const profilePromptHint = this.forkProfile?.promptHint; let rawHistory: Content[] = []; if (geminiClient) { @@ -1790,14 +1800,12 @@ class AgentToolInvocation extends BaseToolInvocation { let promptConfig: PromptConfig; let toolConfig: ToolConfig; - const generationConfig = geminiClient?.getChat().getGenerationConfig(); if (generationConfig?.systemInstruction) { // Keep the parent's current allowlist, but pass names rather than inline // schemas so AgentCore resolves every declaration through the fork's // current ToolRegistry. This preserves the parent's tool surface and // cache prefix when schemas are unchanged without letting a persisted or // stale declaration bypass the live registry. - const parentToolNames = extractParentToolNames(generationConfig); const declaredExecutionToolNames = parentToolNames.length > 0 ? parentToolNames @@ -1816,9 +1824,12 @@ class AgentToolInvocation extends BaseToolInvocation { }; toolConfig = { tools: parentToolNames.length > 0 ? parentToolNames : ['*'], - executionAllowedTools: buildForkExecutionAllowlist( - requestedTools, - declaredExecutionToolNames, + executionAllowedTools: resolveForkExecutionAllowedTools( + parentToolNames, + buildForkExecutionAllowlist( + requestedTools, + declaredExecutionToolNames, + ), ), }; } else { @@ -1832,9 +1843,9 @@ class AgentToolInvocation extends BaseToolInvocation { }; toolConfig = { tools: ['*'], - executionAllowedTools: buildForkExecutionAllowlist( - requestedTools, - registeredToolNames, + executionAllowedTools: resolveForkExecutionAllowedTools( + parentToolNames, + buildForkExecutionAllowlist(requestedTools, registeredToolNames), ), }; } diff --git a/packages/core/src/tools/agent/fork-subagent.test.ts b/packages/core/src/tools/agent/fork-subagent.test.ts index ba354106e82..4d88d4a506f 100644 --- a/packages/core/src/tools/agent/fork-subagent.test.ts +++ b/packages/core/src/tools/agent/fork-subagent.test.ts @@ -6,12 +6,52 @@ import type { Content } from '@google/genai'; import { describe, expect, it } from 'vitest'; +import { ToolNames } from '../tool-names.js'; import { normalizeForkTurns, + resolveForkExecutionAllowedTools, selectForkHistory, validateForkToolList, } from './fork-subagent.js'; +describe('resolveForkExecutionAllowedTools', () => { + const parentTools = [ + ToolNames.READ_FILE, + ToolNames.DISPLAY_IMAGE, + ToolNames.EDIT, + ]; + + it('preserves unrestricted execution when display_image is not advertised', () => { + expect( + resolveForkExecutionAllowedTools([ToolNames.READ_FILE], undefined), + ).toBeUndefined(); + }); + + it('filters display_image from an explicit allowlist', () => { + expect( + resolveForkExecutionAllowedTools(parentTools, [ + ...parentTools, + ToolNames.GLOB, + ]), + ).toEqual([ToolNames.READ_FILE, ToolNames.EDIT, ToolNames.GLOB]); + }); + + it('cannot re-enable display_image through fork_tools', () => { + expect( + resolveForkExecutionAllowedTools(parentTools, [ + ToolNames.DISPLAY_IMAGE, + ToolNames.READ_FILE, + ]), + ).toEqual([ToolNames.READ_FILE]); + }); + + it('fails closed when display_image is advertised without an allowlist', () => { + expect(resolveForkExecutionAllowedTools(parentTools, undefined)).toEqual( + [], + ); + }); +}); + describe('validateForkToolList', () => { it('accepts the inline fork tool contract, including deny-all', () => { expect(validateForkToolList([])).toBeUndefined(); diff --git a/packages/core/src/tools/agent/fork-subagent.ts b/packages/core/src/tools/agent/fork-subagent.ts index 97d674f5c6e..9308478a76c 100644 --- a/packages/core/src/tools/agent/fork-subagent.ts +++ b/packages/core/src/tools/agent/fork-subagent.ts @@ -1,5 +1,6 @@ import { AsyncLocalStorage } from 'node:async_hooks'; import type { Content } from '@google/genai'; +import type { Config } from '../../config/config.js'; import type { SubagentConfig } from '../../subagents/types.js'; import { BUBBLE_APPROVAL_MODE } from '../../subagents/types.js'; import { ToolNames } from '../tool-names.js'; @@ -61,6 +62,47 @@ export function isInForkExecution(): boolean { return forkExecutionStorage.getStore() !== undefined; } +/** + * Keeps the fork's model-visible declarations cache-identical while removing + * the main-session-only image renderer from its execution capability. + */ +export function resolveForkExecutionAllowedTools( + advertisedToolNames: readonly string[], + requestedToolNames: readonly string[] | undefined, +): string[] | undefined { + if (!advertisedToolNames.includes(ToolNames.DISPLAY_IMAGE)) { + return requestedToolNames ? [...requestedToolNames] : undefined; + } + + // display_image is main-session-only. "Unrestricted" (undefined) minus + // display_image cannot be written as a finite allowlist, so fail closed to + // deny-all instead of returning undefined — that would hand the fork + // unrestricted execution, including the very tool this strips. Every live + // caller passes a concrete list (buildForkExecutionAllowlist always returns + // an array); DisplayImageInvocation.execute() also enforces this locally. + return ( + requestedToolNames?.filter((name) => name !== ToolNames.DISPLAY_IMAGE) ?? [] + ); +} + +/** + * Restores the parent's display schema in a fork registry for prompt-cache + * parity. Callers must pair this with resolveForkExecutionAllowedTools(). + */ +export function registerForkDisplayImageForCache( + config: Config, + advertisedToolNames: readonly string[], +): void { + if (!advertisedToolNames.includes(ToolNames.DISPLAY_IMAGE)) return; + + config + .getToolRegistry() + .registerFactory(ToolNames.DISPLAY_IMAGE, async () => { + const { DisplayImageTool } = await import('../display-image.js'); + return new DisplayImageTool(config); + }); +} + export const FORK_PLACEHOLDER_RESULT = 'Fork started — processing in background'; diff --git a/packages/core/src/tools/display-image.test.ts b/packages/core/src/tools/display-image.test.ts new file mode 100644 index 00000000000..84fcc2c5670 --- /dev/null +++ b/packages/core/src/tools/display-image.test.ts @@ -0,0 +1,156 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import type { Config } from '../config/config.js'; +import { runInForkContext } from './agent/fork-subagent.js'; +import { ToolErrorType } from './tool-error.js'; +import { DisplayImageTool } from './display-image.js'; +import { MAX_TERMINAL_IMAGE_BYTES } from './tools.js'; + +const PNG_1X1 = Buffer.from( + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=', + 'base64', +); + +describe('DisplayImageTool', () => { + let workspace: string; + let tool: DisplayImageTool; + + beforeEach(async () => { + workspace = await fs.mkdtemp(path.join(os.tmpdir(), 'display-image-test-')); + tool = new DisplayImageTool({ + getTargetDir: () => workspace, + getWorkspaceContext: () => ({ + isPathWithinWorkspace: (filePath: string) => { + const relative = path.relative(workspace, path.resolve(filePath)); + return ( + relative === '' || + (!relative.startsWith(`..${path.sep}`) && + relative !== '..' && + !path.isAbsolute(relative)) + ); + }, + }), + getTerminalImageRenderSupport: async () => ({ available: true }), + } as unknown as Config); + }); + + afterEach(async () => { + await fs.rm(workspace, { recursive: true, force: true }); + }); + + it('returns a small structured display without image bytes', async () => { + const imagePath = path.join(workspace, 'pixel.png'); + await fs.writeFile(imagePath, PNG_1X1); + + const invocation = tool.build({ file_path: imagePath }); + const result = await invocation.execute(new AbortController().signal); + + expect(result.error).toBeUndefined(); + expect(result.returnDisplay).toEqual({ + type: 'terminal_image', + filePath: imagePath, + mimeType: 'image/png', + }); + expect(result.resultFilePaths).toEqual([imagePath]); + expect(result.llmContent).toContain('does not provide the image contents'); + expect(JSON.stringify(result)).not.toContain(PNG_1X1.toString('base64')); + expect(invocation.toolLocations()).toEqual([{ path: imagePath }]); + }); + + it('refuses to execute inside a fork subagent', async () => { + const imagePath = path.join(workspace, 'pixel.png'); + await fs.writeFile(imagePath, PNG_1X1); + + const invocation = tool.build({ file_path: imagePath }); + const result = await runInForkContext(() => + invocation.execute(new AbortController().signal), + ); + + expect(result.error?.type).toBe(ToolErrorType.EXECUTION_DENIED); + expect(result.llmContent).toContain('main agent'); + expect(result.returnDisplay).not.toHaveProperty('type', 'terminal_image'); + }); + + it('requires an absolute workspace path', () => { + expect(() => tool.build({ file_path: 'pixel.png' })).toThrow( + /must be absolute/, + ); + expect(() => + tool.build({ file_path: path.join(os.tmpdir(), 'outside.png') }), + ).toThrow(/outside the current workspace/); + }); + + it('reports a renderer failure to the model and TUI', async () => { + const imagePath = path.join(workspace, 'pixel.png'); + await fs.writeFile(imagePath, PNG_1X1); + tool = new DisplayImageTool({ + getTargetDir: () => workspace, + getWorkspaceContext: () => ({ + isPathWithinWorkspace: () => true, + }), + getTerminalImageRenderSupport: async () => ({ + available: false, + reason: + 'No compatible native image protocol was detected, and chafa is not installed.', + }), + } as unknown as Config); + + const result = await tool + .build({ file_path: imagePath }) + .execute(new AbortController().signal); + + expect(result.error?.type).toBe(ToolErrorType.EXECUTION_FAILED); + expect(result.llmContent).toContain('The image was not displayed'); + expect(result.returnDisplay).toContain('The image was not displayed'); + expect(result.returnDisplay).not.toHaveProperty('type', 'terminal_image'); + }); + + it('rejects missing files and directories', async () => { + const missing = await tool + .build({ file_path: path.join(workspace, 'missing.png') }) + .execute(new AbortController().signal); + expect(missing.error?.type).toBe(ToolErrorType.FILE_NOT_FOUND); + + const directory = await tool + .build({ file_path: workspace }) + .execute(new AbortController().signal); + expect(directory.error?.type).toBe(ToolErrorType.TARGET_IS_DIRECTORY); + }); + + it('rejects non-PNG and oversized files', async () => { + const textPath = path.join(workspace, 'not-an-image.png'); + await fs.writeFile(textPath, Buffer.from(`GIF89a${'x'.repeat(32)}`)); + const nonPng = await tool + .build({ file_path: textPath }) + .execute(new AbortController().signal); + expect(nonPng.error?.type).toBe(ToolErrorType.INVALID_TOOL_PARAMS); + + const largePath = path.join(workspace, 'large.png'); + const handle = await fs.open(largePath, 'w'); + await handle.truncate(MAX_TERMINAL_IMAGE_BYTES + 1); + await handle.close(); + const oversized = await tool + .build({ file_path: largePath }) + .execute(new AbortController().signal); + expect(oversized.error?.type).toBe(ToolErrorType.FILE_TOO_LARGE); + }); + + it('rejects a truncated PNG header', async () => { + const truncatedPath = path.join(workspace, 'truncated.png'); + await fs.writeFile(truncatedPath, Buffer.from('89504e470d0a1a0a', 'hex')); + + const result = await tool + .build({ file_path: truncatedPath }) + .execute(new AbortController().signal); + + expect(result.error?.type).toBe(ToolErrorType.INVALID_TOOL_PARAMS); + }); +}); diff --git a/packages/core/src/tools/display-image.ts b/packages/core/src/tools/display-image.ts new file mode 100644 index 00000000000..913bbcef95d --- /dev/null +++ b/packages/core/src/tools/display-image.ts @@ -0,0 +1,212 @@ +/** + * @license + * Copyright 2026 Qwen Team + * SPDX-License-Identifier: Apache-2.0 + */ + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import type { Config } from '../config/config.js'; +import { makeRelative, shortenPath, unescapePath } from '../utils/paths.js'; +import { isInForkExecution } from './agent/fork-subagent.js'; +import { ToolErrorType } from './tool-error.js'; +import { ToolDisplayNames, ToolNames } from './tool-names.js'; +import { + BaseDeclarativeTool, + BaseToolInvocation, + Kind, + MAX_TERMINAL_IMAGE_BYTES, + type TerminalImageDisplay, + type ToolInvocation, + type ToolLocation, + type ToolResult, +} from './tools.js'; + +const PNG_SIGNATURE = '89504e470d0a1a0a'; + +export interface DisplayImageToolParams { + file_path: string; +} + +function errorResult(message: string, type: ToolErrorType): ToolResult { + return { + llmContent: message, + returnDisplay: `Error: ${message}`, + error: { message, type }, + }; +} + +class DisplayImageInvocation extends BaseToolInvocation< + DisplayImageToolParams, + ToolResult +> { + constructor( + private readonly config: Config, + params: DisplayImageToolParams, + ) { + super(params); + } + + getDescription(): string { + return shortenPath( + makeRelative(this.params.file_path, this.config.getTargetDir()), + ); + } + + override toolLocations(): ToolLocation[] { + return [{ path: this.params.file_path }]; + } + + async execute(signal: AbortSignal): Promise { + signal.throwIfAborted(); + // The fork registry keeps this tool's declaration for prompt-cache parity, + // so the execution ban cannot live in declaration stripping alone. Enforce + // it here so the invariant holds no matter how the allowlist is wired. + if (isInForkExecution()) { + return errorResult( + 'display_image can only be executed by the main agent.', + ToolErrorType.EXECUTION_DENIED, + ); + } + const filePath = path.resolve(this.params.file_path); + + let stat: Awaited>; + try { + stat = await fs.stat(filePath); + } catch (error) { + const message = + (error as NodeJS.ErrnoException).code === 'ENOENT' + ? `Image file not found: ${filePath}` + : `Unable to access image file "${filePath}": ${ + error instanceof Error ? error.message : String(error) + }`; + return errorResult( + message, + (error as NodeJS.ErrnoException).code === 'ENOENT' + ? ToolErrorType.FILE_NOT_FOUND + : ToolErrorType.READ_CONTENT_FAILURE, + ); + } + + if (stat.isDirectory()) { + return errorResult( + `Image path is a directory: ${filePath}`, + ToolErrorType.TARGET_IS_DIRECTORY, + ); + } + if (!stat.isFile()) { + return errorResult( + `Image path is not a regular file: ${filePath}`, + ToolErrorType.TARGET_NOT_REGULAR_FILE, + ); + } + if (stat.size > MAX_TERMINAL_IMAGE_BYTES) { + return errorResult( + `Image exceeds the ${MAX_TERMINAL_IMAGE_BYTES} byte display limit: ${filePath}`, + ToolErrorType.FILE_TOO_LARGE, + ); + } + + signal.throwIfAborted(); + let fileHandle: Awaited> | undefined; + try { + fileHandle = await fs.open(filePath, 'r'); + // Read the full 24-byte PNG header (signature + IHDR width/height) so a + // truncated file is rejected here, before the renderer support check + // reports success. A valid PNG is always at least 24 bytes. + const header = Buffer.alloc(24); + const { bytesRead } = await fileHandle.read(header, 0, header.length, 0); + if ( + bytesRead !== header.length || + header.subarray(0, 8).toString('hex') !== PNG_SIGNATURE + ) { + return errorResult( + `Only PNG images are supported by display_image: ${filePath}`, + ToolErrorType.INVALID_TOOL_PARAMS, + ); + } + } catch (error) { + return errorResult( + `Unable to read image file "${filePath}": ${ + error instanceof Error ? error.message : String(error) + }`, + ToolErrorType.READ_CONTENT_FAILURE, + ); + } finally { + await fileHandle?.close(); + } + + const renderSupport = await this.config.getTerminalImageRenderSupport(); + if (!renderSupport.available) { + return errorResult( + `The image was not displayed: ${renderSupport.reason}`, + ToolErrorType.EXECUTION_FAILED, + ); + } + + const display: TerminalImageDisplay = { + type: 'terminal_image', + filePath, + mimeType: 'image/png', + }; + return { + llmContent: + `The terminal renderer accepted "${path.basename(filePath)}" for display. ` + + 'This display-only tool does not provide the image contents to you; use read_file if you need to inspect the image.', + returnDisplay: display, + resultFilePaths: [filePath], + }; + } +} + +export class DisplayImageTool extends BaseDeclarativeTool< + DisplayImageToolParams, + ToolResult +> { + static readonly Name = ToolNames.DISPLAY_IMAGE; + + constructor(private readonly config: Config) { + super( + DisplayImageTool.Name, + ToolDisplayNames.DISPLAY_IMAGE, + 'Displays an existing workspace PNG in the user’s interactive terminal. Only the main agent can execute this tool. This is only a user-visible preview: it does not return image contents to you. Use read_file when you need to inspect or understand the image. The file_path must be absolute.', + Kind.Read, + { + type: 'object', + properties: { + file_path: { + type: 'string', + description: 'Absolute path to a PNG inside the current workspace.', + }, + }, + required: ['file_path'], + additionalProperties: false, + }, + false, + ); + } + + protected override validateToolParamValues( + params: DisplayImageToolParams, + ): string | null { + params.file_path = unescapePath(params.file_path.trim()); + if (!params.file_path) { + return 'Missing or empty "file_path".'; + } + if (!path.isAbsolute(params.file_path)) { + return `File path must be absolute: ${params.file_path}`; + } + if ( + !this.config.getWorkspaceContext().isPathWithinWorkspace(params.file_path) + ) { + return `Image path is outside the current workspace: ${params.file_path}`; + } + return null; + } + + protected createInvocation( + params: DisplayImageToolParams, + ): ToolInvocation { + return new DisplayImageInvocation(this.config, params); + } +} diff --git a/packages/core/src/tools/tool-names.ts b/packages/core/src/tools/tool-names.ts index b6478ca4977..66c80dc3ff1 100644 --- a/packages/core/src/tools/tool-names.ts +++ b/packages/core/src/tools/tool-names.ts @@ -69,6 +69,7 @@ export const ToolNames = { RECORD_ARTIFACT: 'record_artifact', GET_GOAL: 'get_goal', UPDATE_GOAL: 'update_goal', + DISPLAY_IMAGE: 'display_image', } as const; /** @@ -123,6 +124,7 @@ export const ToolDisplayNames = { RECORD_ARTIFACT: 'RecordArtifact', GET_GOAL: 'Goal', UPDATE_GOAL: 'UpdateGoal', + DISPLAY_IMAGE: 'DisplayImage', } as const; // Migration from old tool names to new tool names diff --git a/packages/core/src/tools/tools.test.ts b/packages/core/src/tools/tools.test.ts index 244642e8305..1948566e9b4 100644 --- a/packages/core/src/tools/tools.test.ts +++ b/packages/core/src/tools/tools.test.ts @@ -7,7 +7,12 @@ import { describe, it, expect, vi } from 'vitest'; import type { ToolInvocation, ToolResult } from './tools.js'; import type { PermissionDecision } from '../permissions/types.js'; -import { DeclarativeTool, hasCycleInSchema, Kind } from './tools.js'; +import { + DeclarativeTool, + hasCycleInSchema, + isTerminalImageDisplay, + Kind, +} from './tools.js'; import { ToolErrorType } from './tool-error.js'; class TestToolInvocation implements ToolInvocation { @@ -233,3 +238,49 @@ describe('hasCycleInSchema', () => { expect(hasCycleInSchema({})).toBe(false); }); }); + +describe('isTerminalImageDisplay', () => { + it('accepts a well-formed terminal image display', () => { + expect( + isTerminalImageDisplay({ + type: 'terminal_image', + filePath: '/workspace/chart.png', + mimeType: 'image/png', + }), + ).toBe(true); + }); + + it('rejects null, wrong type, non-string filePath, and wrong mimeType', () => { + expect(isTerminalImageDisplay(null)).toBe(false); + expect(isTerminalImageDisplay(undefined)).toBe(false); + expect(isTerminalImageDisplay('terminal_image')).toBe(false); + expect(isTerminalImageDisplay({})).toBe(false); + expect( + isTerminalImageDisplay({ + type: 'something_else', + filePath: '/workspace/chart.png', + mimeType: 'image/png', + }), + ).toBe(false); + expect( + isTerminalImageDisplay({ + type: 'terminal_image', + filePath: 123, + mimeType: 'image/png', + }), + ).toBe(false); + expect( + isTerminalImageDisplay({ + type: 'terminal_image', + filePath: '/workspace/chart.png', + }), + ).toBe(false); + expect( + isTerminalImageDisplay({ + type: 'terminal_image', + filePath: '/workspace/chart.png', + mimeType: 'image/jpeg', + }), + ).toBe(false); + }); +}); diff --git a/packages/core/src/tools/tools.ts b/packages/core/src/tools/tools.ts index 23d0e8efb0c..ead5ea6ad14 100644 --- a/packages/core/src/tools/tools.ts +++ b/packages/core/src/tools/tools.ts @@ -711,6 +711,29 @@ export function isShellProgressData( ); } +export const MAX_TERMINAL_IMAGE_BYTES = 8 * 1024 * 1024; + +export interface TerminalImageDisplay { + type: 'terminal_image'; + filePath: string; + mimeType: 'image/png'; +} + +export function isTerminalImageDisplay( + display: unknown, +): display is TerminalImageDisplay { + return ( + typeof display === 'object' && + display !== null && + 'type' in display && + display.type === 'terminal_image' && + 'filePath' in display && + typeof display.filePath === 'string' && + 'mimeType' in display && + display.mimeType === 'image/png' + ); +} + export type ToolResultDisplay = | string | FileDiff @@ -722,7 +745,8 @@ export type ToolResultDisplay = | AnsiOutputDisplay | McpToolProgressData | VisionBridgeNoticeDisplay - | ShellProgressData; + | ShellProgressData + | TerminalImageDisplay; export interface TeamResultDisplay { type: 'team_result'; diff --git a/packages/web-shell/client/components/messages/toolFormatting.ts b/packages/web-shell/client/components/messages/toolFormatting.ts index d84f40fc127..e449b9f6990 100644 --- a/packages/web-shell/client/components/messages/toolFormatting.ts +++ b/packages/web-shell/client/components/messages/toolFormatting.ts @@ -58,6 +58,7 @@ export const TOOL_DISPLAY_NAMES: Record = { record_artifact: 'RecordArtifact', web_search: 'WebSearch', image_gen: 'ImageGen', + display_image: 'DisplayImage', bash: 'Shell', shell: 'Shell Command', read: 'ReadFile', diff --git a/packages/web-shell/client/i18n.tsx b/packages/web-shell/client/i18n.tsx index e9d14bf36c4..52c49fbb6f1 100644 --- a/packages/web-shell/client/i18n.tsx +++ b/packages/web-shell/client/i18n.tsx @@ -2732,6 +2732,7 @@ const ZH: Messages = { 'toolName.artifact': '制品', 'toolName.record_artifact': '记录制品', 'toolName.image_gen': '生成图片', + 'toolName.display_image': '显示图片', // web-shell-only wire aliases (see TOOL_DISPLAY_NAMES in toolFormatting.ts) 'toolName.bash': '运行命令', 'toolName.shell': 'Shell 命令', diff --git a/patches/ink+7.0.3.patch b/patches/ink+7.0.3.patch index fa594c2a3c4..4f002764556 100644 --- a/patches/ink+7.0.3.patch +++ b/patches/ink+7.0.3.patch @@ -18,7 +18,7 @@ index 1d7bf6b..29401a6 100644 -export default function Text({ color, backgroundColor, dimColor, bold, italic, underline, strikethrough, inverse, wrap, children, 'aria-label': ariaLabel, 'aria-hidden': ariaHidden, }: Props): React.JSX.Element | null; +export default function Text({ color, backgroundColor, dimColor, bold, italic, underline, strikethrough, inverse, wrap, selectable, selectionFlow, selectionBreakAfter, selectionJoiner, children, 'aria-label': ariaLabel, 'aria-hidden': ariaHidden, }: Props): React.JSX.Element | null; diff --git a/node_modules/ink/build/components/Text.js b/node_modules/ink/build/components/Text.js -index 2e4b9ff..77ed0f6 100644 +index 2e4b9ff..1ec2ae6 100644 --- a/node_modules/ink/build/components/Text.js +++ b/node_modules/ink/build/components/Text.js @@ -6,7 +6,7 @@ import { backgroundContext } from './BackgroundContext.js'; @@ -38,12 +38,11 @@ index 2e4b9ff..77ed0f6 100644 + return (React.createElement("ink-text", { style: { flexGrow: 0, flexShrink: 1, flexDirection: 'row', textWrap: wrap }, internal_transform: transform, selectable: selectable, selectionFlow: selectionFlow, selectionBreakAfter: selectionBreakAfter, selectionJoiner: selectionJoiner }, childrenOrAriaLabel)); } //# sourceMappingURL=Text.js.map -\ No newline at end of file diff --git a/node_modules/ink/build/cursor-helpers.d.ts b/node_modules/ink/build/cursor-helpers.d.ts index c6e9466..6dd4c0f 100644 --- a/node_modules/ink/build/cursor-helpers.d.ts +++ b/node_modules/ink/build/cursor-helpers.d.ts -@@ -11,9 +11,11 @@ +@@ -11,9 +11,11 @@ Compare two cursor positions. Returns true if they differ. export declare const cursorPositionChanged: (a: CursorPosition | undefined, b: CursorPosition | undefined) => boolean; /** Build escape sequence to move cursor from bottom of output to the target position and show it. @@ -57,7 +56,7 @@ index c6e9466..6dd4c0f 100644 /** Build escape sequence to move cursor from previousCursorPosition back to the bottom of output. This must be done before eraseLines or any operation that assumes cursor is at the bottom. -@@ -25,6 +27,7 @@ +@@ -25,6 +27,7 @@ export type CursorOnlyInput = { previousCursorPosition: CursorPosition | undefined; visibleLineCount: number; cursorPosition: CursorPosition | undefined; @@ -487,7 +486,7 @@ index 0f4523f..a3ef634 100644 } export {}; diff --git a/node_modules/ink/build/output.js b/node_modules/ink/build/output.js -index c763b77..bfd3d08 100644 +index c763b77..ac81f2e 100644 --- a/node_modules/ink/build/output.js +++ b/node_modules/ink/build/output.js @@ -1,6 +1,26 @@ @@ -777,7 +776,7 @@ index 127c620..6d5a005 100644 }) => void; export default renderNodeToOutput; diff --git a/node_modules/ink/build/render-node-to-output.js b/node_modules/ink/build/render-node-to-output.js -index f00a278..e6bc4ca 100644 +index f00a278..07a98d0 100644 --- a/node_modules/ink/build/render-node-to-output.js +++ b/node_modules/ink/build/render-node-to-output.js @@ -1,7 +1,6 @@