From 1cd955d6d94f905a5d04b916bacccd691949cfbf Mon Sep 17 00:00:00 2001 From: ckrafft Date: Tue, 11 Aug 2026 02:08:52 +0200 Subject: [PATCH 01/16] server: add read_image tool (#25875) Adds a server-tool that allows vision models to analyze server-side images. This tool is reading a single file for now: The image data is base64 encoded and passed to the UI, which decodes it, fills the tag and removes the data URI before passing the tool result back to the model. --- tools/server/server-tools.cpp | 88 +++++++++++++++++++ .../ChatMessageToolCallBlock.svelte | 3 + .../ChatMessageToolCallBlockReadImage.svelte | 80 +++++++++++++++++ .../ChatMessageToolCall/parsers/read-image.ts | 46 ++++++++++ tools/ui/src/lib/constants/built-in-tools.ts | 2 + tools/ui/src/lib/enums/tools.enums.ts | 1 + 6 files changed, 220 insertions(+) create mode 100644 tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadImage.svelte create mode 100644 tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-image.ts diff --git a/tools/server/server-tools.cpp b/tools/server/server-tools.cpp index a4c1059fff49..aea385c42d8e 100644 --- a/tools/server/server-tools.cpp +++ b/tools/server/server-tools.cpp @@ -1,6 +1,7 @@ #include "server-tools.h" #include "subproc.h" +#include "base64.hpp" #include #include @@ -1979,6 +1980,92 @@ static server_tool & find_tool(std::vector> & tools throw std::invalid_argument(string_format("unknown tool \"%s\"", name.c_str())); } +// +// read_image: read an image file and return base64-encoded data with metadata +// + +static constexpr size_t SERVER_TOOL_READ_IMAGE_MAX_SIZE = 16 * 1024 * 1024; // 16 MB + +static std::string get_mime_from_extension(const std::string & path) { + static const std::unordered_map mime_map = { + {".png", "image/png"}, + {".jpg", "image/jpeg"}, + {".jpeg", "image/jpeg"}, + {".webp", "image/webp"}, + {".bmp", "image/bmp"}, + {".tiff", "image/tiff"}, + {".tif", "image/tiff"}, + {".gif", "image/gif"}, + }; + auto ext = fs::path(path).extension().string(); + auto it = mime_map.find(ext); + return (it != mime_map.end()) ? it->second : "application/octet-stream"; +} + +struct server_tool_read_image : server_tool { + server_tool_read_image() { + name = "read_image"; + display_name = "Read image file"; + permission_write = false; + } + + json get_definition() const override { + return { + {"type", "function"}, + {"function", { + {"name", name}, + {"description", "Read an image file from disk and return it as base64-encoded data with metadata."}, + {"parameters", { + {"type", "object"}, + {"properties", { + {"path", {{"type", "string"}, {"description", "Absolute path to the image file."}}}, + }}, + {"required", json::array({"path"})}, + }}, + }}, + }; + } + + json invoke(json params, server_tool::stream *) const override { + std::string path = params.at("path").get(); + + auto io = make_tools_io(params); + + uintmax_t file_size = 0; + if (!io->file_size(path, file_size)) { + return {{"error", "cannot stat file: " + path}}; + } + if (file_size > SERVER_TOOL_READ_IMAGE_MAX_SIZE) { + return {{"error", string_format( + "image too large (%zu bytes, max %zu)", + (size_t)file_size, SERVER_TOOL_READ_IMAGE_MAX_SIZE)}}; + } + + std::string content; + if (!io->read_file(path, content)) { + return {{"error", "failed to open file: " + path}}; + } + + std::string mime = get_mime_from_extension(path); + std::string b64 = base64::encode(content.data(), content.size()); + + // Return as plain_text_response with a data URI line so the UI can + // extract it as an image attachment (via extractBase64Attachments) + std::string data_uri = "data:" + mime + ";base64," + b64; + + return { + {"plain_text_response", + string_format( + "Image: %s\nSize: %zu bytes\nMIME: %s\n%s", + path.c_str(), (size_t)file_size, mime.c_str(), data_uri.c_str())}, + {"path", path}, + {"mime", mime}, + {"size_bytes", (int)file_size}, + }; + } +}; + +// // // public API // @@ -1986,6 +2073,7 @@ static server_tool & find_tool(std::vector> & tools static std::vector> build_tools() { std::vector> tools; tools.push_back(std::make_unique()); + tools.push_back(std::make_unique()); tools.push_back(std::make_unique()); tools.push_back(std::make_unique()); tools.push_back(std::make_unique()); diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte index 5bbaa4ea311b..8be3dc7a75fc 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlock.svelte @@ -7,6 +7,7 @@ import ChatMessageToolCallBlockGetInfo from './ChatMessageToolCallBlockGetInfo.svelte'; import ChatMessageToolCallBlockGrepSearch from './ChatMessageToolCallBlockGrepSearch.svelte'; import ChatMessageToolCallBlockReadFile from './ChatMessageToolCallBlockReadFile.svelte'; + import ChatMessageToolCallBlockReadImage from './ChatMessageToolCallBlockReadImage.svelte'; import ChatMessageToolCallBlockRunJavascript from './ChatMessageToolCallBlockRunJavascript.svelte'; import ChatMessageToolCallBlockSearchResults from './ChatMessageToolCallBlockSearchResults.svelte'; import ChatMessageToolCallBlockWriteFile from './ChatMessageToolCallBlockWriteFile.svelte'; @@ -45,6 +46,8 @@ {:else if section.toolName === BuiltInTool.READ_FILE} +{:else if section.toolName === BuiltInTool.READ_IMAGE} + {:else if section.toolName === BuiltInTool.EDIT_FILE} {:else if section.toolName === BuiltInTool.WRITE_FILE} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadImage.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadImage.svelte new file mode 100644 index 000000000000..b0f1ac5a6296 --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadImage.svelte @@ -0,0 +1,80 @@ + + + + {#snippet titleSnippet()} + Read image + {readImageMeta?.fileName} + {/snippet} + + {#snippet children(_meta, _ctx)} + {#if section.toolResult} + {#if imageAttachment} +
+ {readImageMeta?.fileName +
+ {:else} +
+ Image attachment not found in message extras +
+ {/if} + + {#if readImageMeta?.sizeBytes || readImageMeta?.mimeType} +
+ {#if readImageMeta?.sizeBytes} + Size: {readImageMeta.sizeBytes} bytes + {/if} + {#if readImageMeta?.mimeType} + MIME: {readImageMeta.mimeType} + {/if} +
+ {/if} + + {#if readImageMeta?.path} +
{readImageMeta.path}
+ {/if} + {:else} +
+ Waiting for image data... +
+ {/if} + {/snippet} + diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-image.ts b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-image.ts new file mode 100644 index 000000000000..3a3352348a39 --- /dev/null +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-image.ts @@ -0,0 +1,46 @@ +import type { AgenticSection } from '$lib/utils'; + +export interface ReadImageMeta { + fileName: string; + path: string; + sizeBytes?: number; + mimeType?: string; +} + +/** + * Parse read_image tool result to extract metadata. + * Expected format (after extractBase64Attachments processing): + * Image: /path/to/file.png + * Size: 12345 bytes + * MIME: image/png + * [Attachment saved: mcp-attachment-xxx.png] + * + * The data URI line is replaced by the attachment marker by + * agenticStore.extractBase64Attachments before storage. + */ +export function parseReadImageMeta(section: AgenticSection): ReadImageMeta | null { + if (!section.toolResult) return null; + + const lines = section.toolResult.split('\n'); + let fileName = ''; + let path = ''; + let sizeBytes: number | undefined; + let mimeType: string | undefined; + + for (const line of lines) { + const trimmed = line.trim(); + if (trimmed.startsWith('Image: ')) { + path = trimmed.slice('Image: '.length).trim(); + fileName = path.split('/').pop() ?? path; + } else if (trimmed.startsWith('Size: ')) { + const match = trimmed.match(/Size:\s*(\d+)\s*bytes/); + if (match) sizeBytes = parseInt(match[1], 10); + } else if (trimmed.startsWith('MIME: ')) { + mimeType = trimmed.slice('MIME: '.length).trim(); + } + } + + if (!path) return null; + + return { fileName, path, sizeBytes, mimeType }; +} diff --git a/tools/ui/src/lib/constants/built-in-tools.ts b/tools/ui/src/lib/constants/built-in-tools.ts index 89bcbecb8ae4..4e7031adc5d1 100644 --- a/tools/ui/src/lib/constants/built-in-tools.ts +++ b/tools/ui/src/lib/constants/built-in-tools.ts @@ -10,6 +10,7 @@ import { Braces, Clock, + Eye, FilePen, FilePlus, FileSearch, @@ -47,6 +48,7 @@ export const BUILTIN_TOOL_UI: Readonly> source: ToolSource.BUILTIN }, [BuiltInTool.READ_FILE]: { icon: FileText, label: 'Read file', source: ToolSource.BUILTIN }, + [BuiltInTool.READ_IMAGE]: { icon: Eye, label: 'Read image', source: ToolSource.BUILTIN }, [BuiltInTool.RUN_JAVASCRIPT]: { icon: Braces, label: 'Run JavaScript', diff --git a/tools/ui/src/lib/enums/tools.enums.ts b/tools/ui/src/lib/enums/tools.enums.ts index 7a9751ee7993..b0295a4db2f8 100644 --- a/tools/ui/src/lib/enums/tools.enums.ts +++ b/tools/ui/src/lib/enums/tools.enums.ts @@ -37,6 +37,7 @@ export enum GlobSearchType { */ export enum BuiltInTool { READ_FILE = 'read_file', + READ_IMAGE = 'read_image', EDIT_FILE = 'edit_file', WRITE_FILE = 'write_file', GET_DATETIME = 'get_datetime', From c4e93842f71f6fa0523a02867ed638a1f261e523 Mon Sep 17 00:00:00 2001 From: ckrafft Date: Tue, 11 Aug 2026 02:09:09 +0200 Subject: [PATCH 02/16] cleanup read_image tool: move magic strings to constants * Add dedicated constants file: tools/ui/src/lib/constants/read-image.ts with PREFIX_IMAGE, PREFIX_SIZE, PREFIX_MIME constants * Use ATTACHMENT_SAVED_REGEX from agentic.ts in ChatMessageToolCallBlockReadImage.svelte * Use NEWLINE constant from code.ts instead of hardcoded '\n' * Use PREFIX_SIZE in regex pattern for size parsing * Add SERVER_TOOL_READ_IMAGE_PREFIX_* constants in C++ server-tools.cpp to match the TypeScript PREFIX_* constants for consistency --- tools/server/server-tools.cpp | 9 +++++++-- .../ChatMessageToolCallBlockReadImage.svelte | 3 ++- .../ChatMessageToolCall/parsers/read-image.ts | 16 +++++++++------- tools/ui/src/lib/constants/read-image.ts | 3 +++ 4 files changed, 21 insertions(+), 10 deletions(-) create mode 100644 tools/ui/src/lib/constants/read-image.ts diff --git a/tools/server/server-tools.cpp b/tools/server/server-tools.cpp index aea385c42d8e..02c4f224d09c 100644 --- a/tools/server/server-tools.cpp +++ b/tools/server/server-tools.cpp @@ -1985,6 +1985,9 @@ static server_tool & find_tool(std::vector> & tools // static constexpr size_t SERVER_TOOL_READ_IMAGE_MAX_SIZE = 16 * 1024 * 1024; // 16 MB +static constexpr const char* SERVER_TOOL_READ_IMAGE_PREFIX_IMAGE = "Image: "; +static constexpr const char* SERVER_TOOL_READ_IMAGE_PREFIX_SIZE = "Size: "; +static constexpr const char* SERVER_TOOL_READ_IMAGE_PREFIX_MIME = "MIME: "; static std::string get_mime_from_extension(const std::string & path) { static const std::unordered_map mime_map = { @@ -2056,8 +2059,10 @@ struct server_tool_read_image : server_tool { return { {"plain_text_response", string_format( - "Image: %s\nSize: %zu bytes\nMIME: %s\n%s", - path.c_str(), (size_t)file_size, mime.c_str(), data_uri.c_str())}, + "%s%s\n%s%zu bytes\n%s%s\n%s", + SERVER_TOOL_READ_IMAGE_PREFIX_IMAGE, path.c_str(), + SERVER_TOOL_READ_IMAGE_PREFIX_SIZE, (size_t)file_size, + SERVER_TOOL_READ_IMAGE_PREFIX_MIME, mime.c_str(), data_uri.c_str())}, {"path", path}, {"mime", mime}, {"size_bytes", (int)file_size}, diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadImage.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadImage.svelte index b0f1ac5a6296..a73252fefa7d 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadImage.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadImage.svelte @@ -1,6 +1,7 @@ - + {#snippet titleSnippet()} - Read image - {readImageMeta?.fileName} + Read media + {readMediaMeta?.fileName} {/snippet} {#snippet children(_meta, _ctx)} {#if section.toolResult} - {#if imageAttachment} + {#if mediaAttachment}
{readImageMeta?.fileName
{:else}
- Image attachment not found in message extras + Media attachment not found in message extras
{/if} - {#if readImageMeta?.sizeBytes || readImageMeta?.mimeType} + {#if readMediaMeta?.sizeBytes || readMediaMeta?.mimeType}
- {#if readImageMeta?.sizeBytes} - Size: {readImageMeta.sizeBytes} bytes + {#if readMediaMeta?.sizeBytes} + Size: {readMediaMeta.sizeBytes} bytes {/if} - {#if readImageMeta?.mimeType} - MIME: {readImageMeta.mimeType} + {#if readMediaMeta?.mimeType} + MIME: {readMediaMeta.mimeType} {/if}
{/if} - {#if readImageMeta?.path} -
{readImageMeta.path}
+ {#if readMediaMeta?.path} +
{readMediaMeta.path}
{/if} {:else}
- Waiting for image data... + Waiting for media data...
{/if} {/snippet} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-image.ts b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-media.ts similarity index 74% rename from tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-image.ts rename to tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-media.ts index 61d234a04977..cc3fb634258c 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-image.ts +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/parsers/read-media.ts @@ -1,8 +1,8 @@ import type { AgenticSection } from '$lib/utils'; import { NEWLINE } from '$lib/constants/code'; -import { PREFIX_IMAGE, PREFIX_SIZE, PREFIX_MIME } from '$lib/constants/read-image'; +import { PREFIX_FILE, PREFIX_SIZE, PREFIX_MIME } from '$lib/constants/read-media'; -export interface ReadImageMeta { +export interface ReadMediaMeta { fileName: string; path: string; sizeBytes?: number; @@ -10,9 +10,9 @@ export interface ReadImageMeta { } /** - * Parse read_image tool result to extract metadata. + * Parse read_media tool result to extract metadata. * Expected format (after extractBase64Attachments processing): - * Image: /path/to/file.png + * File: /path/to/file.png * Size: 12345 bytes * MIME: image/png * [Attachment saved: mcp-attachment-xxx.png] @@ -20,7 +20,7 @@ export interface ReadImageMeta { * The data URI line is replaced by the attachment marker by * agenticStore.extractBase64Attachments before storage. */ -export function parseReadImageMeta(section: AgenticSection): ReadImageMeta | null { +export function parseReadMediaMeta(section: AgenticSection): ReadMediaMeta | null { if (!section.toolResult) return null; const lines = section.toolResult.split(NEWLINE); @@ -31,8 +31,8 @@ export function parseReadImageMeta(section: AgenticSection): ReadImageMeta | nul for (const line of lines) { const trimmed = line.trim(); - if (trimmed.startsWith(PREFIX_IMAGE)) { - path = trimmed.slice(PREFIX_IMAGE.length).trim(); + if (trimmed.startsWith(PREFIX_FILE)) { + path = trimmed.slice(PREFIX_FILE.length).trim(); fileName = path.split('/').pop() ?? path; } else if (trimmed.startsWith(PREFIX_SIZE)) { const match = trimmed.match(new RegExp(`${PREFIX_SIZE}\\s*(\\d+)\\s*bytes`)); diff --git a/tools/ui/src/lib/constants/built-in-tools.ts b/tools/ui/src/lib/constants/built-in-tools.ts index 4e7031adc5d1..b5ca5dddbc99 100644 --- a/tools/ui/src/lib/constants/built-in-tools.ts +++ b/tools/ui/src/lib/constants/built-in-tools.ts @@ -48,7 +48,7 @@ export const BUILTIN_TOOL_UI: Readonly> source: ToolSource.BUILTIN }, [BuiltInTool.READ_FILE]: { icon: FileText, label: 'Read file', source: ToolSource.BUILTIN }, - [BuiltInTool.READ_IMAGE]: { icon: Eye, label: 'Read image', source: ToolSource.BUILTIN }, + [BuiltInTool.READ_MEDIA]: { icon: Eye, label: 'Read media', source: ToolSource.BUILTIN }, [BuiltInTool.RUN_JAVASCRIPT]: { icon: Braces, label: 'Run JavaScript', diff --git a/tools/ui/src/lib/constants/read-image.ts b/tools/ui/src/lib/constants/read-media.ts similarity index 65% rename from tools/ui/src/lib/constants/read-image.ts rename to tools/ui/src/lib/constants/read-media.ts index 7346dc013910..7fbda2f1d729 100644 --- a/tools/ui/src/lib/constants/read-image.ts +++ b/tools/ui/src/lib/constants/read-media.ts @@ -1,3 +1,3 @@ -export const PREFIX_IMAGE = 'Image: '; +export const PREFIX_FILE = 'File: '; export const PREFIX_SIZE = 'Size: '; export const PREFIX_MIME = 'MIME: '; diff --git a/tools/ui/src/lib/enums/tools.enums.ts b/tools/ui/src/lib/enums/tools.enums.ts index b0295a4db2f8..31c992fef4f7 100644 --- a/tools/ui/src/lib/enums/tools.enums.ts +++ b/tools/ui/src/lib/enums/tools.enums.ts @@ -37,7 +37,7 @@ export enum GlobSearchType { */ export enum BuiltInTool { READ_FILE = 'read_file', - READ_IMAGE = 'read_image', + READ_MEDIA = 'read_media', EDIT_FILE = 'edit_file', WRITE_FILE = 'write_file', GET_DATETIME = 'get_datetime', From 3cf471fc882320c4d4f017cb6145f184a3112fc5 Mon Sep 17 00:00:00 2001 From: ckrafft Date: Tue, 11 Aug 2026 02:10:07 +0200 Subject: [PATCH 04/16] ui: consolidate audio data URI handling into shared utility * Extract getAudioInputFormat to a shared utility (was duplicated inline) * Store raw base64 in base64Data on the message object * Use base64Data to construct data URIs for audio rendering * Update agentic store to build INPUT_AUDIO parts from base64Data --- tools/server/server-tools.cpp | 15 +++++-- .../ChatMessageToolCallBlockDefault.svelte | 36 +++++++++++------ ...essageToolCallBlockExecShellCommand.svelte | 10 ++--- .../ChatMessageToolCallBlockReadMedia.svelte | 39 +++++++++++++------ tools/ui/src/lib/constants/mcp-resource.ts | 21 +++++++++- tools/ui/src/lib/services/chat.service.ts | 19 +-------- tools/ui/src/lib/stores/agentic.svelte.ts | 32 +++++++++++++-- tools/ui/src/lib/utils/agentic.ts | 19 ++++----- tools/ui/src/lib/utils/audio-format.ts | 22 +++++++++++ tools/ui/src/lib/utils/index.ts | 5 ++- 10 files changed, 154 insertions(+), 64 deletions(-) create mode 100644 tools/ui/src/lib/utils/audio-format.ts diff --git a/tools/server/server-tools.cpp b/tools/server/server-tools.cpp index f9cf335a0859..465c57cc5147 100644 --- a/tools/server/server-tools.cpp +++ b/tools/server/server-tools.cpp @@ -950,13 +950,14 @@ struct server_tool_read_file : server_tool { // read_media: read a media file (image or audio) // -static constexpr size_t SERVER_TOOL_READ_MEDIA_MAX_SIZE = 16 * 1024 * 1024; // 16 MB +static constexpr size_t SERVER_TOOL_READ_MEDIA_MAX_SIZE = 32 * 1024 * 1024; // 32 MB static constexpr const char* SERVER_TOOL_READ_MEDIA_PREFIX_FILE = "File: "; static constexpr const char* SERVER_TOOL_READ_MEDIA_PREFIX_SIZE = "Size: "; static constexpr const char* SERVER_TOOL_READ_MEDIA_PREFIX_MIME = "MIME: "; static std::string get_mime_from_extension(const std::string & path) { static const std::unordered_map mime_map = { + // Images {".png", "image/png"}, {".jpg", "image/jpeg"}, {".jpeg", "image/jpeg"}, @@ -965,6 +966,13 @@ static std::string get_mime_from_extension(const std::string & path) { {".tiff", "image/tiff"}, {".tif", "image/tiff"}, {".gif", "image/gif"}, + // Audio + {".mp3", "audio/mpeg"}, + {".wav", "audio/wav"}, + {".ogg", "audio/ogg"}, + {".flac", "audio/flac"}, + {".m4a", "audio/mp4"}, + {".opus", "audio/opus"}, }; auto ext = fs::path(path).extension().string(); auto it = mime_map.find(ext); @@ -983,11 +991,11 @@ struct server_tool_read_media : server_tool { {"type", "function"}, {"function", { {"name", name}, - {"description", "Read a media file (audio, image) from disk and return it as base64-encoded data with metadata."}, + {"description", "Read the content of a media file (audio, image)."}, {"parameters", { {"type", "object"}, {"properties", { - {"path", {{"type", "string"}, {"description", "Absolute path to the media file."}}}, + {"path", {{"type", "string"}, {"description", "Path to the media file."}}}, }}, {"required", json::array({"path"})}, }}, @@ -2072,7 +2080,6 @@ static server_tool & find_tool(std::vector> & tools throw std::invalid_argument(string_format("unknown tool \"%s\"", name.c_str())); } -// // // public API // diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte index 34dfde78b895..b09c2f24eb9e 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte @@ -9,12 +9,14 @@ import { MAX_HEIGHT_CODE_BLOCK } from '$lib/constants'; import { getBuiltinToolUi } from '$lib/constants/built-in-tools'; import { FileTypeText, ToolResultKind } from '$lib/enums'; - import type { DatabaseMessageExtra } from '$lib/types'; + import type { DatabaseMessageExtra, DatabaseMessageExtraAudioFile } from '$lib/types'; + import { createBase64DataUrl } from '$lib/utils/data-url'; import { type AgenticSection, classifyToolResult, formatJsonPretty, - parseToolResultWithImages + parseToolResultWithMedia, + type ToolResultLine } from '$lib/utils'; interface Props { @@ -29,8 +31,8 @@ const title = $derived(getBuiltinToolUi(section.toolName)?.label ?? section.toolName ?? ''); const outputKind = $derived(classifyToolResult(section.toolResult)); - const parsedLines = $derived( - section.toolResult ? parseToolResultWithImages(section.toolResult, attachments) : [] + const parsedLines: ToolResultLine[] = $derived( + section.toolResult ? parseToolResultWithMedia(section.toolResult, attachments) : [] ); @@ -103,14 +105,24 @@
{line.text}
- {#if line.image} - {line.image.name} - {/if} + {#if line.media} + {#if line.media.type === 'AUDIO'} + {@const audioSrc = createBase64DataUrl(line.media.mimeType ?? 'audio/mpeg', (line.media as DatabaseMessageExtraAudioFile).base64Data)} +
+ +
+ {:else} + {line.media.name} + {/if} + {/if} {/each} {/if} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte index b2fa8b331c3f..907a3cd12ab6 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockExecShellCommand.svelte @@ -23,7 +23,7 @@ isExitCodeSummaryLine, parseExecShellCommandError, parseExecShellCommandExitStatus, - parseToolResultWithImages, + parseToolResultWithMedia, type ToolResultLine } from '$lib/utils'; @@ -53,7 +53,7 @@ ); const parsedLines: ToolResultLine[] = $derived( - section.toolResult ? parseToolResultWithImages(section.toolResult, attachments) : [] + section.toolResult ? parseToolResultWithMedia(section.toolResult, attachments) : [] ); // Drop the trailing "[exit code: N]" line - rendered as a colored @@ -223,10 +223,10 @@ > {#each outputLines as line, i (i)}
{line.text}
- {#if line.image} + {#if line.media} {line.image.name} diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte index e42897112522..7ac0a031114e 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte @@ -2,8 +2,9 @@ import { Eye } from '@lucide/svelte'; import { AttachmentType } from '$lib/enums'; import { ATTACHMENT_SAVED_REGEX } from '$lib/constants/agentic'; - import type { DatabaseMessageExtra, DatabaseMessageExtraImageFile } from '$lib/types'; + import type { DatabaseMessageExtra, DatabaseMessageExtraImageFile, DatabaseMessageExtraAudioFile } from '$lib/types'; import { type AgenticSection } from '$lib/utils'; + import { createBase64DataUrl } from '$lib/utils/data-url'; import { parseReadMediaMeta } from './parsers/read-media'; import ToolCallBlock from './ToolCallBlock.svelte'; @@ -29,10 +30,13 @@ if (!match) return null; const attachmentName = match[1]; return extras.find( - (e): e is DatabaseMessageExtraImageFile => - e.type === AttachmentType.IMAGE && e.name === attachmentName + (e): e is DatabaseMessageExtraImageFile | DatabaseMessageExtraAudioFile => + (e.type === AttachmentType.IMAGE || e.type === AttachmentType.AUDIO) && + e.name === attachmentName ) ?? null; }); + + const isAudio = $derived(mediaAttachment?.type === AttachmentType.AUDIO); @@ -44,14 +48,27 @@ {#snippet children(_meta, _ctx)} {#if section.toolResult} {#if mediaAttachment} -
- {readMediaMeta?.fileName -
+ {#if isAudio} + {@const audioSrc = createBase64DataUrl( + readMediaMeta?.mimeType ?? 'audio/mpeg', + (mediaAttachment as DatabaseMessageExtraAudioFile).base64Data + )} +
+ +
+ {:else} +
+ {readMediaMeta?.fileName +
+ {/if} {:else}
Media attachment not found in message extras diff --git a/tools/ui/src/lib/constants/mcp-resource.ts b/tools/ui/src/lib/constants/mcp-resource.ts index 0ef8d9ec13b6..bc8a7e8fa5e9 100644 --- a/tools/ui/src/lib/constants/mcp-resource.ts +++ b/tools/ui/src/lib/constants/mcp-resource.ts @@ -1,7 +1,8 @@ -import { MimeTypeImage } from '$lib/enums'; +import { MimeTypeAudio, MimeTypeImage } from '$lib/enums'; // File extension patterns for resource type detection export const IMAGE_FILE_EXTENSION_REGEX = /\.(png|jpg|jpeg|gif|svg|webp)$/i; +export const AUDIO_FILE_EXTENSION_REGEX = /\.(mp3|wav|ogg|flac|m4a|opus)$/i; export const CODE_FILE_EXTENSION_REGEX = /\.(js|ts|json|yaml|yml|xml|html|css|py|rs|go|java|cpp|c|h|rb|sh|toml)$/i; export const TEXT_FILE_EXTENSION_REGEX = /\.(txt|md|log)$/i; @@ -53,3 +54,21 @@ export const IMAGE_MIME_TO_EXTENSION: Record = { [MimeTypeImage.PNG]: 'png', [MimeTypeImage.WEBP]: 'webp' } as const; + +/** + * Mapping from audio MIME types to file extensions. + * Used for generating attachment filenames from MIME types. + */ +export const AUDIO_MIME_TO_EXTENSION: Record = { + [MimeTypeAudio.MP3_MPEG]: 'mp3', + [MimeTypeAudio.MP3]: 'mp3', + [MimeTypeAudio.MP4]: 'm4a', + [MimeTypeAudio.WAV]: 'wav', + [MimeTypeAudio.WAVE]: 'wav', + [MimeTypeAudio.X_WAV]: 'wav', + [MimeTypeAudio.X_WAVE]: 'wav', + [MimeTypeAudio.VND_WAVE]: 'wav', + [MimeTypeAudio.X_PN_WAV]: 'wav', + [MimeTypeAudio.WEBM]: 'webm', + [MimeTypeAudio.WEBM_OPUS]: 'opus' +} as const; diff --git a/tools/ui/src/lib/services/chat.service.ts b/tools/ui/src/lib/services/chat.service.ts index 775a5d6e060e..2b03a471d5ed 100644 --- a/tools/ui/src/lib/services/chat.service.ts +++ b/tools/ui/src/lib/services/chat.service.ts @@ -22,7 +22,6 @@ import { ContentPartType, FileTypeAudio, MessageRole, - MimeTypeAudio, ReasoningFormat, StreamConnectionState } from '$lib/enums'; @@ -42,23 +41,7 @@ import { isAbortError } from '$lib/utils/abort'; import { getAuthHeaders, getJsonHeaders } from '$lib/utils/api-headers'; import { formatAttachmentText } from '$lib/utils/formatters'; import { streamIdentity } from '$lib/utils/stream-identity'; - -function getAudioInputFormat(mimeType: string): AudioInputFormat { - const normalizedMimeType = mimeType.trim().toLowerCase(); - - if ( - normalizedMimeType === MimeTypeAudio.WAV || - normalizedMimeType === MimeTypeAudio.WAVE || - normalizedMimeType === MimeTypeAudio.X_WAV || - normalizedMimeType === MimeTypeAudio.X_WAVE || - normalizedMimeType === MimeTypeAudio.VND_WAVE || - normalizedMimeType === MimeTypeAudio.X_PN_WAV - ) { - return FileTypeAudio.WAV; - } - - return FileTypeAudio.MP3; -} +import { getAudioInputFormat } from '../utils/audio-format'; interface ResumableStreamState { bytesReceived: number; diff --git a/tools/ui/src/lib/stores/agentic.svelte.ts b/tools/ui/src/lib/stores/agentic.svelte.ts index 080ed7c89ab7..63564c95358b 100644 --- a/tools/ui/src/lib/stores/agentic.svelte.ts +++ b/tools/ui/src/lib/stores/agentic.svelte.ts @@ -75,9 +75,10 @@ import type { import type { DatabaseMessage, DatabaseMessageExtra, - DatabaseMessageExtraImageFile + DatabaseMessageExtraImageFile, + DatabaseMessageExtraAudioFile } from '$lib/types/database'; -import { isAbortError } from '$lib/utils'; +import { isAbortError, getAudioInputFormat } from '$lib/utils'; import { SvelteMap } from 'svelte/reactivity'; function createDefaultSession(): AgenticSession { @@ -990,7 +991,15 @@ class AgenticStore { ]; for (const attachment of attachments) { - if (attachment.type === AttachmentType.IMAGE) { + if (attachment.type === AttachmentType.AUDIO) { + contentParts.push({ + type: ContentPartType.INPUT_AUDIO, + input_audio: { + data: (attachment as DatabaseMessageExtraAudioFile).base64Data, + format: getAudioInputFormat((attachment as DatabaseMessageExtraAudioFile).mimeType) + } + }); + } else if (attachment.type === AttachmentType.IMAGE) { if (modelsStore.modelSupportsVision(effectiveModel)) { contentParts.push({ image_url: { @@ -1092,6 +1101,7 @@ class AgenticStore { return line; } + // Audio extras require raw base64 in base64Data (not full data URI). attachmentIndex += 1; const name = this.buildAttachmentName(mimeType, attachmentIndex); @@ -1101,6 +1111,17 @@ class AgenticStore { return `[Attachment saved: ${name}]`; } + if (mimeType.startsWith('audio/')) { + attachments.push({ + type: AttachmentType.AUDIO, + name, + mimeType, + base64Data: base64Data + }); + + return `[Attachment saved: ${name}]`; + } + return line; }); @@ -1108,6 +1129,11 @@ class AgenticStore { } private buildAttachmentName(mimeType: string, index: number): string { + if (mimeType.startsWith('audio/')) { + const extension = AUDIO_MIME_TO_EXTENSION[mimeType] ?? 'mp3'; + return `${MCP_ATTACHMENT_NAME_PREFIX}-${Date.now()}-${index}.${extension}`; + } + const extension = IMAGE_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_IMAGE_EXTENSION; return `${MCP_ATTACHMENT_NAME_PREFIX}-${Date.now()}-${index}.${extension}`; diff --git a/tools/ui/src/lib/utils/agentic.ts b/tools/ui/src/lib/utils/agentic.ts index 9c92a42d40ea..918b124078e9 100644 --- a/tools/ui/src/lib/utils/agentic.ts +++ b/tools/ui/src/lib/utils/agentic.ts @@ -50,11 +50,11 @@ export interface AgenticSection { } /** - * Represents a tool result line that may reference an image attachment + * Represents a tool result line that may reference a media attachment (image or audio) */ export type ToolResultLine = { text: string; - image?: DatabaseMessageExtraImageFile; + media?: DatabaseMessageExtraImageFile | DatabaseMessageExtraAudioFile; }; /** @@ -306,11 +306,11 @@ const TOOL_RESULT_LINES_CACHE_MAX_SIZE = 32; const toolResultLinesCache = new Map(); /** - * Parse tool result text into lines, matching image attachments by name. - * Memoized: called per render during streaming on unchanged tool result + * Parse tool result text into lines, matching media attachments (images and audio) by name. + * Memoized: called per render during streaming on unchanged tool result * strings with unchanged extras. */ -export function parseToolResultWithImages( +export function parseToolResultWithMedia( toolResult: string, extras?: DatabaseMessageExtra[] ): ToolResultLine[] { @@ -332,12 +332,13 @@ export function parseToolResultWithImages( if (!match || !extras) return { text: line }; const attachmentName = match[1]; - const image = extras.find( - (e): e is DatabaseMessageExtraImageFile => - e.type === AttachmentType.IMAGE && e.name === attachmentName + const media = extras.find( + (e): e is DatabaseMessageExtraImageFile | DatabaseMessageExtraAudioFile => + (e.type === AttachmentType.IMAGE || e.type === AttachmentType.AUDIO) && + e.name === attachmentName ); - return { image, text: line }; + return { media, text: line }; }); if (toolResultLinesCache.size >= TOOL_RESULT_LINES_CACHE_MAX_SIZE) { diff --git a/tools/ui/src/lib/utils/audio-format.ts b/tools/ui/src/lib/utils/audio-format.ts new file mode 100644 index 000000000000..3d345bb4eacc --- /dev/null +++ b/tools/ui/src/lib/utils/audio-format.ts @@ -0,0 +1,22 @@ +import type { AudioInputFormat } from '$lib/types/api'; +import { MimeTypeAudio, FileTypeAudio } from '$lib/enums'; + +/** + * Map a MIME type to the AudioInputFormat expected by the API. + */ +export function getAudioInputFormat(mimeType: string): AudioInputFormat { + const normalizedMimeType = mimeType.trim().toLowerCase(); + + if ( + normalizedMimeType === MimeTypeAudio.WAV || + normalizedMimeType === MimeTypeAudio.WAVE || + normalizedMimeType === MimeTypeAudio.X_WAV || + normalizedMimeType === MimeTypeAudio.X_WAVE || + normalizedMimeType === MimeTypeAudio.VND_WAVE || + normalizedMimeType === MimeTypeAudio.X_PN_WAV + ) { + return FileTypeAudio.WAV; + } + + return FileTypeAudio.MP3; +} diff --git a/tools/ui/src/lib/utils/index.ts b/tools/ui/src/lib/utils/index.ts index 5762e8e4bb92..c3585bd5f85d 100644 --- a/tools/ui/src/lib/utils/index.ts +++ b/tools/ui/src/lib/utils/index.ts @@ -248,7 +248,7 @@ export { export { deriveAgenticSections, buildAssistantRawOutput, - parseToolResultWithImages, + parseToolResultWithMedia, splitSearchSummaryList, hasAgenticContent, classifyToolResult, @@ -325,3 +325,6 @@ export { uuid } from './uuid'; // CSS utilities export { remToPx } from './css'; + +// Audio format helper (used by agentic store and chat service) +export { getAudioInputFormat } from './audio-format'; From 6babc0253a551bd77b285920d6425f61f5b176c4 Mon Sep 17 00:00:00 2001 From: ckrafft Date: Tue, 11 Aug 2026 02:10:18 +0200 Subject: [PATCH 05/16] server: read_media: restrict audio to wav/mp3 and minor fixes * Server get_mime_from_extension now only advertises audio/wav and audio/mpeg (the only formats the model's input_audio API accepts) * Case-insensitive extension matching (fixes .MP3, .Wav, etc.) * Unknown extensions return an error instead of a multi-MB data URI that inflates model context with garbage * Updated tool description to document supported formats * Frontend AUDIO_MIME_TO_EXTENSION trimmed to match server * fix a missing import in tools/ui/src/lib/stores/agentic.svelte.ts --- tools/server/server-tools.cpp | 26 ++++++++++++++++------ tools/ui/src/lib/constants/mcp-resource.ts | 3 --- tools/ui/src/lib/stores/agentic.svelte.ts | 4 +++- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/tools/server/server-tools.cpp b/tools/server/server-tools.cpp index 465c57cc5147..e420d1125387 100644 --- a/tools/server/server-tools.cpp +++ b/tools/server/server-tools.cpp @@ -966,16 +966,18 @@ static std::string get_mime_from_extension(const std::string & path) { {".tiff", "image/tiff"}, {".tif", "image/tiff"}, {".gif", "image/gif"}, - // Audio + // Audio — only wav and mp3: the model's input_audio API + // only accepts these two formats (FileTypeAudio.WAV | MP3). {".mp3", "audio/mpeg"}, {".wav", "audio/wav"}, - {".ogg", "audio/ogg"}, - {".flac", "audio/flac"}, - {".m4a", "audio/mp4"}, - {".opus", "audio/opus"}, }; + auto ext = fs::path(path).extension().string(); - auto it = mime_map.find(ext); + std::string ext_lower; + ext_lower.reserve(ext.size()); + for (char c : ext) ext_lower += static_cast(std::tolower(static_cast(c))); + + auto it = mime_map.find(ext_lower); return (it != mime_map.end()) ? it->second : "application/octet-stream"; } @@ -991,7 +993,10 @@ struct server_tool_read_media : server_tool { {"type", "function"}, {"function", { {"name", name}, - {"description", "Read the content of a media file (audio, image)."}, + {"description", + "Read the content of a media file (image or audio).\n" + " Audio: .wav, .mp3.\n" + " Images: .png, .jpg, .jpeg, .webp, .bmp, .tiff, .gif."}, {"parameters", { {"type", "object"}, {"properties", { @@ -1024,6 +1029,13 @@ struct server_tool_read_media : server_tool { } std::string mime = get_mime_from_extension(path); + + // Reject unknown/unrecognized file extensions instead of producing + // a multi-MB data URI that inflates the model context with garbage. + if (mime == "application/octet-stream") { + return {{"error", "unrecognized or unsupported extension: " + path}}; + } + std::string b64 = base64::encode(content.data(), content.size()); // Return as plain_text_response with a data URI line so the UI can diff --git a/tools/ui/src/lib/constants/mcp-resource.ts b/tools/ui/src/lib/constants/mcp-resource.ts index bc8a7e8fa5e9..db6b9883e604 100644 --- a/tools/ui/src/lib/constants/mcp-resource.ts +++ b/tools/ui/src/lib/constants/mcp-resource.ts @@ -62,13 +62,10 @@ export const IMAGE_MIME_TO_EXTENSION: Record = { export const AUDIO_MIME_TO_EXTENSION: Record = { [MimeTypeAudio.MP3_MPEG]: 'mp3', [MimeTypeAudio.MP3]: 'mp3', - [MimeTypeAudio.MP4]: 'm4a', [MimeTypeAudio.WAV]: 'wav', [MimeTypeAudio.WAVE]: 'wav', [MimeTypeAudio.X_WAV]: 'wav', [MimeTypeAudio.X_WAVE]: 'wav', [MimeTypeAudio.VND_WAVE]: 'wav', [MimeTypeAudio.X_PN_WAV]: 'wav', - [MimeTypeAudio.WEBM]: 'webm', - [MimeTypeAudio.WEBM_OPUS]: 'opus' } as const; diff --git a/tools/ui/src/lib/stores/agentic.svelte.ts b/tools/ui/src/lib/stores/agentic.svelte.ts index 63564c95358b..6573d74466f8 100644 --- a/tools/ui/src/lib/stores/agentic.svelte.ts +++ b/tools/ui/src/lib/stores/agentic.svelte.ts @@ -25,7 +25,9 @@ import { DATA_URI_BASE64_REGEX, DEFAULT_IMAGE_EXTENSION, IMAGE_MIME_TO_EXTENSION, - MCP_ATTACHMENT_NAME_PREFIX + AUDIO_MIME_TO_EXTENSION, + MCP_ATTACHMENT_NAME_PREFIX, + AUDIO_FILE_EXTENSION_REGEX } from '$lib/constants'; import { BuiltInTool, ToolPermissionDecision, ToolSource } from '$lib/enums'; import { From 75bb2f353f292f124e85675dcf955271da258d15 Mon Sep 17 00:00:00 2001 From: ckrafft Date: Tue, 11 Aug 2026 02:10:32 +0200 Subject: [PATCH 06/16] server: read_media: add to --tools help text and README tool list --- common/arg.cpp | 2 +- tools/server/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index c37d5cd0aa4b..01689da8d25d 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -3302,7 +3302,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex {"--tools"}, "TOOL1,TOOL2,...", "experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)\n" "specify \"all\" to enable all tools\n" - "available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime, get_info\n" + "available tools: read_file, read_media, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime, get_info\n" "note: for security reasons, this will limit --cors-origins to localhost by default", [](common_params & params, const std::string & value) { params.server_tools = parse_csv_row(value); diff --git a/tools/server/README.md b/tools/server/README.md index 6927caddbb6c..8054400960ce 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -196,7 +196,7 @@ For the full list of features, please refer to [server's changelog](https://gith | `--ui-config, --webui-config JSON` | JSON that provides default UI settings (overrides UI defaults)
(env: LLAMA_ARG_UI_CONFIG) | | `--ui-config-file, --webui-config-file PATH` | JSON file that provides default UI settings (overrides UI defaults)
(env: LLAMA_ARG_UI_CONFIG_FILE) | | `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)
(env: LLAMA_ARG_UI_MCP_PROXY) | -| `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)
specify "all" to enable all tools
available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime, get_info
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_TOOLS) | +| `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)
specify "all" to enable all tools
available tools: read_file, read_media, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime, get_info
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_TOOLS) | | `--tools-runtime OPTION` | experimental: run tools in a separate runtime environment (default: none, use host environment)
available options:
'docker:', 'podman:': spin up a new container and reuse it for all invocations, clean up on server exit
'docker-container:', 'podman-container:': use an existing container by ID, won't stop on server exit
'ssh:': run tools on a remote POSIX host over SSH, key-based auth and a trusted host key are required

(env: LLAMA_ARG_TOOLS_RUNTIME) | | `--mcp-servers-config PATH` | experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_MCP_SERVERS_CONFIG) | | `--mcp-servers-json JSON` | experimental: inline JSON with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)
note: for security reasons, this will limit --cors-origins to localhost by default
(env: LLAMA_ARG_MCP_SERVERS_JSON) | From 2c571dbb983952168b2b884bd11c409e3912ce58 Mon Sep 17 00:00:00 2001 From: ckrafft Date: Tue, 11 Aug 2026 02:10:45 +0200 Subject: [PATCH 07/16] ui: fix indentation in ChatMessageToolCallBlockDefault.svelte --- .../ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte index b09c2f24eb9e..a443bec5df8f 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte @@ -121,8 +121,8 @@ class="mt-2 mb-2 h-auto max-w-full rounded-lg" loading="lazy" /> - {/if} {/if} + {/if} {/each}
{/if} From 4fc3e5c92b7aeff37fa79170a998337086405848 Mon Sep 17 00:00:00 2001 From: ckrafft Date: Tue, 11 Aug 2026 02:11:08 +0200 Subject: [PATCH 08/16] server: read_media tool: fix a cast to use the correct type --- tools/server/server-tools.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/server/server-tools.cpp b/tools/server/server-tools.cpp index e420d1125387..8a706e4b9e16 100644 --- a/tools/server/server-tools.cpp +++ b/tools/server/server-tools.cpp @@ -1053,7 +1053,7 @@ struct server_tool_read_media : server_tool { data_uri.c_str())}, {"path", path}, {"mime", mime}, - {"size_bytes", (int)file_size}, + {"size_bytes", (size_t)file_size}, }; } }; From 0a35e7dc718015237cf16303533a211a4771fd12 Mon Sep 17 00:00:00 2001 From: ckrafft Date: Tue, 11 Aug 2026 02:11:20 +0200 Subject: [PATCH 09/16] server: read_media: multiple fixes * server-tools.cpp import cctype, remove UTF-8 char, check mime before reading file * ui: add MimeTypePrefix.AUDIO and use it in agentic.svelte.ts --- tools/server/server-tools.cpp | 13 +++++++------ tools/ui/src/lib/enums/files.enums.ts | 1 + tools/ui/src/lib/stores/agentic.svelte.ts | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/tools/server/server-tools.cpp b/tools/server/server-tools.cpp index 8a706e4b9e16..59305f1a8f3f 100644 --- a/tools/server/server-tools.cpp +++ b/tools/server/server-tools.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -966,7 +967,7 @@ static std::string get_mime_from_extension(const std::string & path) { {".tiff", "image/tiff"}, {".tif", "image/tiff"}, {".gif", "image/gif"}, - // Audio — only wav and mp3: the model's input_audio API + // Audio - only wav and mp3: the model's input_audio API // only accepts these two formats (FileTypeAudio.WAV | MP3). {".mp3", "audio/mpeg"}, {".wav", "audio/wav"}, @@ -1023,11 +1024,6 @@ struct server_tool_read_media : server_tool { (size_t)file_size, SERVER_TOOL_READ_MEDIA_MAX_SIZE)}}; } - std::string content; - if (!io->read_file(path, content)) { - return {{"error", "failed to open file: " + path}}; - } - std::string mime = get_mime_from_extension(path); // Reject unknown/unrecognized file extensions instead of producing @@ -1036,6 +1032,11 @@ struct server_tool_read_media : server_tool { return {{"error", "unrecognized or unsupported extension: " + path}}; } + std::string content; + if (!io->read_file(path, content)) { + return {{"error", "failed to open file: " + path}}; + } + std::string b64 = base64::encode(content.data(), content.size()); // Return as plain_text_response with a data URI line so the UI can diff --git a/tools/ui/src/lib/enums/files.enums.ts b/tools/ui/src/lib/enums/files.enums.ts index eecb36c23e63..5785428cf032 100644 --- a/tools/ui/src/lib/enums/files.enums.ts +++ b/tools/ui/src/lib/enums/files.enums.ts @@ -163,6 +163,7 @@ export enum FileExtensionText { // MIME type prefixes and includes for content detection export enum MimeTypePrefix { IMAGE = 'image/', + AUDIO = 'audio/', TEXT = 'text' } diff --git a/tools/ui/src/lib/stores/agentic.svelte.ts b/tools/ui/src/lib/stores/agentic.svelte.ts index 6573d74466f8..47ea1f04971f 100644 --- a/tools/ui/src/lib/stores/agentic.svelte.ts +++ b/tools/ui/src/lib/stores/agentic.svelte.ts @@ -1113,7 +1113,7 @@ class AgenticStore { return `[Attachment saved: ${name}]`; } - if (mimeType.startsWith('audio/')) { + if (mimeType.startsWith(MimeTypePrefix.AUDIO)) { attachments.push({ type: AttachmentType.AUDIO, name, From 036f13bb3f276b3e4c4293f9bba7c2d2a9e4c886 Mon Sep 17 00:00:00 2001 From: ckrafft Date: Tue, 11 Aug 2026 02:11:33 +0200 Subject: [PATCH 10/16] server: make read_media inherit from read_file and add uses_cwd --- tools/server/server-tools.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/server/server-tools.cpp b/tools/server/server-tools.cpp index 59305f1a8f3f..2a678b08d2bb 100644 --- a/tools/server/server-tools.cpp +++ b/tools/server/server-tools.cpp @@ -982,10 +982,11 @@ static std::string get_mime_from_extension(const std::string & path) { return (it != mime_map.end()) ? it->second : "application/octet-stream"; } -struct server_tool_read_media : server_tool { +struct server_tool_read_media : server_tool_read_file { server_tool_read_media() { name = "read_media"; display_name = "Read media file"; + uses_cwd = true; permission_write = false; } From a64c1e2790983b4585814b2df1226b1fbcc43010 Mon Sep 17 00:00:00 2001 From: ckrafft Date: Tue, 11 Aug 2026 02:11:46 +0200 Subject: [PATCH 11/16] ui: fix formating issues --- .../ChatMessageToolCallBlockDefault.svelte | 5 ++++- .../ChatMessageToolCallBlockReadMedia.svelte | 18 ++++++++++++------ tools/ui/src/lib/constants/mcp-resource.ts | 2 +- tools/ui/src/lib/stores/agentic.svelte.ts | 18 +++++++++++------- tools/ui/src/lib/utils/agentic.ts | 2 +- 5 files changed, 29 insertions(+), 16 deletions(-) diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte index a443bec5df8f..42dac50216c7 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte @@ -107,7 +107,10 @@ {#if line.media} {#if line.media.type === 'AUDIO'} - {@const audioSrc = createBase64DataUrl(line.media.mimeType ?? 'audio/mpeg', (line.media as DatabaseMessageExtraAudioFile).base64Data)} + {@const audioSrc = createBase64DataUrl( + line.media.mimeType ?? 'audio/mpeg', + (line.media as DatabaseMessageExtraAudioFile).base64Data + )}
{#if line.media} - {#if line.media.type === 'AUDIO'} - {@const audioSrc = createBase64DataUrl( - line.media.mimeType ?? 'audio/mpeg', - (line.media as DatabaseMessageExtraAudioFile).base64Data - )} + {#if line.media.type === AttachmentType.AUDIO} + {@const audioMimeType = line.media.mimeType ?? MimeTypeAudio.MP3_MPEG}
diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte index eec32424ad7b..98fcbd1a9df2 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte @@ -1,12 +1,7 @@ @@ -53,32 +51,29 @@ {#snippet children(_meta, _ctx)} {#if section.toolResult} - {#if mediaAttachment} - {#if isAudio} - {@const audioSrc = createBase64DataUrl( - readMediaMeta?.mimeType ?? 'audio/mpeg', - (mediaAttachment as DatabaseMessageExtraAudioFile).base64Data - )} -
- -
- {:else} -
- {readMediaMeta?.fileName -
- {/if} - {:else} + {#if !mediaAttachment}
Media attachment not found in message extras
+ {:else if mediaAttachment.type === AttachmentType.AUDIO} +
+ +
+ {:else} +
+ {readMediaMeta?.fileName +
{/if} {#if readMediaMeta?.sizeBytes || readMediaMeta?.mimeType} @@ -93,7 +88,7 @@ {/if} {#if readMediaMeta?.path} -
{readMediaMeta.path}
+
{readMediaMeta.path}
{/if} {:else}
diff --git a/tools/ui/src/lib/constants/built-in-tools.ts b/tools/ui/src/lib/constants/built-in-tools.ts index b5ca5dddbc99..5bd24ffe8397 100644 --- a/tools/ui/src/lib/constants/built-in-tools.ts +++ b/tools/ui/src/lib/constants/built-in-tools.ts @@ -48,7 +48,7 @@ export const BUILTIN_TOOL_UI: Readonly> source: ToolSource.BUILTIN }, [BuiltInTool.READ_FILE]: { icon: FileText, label: 'Read file', source: ToolSource.BUILTIN }, - [BuiltInTool.READ_MEDIA]: { icon: Eye, label: 'Read media', source: ToolSource.BUILTIN }, + [BuiltInTool.READ_MEDIA]: { icon: Eye, label: 'Read media', source: ToolSource.FRONTEND }, [BuiltInTool.RUN_JAVASCRIPT]: { icon: Braces, label: 'Run JavaScript', diff --git a/tools/ui/src/lib/constants/index.ts b/tools/ui/src/lib/constants/index.ts index 09006cf1e991..357a33a6241e 100644 --- a/tools/ui/src/lib/constants/index.ts +++ b/tools/ui/src/lib/constants/index.ts @@ -49,6 +49,7 @@ export * from './sse'; export * from './precision'; export * from './processing-info'; export * from './pwa'; +export * from './read-media'; export * from './routes'; export * from './sandbox'; export * from './settings-keys'; diff --git a/tools/ui/src/lib/constants/mcp-resource.ts b/tools/ui/src/lib/constants/mcp-resource.ts index cfa682d58173..7a1bdb91f974 100644 --- a/tools/ui/src/lib/constants/mcp-resource.ts +++ b/tools/ui/src/lib/constants/mcp-resource.ts @@ -2,7 +2,6 @@ import { MimeTypeAudio, MimeTypeImage } from '$lib/enums'; // File extension patterns for resource type detection export const IMAGE_FILE_EXTENSION_REGEX = /\.(png|jpg|jpeg|gif|svg|webp)$/i; -export const AUDIO_FILE_EXTENSION_REGEX = /\.(mp3|wav|ogg|flac|m4a|opus)$/i; export const CODE_FILE_EXTENSION_REGEX = /\.(js|ts|json|yaml|yml|xml|html|css|py|rs|go|java|cpp|c|h|rb|sh|toml)$/i; export const TEXT_FILE_EXTENSION_REGEX = /\.(txt|md|log)$/i; @@ -28,6 +27,9 @@ export const MCP_RESOURCE_ATTACHMENT_ID_PREFIX = 'res'; // Default file extension for unknown image types export const DEFAULT_IMAGE_EXTENSION = 'img'; +// Default file extension for unknown audio types +export const DEFAULT_AUDIO_EXTENSION = 'mp3'; + // Default filename for resource content downloads export const DEFAULT_RESOURCE_FILENAME = 'resource.txt'; diff --git a/tools/ui/src/lib/constants/read-media.ts b/tools/ui/src/lib/constants/read-media.ts index 7fbda2f1d729..eedaa8a543ad 100644 --- a/tools/ui/src/lib/constants/read-media.ts +++ b/tools/ui/src/lib/constants/read-media.ts @@ -1,3 +1,63 @@ +import { + BuiltInTool, + JsonSchemaType, + MimeTypeAudio, + MimeTypeImage, + ToolCallType +} from '$lib/enums'; +import type { OpenAIToolDefinition } from '$lib/types'; + +export const READ_MEDIA_TOOL_NAME = BuiltInTool.READ_MEDIA; + +// header lines of the tool result, parsed back by the read_media renderer export const PREFIX_FILE = 'File: '; export const PREFIX_SIZE = 'Size: '; export const PREFIX_MIME = 'MIME: '; + +/** Image extensions the tool accepts. The server decodes images with stb_image, which has no webp or tiff. */ +export const READ_MEDIA_IMAGE_MIME: Record = { + gif: MimeTypeImage.GIF, + jpeg: MimeTypeImage.JPEG, + jpg: MimeTypeImage.JPEG, + png: MimeTypeImage.PNG +} as const; + +/** Audio extensions the tool accepts. The `input_audio` API only takes wav and mp3. */ +export const READ_MEDIA_AUDIO_MIME: Record = { + mp3: MimeTypeAudio.MP3_MPEG, + wav: MimeTypeAudio.WAV +} as const; + +/** + * Build the read_media tool definition for the modalities the active model has. + * At least one of the two flags must be true, otherwise the tool is not offered + * at all - a model that cannot see or hear has nothing to do with the bytes. + */ +export function buildReadMediaToolDefinition( + supportsVision: boolean, + supportsAudio: boolean +): OpenAIToolDefinition { + const kinds: string[] = []; + + if (supportsVision) kinds.push(`images (${Object.keys(READ_MEDIA_IMAGE_MIME).join(', ')})`); + + if (supportsAudio) kinds.push(`audio (${Object.keys(READ_MEDIA_AUDIO_MIME).join(', ')})`); + + return { + function: { + description: `Read a media file and attach it to the conversation so it can be perceived directly. Supports ${kinds.join(' and ')}.`, + name: READ_MEDIA_TOOL_NAME, + parameters: { + properties: { + path: { + description: 'Path to the media file', + type: JsonSchemaType.STRING + } + }, + required: ['path'], + type: JsonSchemaType.OBJECT + } + }, + type: ToolCallType.FUNCTION + }; +} diff --git a/tools/ui/src/lib/constants/tools.ts b/tools/ui/src/lib/constants/tools.ts index 65f4457c969d..a467cda2b97a 100644 --- a/tools/ui/src/lib/constants/tools.ts +++ b/tools/ui/src/lib/constants/tools.ts @@ -3,6 +3,12 @@ import { ToolSource } from '$lib/enums/tools.enums'; /** HTTP header carrying the working directory a tool call runs in. The server resolves relative paths against it; the model cannot override it. */ export const X_TOOL_CWD_HEADER = 'x-tool-cwd'; +/** HTTP header asking the server to encode a tool's output differently, e.g. read_file returning base64. Not a tool parameter, so it stays out of the definition the model sees. */ +export const X_RESP_TYPE_HEADER = 'x-resp-type'; + +/** `X_RESP_TYPE_HEADER` value that makes read_file return the raw bytes as base64 instead of text. */ +export const RESP_TYPE_BASE64 = 'base64'; + export const TOOL_GROUP_LABELS = { [ToolSource.BUILTIN]: 'Built-in', [ToolSource.CUSTOM]: 'JSON Schema', diff --git a/tools/ui/src/lib/services/read-media.service.ts b/tools/ui/src/lib/services/read-media.service.ts new file mode 100644 index 000000000000..91e2b4959a41 --- /dev/null +++ b/tools/ui/src/lib/services/read-media.service.ts @@ -0,0 +1,110 @@ +import { ToolsService } from './tools.service'; +import { + NEWLINE, + PREFIX_FILE, + PREFIX_MIME, + PREFIX_SIZE, + READ_MEDIA_AUDIO_MIME, + READ_MEDIA_IMAGE_MIME, + RESP_TYPE_BASE64 +} from '$lib/constants'; +import { BuiltInTool, ToolResponseField } from '$lib/enums'; +import type { ToolExecutionResult } from '$lib/types'; + +/** Modalities of the model the tool call runs for. */ +export interface ReadMediaCapabilities { + audio: boolean; + vision: boolean; +} + +/** Lowercase extension of a path, without the dot. Empty when the file name has none. */ +function fileExtension(path: string): string { + const name = path.split(/[\\/]/).pop() ?? ''; + const dot = name.lastIndexOf('.'); + + return dot > 0 ? name.slice(dot + 1).toLowerCase() : ''; +} + +/** + * **ReadMediaService** - frontend executor for the `read_media` tool + * + * The tool is synthetic: no such tool exists on the server. It reads the file + * through the built-in `read_file` tool with the `base64` response type, then + * turns the bytes into a data URI line. The agentic store lifts that line into + * an image or audio attachment on the tool result message, which is what makes + * the model perceive the file instead of reading a wall of base64. + * + * Living in the frontend is what lets it exist only for models that can + * actually use the result - the server has no idea which model is selected. + * + * @see buildReadMediaToolDefinition in constants/read-media.ts - tool schema sent to the LLM + * @see agenticStore in stores/agentic.svelte.ts - tool dispatch and attachment extraction + */ +export class ReadMediaService { + static async executeTool( + params: Record, + capabilities: ReadMediaCapabilities, + signal?: AbortSignal, + cwd?: string + ): Promise { + const path = typeof params.path === 'string' ? params.path : ''; + + if (!path) { + return { content: 'Error: missing "path" argument.', isError: true }; + } + + const extension = fileExtension(path); + const imageMime = READ_MEDIA_IMAGE_MIME[extension]; + const audioMime = READ_MEDIA_AUDIO_MIME[extension]; + let resolvedMime: string | undefined; + + if (imageMime && capabilities.vision) resolvedMime = imageMime; + else if (audioMime && capabilities.audio) resolvedMime = audioMime; + + if (!resolvedMime) { + const supported = [ + ...(capabilities.vision ? Object.keys(READ_MEDIA_IMAGE_MIME) : []), + ...(capabilities.audio ? Object.keys(READ_MEDIA_AUDIO_MIME) : []) + ]; + + // an unreadable-by-this-model file is a dead end, so say why instead of failing silently + const reason = + imageMime || audioMime + ? `the current model cannot perceive ".${extension}" files` + : `".${extension}" is not a supported media type`; + + return { + content: `Error: ${reason}. Supported: ${supported.join(', ')}.`, + isError: true + }; + } + + const raw = await ToolsService.executeToolRaw( + BuiltInTool.READ_FILE, + { path }, + signal, + cwd, + RESP_TYPE_BASE64 + ); + + if (ToolResponseField.ERROR in raw) { + return { content: String(raw[ToolResponseField.ERROR]), isError: true }; + } + + const base64 = typeof raw.base64 === 'string' ? raw.base64 : ''; + + if (!base64) { + return { content: `Error: no data returned for ${path}.`, isError: true }; + } + + const sizeBytes = typeof raw.size_bytes === 'number' ? raw.size_bytes : 0; + const content = [ + `${PREFIX_FILE}${path}`, + `${PREFIX_SIZE}${sizeBytes} bytes`, + `${PREFIX_MIME}${resolvedMime}`, + `data:${resolvedMime};base64,${base64}` + ].join(NEWLINE); + + return { content, isError: false }; + } +} diff --git a/tools/ui/src/lib/services/tools.service.ts b/tools/ui/src/lib/services/tools.service.ts index cd6b12cebe06..9cb5ecb46dbf 100644 --- a/tools/ui/src/lib/services/tools.service.ts +++ b/tools/ui/src/lib/services/tools.service.ts @@ -1,5 +1,5 @@ import { base } from '$app/paths'; -import { API_TOOLS, X_TOOL_CWD_HEADER } from '$lib/constants'; +import { API_TOOLS, X_RESP_TYPE_HEADER, X_TOOL_CWD_HEADER } from '$lib/constants'; import { ToolResponseField } from '$lib/enums'; import type { ServerBuiltinToolInfo, ToolExecutionResult } from '$lib/types'; import { apiFetch } from '$lib/utils'; @@ -51,16 +51,26 @@ export class ToolsService { * Execute a built-in tool and return the raw JSON response. Unlike * executeTool, this preserves structured fields (e.g. file_glob_search's * `entries` and `base`) that the flattened ToolExecutionResult drops. + * + * @param respType - sent as the x-resp-type request header. Only read_file + * honors it, with `base64` to get the raw bytes instead of decoded text. */ static async executeToolRaw( toolName: string, params: Record, signal?: AbortSignal, - cwd?: string + cwd?: string, + respType?: string ): Promise> { + const headers: Record = {}; + + if (cwd) headers[X_TOOL_CWD_HEADER] = cwd; + + if (respType) headers[X_RESP_TYPE_HEADER] = respType; + return apiFetch>(API_TOOLS.EXECUTE, { body: JSON.stringify({ params, tool: toolName }), - headers: cwd ? { [X_TOOL_CWD_HEADER]: cwd } : undefined, + headers: Object.keys(headers).length > 0 ? headers : undefined, method: 'POST', signal }); diff --git a/tools/ui/src/lib/stores/agentic.svelte.ts b/tools/ui/src/lib/stores/agentic.svelte.ts index 4a541eef41ca..7459d80e49ab 100644 --- a/tools/ui/src/lib/stores/agentic.svelte.ts +++ b/tools/ui/src/lib/stores/agentic.svelte.ts @@ -22,12 +22,12 @@ import { DEFAULT_AGENTIC_CONFIG, NEWLINE } from '$lib/constants'; import { + AUDIO_MIME_TO_EXTENSION, DATA_URI_BASE64_REGEX, + DEFAULT_AUDIO_EXTENSION, DEFAULT_IMAGE_EXTENSION, IMAGE_MIME_TO_EXTENSION, - AUDIO_MIME_TO_EXTENSION, - MCP_ATTACHMENT_NAME_PREFIX, - AUDIO_FILE_EXTENSION_REGEX + MCP_ATTACHMENT_NAME_PREFIX } from '$lib/constants'; import { BuiltInTool, ToolPermissionDecision, ToolSource } from '$lib/enums'; import { @@ -38,6 +38,7 @@ import { ToolCallType } from '$lib/enums'; import { ChatService } from '$lib/services'; +import { ReadMediaService } from '$lib/services/read-media.service'; import { SandboxService } from '$lib/services/sandbox.service'; import { ToolsService } from '$lib/services/tools.service'; import { conversationsStore } from '$lib/stores/conversations.svelte'; @@ -903,7 +904,18 @@ class AgenticStore { if (executionResult.isError) toolSuccess = false; } else if (toolSource === ToolSource.FRONTEND) { const args = this.parseToolArguments(toolCall.function.arguments); - const executionResult = await SandboxService.executeTool(toolName, args, signal); + const executionResult = + toolName === BuiltInTool.READ_MEDIA + ? await ReadMediaService.executeTool( + args, + { + audio: modelsStore.modelSupportsAudio(effectiveModel), + vision: modelsStore.modelSupportsVision(effectiveModel) + }, + signal, + conversationsStore.activeConversation?.cwd + ) + : await SandboxService.executeTool(toolName, args, signal); result = executionResult.content; @@ -1107,7 +1119,6 @@ class AgenticStore { return line; } - // Audio extras require raw base64 in base64Data (not full data URI). attachmentIndex += 1; const name = this.buildAttachmentName(mimeType, attachmentIndex); @@ -1118,11 +1129,12 @@ class AgenticStore { } if (mimeType.startsWith(MimeTypePrefix.AUDIO)) { + // audio extras hold the bare base64, the input_audio part has no room for a data URI attachments.push({ - type: AttachmentType.AUDIO, - name, + base64Data, mimeType, - base64Data: base64Data + name, + type: AttachmentType.AUDIO }); return `[Attachment saved: ${name}]`; @@ -1135,12 +1147,9 @@ class AgenticStore { } private buildAttachmentName(mimeType: string, index: number): string { - if (mimeType.startsWith('audio/')) { - const extension = AUDIO_MIME_TO_EXTENSION[mimeType] ?? 'mp3'; - return `${MCP_ATTACHMENT_NAME_PREFIX}-${Date.now()}-${index}.${extension}`; - } - - const extension = IMAGE_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_IMAGE_EXTENSION; + const extension = mimeType.startsWith(MimeTypePrefix.AUDIO) + ? (AUDIO_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_AUDIO_EXTENSION) + : (IMAGE_MIME_TO_EXTENSION[mimeType] ?? DEFAULT_IMAGE_EXTENSION); return `${MCP_ATTACHMENT_NAME_PREFIX}-${Date.now()}-${index}.${extension}`; } diff --git a/tools/ui/src/lib/stores/models.svelte.ts b/tools/ui/src/lib/stores/models.svelte.ts index 145de119ce7f..150fb3800f64 100644 --- a/tools/ui/src/lib/stores/models.svelte.ts +++ b/tools/ui/src/lib/stores/models.svelte.ts @@ -18,7 +18,10 @@ import { ModelsService } from '$lib/services/models.service'; import { PropsService } from '$lib/services/props.service'; import { conversationsStore } from '$lib/stores/conversations.svelte'; import { isRouterMode, serverStore } from '$lib/stores/server.svelte'; -import { getAuthHeaders, TTLCache } from '$lib/utils'; +// deep imports, not the '$lib/utils' barrel: it re-exports modules that reach back +// into the stores, and going through it here would read a half-built module +import { getAuthHeaders } from '$lib/utils/api-headers'; +import { TTLCache } from '$lib/utils/cache-ttl'; import { detectThinkingSupport, detectThinkingSupportWithReason diff --git a/tools/ui/src/lib/stores/tools.svelte.ts b/tools/ui/src/lib/stores/tools.svelte.ts index b1a946391a0b..3984dd2bc8d8 100644 --- a/tools/ui/src/lib/stores/tools.svelte.ts +++ b/tools/ui/src/lib/stores/tools.svelte.ts @@ -1,4 +1,5 @@ import { + buildReadMediaToolDefinition, buildSandboxToolDefinition, DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY, HOME_TILDE, @@ -15,6 +16,7 @@ import { } from '$lib/enums'; import { ToolsService } from '$lib/services/tools.service'; import { mcpStore } from '$lib/stores/mcp.svelte'; +import { modelsStore, selectedModelName } from '$lib/stores/models.svelte'; import { config } from '$lib/stores/settings.svelte'; import type { OpenAIToolDefinition, ToolEntry, ToolGroup } from '$lib/types'; import { SvelteMap, SvelteSet } from 'svelte/reactivity'; @@ -168,9 +170,42 @@ class ToolsStore { } get frontendTools(): OpenAIToolDefinition[] { - return config().jsSandboxEnabled - ? [buildSandboxToolDefinition(!!config().symbolicMathEnabled)] - : []; + const tools: OpenAIToolDefinition[] = []; + + if (config().jsSandboxEnabled) { + tools.push(buildSandboxToolDefinition(!!config().symbolicMathEnabled)); + } + + const readMedia = this.readMediaTool(); + + if (readMedia) tools.push(readMedia); + + return tools; + } + + /** + * `read_media` runs in the frontend on top of the server's `read_file`, so it + * exists only when that tool is served and the active model can perceive the + * bytes. The server cannot make this call - it does not know which model the + * conversation uses. + */ + private readMediaTool(): OpenAIToolDefinition | null { + const hasReadFile = this._builtinTools.some( + (def) => def.function.name === BuiltInTool.READ_FILE + ); + + if (!hasReadFile) return null; + + const model = selectedModelName() ?? modelsStore.models[0]?.model ?? ''; + + if (!model) return null; + + const vision = modelsStore.modelSupportsVision(model); + const audio = modelsStore.modelSupportsAudio(model); + + if (!vision && !audio) return null; + + return buildReadMediaToolDefinition(vision, audio); } get customTools(): OpenAIToolDefinition[] { diff --git a/tools/ui/src/lib/utils/agentic.ts b/tools/ui/src/lib/utils/agentic.ts index 71f118c7c77a..b2dd2cd9ed10 100644 --- a/tools/ui/src/lib/utils/agentic.ts +++ b/tools/ui/src/lib/utils/agentic.ts @@ -301,7 +301,7 @@ export function splitSearchSummaryList( return { lines }; } -/** Bounded cache for parseToolResultWithImages results. */ +/** Bounded cache for parseToolResultWithMedia results. */ const TOOL_RESULT_LINES_CACHE_MAX_SIZE = 32; const toolResultLinesCache = new Map(); diff --git a/tools/ui/tests/client/README-perf.md b/tools/ui/tests/client/README-perf.md index a4e3e4c6446a..198b21956832 100644 --- a/tools/ui/tests/client/README-perf.md +++ b/tools/ui/tests/client/README-perf.md @@ -33,12 +33,12 @@ npx vitest --project=client --run tests/client/agentic-stream.perf.svelte.test.t The point of the harness is the _scaling curve_, not any single number. -| Knob | Reads on | -| --------------------------- | ---------------------------------------------------------------------------------------------------- | -| `priorToolCalls` (0/1/5/20) | the reactive fan-out. Flat => no fan-out. Linear => confirmed. | -| `toolResultBytes` | whole-blob string scans (`extractSearchResults`, `parseToolResultWithImages`, `classifyToolResult`). | -| `editFileEdits` | `computeLineDiff`, the O(m\*n) LCS. | -| `openCodeFence` | `hljs.highlightAuto` on partial code. | +| Knob | Reads on | +| --------------------------- | --------------------------------------------------------------------------------------------------- | +| `priorToolCalls` (0/1/5/20) | the reactive fan-out. Flat => no fan-out. Linear => confirmed. | +| `toolResultBytes` | whole-blob string scans (`extractSearchResults`, `parseToolResultWithMedia`, `classifyToolResult`). | +| `editFileEdits` | `computeLineDiff`, the O(m\*n) LCS. | +| `openCodeFence` | `hljs.highlightAuto` on partial code. | Deliberately no hard assertions: CI timing is noisy and the value here is the before/after delta, not a gate. diff --git a/tools/ui/tests/unit/agentic-hotpath.bench.ts b/tools/ui/tests/unit/agentic-hotpath.bench.ts index 8bdaa095a89d..24dcb48911f8 100644 --- a/tools/ui/tests/unit/agentic-hotpath.bench.ts +++ b/tools/ui/tests/unit/agentic-hotpath.bench.ts @@ -6,7 +6,7 @@ // // Run: npx vitest bench --project=unit tests/unit/agentic-hotpath.bench.ts -import { classifyToolResult, parseToolResultWithImages } from '$lib/utils/agentic'; +import { classifyToolResult, parseToolResultWithMedia } from '$lib/utils/agentic'; import { detectIncompleteCodeBlock, highlightCode } from '$lib/utils/code'; import { computeLineDiff } from '$lib/utils/compute-line-diff'; import { preprocessLaTeX } from '$lib/utils/latex-protection'; @@ -200,17 +200,17 @@ describe('exit-code regex', () => { // --- per-line result parsers ---------------------------------------------- -describe('parseToolResultWithImages', () => { +describe('parseToolResultWithMedia', () => { bench('1KB', () => { - parseToolResultWithImages(SHELL_OUTPUT_1KB, []); + parseToolResultWithMedia(SHELL_OUTPUT_1KB, []); }); bench('200KB', () => { - parseToolResultWithImages(SHELL_OUTPUT_200KB, []); + parseToolResultWithMedia(SHELL_OUTPUT_200KB, []); }); bench('2MB', () => { - parseToolResultWithImages(SHELL_OUTPUT_2MB, []); + parseToolResultWithMedia(SHELL_OUTPUT_2MB, []); }); }); From bf610f1080a29d45b1949bf88d2121e8baeba53a Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Wed, 12 Aug 2026 00:15:35 +0200 Subject: [PATCH 14/16] correct partial commit --- .../ChatMessageToolCallBlockDefault.svelte | 2 +- .../ChatMessageToolCallBlockReadMedia.svelte | 6 +++--- .../ChatMessageToolCall/parsers/read-media.ts | 9 ++++++--- tools/ui/src/lib/constants/mcp-resource.ts | 8 ++++---- tools/ui/src/lib/services/chat.service.ts | 2 +- tools/ui/src/lib/services/read-media.service.ts | 2 +- tools/ui/src/lib/stores/agentic.svelte.ts | 11 +++++------ tools/ui/src/lib/utils/audio-format.ts | 2 +- 8 files changed, 22 insertions(+), 20 deletions(-) diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte index 64f07a2e6515..87208e2daa61 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockDefault.svelte @@ -10,7 +10,6 @@ import { getBuiltinToolUi } from '$lib/constants/built-in-tools'; import { AttachmentType, FileTypeText, MimeTypeAudio, ToolResultKind } from '$lib/enums'; import type { DatabaseMessageExtra } from '$lib/types'; - import { createBase64DataUrl } from '$lib/utils/data-url'; import { type AgenticSection, classifyToolResult, @@ -18,6 +17,7 @@ parseToolResultWithMedia, type ToolResultLine } from '$lib/utils'; + import { createBase64DataUrl } from '$lib/utils/data-url'; interface Props { section: AgenticSection; diff --git a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte index 98fcbd1a9df2..424c794d0bae 100644 --- a/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte +++ b/tools/ui/src/lib/components/app/chat/ChatMessages/ChatMessage/ChatMessageToolCall/ChatMessageToolCallBlockReadMedia.svelte @@ -1,11 +1,11 @@