ngxson · Aug 12, 2026 · Aug 11, 2026 · Aug 11, 2026 · Aug 11, 2026 · Aug 11, 2026
diff --git a/common/arg.cpp b/common/arg.cpp
@@ -3302,7 +3302,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         {"--tools"}, "TOOL1,TOOL2,...",
         "experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)\n"
         "specify \"all\" to enable all tools\n"
-        "available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime, get_info\n"
+        "available tools: read_file, read_media, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime, get_info\n"
         "note: for security reasons, this will limit --cors-origins to localhost by default",
         [](common_params & params, const std::string & value) {
             params.server_tools = parse_csv_row(value);

diff --git a/tools/server/README.md b/tools/server/README.md
@@ -196,7 +196,7 @@ For the full list of features, please refer to [server's changelog](https://gith
 | `--ui-config, --webui-config JSON` | JSON that provides default UI settings (overrides UI defaults)<br/>(env: LLAMA_ARG_UI_CONFIG) |
 | `--ui-config-file, --webui-config-file PATH` | JSON file that provides default UI settings (overrides UI defaults)<br/>(env: LLAMA_ARG_UI_CONFIG_FILE) |
 | `--ui-mcp-proxy, --webui-mcp-proxy, --no-ui-mcp-proxy, --no-webui-mcp-proxy` | experimental: whether to enable MCP CORS proxy - do not enable in untrusted environments (default: disabled)<br/>(env: LLAMA_ARG_UI_MCP_PROXY) |
-| `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)<br/>specify "all" to enable all tools<br/>available tools: read_file, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime, get_info<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_TOOLS) |
+| `--tools TOOL1,TOOL2,...` | experimental: whether to enable built-in tools for AI agents - do not enable in untrusted environments (default: no tools)<br/>specify "all" to enable all tools<br/>available tools: read_file, read_media, file_glob_search, grep_search, exec_shell_command, write_file, edit_file, get_datetime, get_info<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_TOOLS) |
 | `--tools-runtime OPTION` | experimental: run tools in a separate runtime environment (default: none, use host environment)<br/>available options:<br/>  'docker:<image>', 'podman:<image>': spin up a new container and reuse it for all invocations, clean up on server exit<br/>  'docker-container:<id>', 'podman-container:<id>': use an existing container by ID, won't stop on server exit<br/>  'ssh:<target>': run tools on a remote POSIX host over SSH, key-based auth and a trusted host key are required<br/><br/>(env: LLAMA_ARG_TOOLS_RUNTIME) |
 | `--mcp-servers-config PATH` | experimental: path to JSON file with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_MCP_SERVERS_CONFIG) |
 | `--mcp-servers-json JSON` | experimental: inline JSON with MCP server definitions (Cursor-compatible format) - do not enable in untrusted environments (default: none)<br/>note: for security reasons, this will limit --cors-origins to localhost by default<br/>(env: LLAMA_ARG_MCP_SERVERS_JSON) |

@@ -1,6 +1,7 @@
 #include "server-tools.h"
 
 #include "subproc.h"
+#include "base64.hpp"
 
 #include <filesystem>
 #include <fstream>
@@ -9,6 +10,7 @@
 #include <chrono>
 #include <ctime>
 #include <atomic>
+#include <cctype>
 #include <cstring>
 #include <cctype>
 #include <cstdint>
@@ -945,6 +947,119 @@ struct server_tool_read_file : server_tool {
     }
 };
 
+//
+// read_media: read a media file (image or audio)
+//
+
+static constexpr size_t SERVER_TOOL_READ_MEDIA_MAX_SIZE = 32 * 1024 * 1024; // 32 MB
+static constexpr const char* SERVER_TOOL_READ_MEDIA_PREFIX_FILE = "File: ";
+static constexpr const char* SERVER_TOOL_READ_MEDIA_PREFIX_SIZE = "Size: ";
+static constexpr const char* SERVER_TOOL_READ_MEDIA_PREFIX_MIME = "MIME: ";
+
+static std::string get_mime_from_extension(const std::string & path) {
+    static const std::unordered_map<std::string, std::string> mime_map = {
+        // Images
+        {".png",  "image/png"},
+        {".jpg",  "image/jpeg"},
+        {".jpeg", "image/jpeg"},
+        {".webp", "image/webp"},
+        {".bmp",  "image/bmp"},
+        {".tiff", "image/tiff"},
+        {".tif",  "image/tiff"},
+        {".gif",  "image/gif"},
+        // Audio - only wav and mp3: the model's input_audio API
+        // only accepts these two formats (FileTypeAudio.WAV | MP3).
+        {".mp3",  "audio/mpeg"},
+        {".wav",  "audio/wav"},
+    };
+
+    auto ext = fs::path(path).extension().string();
+    std::string ext_lower;
+    ext_lower.reserve(ext.size());
+    for (char c : ext) ext_lower += static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
+
+    auto it = mime_map.find(ext_lower);
+    return (it != mime_map.end()) ? it->second : "application/octet-stream";
+}
+
+struct server_tool_read_media : server_tool_read_file {
+    server_tool_read_media() {
+        name = "read_media";
+        display_name = "Read media file";
+        uses_cwd = true;
+        permission_write = false;
+    }
+
+    json get_definition() const override {
+        return {
+            {"type", "function"},
+            {"function", {
+                {"name", name},
+                {"description",
+                    "Read the content of a media file (image or audio).\n"
+                    " Audio: .wav, .mp3.\n"
+                    " Images: .png, .jpg, .jpeg, .webp, .bmp, .tiff, .gif."},
+                {"parameters", {
+                    {"type", "object"},
+                    {"properties", {
+                        {"path", {{"type", "string"}, {"description", "Path to the media file."}}},
+                    }},
+                    {"required", json::array({"path"})},
+                }},
+            }},
+        };
+    }
+
+    json invoke(json params, server_tool::stream *) const override {
+        std::string path = params.at("path").get<std::string>();
+
+        auto io = make_tools_io(params);
+
+        uintmax_t file_size = 0;
+        if (!io->file_size(path, file_size)) {
+            return {{"error", "cannot stat file: " + path}};
+        }
+        if (file_size > SERVER_TOOL_READ_MEDIA_MAX_SIZE) {
+            return {{"error", string_format(
+                "media file too large (%zu bytes, max %zu)",
+                (size_t)file_size, SERVER_TOOL_READ_MEDIA_MAX_SIZE)}};
+        }
+
+        std::string mime = get_mime_from_extension(path);
+
+        // Reject unknown/unrecognized file extensions instead of producing
+        // a multi-MB data URI that inflates the model context with garbage.
+        if (mime == "application/octet-stream") {
+            return {{"error", "unrecognized or unsupported extension: " + path}};
+        }
+
+        std::string content;
+        if (!io->read_file(path, content)) {
+            return {{"error", "failed to open file: " + path}};
+        }
+
+        std::string b64  = base64::encode(content.data(), content.size());
+
+        // Return as plain_text_response with a data URI line so the UI can
+        // extract it as an attachment (via extractBase64Attachments)
+        // and replace it with some placeholder
+        std::string data_uri = "data:" + mime + ";base64," + b64;
+
+        return {
+            {"plain_text_response",
+                string_format(
+                    "%s%s\n%s%zu bytes\n%s%s\n%s",
+                    SERVER_TOOL_READ_MEDIA_PREFIX_FILE, path.c_str(),
+                    SERVER_TOOL_READ_MEDIA_PREFIX_SIZE, (size_t)file_size,
+                    SERVER_TOOL_READ_MEDIA_PREFIX_MIME, mime.c_str(),
+                    data_uri.c_str())},
+            {"path", path},
+            {"mime", mime},
+            {"size_bytes", (size_t)file_size},
+        };
+    }
+};
+
 //
 // file_glob_search: find files matching a glob pattern under a base directory
 //
@@ -1986,6 +2101,7 @@ static server_tool & find_tool(std::vector<std::unique_ptr<server_tool>> & tools
 static std::vector<std::unique_ptr<server_tool>> build_tools() {
     std::vector<std::unique_ptr<server_tool>> tools;
     tools.push_back(std::make_unique<server_tool_read_file>());
+    tools.push_back(std::make_unique<server_tool_read_media>());
     tools.push_back(std::make_unique<server_tool_file_glob_search>());
     tools.push_back(std::make_unique<server_tool_grep_search>());
     tools.push_back(std::make_unique<server_tool_exec_shell_command>());

@@ -7,6 +7,7 @@
 	import ChatMessageToolCallBlockGetInfo from './ChatMessageToolCallBlockGetInfo.svelte';
 	import ChatMessageToolCallBlockGrepSearch from './ChatMessageToolCallBlockGrepSearch.svelte';
 	import ChatMessageToolCallBlockReadFile from './ChatMessageToolCallBlockReadFile.svelte';
+	import ChatMessageToolCallBlockReadMedia from './ChatMessageToolCallBlockReadMedia.svelte';
 	import ChatMessageToolCallBlockRunJavascript from './ChatMessageToolCallBlockRunJavascript.svelte';
 	import ChatMessageToolCallBlockSearchResults from './ChatMessageToolCallBlockSearchResults.svelte';
 	import ChatMessageToolCallBlockWriteFile from './ChatMessageToolCallBlockWriteFile.svelte';
@@ -45,6 +46,8 @@
 	<ChatMessageToolCallBlockGetInfo {section} {isStreaming} />
 {:else if section.toolName === BuiltInTool.READ_FILE}
 	<ChatMessageToolCallBlockReadFile {section} {open} {isStreaming} {onToggle} />
+{:else if section.toolName === BuiltInTool.READ_MEDIA}
+	<ChatMessageToolCallBlockReadMedia {section} {open} {isStreaming} {onToggle} />
 {:else if section.toolName === BuiltInTool.EDIT_FILE}
 	<ChatMessageToolCallBlockEditFile {section} {open} {isStreaming} {onToggle} />
 {:else if section.toolName === BuiltInTool.WRITE_FILE}

@@ -9,12 +9,14 @@
 	import { MAX_HEIGHT_CODE_BLOCK } from '$lib/constants';
 	import { getBuiltinToolUi } from '$lib/constants/built-in-tools';
 	import { FileTypeText, ToolResultKind } from '$lib/enums';
-	import type { DatabaseMessageExtra } from '$lib/types';
+	import type { DatabaseMessageExtra, DatabaseMessageExtraAudioFile } from '$lib/types';
+	import { createBase64DataUrl } from '$lib/utils/data-url';
 	import {
 		type AgenticSection,
 		classifyToolResult,
 		formatJsonPretty,
-		parseToolResultWithImages
+		parseToolResultWithMedia,
+		type ToolResultLine
 	} from '$lib/utils';
 
 	interface Props {
@@ -29,8 +31,8 @@
 
 	const title = $derived(getBuiltinToolUi(section.toolName)?.label ?? section.toolName ?? '');
 	const outputKind = $derived(classifyToolResult(section.toolResult));
-	const parsedLines = $derived(
-		section.toolResult ? parseToolResultWithImages(section.toolResult, attachments) : []
+	const parsedLines: ToolResultLine[] = $derived(
+		section.toolResult ? parseToolResultWithMedia(section.toolResult, attachments) : []
 	);
 </script>
 
@@ -103,13 +105,26 @@
 							<div class="font-mono text-[11px] leading-relaxed whitespace-pre-wrap">
 								{line.text}
 							</div>
-							{#if line.image}
-								<img
-									src={line.image.base64Url}
-									alt={line.image.name}
-									class="mt-2 mb-2 h-auto max-w-full rounded-lg"
-									loading="lazy"
-								/>
+							{#if line.media}
+								{#if line.media.type === 'AUDIO'}
+									{@const audioSrc = createBase64DataUrl(
+										line.media.mimeType ?? 'audio/mpeg',
+										(line.media as DatabaseMessageExtraAudioFile).base64Data
+									)}
+									<div class="mt-2 mb-2">
+										<audio controls class="w-full rounded-lg">
+											<source src={audioSrc} type={line.media.mimeType ?? 'audio/mpeg'} />
+											Your browser does not support the audio element.
+										</audio>
+									</div>
+								{:else}
+									<img
+										src={line.media.base64Url}
+										alt={line.media.name}
+										class="mt-2 mb-2 h-auto max-w-full rounded-lg"
+										loading="lazy"
+									/>
+								{/if}
 							{/if}
 						{/each}
 					</div>

@@ -23,7 +23,7 @@
 		isExitCodeSummaryLine,
 		parseExecShellCommandError,
 		parseExecShellCommandExitStatus,
-		parseToolResultWithImages,
+		parseToolResultWithMedia,
 		type ToolResultLine
 	} from '$lib/utils';
 
@@ -53,7 +53,7 @@
 	);
 
 	const parsedLines: ToolResultLine[] = $derived(
-		section.toolResult ? parseToolResultWithImages(section.toolResult, attachments) : []
+		section.toolResult ? parseToolResultWithMedia(section.toolResult, attachments) : []
 	);
 
 	// Drop the trailing "[exit code: N]" line - rendered as a colored
@@ -223,10 +223,10 @@
 			>
 				{#each outputLines as line, i (i)}
 					<div class="font-mono text-[11px] leading-relaxed whitespace-pre-wrap">{line.text}</div>
-					{#if line.image}
+					{#if line.media}
 						<img
-							src={line.image.base64Url}
-							alt={line.image.name}
+							src={line.media.base64Url}
+							alt={line.media.name}
 							class="mt-2 mb-2 h-auto max-w-full rounded-lg"
 							loading="lazy"
 						/>

@@ -0,0 +1,104 @@
+<script lang="ts">
+	import { Eye } from '@lucide/svelte';
+	import { AttachmentType } from '$lib/enums';
+	import { ATTACHMENT_SAVED_REGEX } from '$lib/constants/agentic';
+	import type {
+		DatabaseMessageExtra,
+		DatabaseMessageExtraImageFile,
+		DatabaseMessageExtraAudioFile
+	} from '$lib/types';
+	import { type AgenticSection } from '$lib/utils';
+	import { createBase64DataUrl } from '$lib/utils/data-url';
+	import { parseReadMediaMeta } from './parsers/read-media';
+	import ToolCallBlock from './ToolCallBlock.svelte';
+
+	interface Props {
+		section: AgenticSection;
+		open: boolean;
+		isStreaming: boolean;
+		onToggle?: () => void;
+	}
+
+	let { section, open, isStreaming, onToggle }: Props = $props();
+
+	const readMediaMeta = $derived(parseReadMediaMeta(section));
+
+	// Find the attachment from toolResultExtras (attached to the tool result message.
+	// The extractBase64Attachments function in agentic.svelte.ts replaces the data URI line
+	// with [Attachment saved: name] and stores the base64 as an extra.
+	const mediaAttachment = $derived.by(() => {
+		const extras = section.toolResultExtras;
+		if (!extras || extras.length === 0) return null;
+		// Extract the attachment name from the cleaned result text
+		const match = section.toolResult?.match(ATTACHMENT_SAVED_REGEX);
+		if (!match) return null;
+		const attachmentName = match[1];
+		return (
+			extras.find(
+				(e): e is DatabaseMessageExtraImageFile | DatabaseMessageExtraAudioFile =>
+					(e.type === AttachmentType.IMAGE || e.type === AttachmentType.AUDIO) &&
+					e.name === attachmentName
+			) ?? null
+		);
+	});
+
+	const isAudio = $derived(mediaAttachment?.type === AttachmentType.AUDIO);
+</script>
+
+<ToolCallBlock {section} {open} {isStreaming} meta={readMediaMeta} {onToggle}>
+	{#snippet titleSnippet()}
+		<span class="text-muted-foreground">Read media </span>
+		<span class="font-mono">{readMediaMeta?.fileName}</span>
+	{/snippet}
+
+	{#snippet children(_meta, _ctx)}
+		{#if section.toolResult}
+			{#if mediaAttachment}
+				{#if isAudio}
+					{@const audioSrc = createBase64DataUrl(
+						readMediaMeta?.mimeType ?? 'audio/mpeg',
+						(mediaAttachment as DatabaseMessageExtraAudioFile).base64Data
+					)}
+					<div class="mt-2">
+						<audio controls class="w-full rounded-lg">
+							<source src={audioSrc} type={readMediaMeta?.mimeType ?? 'audio/mpeg'} />
+							Your browser does not support the audio element.
+						</audio>
+					</div>
+				{:else}
+					<div class="mt-2">
+						<img
+							src={mediaAttachment.base64Url}
+							alt={readMediaMeta?.fileName ?? 'media'}
+							class="max-h-[60vh] max-w-full rounded-lg object-contain shadow-lg"
+							loading="lazy"
+						/>
+					</div>
+				{/if}
+			{:else}
+				<div class="rounded bg-muted/20 p-2 text-xs text-muted-foreground/70 italic">
+					Media attachment not found in message extras
+				</div>
+			{/if}
+
+			{#if readMediaMeta?.sizeBytes || readMediaMeta?.mimeType}
+				<div class="mt-2 flex gap-4 text-xs text-muted-foreground">
+					{#if readMediaMeta?.sizeBytes}
+						<span>Size: {readMediaMeta.sizeBytes} bytes</span>
+					{/if}
+					{#if readMediaMeta?.mimeType}
+						<span>MIME: {readMediaMeta.mimeType}</span>
+					{/if}
+				</div>
+			{/if}
+
+			{#if readMediaMeta?.path}
+				<div class="mt-1 text-xs text-muted-foreground/60 font-mono">{readMediaMeta.path}</div>
+			{/if}
+		{:else}
+			<div class="rounded bg-muted/20 p-2 text-xs text-muted-foreground/70 italic">
+				Waiting for media data...
+			</div>
+		{/if}
+	{/snippet}
+</ToolCallBlock>