diff --git a/tools/ui/src/lib/constants/browser-info.ts b/tools/ui/src/lib/constants/browser-info.ts new file mode 100644 index 000000000000..86d999643ab1 --- /dev/null +++ b/tools/ui/src/lib/constants/browser-info.ts @@ -0,0 +1,36 @@ +import { CLI_FLAGS } from './cli-flags.constants'; +import { BuiltInTool, JsonSchemaType, ToolCallType } from '$lib/enums'; +import type { OpenAIToolDefinition } from '$lib/types'; + +export const BROWSER_INFO_TOOL_NAME = BuiltInTool.GET_INFO; + +/** UA token to OS name, first match wins - Android and iOS UAs also carry the Linux / Mac OS X tokens */ +export const BROWSER_INFO_OS_UA_PATTERNS: readonly [RegExp, string][] = [ + [/Windows NT/, 'Windows'], + [/Android/, 'Android'], + [/iPhone|iPad|iPod/, 'iOS'], + [/CrOS/, 'ChromeOS'], + [/Mac OS X/, 'macOS'], + [/Linux/, 'Linux'] +]; + +export const BROWSER_INFO_OS_UNKNOWN = 'unknown'; + +/** Sent to the model as the `note` field of the tool result, next to the OS name */ +export const BROWSER_INFO_NOTE = `This environment is browser-only, it cannot read or modify local files, and it cannot run shell commands. To get local file access, tell user to launch llama-server with the ${CLI_FLAGS.AGENT} argument.`; + +export function buildBrowserInfoToolDefinition(): OpenAIToolDefinition { + return { + function: { + description: + 'Get runtime info (OS name), may call when user asks about local files or shell commands', + name: BROWSER_INFO_TOOL_NAME, + parameters: { + properties: {}, + required: [], + type: JsonSchemaType.OBJECT + } + }, + type: ToolCallType.FUNCTION + }; +} diff --git a/tools/ui/src/lib/constants/cli-flags.constants.ts b/tools/ui/src/lib/constants/cli-flags.constants.ts index 4fbee8a36973..c4af2b6f4629 100644 --- a/tools/ui/src/lib/constants/cli-flags.constants.ts +++ b/tools/ui/src/lib/constants/cli-flags.constants.ts @@ -1,4 +1,5 @@ export const CLI_FLAGS = { + AGENT: '--agent', API_KEY: '--api-key', MCP_PROXY: '--ui-mcp-proxy', SLOTS: '--slots', diff --git a/tools/ui/src/lib/constants/index.ts b/tools/ui/src/lib/constants/index.ts index 17e5f4d00ab5..cdd91a0738c0 100644 --- a/tools/ui/src/lib/constants/index.ts +++ b/tools/ui/src/lib/constants/index.ts @@ -59,3 +59,4 @@ export * from './uri-template.constants'; export * from './url.constants'; export * from './working-directory.constants'; export * from './read-media'; +export * from './browser-info'; diff --git a/tools/ui/src/lib/stores/agentic.svelte.ts b/tools/ui/src/lib/stores/agentic.svelte.ts index 075c46c97504..d6b57ffa80d9 100644 --- a/tools/ui/src/lib/stores/agentic.svelte.ts +++ b/tools/ui/src/lib/stores/agentic.svelte.ts @@ -56,7 +56,8 @@ import type { AgenticSession, McpServerOverride, MCPToolCall, - SettingsConfigType + SettingsConfigType, + ToolExecutionResult } from '$lib/types'; import type { AgenticFlowCallbacks, @@ -83,7 +84,7 @@ import type { DatabaseMessageExtraAudioFile, DatabaseMessageExtraImageFile } from '$lib/types/database'; -import { getAudioInputFormat, isAbortError } from '$lib/utils'; +import { executeBrowserInfoTool, getAudioInputFormat, isAbortError } from '$lib/utils'; import { SvelteMap } from 'svelte/reactivity'; function createDefaultSession(): AgenticSession { @@ -942,18 +943,24 @@ class AgenticStore { if (executionResult.isError) toolSuccess = false; } else if (toolSource === ToolSource.FRONTEND) { const args = this.parseToolArguments(toolCall.function.arguments); - const executionResult = - toolName === BuiltInTool.READ_MEDIA - ? await ReadMediaService.executeTool( - args, - { - audio: modelsStore.modelSupportsAudio(effectiveModel), - vision: modelsStore.modelSupportsVision(effectiveModel) - }, - signal, - conversationsStore.activeConversation?.cwd - ) - : await SandboxService.executeTool(toolName, args, signal); + + let executionResult: ToolExecutionResult; + + if (toolName === BuiltInTool.GET_INFO) { + executionResult = executeBrowserInfoTool(); + } else if (toolName === BuiltInTool.READ_MEDIA) { + executionResult = await ReadMediaService.executeTool( + args, + { + audio: modelsStore.modelSupportsAudio(effectiveModel), + vision: modelsStore.modelSupportsVision(effectiveModel) + }, + signal, + conversationsStore.activeConversation?.cwd + ); + } else { + executionResult = await SandboxService.executeTool(toolName, args, signal); + } result = executionResult.content; diff --git a/tools/ui/src/lib/stores/tools.svelte.ts b/tools/ui/src/lib/stores/tools.svelte.ts index 642662928831..fb3662025dac 100644 --- a/tools/ui/src/lib/stores/tools.svelte.ts +++ b/tools/ui/src/lib/stores/tools.svelte.ts @@ -1,5 +1,6 @@ import { browser } from '$app/environment'; import { + buildBrowserInfoToolDefinition, buildReadMediaToolDefinition, DISABLED_TOOL_KEYS_LOCALSTORAGE_KEY, HOME_TILDE, @@ -185,9 +186,18 @@ class ToolsStore { if (readMedia) tools.push(readMedia); + // provide browser's get_info tool if server doesn't provide one + if (!this.hasBuiltinTool(BuiltInTool.GET_INFO)) { + tools.push(buildBrowserInfoToolDefinition()); + } + return tools; } + private hasBuiltinTool(name: BuiltInTool): boolean { + return this._builtinTools.some((def) => def.function.name === name); + } + /** * `read_media` runs in the frontend on top of the server's `read_file`, so it * exists only when that tool is served and the active model can perceive the @@ -195,11 +205,7 @@ class ToolsStore { * conversation uses. */ private readMediaTool(): OpenAIToolDefinition | null { - const hasReadFile = this._builtinTools.some( - (def) => def.function.name === BuiltInTool.READ_FILE - ); - - if (!hasReadFile) return null; + if (!this.hasBuiltinTool(BuiltInTool.READ_FILE)) return null; const model = modelsStore.selectedModelName ?? modelsStore.models[0]?.model ?? ''; diff --git a/tools/ui/src/lib/utils/browser-info.ts b/tools/ui/src/lib/utils/browser-info.ts new file mode 100644 index 000000000000..c96abb01e028 --- /dev/null +++ b/tools/ui/src/lib/utils/browser-info.ts @@ -0,0 +1,39 @@ +/** + * Browser fallback for the server's `get_info` tool, offered only when the + * server does not serve one (llama-server without --agent). It tells the model + * which OS the browser runs on and that there is no local file or shell access, + * so it does not plan around tools that are not there. + * + * @see server_tool_get_info in tools/server/server-tools.cpp - the served variant + * @see buildBrowserInfoToolDefinition in constants/browser-info.ts - tool schema sent to the LLM + */ + +import { browser } from '$app/environment'; +import { + BROWSER_INFO_NOTE, + BROWSER_INFO_OS_UA_PATTERNS, + BROWSER_INFO_OS_UNKNOWN +} from '$lib/constants'; +import type { ToolExecutionResult } from '$lib/types'; + +function detectOs(userAgent: string): string { + for (const [pattern, os] of BROWSER_INFO_OS_UA_PATTERNS) { + if (pattern.test(userAgent)) return os; + } + + return BROWSER_INFO_OS_UNKNOWN; +} + +/** + * Result shape mirrors the server tool's JSON so the `get_info` renderer reads + * `os` the same way, minus `cwd` - there is no working directory to report. + */ +export function executeBrowserInfoTool(): ToolExecutionResult { + return { + content: JSON.stringify({ + note: BROWSER_INFO_NOTE, + os: browser ? detectOs(navigator.userAgent) : BROWSER_INFO_OS_UNKNOWN + }), + isError: false + }; +} diff --git a/tools/ui/src/lib/utils/index.ts b/tools/ui/src/lib/utils/index.ts index 3fb701b323bd..d991363fc52c 100644 --- a/tools/ui/src/lib/utils/index.ts +++ b/tools/ui/src/lib/utils/index.ts @@ -331,6 +331,9 @@ export { getChatCommands } from './chat-commands'; // SANDBOX_TOOL_DEFINITION is deprecated; kept for backward compatibility. export { buildSandboxToolDefinition, SANDBOX_TOOL_DEFINITION } from './sandbox-tool'; +// Browser fallback for the server's get_info tool +export { executeBrowserInfoTool } from './browser-info'; + // Cryptography utilities export { uuid } from './uuid';