diff --git a/agent/agent_init.py b/agent/agent_init.py index 649d5338a996..6f89ed237dca 100644 --- a/agent/agent_init.py +++ b/agent/agent_init.py @@ -492,6 +492,7 @@ def init_agent( reasoning_callback: callable = None, clarify_callback: callable = None, read_terminal_callback: callable = None, + read_preview_callback: callable = None, step_callback: callable = None, stream_delta_callback: callable = None, interim_assistant_callback: callable = None, @@ -757,6 +758,7 @@ def init_agent( agent.reasoning_callback = reasoning_callback agent.clarify_callback = clarify_callback agent.read_terminal_callback = read_terminal_callback + agent.read_preview_callback = read_preview_callback agent.step_callback = step_callback agent.stream_delta_callback = stream_delta_callback agent.interim_assistant_callback = interim_assistant_callback diff --git a/agent/agent_runtime_helpers.py b/agent/agent_runtime_helpers.py index 8ddf1b95fd80..062549ed7f65 100644 --- a/agent/agent_runtime_helpers.py +++ b/agent/agent_runtime_helpers.py @@ -98,7 +98,7 @@ def _ra(): AGENT_RUNTIME_POST_HOOK_TOOL_NAMES = frozenset( - {"todo", "session_search", "memory", "clarify", "read_terminal", "delegate_task"} + {"todo", "session_search", "memory", "clarify", "read_terminal", "read_preview", "delegate_task"} ) @@ -2979,6 +2979,17 @@ def _execute(next_args: dict) -> Any: ), next_args, ) + elif function_name == "read_preview": + def _execute(next_args: dict) -> Any: + from tools.read_preview_tool import read_preview_tool as _read_preview_tool + return _finish_agent_tool( + _read_preview_tool( + start=next_args.get("start"), + count=next_args.get("count"), + callback=getattr(agent, "read_preview_callback", None), + ), + next_args, + ) elif function_name == "delegate_task": def _execute(next_args: dict) -> Any: return _finish_agent_tool(agent._dispatch_delegate_task(next_args), next_args) diff --git a/agent/tool_executor.py b/agent/tool_executor.py index e1a3f3013df6..5e6e96311db0 100644 --- a/agent/tool_executor.py +++ b/agent/tool_executor.py @@ -1677,6 +1677,27 @@ def _execute(next_args: dict) -> Any: tool_duration = time.time() - tool_start_time if agent._should_emit_quiet_tool_messages(): agent._vprint(f" {_get_cute_tool_message_impl('read_terminal', function_args, tool_duration, result=function_result)}") + elif function_name == "read_preview": + def _execute(next_args: dict) -> Any: + from tools.read_preview_tool import read_preview_tool as _read_preview_tool + return _read_preview_tool( + start=next_args.get("start"), + count=next_args.get("count"), + callback=getattr(agent, "read_preview_callback", None), + ) + function_result, function_args, middleware_trace, _execution_blocked, _execution_dispatched = _managed_values(_run_agent_tool_execution_middleware( + agent, + function_name=function_name, + function_args=function_args, + effective_task_id=effective_task_id, + tool_call_id=getattr(tool_call, "id", "") or "", + execute=_execute, + scope_block=_ts_scope_block, + display_index=i, + )) + tool_duration = time.time() - tool_start_time + if agent._should_emit_quiet_tool_messages(): + agent._vprint(f" {_get_cute_tool_message_impl('read_preview', function_args, tool_duration, result=function_result)}") elif function_name == "delegate_task": tasks_arg = function_args.get("tasks") if tasks_arg and isinstance(tasks_arg, list): diff --git a/apps/desktop/src/app/chat/right-rail/preview-pane.tsx b/apps/desktop/src/app/chat/right-rail/preview-pane.tsx index 054600b5e2ae..cc088bf2e186 100644 --- a/apps/desktop/src/app/chat/right-rail/preview-pane.tsx +++ b/apps/desktop/src/app/chat/right-rail/preview-pane.tsx @@ -21,10 +21,13 @@ import { } from './preview-console' import { type ConsoleEntry } from './preview-console-state' import { LocalFilePreview, PreviewEmptyState } from './preview-file' +import { registerPreviewPageReader } from './preview-reader' import { previewConsoleState, registerPreviewDevTools } from './preview-strip-tools' type PreviewWebview = HTMLElement & { closeDevTools?: () => void + executeJavaScript?: (code: string) => Promise + getTitle?: () => string getURL?: () => string isDevToolsOpened?: () => boolean openDevTools?: () => void @@ -317,6 +320,32 @@ export function PreviewPane({ embedded = false, onRestartServer, reloadRequest = return () => registerPreviewDevTools(tabId, null) }, [devtoolsOpen, isRemoteHtml, isWebPreview, tabId, toggleDevTools]) + // Publish the PAGE reader for this tab (the read_preview tool): extract the + // rendered page's title + visible text from the webview. innerText (not + // textContent) so hidden nodes and script/style bodies stay out, matching + // what the user actually sees. + useEffect(() => { + if (!isWebPreview || !tabId) { + return + } + + return registerPreviewPageReader(tabId, async () => { + const webview = webviewRef.current + + if (!webview?.executeJavaScript) { + throw new Error('preview webview is not ready') + } + + const text = await webview.executeJavaScript('document.body ? document.body.innerText : ""') + + return { + text: typeof text === 'string' ? text : '', + title: webview.getTitle?.() ?? '', + url: webview.getURL?.() ?? '' + } + }) + }, [isWebPreview, tabId]) + // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment) useEffect(() => { if (!consoleOpen) { diff --git a/apps/desktop/src/app/chat/right-rail/preview-reader.test.ts b/apps/desktop/src/app/chat/right-rail/preview-reader.test.ts new file mode 100644 index 000000000000..9e9d6359e26b --- /dev/null +++ b/apps/desktop/src/app/chat/right-rail/preview-reader.test.ts @@ -0,0 +1,140 @@ +import { beforeEach, describe, expect, it } from 'vitest' + +import { $rightRailActiveTabId, selectRightRailTab } from '@/store/layout' +import { closeRightRail, openPreview, type PreviewTarget } from '@/store/preview' + +import { PREVIEW_READ_MAX_CHARS, readActivePreview, registerPreviewPageReader } from './preview-reader' + +function urlTarget(url: string): PreviewTarget { + return { kind: 'url', label: 'Browser', source: url, url } +} + +function fileTarget(path: string): PreviewTarget { + return { kind: 'file', label: path, path, previewKind: 'text', source: path, url: `file://${path}` } +} + +describe('readActivePreview (read_preview tool)', () => { + // All URL targets share the singleton Browser tab id, so a reader registered + // in one test would answer the next — unregister whatever a test installed. + let cleanups: Array<() => void> = [] + + const register = (tabId: string, reader: Parameters[1]) => { + const unregister = registerPreviewPageReader(tabId, reader) + + cleanups.push(unregister) + + return unregister + } + + beforeEach(() => { + for (const cleanup of cleanups) { + cleanup() + } + + cleanups = [] + closeRightRail() + window.localStorage.clear() + }) + + it('answers null when nothing is open, so the tool reports it cleanly', async () => { + expect(await readActivePreview()).toBeNull() + }) + + it('serializes the Browser tab through its registered page reader', async () => { + openPreview(urlTarget('https://news.ycombinator.com'), 'tool-result') + register($rightRailActiveTabId.get()!, async () => ({ + text: 'Top stories…', + title: 'Hacker News', + url: 'https://news.ycombinator.com/news' + })) + + expect(await readActivePreview()).toMatchObject({ + kind: 'url', + text: 'Top stories…', + title: 'Hacker News', + total_chars: 12, + // The live address wins over the target (in-page navigation). + url: 'https://news.ycombinator.com/news' + }) + }) + + it('windows long pages with start/count and reports the full length', async () => { + openPreview(urlTarget('https://example.com'), 'tool-result') + register($rightRailActiveTabId.get()!, async () => ({ + text: 'abcdefghij', + title: 't', + url: '' + })) + + expect(await readActivePreview({ count: 4, start: 2 })).toMatchObject({ + end: 6, + start: 2, + text: 'cdef', + total_chars: 10 + }) + }) + + it('caps a single read at PREVIEW_READ_MAX_CHARS even when asked for more', async () => { + openPreview(urlTarget('https://example.com'), 'tool-result') + register($rightRailActiveTabId.get()!, async () => ({ + text: 'x'.repeat(PREVIEW_READ_MAX_CHARS + 5000), + title: 't', + url: '' + })) + + const result = await readActivePreview({ count: PREVIEW_READ_MAX_CHARS + 5000 }) + + expect(result?.text).toHaveLength(PREVIEW_READ_MAX_CHARS) + expect(result?.total_chars).toBe(PREVIEW_READ_MAX_CHARS + 5000) + }) + + it('answers identity + retry note for a Browser tab whose pane is not mounted', async () => { + openPreview(urlTarget('https://example.com'), 'tool-result') + + expect(await readActivePreview()).toMatchObject({ + kind: 'url', + note: expect.stringContaining('retry') as string, + text: '', + url: 'https://example.com' + }) + }) + + it('answers a file tab with its identity and points at read_file', async () => { + openPreview(fileTarget('/work/notes.md'), 'file-browser') + + expect(await readActivePreview()).toMatchObject({ + kind: 'file', + note: expect.stringContaining('read_file') as string, + path: '/work/notes.md' + }) + }) + + it('reads the tab the user is LOOKING at, not the last one opened', async () => { + openPreview(fileTarget('/work/one.md'), 'file-browser') + openPreview(fileTarget('/work/two.md'), 'file-browser') + selectRightRailTab('file:file:///work/one.md') + + expect(await readActivePreview()).toMatchObject({ path: '/work/one.md' }) + }) + + it('falls back to the identity answer when the reader throws (webview booting)', async () => { + openPreview(urlTarget('https://example.com'), 'tool-result') + register($rightRailActiveTabId.get()!, async () => { + throw new Error('webview gone') + }) + + expect(await readActivePreview()).toMatchObject({ note: expect.stringContaining('retry') as string, text: '' }) + }) + + it('unregister is idempotent and scoped to the same reader', async () => { + openPreview(urlTarget('https://example.com'), 'tool-result') + const tabId = $rightRailActiveTabId.get()! + const first = register(tabId, async () => ({ text: 'first', title: '', url: '' })) + + register(tabId, async () => ({ text: 'second', title: '', url: '' })) + // Unregistering the STALE reader must not evict the live one. + first() + + expect(await readActivePreview()).toMatchObject({ text: 'second' }) + }) +}) diff --git a/apps/desktop/src/app/chat/right-rail/preview-reader.ts b/apps/desktop/src/app/chat/right-rail/preview-reader.ts new file mode 100644 index 000000000000..7f48c8cbb158 --- /dev/null +++ b/apps/desktop/src/app/chat/right-rail/preview-reader.ts @@ -0,0 +1,122 @@ +/** + * PREVIEW READER — the read_preview tool's window into the preview pane, the + * preview analog of the terminal's buffer registry (see right-sidebar/ + * terminal/buffer.ts). + * + * A URL/HTML preview renders in a sandboxed owned by PreviewPane; + * that pane registers a PAGE READER here (url + title + rendered text), keyed + * by tab id. `readActivePreview` resolves the ACTIVE tab from the store and + * owns the windowing: a registered reader answers with the live page's text; + * a tab with no reader (a file peek, an artifact) still answers with its + * identity and a note pointing the agent at the tool that reads that content + * directly (read_file / the conversation's artifact). + */ + +import { $rightRailActiveTabId } from '@/store/layout' +import { $previewTabs } from '@/store/preview' + +export interface PreviewReadOptions { + /** Characters to return from `start` (capped at PREVIEW_READ_MAX_CHARS). */ + count?: number + /** 0-indexed character offset into the page text. */ + start?: number +} + +export interface PreviewReadResult { + end: number + kind: string + note?: string + path?: string + start: number + text: string + title: string + total_chars: number + url: string +} + +/** What a pane's page reader extracts — the reader module owns the windowing. */ +interface PreviewPage { + text: string + title: string + url: string +} + +type PageReader = () => Promise + +/** Default + hard cap on one read — a page's innerText can be megabytes, and + * this crosses the gateway into model context. Page with start/count. */ +export const PREVIEW_READ_MAX_CHARS = 24_000 + +const readers = new Map() + +/** Register a live preview's page reader; returns an idempotent unregister. */ +export function registerPreviewPageReader(tabId: string, reader: PageReader): () => void { + readers.set(tabId, reader) + + return () => { + if (readers.get(tabId) === reader) { + readers.delete(tabId) + } + } +} + +function windowText( + base: Omit, + text: string, + opts: PreviewReadOptions +): PreviewReadResult { + const total = text.length + const from = Math.max(0, Math.min(opts.start ?? 0, total)) + const want = Math.min(Math.max(1, opts.count ?? PREVIEW_READ_MAX_CHARS), PREVIEW_READ_MAX_CHARS) + const to = Math.max(from, Math.min(from + want, total)) + + return { ...base, end: to, start: from, text: text.slice(from, to), total_chars: total } +} + +/** Read the ACTIVE preview tab. Null only when no tab is open at all. */ +export async function readActivePreview(opts: PreviewReadOptions = {}): Promise { + const tabs = $previewTabs.get() + const tab = tabs.find(t => t.id === $rightRailActiveTabId.get()) ?? tabs[0] + + if (!tab) { + return null + } + + const { target } = tab + const reader = readers.get(tab.id) + + if (reader) { + try { + const page = await reader() + + return windowText( + { kind: target.kind, path: target.path, title: page.title || target.label, url: page.url || target.url }, + page.text, + opts + ) + } catch { + // Webview not ready (still booting / just navigated) — fall through to + // the identity answer, whose note says to retry. + } + } + + // No live webview behind the tab (a file peek, an artifact, or a page still + // booting): answer with the tab's identity so the agent knows what's on + // screen and which of its own tools reads the content directly. + return windowText( + { + kind: target.kind, + note: + target.kind === 'file' + ? 'File preview — read the file itself with read_file.' + : target.kind === 'artifact' + ? 'Generated artifact — its content is in the conversation that produced it.' + : 'The page has not finished loading — retry in a moment.', + path: target.path, + title: target.label, + url: target.url + }, + '', + opts + ) +} diff --git a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event.ts b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event.ts index 7493cf19e673..35bf230381a2 100644 --- a/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event.ts +++ b/apps/desktop/src/app/session/hooks/use-message-stream/gateway-event.ts @@ -3,6 +3,7 @@ import type { HermesSkin } from '@hermes/shared/skin' import type { QueryClient } from '@tanstack/react-query' import { type MutableRefObject, useCallback, useEffect, useRef } from 'react' +import { readActivePreview } from '@/app/chat/right-rail/preview-reader' import { writeAgentTerminalChunk } from '@/app/right-sidebar/terminal/agent-terminal-stream' import { readActiveTerminal } from '@/app/right-sidebar/terminal/buffer' import { closeAgentTerminalByProc } from '@/app/right-sidebar/terminal/terminals' @@ -1010,6 +1011,22 @@ export function useGatewayEventHandler(deps: GatewayEventDeps) { text: result ? JSON.stringify(result) : '' }) } + } else if (event.type === 'preview.read.request') { + // read_preview tool: serialize the active preview tab (a Browser + // webview's page text is async) and answer. Empty text = nothing open. + const requestId = typeof payload?.request_id === 'string' ? payload.request_id : '' + + if (requestId) { + const start = typeof payload?.start === 'number' ? payload.start : undefined + const count = typeof payload?.count === 'number' ? payload.count : undefined + + void readActivePreview({ count, start }).then(result => { + void $gateway.get()?.request('preview.read.respond', { + request_id: requestId, + text: result ? JSON.stringify(result) : '' + }) + }) + } } else if (event.type === 'agent.terminal.output') { // Live chunk from a background process → its read-only agent terminal tab. writeAgentTerminalChunk(payload?.process_id ?? '', payload?.chunk ?? '') diff --git a/apps/desktop/src/lib/chat-messages.ts b/apps/desktop/src/lib/chat-messages.ts index f50f4ed55080..1e0bd1b0a454 100644 --- a/apps/desktop/src/lib/chat-messages.ts +++ b/apps/desktop/src/lib/chat-messages.ts @@ -81,7 +81,8 @@ export type GatewayEventPayload = { // secret.request (skill credential capture) env_var?: string prompt?: string - // terminal.read.request (GUI agent reading the in-app terminal pane) + // terminal.read.request / preview.read.request (GUI agent reading the + // in-app terminal pane or the browser/preview pane) start?: number count?: number // status.update (kind=process → background process completion/watch-match) diff --git a/run_agent.py b/run_agent.py index 23eb9777b532..3e13c6493f66 100644 --- a/run_agent.py +++ b/run_agent.py @@ -469,6 +469,7 @@ def __init__( reasoning_callback: callable = None, clarify_callback: callable = None, read_terminal_callback: callable = None, + read_preview_callback: callable = None, step_callback: callable = None, stream_delta_callback: callable = None, interim_assistant_callback: callable = None, @@ -553,6 +554,7 @@ def __init__( reasoning_callback=reasoning_callback, clarify_callback=clarify_callback, read_terminal_callback=read_terminal_callback, + read_preview_callback=read_preview_callback, step_callback=step_callback, stream_delta_callback=stream_delta_callback, interim_assistant_callback=interim_assistant_callback, diff --git a/tests/run_agent/test_run_agent.py b/tests/run_agent/test_run_agent.py index 721dafbeae4f..a4d5fea1e7ba 100644 --- a/tests/run_agent/test_run_agent.py +++ b/tests/run_agent/test_run_agent.py @@ -2283,6 +2283,7 @@ class TestAgentRuntimePostHookOwnershipSync: ("memory", {"action": "view", "target": "memory"}), ("clarify", {"question": "Continue?"}), ("read_terminal", {}), + ("read_preview", {}), ("delegate_task", {"goal": "Check the child path"}), ) @@ -2322,6 +2323,10 @@ def test_agent_runtime_tools_emit_once_per_executor_path( "tools.read_terminal_tool.read_terminal_tool", lambda **kwargs: '{"ok":true}', ) + monkeypatch.setattr( + "tools.read_preview_tool.read_preview_tool", + lambda **kwargs: '{"ok":true}', + ) monkeypatch.setattr(agent, "_get_session_db_for_recall", lambda: None) monkeypatch.setattr( agent, diff --git a/tests/tools/test_read_preview_tool.py b/tests/tools/test_read_preview_tool.py new file mode 100644 index 000000000000..3bbc03fb47c6 --- /dev/null +++ b/tests/tools/test_read_preview_tool.py @@ -0,0 +1,63 @@ +"""Tests for the desktop-gated ``read_preview`` tool.""" + +import json + +from tools import read_preview_tool as rp + + +def test_gated_on_desktop(monkeypatch): + """Hidden unless HERMES_DESKTOP is set (mirrors read_terminal).""" + monkeypatch.delenv("HERMES_DESKTOP", raising=False) + assert rp.check_read_preview_requirements() is False + + monkeypatch.setenv("HERMES_DESKTOP", "1") + assert rp.check_read_preview_requirements() is True + + +def test_requires_callback(): + """Outside the desktop GUI there is no bridge — a clear error, no crash.""" + result = json.loads(rp.read_preview_tool(callback=None)) + assert "desktop" in result["error"] + + +def test_empty_answer_means_nothing_open(): + result = json.loads(rp.read_preview_tool(callback=lambda **_: "")) + assert "error" in result + + +def test_passes_json_through(): + payload = {"kind": "url", "url": "https://news.ycombinator.com", "title": "HN", "text": "hello"} + result = json.loads(rp.read_preview_tool(callback=lambda **_: json.dumps(payload))) + assert result == payload + + +def test_wraps_non_json_text(): + result = json.loads(rp.read_preview_tool(callback=lambda **_: "plain words")) + assert result == {"text": "plain words"} + + +def test_window_forwarded_and_validated(): + seen = {} + + def cb(**kwargs): + seen.update(kwargs) + return json.dumps({"kind": "url"}) + + rp.read_preview_tool(start=100, count=500, callback=cb) + assert seen == {"start": 100, "count": 500} + + # Floors mirror read_terminal: start >= 0, count >= 1. + seen.clear() + rp.read_preview_tool(start=-5, count=0, callback=cb) + assert seen == {"start": 0, "count": 1} + + result = json.loads(rp.read_preview_tool(start="lots", callback=cb)) + assert "integers" in result["error"] + + +def test_callback_failure_is_reported(): + def _boom(**_kwargs): + raise RuntimeError("renderer went away") + + result = json.loads(rp.read_preview_tool(callback=_boom)) + assert "renderer went away" in result["error"] diff --git a/tools/read_preview_tool.py b/tools/read_preview_tool.py new file mode 100644 index 000000000000..cbe32d74f99f --- /dev/null +++ b/tools/read_preview_tool.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Read the in-app browser / preview pane in the Hermes desktop GUI. + +The preview's content lives in the desktop renderer (a sandboxed ```` +for URL tabs), so this tool round-trips through the gateway's blocking-prompt +bridge — the same one ``read_terminal`` uses: tui_gateway emits +``preview.read.request``, the renderer serializes the active preview tab and +answers with ``preview.read.respond``. This module is just schema + a thin +dispatcher over the platform-injected callback. +""" + +import json +from typing import Callable, Optional + +from tools.registry import registry, tool_error +from utils import env_var_enabled + + +def read_preview_tool( + start: Optional[int] = None, + count: Optional[int] = None, + callback: Optional[Callable] = None, +) -> str: + """Return the active preview tab's contents (+ metadata) as a JSON string.""" + if callback is None: + return tool_error("read_preview is only available in the Hermes desktop app.") + + try: + window = { + key: max(floor, int(val)) + for key, val, floor in (("start", start, 0), ("count", count, 1)) + if val is not None + } + except (TypeError, ValueError): + return tool_error("start and count must be integers.") + + try: + raw = callback(**window) + except Exception as exc: + return tool_error(f"Failed to read the preview pane: {exc}") + + if not raw: + return tool_error("No preview tab is open, or the read timed out.") + + # Desktop answers with a JSON object; pass it through, else wrap the raw text. + try: + return json.dumps(json.loads(raw), ensure_ascii=False) + except (TypeError, ValueError): + return json.dumps({"text": str(raw)}, ensure_ascii=False) + + +def check_read_preview_requirements() -> bool: + """Desktop GUI only — HERMES_DESKTOP is set on the gateway the app spawns.""" + return env_var_enabled("HERMES_DESKTOP") + + +READ_PREVIEW_SCHEMA = { + "name": "read_preview", + "description": ( + "Read what's currently shown in the in-app browser / preview pane of the " + "Hermes desktop GUI (the pane open_preview opens beside this chat). Call " + "with no arguments for the first window of the active tab's content. " + "Returns JSON {kind, url, title, text, start, end, total_chars, note?}: " + "a URL (Browser) tab's text is the rendered page's visible text — page " + "through longer pages with `start`/`count` (character offsets, capped " + "per read); a file tab answers identity only (read the file with " + "read_file); an artifact tab points back at the conversation. Use after " + "open_preview, or whenever the user refers to what's on screen in the " + "browser ('what does this page say?')." + ), + "parameters": { + "type": "object", + "properties": { + "start": { + "type": "integer", + "description": "0-indexed character offset into the page text. Omit for the start.", + }, + "count": { + "type": "integer", + "description": "Characters to return from start. Defaults to (and is capped at) the per-read maximum.", + }, + }, + }, +} + + +registry.register( + name="read_preview", + toolset="terminal", + schema=READ_PREVIEW_SCHEMA, + handler=lambda args, **kw: read_preview_tool( + start=args.get("start"), + count=args.get("count"), + callback=kw.get("callback"), + ), + check_fn=check_read_preview_requirements, + emoji="🔍", +) diff --git a/toolsets.py b/toolsets.py index f4bb3c3343a9..41f31426cb13 100644 --- a/toolsets.py +++ b/toolsets.py @@ -37,7 +37,7 @@ # read-only terminal tab, open a URL/file in the preview pane, focus a # pane, and react to a message with an emoji (all gated on HERMES_DESKTOP # via check_fn — hidden outside the GUI). - "read_terminal", "close_terminal", "open_preview", "focus_pane", "react_to_message", + "read_terminal", "close_terminal", "open_preview", "read_preview", "focus_pane", "react_to_message", # File manipulation "read_file", "write_file", "patch", "search_files", # Vision + image generation @@ -376,6 +376,7 @@ "tools": [ "web_search", "web_extract", "terminal", "process", "read_terminal", "close_terminal", + "open_preview", "read_preview", "read_file", "write_file", "patch", "search_files", "vision_analyze", "skills_list", "skill_view", "skill_manage", diff --git a/tui_gateway/methods_prompt.py b/tui_gateway/methods_prompt.py index 7e3157544e0d..e662ff2f36bc 100644 --- a/tui_gateway/methods_prompt.py +++ b/tui_gateway/methods_prompt.py @@ -894,6 +894,14 @@ def _(rid, params: dict) -> dict: return _respond(rid, params, "text", allow_expired=True) +@method("preview.read.respond") +def _(rid, params: dict) -> dict: + # `text` is a JSON string of the active preview tab's serialized contents. + # allow_expired=True for the same reason as terminal.read: the tool's + # bounded wait can expire while a slow page extraction is still running. + return _respond(rid, params, "text", allow_expired=True) + + @method("sudo.respond") def _(rid, params: dict) -> dict: return _respond(rid, params, "password", allow_expired=True) diff --git a/tui_gateway/server.py b/tui_gateway/server.py index a36a539408b1..76a7f585cfef 100644 --- a/tui_gateway/server.py +++ b/tui_gateway/server.py @@ -3138,6 +3138,7 @@ def _block(event: str, sid: str, payload: dict, timeout: float | None = 300) -> "sudo.request", "clarify.request", "terminal.read.request", + "preview.read.request", }: _emit( f"{event.removesuffix('.request')}.expire", @@ -5674,6 +5675,16 @@ def _agent_cbs(sid: str) -> dict: {k: v for k, v in (("start", start), ("count", count)) if v is not None}, timeout=30, ), + # read_preview tool (desktop GUI): the renderer serializes the active + # preview tab (a Browser webview's readable text, a file's identity) + # and answers preview.read.respond. Longer timeout than the terminal + # read — a URL tab extracts text from a live page. + "read_preview_callback": lambda start=None, count=None: _block( + "preview.read.request", + sid, + {k: v for k, v in (("start", start), ("count", count)) if v is not None}, + timeout=45, + ), } # Interim assistant commentary (text alongside tool calls, or the attempted diff --git a/website/docs/reference/tools-reference.md b/website/docs/reference/tools-reference.md index fee732350aa8..40804584f735 100644 --- a/website/docs/reference/tools-reference.md +++ b/website/docs/reference/tools-reference.md @@ -8,7 +8,7 @@ description: "Authoritative reference for Hermes built-in tools, grouped by tool This page documents Hermes' built-in tools, grouped by toolset. Availability varies by platform, credentials, and enabled toolsets. -**Quick counts (current registry):** ~81 tools — 10 browser tools (core) + 2 CDP-gated browser tools, 4 file tools, 4 Home Assistant tools, 6 terminal tools (`terminal`, `process`, plus desktop-GUI-gated `read_terminal`, `close_terminal`, `open_preview`, `focus_pane`), 2 web tools, 5 Feishu tools, 7 Spotify tools (registered by the bundled `spotify` plugin), 5 Yuanbao tools, 12 kanban tools (registered when the kanban dispatcher spawns the agent), 3 project tools (desktop/GUI sessions), 2 Discord tools, 3 video tools (`video_generate`, `xai_video_edit`, `xai_video_extend`), and a handful of standalone tools (`memory`, `clarify`, `delegate_task`, `execute_code`, `cronjob`, `session_search`, `skill_view`/`skill_manage`/`skills_list`, `text_to_speech`, `image_generate`, `vision_analyze`, `video_analyze`, `todo`, `computer_use`, `x_search`). +**Quick counts (current registry):** ~82 tools — 10 browser tools (core) + 2 CDP-gated browser tools, 4 file tools, 4 Home Assistant tools, 7 terminal tools (`terminal`, `process`, plus desktop-GUI-gated `read_terminal`, `close_terminal`, `open_preview`, `read_preview`, `focus_pane`), 2 web tools, 5 Feishu tools, 7 Spotify tools (registered by the bundled `spotify` plugin), 5 Yuanbao tools, 12 kanban tools (registered when the kanban dispatcher spawns the agent), 3 project tools (desktop/GUI sessions), 2 Discord tools, 3 video tools (`video_generate`, `xai_video_edit`, `xai_video_extend`), and a handful of standalone tools (`memory`, `clarify`, `delegate_task`, `execute_code`, `cronjob`, `session_search`, `skill_view`/`skill_manage`/`skills_list`, `text_to_speech`, `image_generate`, `vision_analyze`, `video_analyze`, `todo`, `computer_use`, `x_search`). :::tip MCP Tools In addition to built-in tools, Hermes can load tools dynamically from MCP servers. MCP tools appear with the prefix `mcp____` (e.g., `mcp__github__create_issue` for the `github` MCP server). See [MCP Integration](/user-guide/features/mcp) for configuration. @@ -174,6 +174,7 @@ Tools for driving desktop [Projects](../user-guide/cli.md) — named, multi-fold | `read_terminal` | Read what's currently shown in the in-app terminal pane of the Hermes desktop GUI (the embedded shell beside this chat). Desktop-app only. | — | | `close_terminal` | Close the read-only terminal tab for a background process in the Hermes desktop GUI. Does NOT kill the process — only drops the tab/view; use process(action='kill') to stop it. Desktop-app only. | — | | `open_preview` | Open a web URL, localhost dev-server URL, or file path in the preview pane beside the chat in the Hermes desktop app. Desktop-app only. | — | +| `read_preview` | Read what's currently shown in the preview pane of the Hermes desktop GUI — the in-app Browser's page text (URL + title + rendered text, pageable with `start`/`count`), or a file/artifact tab's identity. Desktop-app only. | — | | `focus_pane` | Reveal and focus a pane in the Hermes desktop app (chat, files, terminal, review, sessions). Desktop-app only. | — | ## `todo` toolset diff --git a/website/docs/reference/toolsets-reference.md b/website/docs/reference/toolsets-reference.md index 05b24828633c..85328073864c 100644 --- a/website/docs/reference/toolsets-reference.md +++ b/website/docs/reference/toolsets-reference.md @@ -77,7 +77,7 @@ Or in-session: | `session_search` | `session_search` | Search past conversation sessions. | | `skills` | `skill_manage`, `skill_view`, `skills_list` | Skill CRUD and browsing. | | `spotify` | `spotify_albums`, `spotify_devices`, `spotify_library`, `spotify_playback`, `spotify_playlists`, `spotify_queue`, `spotify_search` | Native Spotify control (playback, queue, search, playlists, albums, library). Registered by the bundled `spotify` plugin. | -| `terminal` | `close_terminal`, `focus_pane`, `open_preview`, `process`, `read_terminal`, `terminal` | Shell command execution and background process management. `read_terminal`, `close_terminal`, `open_preview`, and `focus_pane` drive the desktop GUI's embedded panes and are check_fn-gated — they only register in desktop-app sessions. | +| `terminal` | `close_terminal`, `focus_pane`, `open_preview`, `process`, `read_preview`, `read_terminal`, `terminal` | Shell command execution and background process management. `read_terminal`, `close_terminal`, `open_preview`, `read_preview`, and `focus_pane` drive the desktop GUI's embedded panes and are check_fn-gated — they only register in desktop-app sessions. | | `todo` | `todo` | Task list management within a session. | | `tts` | `text_to_speech` | Text-to-speech audio generation. | | `vision` | `vision_analyze` | Image analysis via vision-capable models. | @@ -92,7 +92,7 @@ Platform toolsets define the complete tool configuration for a deployment target | Toolset | Differences from `hermes-cli` | |---------|-------------------------------| -| `hermes-cli` | Full toolset — the default for interactive CLI sessions. Includes file, terminal (plus the desktop-GUI pane tools `read_terminal`, `close_terminal`, `open_preview`, `focus_pane`), web, browser, memory, skills, vision, image_gen, todo, tts, delegation, code_execution, cronjob, session_search, clarify, computer_use, Home Assistant, and the kanban tools (all check_fn-gated at runtime). | +| `hermes-cli` | Full toolset — the default for interactive CLI sessions. Includes file, terminal (plus the desktop-GUI pane tools `read_terminal`, `close_terminal`, `open_preview`, `read_preview`, `focus_pane`), web, browser, memory, skills, vision, image_gen, todo, tts, delegation, code_execution, cronjob, session_search, clarify, computer_use, Home Assistant, and the kanban tools (all check_fn-gated at runtime). | | `hermes-acp` | Drops `clarify`, `cronjob`, `image_generate`, `text_to_speech`, `computer_use`, all four Home Assistant tools, the kanban tools, and the desktop-GUI pane tools. Focused on coding tasks in IDE context. | | `hermes-api-server` | Drops `clarify`, `text_to_speech`, `computer_use`, the kanban tools, and the desktop-GUI pane tools. Keeps everything else — suitable for programmatic access where user interaction isn't possible. | | `hermes-cron` | Same as `hermes-cli`. |