Hermes can read the in-app browser

agent/agent_init.py

@@ -492,6 +492,7 @@ def init_agent(
     reasoning_callback: callable = None,
     clarify_callback: callable = None,
     read_terminal_callback: callable = None,
+    read_preview_callback: callable = None,
     step_callback: callable = None,
     stream_delta_callback: callable = None,
     interim_assistant_callback: callable = None,
@@ -757,6 +758,7 @@ def init_agent(
     agent.reasoning_callback = reasoning_callback
     agent.clarify_callback = clarify_callback
     agent.read_terminal_callback = read_terminal_callback
+    agent.read_preview_callback = read_preview_callback
     agent.step_callback = step_callback
     agent.stream_delta_callback = stream_delta_callback
     agent.interim_assistant_callback = interim_assistant_callback

agent/agent_runtime_helpers.py

@@ -98,7 +98,7 @@ def _ra():
 
 
 AGENT_RUNTIME_POST_HOOK_TOOL_NAMES = frozenset(
-    {"todo", "session_search", "memory", "clarify", "read_terminal", "delegate_task"}
+    {"todo", "session_search", "memory", "clarify", "read_terminal", "read_preview", "delegate_task"}
 )
 
 
@@ -2979,6 +2979,17 @@ def _execute(next_args: dict) -> Any:
                 ),
                 next_args,
             )
+    elif function_name == "read_preview":
+        def _execute(next_args: dict) -> Any:
+            from tools.read_preview_tool import read_preview_tool as _read_preview_tool
+            return _finish_agent_tool(
+                _read_preview_tool(
+                    start=next_args.get("start"),
+                    count=next_args.get("count"),
+                    callback=getattr(agent, "read_preview_callback", None),
+                ),
+                next_args,
+            )
     elif function_name == "delegate_task":
         def _execute(next_args: dict) -> Any:
             return _finish_agent_tool(agent._dispatch_delegate_task(next_args), next_args)

agent/tool_executor.py

@@ -1677,6 +1677,27 @@ def _execute(next_args: dict) -> Any:
             tool_duration = time.time() - tool_start_time
             if agent._should_emit_quiet_tool_messages():
                 agent._vprint(f"  {_get_cute_tool_message_impl('read_terminal', function_args, tool_duration, result=function_result)}")
+        elif function_name == "read_preview":
+            def _execute(next_args: dict) -> Any:
+                from tools.read_preview_tool import read_preview_tool as _read_preview_tool
+                return _read_preview_tool(
+                    start=next_args.get("start"),
+                    count=next_args.get("count"),
+                    callback=getattr(agent, "read_preview_callback", None),
+                )
+            function_result, function_args, middleware_trace, _execution_blocked, _execution_dispatched = _managed_values(_run_agent_tool_execution_middleware(
+                agent,
+                function_name=function_name,
+                function_args=function_args,
+                effective_task_id=effective_task_id,
+                tool_call_id=getattr(tool_call, "id", "") or "",
+                execute=_execute,
+                scope_block=_ts_scope_block,
+                display_index=i,
+            ))
+            tool_duration = time.time() - tool_start_time
+            if agent._should_emit_quiet_tool_messages():
+                agent._vprint(f"  {_get_cute_tool_message_impl('read_preview', function_args, tool_duration, result=function_result)}")
         elif function_name == "delegate_task":
             tasks_arg = function_args.get("tasks")
             if tasks_arg and isinstance(tasks_arg, list):

apps/desktop/src/app/chat/right-rail/preview-pane.tsx

@@ -21,10 +21,13 @@ import {
 } from './preview-console'
 import { type ConsoleEntry } from './preview-console-state'
 import { LocalFilePreview, PreviewEmptyState } from './preview-file'
+import { registerPreviewPageReader } from './preview-reader'
 import { previewConsoleState, registerPreviewDevTools } from './preview-strip-tools'
 
 type PreviewWebview = HTMLElement & {
   closeDevTools?: () => void
+  executeJavaScript?: (code: string) => Promise<unknown>
+  getTitle?: () => string
   getURL?: () => string
   isDevToolsOpened?: () => boolean
   openDevTools?: () => void
@@ -317,6 +320,32 @@ export function PreviewPane({ embedded = false, onRestartServer, reloadRequest =
     return () => registerPreviewDevTools(tabId, null)
   }, [devtoolsOpen, isRemoteHtml, isWebPreview, tabId, toggleDevTools])
 
+  // Publish the PAGE reader for this tab (the read_preview tool): extract the
+  // rendered page's title + visible text from the webview. innerText (not
+  // textContent) so hidden nodes and script/style bodies stay out, matching
+  // what the user actually sees.
+  useEffect(() => {
+    if (!isWebPreview || !tabId) {
+      return
+    }
+
+    return registerPreviewPageReader(tabId, async () => {
+      const webview = webviewRef.current
+
+      if (!webview?.executeJavaScript) {
+        throw new Error('preview webview is not ready')
+      }
+
+      const text = await webview.executeJavaScript('document.body ? document.body.innerText : ""')
+
+      return {
+        text: typeof text === 'string' ? text : '',
+        title: webview.getTitle?.() ?? '',
+        url: webview.getURL?.() ?? ''
+      }
+    })
+  }, [isWebPreview, tabId])
+
   // eslint-disable-next-line no-restricted-syntax -- legitimate non-atom ref write (see eslint rule comment)
   useEffect(() => {
     if (!consoleOpen) {

apps/desktop/src/app/chat/right-rail/preview-reader.test.ts

@@ -0,0 +1,140 @@
+import { beforeEach, describe, expect, it } from 'vitest'
+
+import { $rightRailActiveTabId, selectRightRailTab } from '@/store/layout'
+import { closeRightRail, openPreview, type PreviewTarget } from '@/store/preview'
+
+import { PREVIEW_READ_MAX_CHARS, readActivePreview, registerPreviewPageReader } from './preview-reader'
+
+function urlTarget(url: string): PreviewTarget {
+  return { kind: 'url', label: 'Browser', source: url, url }
+}
+
+function fileTarget(path: string): PreviewTarget {
+  return { kind: 'file', label: path, path, previewKind: 'text', source: path, url: `file://${path}` }
+}
+
+describe('readActivePreview (read_preview tool)', () => {
+  // All URL targets share the singleton Browser tab id, so a reader registered
+  // in one test would answer the next — unregister whatever a test installed.
+  let cleanups: Array<() => void> = []
+
+  const register = (tabId: string, reader: Parameters<typeof registerPreviewPageReader>[1]) => {
+    const unregister = registerPreviewPageReader(tabId, reader)
+
+    cleanups.push(unregister)
+
+    return unregister
+  }
+
+  beforeEach(() => {
+    for (const cleanup of cleanups) {
+      cleanup()
+    }
+
+    cleanups = []
+    closeRightRail()
+    window.localStorage.clear()
+  })
+
+  it('answers null when nothing is open, so the tool reports it cleanly', async () => {
+    expect(await readActivePreview()).toBeNull()
+  })
+
+  it('serializes the Browser tab through its registered page reader', async () => {
+    openPreview(urlTarget('https://news.ycombinator.com'), 'tool-result')
+    register($rightRailActiveTabId.get()!, async () => ({
+      text: 'Top stories…',
+      title: 'Hacker News',
+      url: 'https://news.ycombinator.com/news'
+    }))
+
+    expect(await readActivePreview()).toMatchObject({
+      kind: 'url',
+      text: 'Top stories…',
+      title: 'Hacker News',
+      total_chars: 12,
+      // The live address wins over the target (in-page navigation).
+      url: 'https://news.ycombinator.com/news'
+    })
+  })
+
+  it('windows long pages with start/count and reports the full length', async () => {
+    openPreview(urlTarget('https://example.com'), 'tool-result')
+    register($rightRailActiveTabId.get()!, async () => ({
+      text: 'abcdefghij',
+      title: 't',
+      url: ''
+    }))
+
+    expect(await readActivePreview({ count: 4, start: 2 })).toMatchObject({
+      end: 6,
+      start: 2,
+      text: 'cdef',
+      total_chars: 10
+    })
+  })
+
+  it('caps a single read at PREVIEW_READ_MAX_CHARS even when asked for more', async () => {
+    openPreview(urlTarget('https://example.com'), 'tool-result')
+    register($rightRailActiveTabId.get()!, async () => ({
+      text: 'x'.repeat(PREVIEW_READ_MAX_CHARS + 5000),
+      title: 't',
+      url: ''
+    }))
+
+    const result = await readActivePreview({ count: PREVIEW_READ_MAX_CHARS + 5000 })
+
+    expect(result?.text).toHaveLength(PREVIEW_READ_MAX_CHARS)
+    expect(result?.total_chars).toBe(PREVIEW_READ_MAX_CHARS + 5000)
+  })
+
+  it('answers identity + retry note for a Browser tab whose pane is not mounted', async () => {
+    openPreview(urlTarget('https://example.com'), 'tool-result')
+
+    expect(await readActivePreview()).toMatchObject({
+      kind: 'url',
+      note: expect.stringContaining('retry') as string,
+      text: '',
+      url: 'https://example.com'
+    })
+  })
+
+  it('answers a file tab with its identity and points at read_file', async () => {
+    openPreview(fileTarget('/work/notes.md'), 'file-browser')
+
+    expect(await readActivePreview()).toMatchObject({
+      kind: 'file',
+      note: expect.stringContaining('read_file') as string,
+      path: '/work/notes.md'
+    })
+  })
+
+  it('reads the tab the user is LOOKING at, not the last one opened', async () => {
+    openPreview(fileTarget('/work/one.md'), 'file-browser')
+    openPreview(fileTarget('/work/two.md'), 'file-browser')
+    selectRightRailTab('file:file:///work/one.md')
+
+    expect(await readActivePreview()).toMatchObject({ path: '/work/one.md' })
+  })
+
+  it('falls back to the identity answer when the reader throws (webview booting)', async () => {
+    openPreview(urlTarget('https://example.com'), 'tool-result')
+    register($rightRailActiveTabId.get()!, async () => {
+      throw new Error('webview gone')
+    })
+
+    expect(await readActivePreview()).toMatchObject({ note: expect.stringContaining('retry') as string, text: '' })
+  })
+
+  it('unregister is idempotent and scoped to the same reader', async () => {
+    openPreview(urlTarget('https://example.com'), 'tool-result')
+    const tabId = $rightRailActiveTabId.get()!
+    const first = register(tabId, async () => ({ text: 'first', title: '', url: '' }))
+
+    register(tabId, async () => ({ text: 'second', title: '', url: '' }))
+    // Unregistering the STALE reader must not evict the live one.
+    first()
+
+    expect(await readActivePreview()).toMatchObject({ text: 'second' })
+  })
+})

apps/desktop/src/app/chat/right-rail/preview-reader.ts

@@ -0,0 +1,122 @@
+/**
+ * PREVIEW READER — the read_preview tool's window into the preview pane, the
+ * preview analog of the terminal's buffer registry (see right-sidebar/
+ * terminal/buffer.ts).
+ *
+ * A URL/HTML preview renders in a sandboxed <webview> owned by PreviewPane;
+ * that pane registers a PAGE READER here (url + title + rendered text), keyed
+ * by tab id. `readActivePreview` resolves the ACTIVE tab from the store and
+ * owns the windowing: a registered reader answers with the live page's text;
+ * a tab with no reader (a file peek, an artifact) still answers with its
+ * identity and a note pointing the agent at the tool that reads that content
+ * directly (read_file / the conversation's artifact).
+ */
+
+import { $rightRailActiveTabId } from '@/store/layout'
+import { $previewTabs } from '@/store/preview'
+
+export interface PreviewReadOptions {
+  /** Characters to return from `start` (capped at PREVIEW_READ_MAX_CHARS). */
+  count?: number
+  /** 0-indexed character offset into the page text. */
+  start?: number
+}
+
+export interface PreviewReadResult {
+  end: number
+  kind: string
+  note?: string
+  path?: string
+  start: number
+  text: string
+  title: string
+  total_chars: number
+  url: string
+}
+
+/** What a pane's page reader extracts — the reader module owns the windowing. */
+interface PreviewPage {
+  text: string
+  title: string
+  url: string
+}
+
+type PageReader = () => Promise<PreviewPage>
+
+/** Default + hard cap on one read — a page's innerText can be megabytes, and
+ *  this crosses the gateway into model context. Page with start/count. */
+export const PREVIEW_READ_MAX_CHARS = 24_000
+
+const readers = new Map<string, PageReader>()
+
+/** Register a live preview's page reader; returns an idempotent unregister. */
+export function registerPreviewPageReader(tabId: string, reader: PageReader): () => void {
+  readers.set(tabId, reader)
+
+  return () => {
+    if (readers.get(tabId) === reader) {
+      readers.delete(tabId)
+    }
+  }
+}
+
+function windowText(
+  base: Omit<PreviewReadResult, 'end' | 'start' | 'text' | 'total_chars'>,
+  text: string,
+  opts: PreviewReadOptions
+): PreviewReadResult {
+  const total = text.length
+  const from = Math.max(0, Math.min(opts.start ?? 0, total))
+  const want = Math.min(Math.max(1, opts.count ?? PREVIEW_READ_MAX_CHARS), PREVIEW_READ_MAX_CHARS)
+  const to = Math.max(from, Math.min(from + want, total))
+
+  return { ...base, end: to, start: from, text: text.slice(from, to), total_chars: total }
+}
+
+/** Read the ACTIVE preview tab. Null only when no tab is open at all. */
+export async function readActivePreview(opts: PreviewReadOptions = {}): Promise<PreviewReadResult | null> {
+  const tabs = $previewTabs.get()
+  const tab = tabs.find(t => t.id === $rightRailActiveTabId.get()) ?? tabs[0]
+
+  if (!tab) {
+    return null
+  }
+
+  const { target } = tab
+  const reader = readers.get(tab.id)
+
+  if (reader) {
+    try {
+      const page = await reader()
+
+      return windowText(
+        { kind: target.kind, path: target.path, title: page.title || target.label, url: page.url || target.url },
+        page.text,
+        opts
+      )
+    } catch {
+      // Webview not ready (still booting / just navigated) — fall through to
+      // the identity answer, whose note says to retry.
+    }
+  }
+
+  // No live webview behind the tab (a file peek, an artifact, or a page still
+  // booting): answer with the tab's identity so the agent knows what's on
+  // screen and which of its own tools reads the content directly.
+  return windowText(
+    {
+      kind: target.kind,
+      note:
+        target.kind === 'file'
+          ? 'File preview — read the file itself with read_file.'
+          : target.kind === 'artifact'
+            ? 'Generated artifact — its content is in the conversation that produced it.'
+            : 'The page has not finished loading — retry in a moment.',
+      path: target.path,
+      title: target.label,
+      url: target.url
+    },
+    '',
+    opts
+  )
+}