feat(app): add embedded-browser automation tools for the agent (#1186)

packages/app/e2e/snap/browser-tools.snap.ts

@@ -0,0 +1,60 @@
+import { expect, type Locator } from "@playwright/test"
+import { fileURLToPath } from "node:url"
+import { test } from "../fixtures"
+import { composeGrid, snapOutputPath, type Shot } from "./_compose"
+
+test.use({ viewport: { width: 520, height: 420 }, deviceScaleFactor: 2 })
+
+const LANGUAGE_KEY = "pawwork.global.dat:language"
+const fixturePath = fileURLToPath(new URL("./fixtures/browser-tools-fixture.tsx", import.meta.url))
+
+async function captureBlock(name: string, block: Locator): Promise<Shot> {
+  await expect(block).toBeVisible({ timeout: 30_000 })
+  return { name, buf: await block.screenshot() }
+}
+
+async function waitForThemeBoot(page: import("@playwright/test").Page): Promise<void> {
+  await page.waitForFunction(
+    () => getComputedStyle(document.documentElement).getPropertyValue("--bg-base").trim().length > 0,
+    null,
+    { timeout: 30_000 },
+  )
+}
+
+test("browser-tools", async ({ page }) => {
+  test.setTimeout(120_000)
+
+  await page.addInitScript((key) => {
+    localStorage.setItem(key, JSON.stringify({ locale: "zh" }))
+  }, LANGUAGE_KEY)
+
+  await page.goto("/")
+  await waitForThemeBoot(page)
+  await page.evaluate(async (path) => {
+    const mod = await import(path)
+    mod.mountBrowserToolsFixture(document.body)
+  }, `/@fs/${fixturePath}`)
+
+  const cards = page.locator('[data-snap="browser-tool-cards"]')
+  await expect(cards).toBeVisible({ timeout: 30_000 })
+
+  // Each tool renders its localized title via the shared toolInfoForInput map,
+  // not the generic "调用 browser_navigate" fallback.
+  await expect(cards).toContainText("打开网页", { timeout: 30_000 })
+  await expect(cards).toContainText("网页截图", { timeout: 30_000 })
+  await expect(cards).toContainText("提取文本", { timeout: 30_000 })
+  await expect(cards).toContainText("等待", { timeout: 30_000 })
+  await expect(cards).toContainText("点击", { timeout: 30_000 })
+  await expect(cards).toContainText("输入", { timeout: 30_000 })
+  // Subtitle carries the call's target (url / selector).
+  await expect(cards).toContainText("https://news.ycombinator.com/", { timeout: 30_000 })
+  await expect(cards).toContainText("main article", { timeout: 30_000 })
+  // The leading "browser" family icon is supplied by the trow summary via
+  // toolIcon() (locked separately in tool-info.test.ts); these standalone cards
+  // exercise the card title/subtitle content the timeline shows.
+
+  const shots: Shot[] = [await captureBlock("browser-tool-cards", cards)]
+  const out = snapOutputPath("browser-tools")
+  await composeGrid(shots, out, { cols: 1 })
+  process.stdout.write(`\n[snap] browser-tools grid -> ${out}\n\n`)
+})

packages/app/e2e/snap/fixtures/browser-tools-fixture.tsx

@@ -0,0 +1,55 @@
+import { For } from "solid-js"
+import { Dynamic, render } from "solid-js/web"
+import { I18nProvider } from "@opencode-ai/ui/context"
+import { ToolRegistry } from "@opencode-ai/ui/message-part"
+import { zhI18n } from "./trow-snap-fixture-data"
+
+// One representative call per embedded-browser tool, so the snap shows each
+// card's icon, title, and subtitle exactly as the timeline renders them.
+const CARDS: Array<{ tool: string; input: Record<string, unknown> }> = [
+  { tool: "browser_navigate", input: { url: "https://news.ycombinator.com/" } },
+  { tool: "browser_screenshot", input: {} },
+  { tool: "browser_extract", input: { selector: "main article" } },
+  { tool: "browser_wait", input: { selector: ".results" } },
+  { tool: "browser_click", input: { selector: "button[type=submit]" } },
+  { tool: "browser_type", input: { selector: "input[name=q]", text: "pawwork" } },
+]
+
+function BrowserToolsFixture() {
+  return (
+    <div
+      data-snap="browser-tool-cards"
+      style={{
+        display: "grid",
+        gap: "8px",
+        padding: "24px",
+        background: "var(--bg-base)",
+        color: "var(--fg-base)",
+        width: "440px",
+      }}
+    >
+      <For each={CARDS}>
+        {(card) => {
+          const component = ToolRegistry.render(card.tool)
+          return (
+            <div data-slot="trow-result-body" data-timeline-anchor={`tool:${card.tool}`}>
+              <Dynamic component={component} tool={card.tool} input={card.input} metadata={{}} status="completed" />
+            </div>
+          )
+        }}
+      </For>
+    </div>
+  )
+}
+
+export function mountBrowserToolsFixture(root: HTMLElement) {
+  root.innerHTML = ""
+  render(
+    () => (
+      <I18nProvider value={zhI18n}>
+        <BrowserToolsFixture />
+      </I18nProvider>
+    ),
+    root,
+  )
+}

packages/desktop-electron/src/main/browser/automation-bridge.ts

@@ -0,0 +1,25 @@
+import { resolveAutomationController } from "../ipc/browser"
+
+/**
+ * Inject the controller-backed browser automation into the in-process opencode
+ * server. The server's browser_* tools call BrowserBridge (a registrable port);
+ * here in main — where the WebContentsView controllers live — we register the
+ * concrete implementation. This is the inverse of the usual main -> server
+ * direction: main hands an implementation down into the server.
+ *
+ * Each call resolves the focused (or only) window fresh, so the agent always
+ * drives the window the user is looking at, and a window opened or closed later
+ * needs no re-registration. Tools are gated out of the registry on non-desktop
+ * clients, so this is the only place an implementation is ever registered.
+ */
+export async function registerBrowserAutomationBridge(): Promise<void> {
+  const { BrowserBridge } = await import("virtual:opencode-server")
+  BrowserBridge.register({
+    navigate: ({ url }) => resolveAutomationController().navigateAndReport(url),
+    screenshot: () => resolveAutomationController().captureScreenshot(),
+    extract: ({ selector, maxChars }) => resolveAutomationController().extractText(selector, maxChars),
+    waitFor: ({ selector, text, timeoutMs }) => resolveAutomationController().waitFor(selector, text, timeoutMs),
+    click: ({ selector }) => resolveAutomationController().clickSelector(selector),
+    type: ({ selector, text, submit }) => resolveAutomationController().typeText(selector, text, submit),
+  })
+}

packages/desktop-electron/src/main/browser/controller.ts

@@ -1,10 +1,27 @@
 import { WebContentsView, shell, type BrowserWindow } from "electron"
 import type { BrowserState, BrowserViewLayout } from "@opencode-ai/app/desktop-api"
 import { browserViewWebPreferences } from "./options"
-import { clearDataReloadAction, computeViewBounds, deriveBrowserState, parseNavigable, safeExternalUrl } from "./logic"
+import {
+  buildClickRectScript,
+  buildExtractScript,
+  buildFocusScript,
+  buildWaitScript,
+  clearDataReloadAction,
+  clickPointFromRect,
+  computeViewBounds,
+  deriveBrowserState,
+  parseNavigable,
+  safeExternalUrl,
+} from "./logic"
 
 export const BROWSER_STATE_CHANNEL = "browser:state"
 
+// How often browser_wait re-checks its page predicate. Human-paced page loads
+// don't need tighter polling, and each tick is a round-trip into the page.
+const WAIT_POLL_INTERVAL_MS = 200
+
+const delay = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms))
+
 /**
  * Owns one embedded browser per window: a WebContentsView painted over the
  * panel's content region. The view is a native layer above the DOM, so the
@@ -156,6 +173,83 @@ export class BrowserViewController {
     }
   }
 
+  // --- Agent automation (BrowserBridge) ---
+  // These drive the same WebContentsView the user sees: the agent acts on the
+  // live, logged-in page (page scripts via executeJavaScript, synthetic input via
+  // sendInputEvent), not a separate headless fetch. Return shapes are plain JSON so
+  // they cross the in-process bridge into the opencode server unchanged.
+
+  async navigateAndReport(input: string): Promise<{ url: string; title: string }> {
+    const url = parseNavigable(input)
+    if (!url) throw new Error("URL must start with http:// or https://")
+    await this.loadInternal(url)
+    return { url: this.wc.getURL(), title: this.wc.getTitle() }
+  }
+
+  async captureScreenshot(): Promise<{ mime: string; base64: string; width: number; height: number }> {
+    const image = await this.wc.capturePage()
+    const size = image.getSize()
+    return { mime: "image/png", base64: image.toPNG().toString("base64"), width: size.width, height: size.height }
+  }
+
+  async extractText(
+    selector: string | undefined,
+    maxChars: number,
+  ): Promise<{ url: string; title: string; text: string; truncated: boolean }> {
+    const raw = await this.wc.executeJavaScript(buildExtractScript(selector), true)
+    const text = typeof raw === "string" ? raw : ""
+    const truncated = text.length > maxChars
+    return {
+      url: this.wc.getURL(),
+      title: this.wc.getTitle(),
+      text: truncated ? text.slice(0, maxChars) : text,
+      truncated,
+    }
+  }
+
+  async waitFor(
+    selector: string | undefined,
+    text: string | undefined,
+    timeoutMs: number,
+  ): Promise<{ found: boolean; waitedMs: number; reason: "selector" | "text" | "timeout" }> {
+    const script = buildWaitScript(selector, text)
+    const start = Date.now()
+    const deadline = start + timeoutMs
+    while (Date.now() < deadline) {
+      if (this.destroyed || this.wc.isDestroyed()) break
+      const hit = await this.wc.executeJavaScript(script, true).catch(() => false)
+      if (hit === true) return { found: true, waitedMs: Date.now() - start, reason: selector ? "selector" : "text" }
+      await delay(WAIT_POLL_INTERVAL_MS)
+    }
+    return { found: false, waitedMs: Date.now() - start, reason: "timeout" }
+  }
+
+  async clickSelector(selector: string): Promise<{ matched: boolean; x: number; y: number }> {
+    const rect = await this.wc.executeJavaScript(buildClickRectScript(selector), true)
+    const point = clickPointFromRect(rect)
+    if (!point) return { matched: false, x: 0, y: 0 }
+    this.wc.sendInputEvent({ type: "mouseDown", x: point.x, y: point.y, button: "left", clickCount: 1 })
+    this.wc.sendInputEvent({ type: "mouseUp", x: point.x, y: point.y, button: "left", clickCount: 1 })
+    return { matched: true, x: point.x, y: point.y }
+  }
+
+  async typeText(
+    selector: string | undefined,
+    text: string,
+    submit: boolean,
+  ): Promise<{ matched: boolean; submitted: boolean }> {
+    if (selector) {
+      const focused = await this.wc.executeJavaScript(buildFocusScript(selector), true).catch(() => false)
+      if (focused !== true) return { matched: false, submitted: false }
+    }
+    for (const char of text) this.wc.sendInputEvent({ type: "char", keyCode: char })
+    if (submit) {
+      this.wc.sendInputEvent({ type: "keyDown", keyCode: "Return" })
+      this.wc.sendInputEvent({ type: "keyUp", keyCode: "Return" })
+    }
+    return { matched: true, submitted: submit }
+  }
+
   destroy() {
     if (this.destroyed) return
     this.destroyed = true

packages/desktop-electron/src/main/browser/logic.test.ts

@@ -1,6 +1,11 @@
 import { describe, expect, test } from "bun:test"
 import {
+  buildClickRectScript,
+  buildExtractScript,
+  buildFocusScript,
+  buildWaitScript,
   clearDataReloadAction,
+  clickPointFromRect,
   computeViewBounds,
   deriveBrowserState,
   parseNavigable,
@@ -111,3 +116,63 @@ describe("clearDataReloadAction", () => {
     expect(clearDataReloadAction({ hasPage: false, loading: false })).toBe("none")
   })
 })
+
+describe("buildExtractScript", () => {
+  test("reads the whole body when no selector is given", () => {
+    const script = buildExtractScript()
+    expect(script).toContain("const sel = null;")
+    expect(script).toContain("document.body")
+  })
+
+  test("JSON-encodes the selector so page-supplied input can't escape the literal", () => {
+    const hostile = 'a"]); alert(1); ("'
+    const script = buildExtractScript(hostile)
+    // The selector only ever appears as a fully-quoted JSON string, never spliced
+    // in raw (which would let a stray `"` close the literal and inject code).
+    expect(script).toContain(`const sel = ${JSON.stringify(hostile)};`)
+    expect(script).not.toContain('const sel = a"')
+  })
+})
+
+describe("buildWaitScript", () => {
+  test("uses a selector predicate when a selector is given", () => {
+    const script = buildWaitScript(".ready")
+    expect(script).toContain(`const sel = ${JSON.stringify(".ready")};`)
+    expect(script).toContain("querySelector(sel)")
+  })
+
+  test("falls back to a body-text predicate when only text is given", () => {
+    const script = buildWaitScript(undefined, "Done")
+    expect(script).toContain("const sel = null;")
+    expect(script).toContain(`const txt = ${JSON.stringify("Done")};`)
+    expect(script).toContain("includes(txt)")
+  })
+})
+
+describe("buildClickRectScript / buildFocusScript", () => {
+  test("encode the selector and return the rect / focus result", () => {
+    expect(buildClickRectScript("#go")).toContain(`querySelector(${JSON.stringify("#go")})`)
+    expect(buildClickRectScript("#go")).toContain("getBoundingClientRect()")
+    expect(buildFocusScript("#field")).toContain(`querySelector(${JSON.stringify("#field")})`)
+    expect(buildFocusScript("#field")).toContain("document.activeElement === el")
+  })
+
+  test("click scroll is instant so the rect is read after the scroll settles", () => {
+    // A page's CSS `scroll-behavior: smooth` would otherwise make scrollIntoView
+    // async, leaving getBoundingClientRect on a stale rect.
+    expect(buildClickRectScript("#go")).toContain('behavior: "instant"')
+  })
+})
+
+describe("clickPointFromRect", () => {
+  test("returns the rounded center of a real rect", () => {
+    expect(clickPointFromRect({ x: 10, y: 20, width: 30, height: 40 })).toEqual({ x: 25, y: 40 })
+    expect(clickPointFromRect({ x: 0.5, y: 0.5, width: 3, height: 3 })).toEqual({ x: 2, y: 2 })
+  })
+
+  test("returns null for a missing or zero-area rect", () => {
+    expect(clickPointFromRect(null)).toBeNull()
+    expect(clickPointFromRect({ x: 0, y: 0, width: 0, height: 10 })).toBeNull()
+    expect(clickPointFromRect({ x: 0, y: 0, width: 10, height: 0 })).toBeNull()
+  })
+})