From c4d81f0f05c76700055a18ec80af5ab34af2fdbb Mon Sep 17 00:00:00 2001 From: AarynLu Date: Tue, 4 Aug 2026 21:48:05 +0800 Subject: [PATCH 01/43] feat(browser-use): embedded APP-053 host plane + Composer slash command Wire Atmos in-app browser into Browser Use: loopback control plane on Desktop Electron (guest CDP/DOM), crates/browser-use with cua+embedded backends, atmos browser-use CLI, system skill, and / Browser Use slash in Welcome + Terminal composer. --- Cargo.lock | 11 + apps/cli/Cargo.toml | 1 + apps/cli/src/commands/browser_use.rs | 176 ++++++++ apps/cli/src/commands/mod.rs | 1 + apps/cli/src/main.rs | 8 + apps/cli/src/output.rs | 7 +- apps/desktop-electron/src/app-state.ts | 4 + .../browser-use-control.structural.test.ts | 28 ++ .../src/browser/browser-use-control.ts | 419 ++++++++++++++++++ .../src/browser/surface-manager.ts | 39 ++ apps/desktop-electron/src/main.ts | 10 + apps/web/messages/en.json | 8 + apps/web/messages/zh.json | 8 + .../components/TerminalAgentInputOverlay.tsx | 34 +- .../welcome/components/WelcomePage.tsx | 56 ++- .../welcome/hooks/use-welcome-slash-search.ts | 10 +- .../lib/__tests__/slash-browser-use.test.ts | 42 ++ .../features/welcome/lib/slash-browser-use.ts | 57 +++ crates/browser-use/Cargo.toml | 13 + crates/browser-use/src/backends/cua.rs | 226 ++++++++++ crates/browser-use/src/backends/embedded.rs | 324 ++++++++++++++ crates/browser-use/src/backends/mod.rs | 13 + crates/browser-use/src/engine_client.rs | 100 +++++ crates/browser-use/src/lib.rs | 91 ++++ crates/browser-use/src/types.rs | 107 +++++ crates/infra/src/utils/system_skill_sync.rs | 3 + skills/atmos-browser-use/SKILL.md | 67 +++ skills/system-skills-manifest.json | 3 + .../APP-053_desktop-browser-webview/TECH.md | 1 + 29 files changed, 1846 insertions(+), 21 deletions(-) create mode 100644 apps/cli/src/commands/browser_use.rs create mode 100644 apps/desktop-electron/src/browser/browser-use-control.structural.test.ts create mode 100644 apps/desktop-electron/src/browser/browser-use-control.ts create mode 100644 apps/web/src/features/welcome/lib/__tests__/slash-browser-use.test.ts create mode 100644 apps/web/src/features/welcome/lib/slash-browser-use.ts create mode 100644 crates/browser-use/Cargo.toml create mode 100644 crates/browser-use/src/backends/cua.rs create mode 100644 crates/browser-use/src/backends/embedded.rs create mode 100644 crates/browser-use/src/backends/mod.rs create mode 100644 crates/browser-use/src/engine_client.rs create mode 100644 crates/browser-use/src/lib.rs create mode 100644 crates/browser-use/src/types.rs create mode 100644 skills/atmos-browser-use/SKILL.md diff --git a/Cargo.lock b/Cargo.lock index cae762086..a83088a78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -490,6 +490,7 @@ name = "atmos" version = "2026.7.27" dependencies = [ "base64 0.22.1", + "browser-use", "chrono", "clap", "desktop-use", @@ -827,6 +828,16 @@ dependencies = [ "thiserror 2.0.17", ] +[[package]] +name = "browser-use" +version = "0.1.0" +dependencies = [ + "dirs 5.0.1", + "serde", + "serde_json", + "tempfile", +] + [[package]] name = "bs58" version = "0.5.1" diff --git a/apps/cli/Cargo.toml b/apps/cli/Cargo.toml index bcea658e3..c23a8a5a9 100644 --- a/apps/cli/Cargo.toml +++ b/apps/cli/Cargo.toml @@ -16,3 +16,4 @@ sysinfo = "0.30" tokio = { workspace = true } runtime-manager = { path = "../../crates/runtime-manager", features = ["supervisor"] } desktop-use = { path = "../../crates/desktop-use" } +browser-use = { path = "../../crates/browser-use" } diff --git a/apps/cli/src/commands/browser_use.rs b/apps/cli/src/commands/browser_use.rs new file mode 100644 index 000000000..a2d178c6b --- /dev/null +++ b/apps/cli/src/commands/browser_use.rs @@ -0,0 +1,176 @@ +//! `atmos browser-use` — page CDP control (separate from Desktop Use). + +use browser_use::{execute, BrowserAction, BrowserBackendKind, BrowserRequest}; +use clap::{Args, Subcommand}; +use serde_json::{json, Value}; + +pub async fn execute_cmd(command: BrowserUseCommand) -> Result { + let req = match command { + BrowserUseCommand::Prepare(a) => BrowserRequest { + backend: parse_backend(&a.backend)?, + action: BrowserAction::Prepare, + pid: a.pid, + window_id: a.window_id, + session: a.session, + profile_strategy: a.strategy, + ..Default::default() + }, + BrowserUseCommand::State(a) => BrowserRequest { + backend: parse_backend(&a.backend)?, + action: BrowserAction::State, + pid: a.pid, + window_id: a.window_id, + target_id: a.target_id, + tab_id: a.tab_id, + session: a.session, + ..Default::default() + }, + BrowserUseCommand::Click(a) => BrowserRequest { + backend: parse_backend(&a.backend)?, + action: BrowserAction::Click, + target_id: Some(a.target_id), + tab_id: Some(a.tab_id), + element_ref: Some(a.element_ref), + session: a.session, + ..Default::default() + }, + BrowserUseCommand::Type(a) => BrowserRequest { + backend: parse_backend(&a.backend)?, + action: BrowserAction::Type, + target_id: Some(a.target_id), + tab_id: Some(a.tab_id), + text: Some(a.text), + element_ref: a.element_ref, + session: a.session, + ..Default::default() + }, + BrowserUseCommand::Navigate(a) => BrowserRequest { + backend: parse_backend(&a.backend)?, + action: BrowserAction::Navigate, + target_id: Some(a.target_id), + tab_id: Some(a.tab_id), + url: Some(a.url), + session: a.session, + ..Default::default() + }, + }; + let res = execute(req); + serde_json::to_value(res).map_err(|e| e.to_string()) +} + +fn parse_backend(s: &str) -> Result { + BrowserBackendKind::parse(s).ok_or_else(|| { + format!( + "invalid --backend {:?} (use cua|external|embedded|atmos)", + s + ) + }) +} + +#[derive(Debug, Subcommand)] +pub enum BrowserUseCommand { + /// Attach / prepare a browser CDP endpoint (CUA external Chromium). + Prepare(PrepareArgs), + /// Read browser state / tabs / snapshot refs. + State(StateArgs), + /// Click a page element by ref in a bound tab. + Click(ClickArgs), + /// Type into the bound tab (optional ref). + Type(TypeArgs), + /// Navigate a bound tab to a URL. + Navigate(NavigateArgs), +} + +#[derive(Debug, Args)] +pub struct PrepareArgs { + /// Backend: `cua` (system Chromium) | `embedded` (Atmos in-app browser / APP-053). + #[arg(long, default_value = "cua")] + pub backend: String, + #[arg(long)] + pub pid: Option, + /// Required when --strategy existing_profile (engine 0.17). + #[arg(long)] + pub window_id: Option, + #[arg(long)] + pub session: Option, + /// existing_profile (needs --window-id) | isolated_new | isolated_named. Default: omit strategy. + #[arg(long)] + pub strategy: Option, +} + +#[derive(Debug, Args)] +pub struct StateArgs { + #[arg(long, default_value = "cua")] + pub backend: String, + /// Bind mode: native browser pid (with --window-id). + #[arg(long)] + pub pid: Option, + /// Bind mode: native window id (with --pid). + #[arg(long)] + pub window_id: Option, + /// Snapshot mode: opaque target id from prior bind. + #[arg(long)] + pub target_id: Option, + /// Snapshot mode: opaque tab id from prior bind. + #[arg(long)] + pub tab_id: Option, + #[arg(long)] + pub session: Option, +} + +#[derive(Debug, Args)] +pub struct ClickArgs { + #[arg(long, default_value = "cua")] + pub backend: String, + #[arg(long)] + pub target_id: String, + #[arg(long)] + pub tab_id: String, + #[arg(long = "ref")] + pub element_ref: String, + #[arg(long)] + pub session: Option, +} + +#[derive(Debug, Args)] +pub struct TypeArgs { + #[arg(long, default_value = "cua")] + pub backend: String, + #[arg(long)] + pub target_id: String, + #[arg(long)] + pub tab_id: String, + #[arg(long)] + pub text: String, + /// Element ref from state (required for CUA; optional for embedded active element). + #[arg(long = "ref")] + pub element_ref: Option, + #[arg(long)] + pub session: Option, +} + +#[derive(Debug, Args)] +pub struct NavigateArgs { + #[arg(long, default_value = "cua")] + pub backend: String, + #[arg(long)] + pub target_id: String, + #[arg(long)] + pub tab_id: String, + #[arg(long)] + pub url: String, + #[arg(long)] + pub session: Option, +} + +/// Helpful static note for agents (not a runtime command). +#[allow(dead_code)] +pub fn product_note() -> Value { + json!({ + "product": "Browser Use", + "no_mcp": true, + "desktop_use": "separate — OS windows / keys / AX", + "backends": ["cua", "embedded"], + "embedded": "Atmos Desktop in-app webview (persist:atmos-browser) via host control plane", + }) +} diff --git a/apps/cli/src/commands/mod.rs b/apps/cli/src/commands/mod.rs index 000db5418..dec722b89 100644 --- a/apps/cli/src/commands/mod.rs +++ b/apps/cli/src/commands/mod.rs @@ -1,3 +1,4 @@ +pub mod browser_use; pub mod canvas; pub mod computer; pub mod desktop_use; diff --git a/apps/cli/src/main.rs b/apps/cli/src/main.rs index 39a65d97f..2c2335e3f 100644 --- a/apps/cli/src/main.rs +++ b/apps/cli/src/main.rs @@ -4,6 +4,7 @@ mod output; use api_client::ApiClientArgs; use clap::{Parser, Subcommand}; +use commands::browser_use::{execute_cmd as execute_browser_use, BrowserUseCommand}; use commands::canvas::{execute as execute_canvas, CanvasCommand, CanvasOpts}; use commands::computer::{execute as execute_computer, ComputerCommand}; use commands::desktop_use::{execute as execute_desktop_use, DesktopUseCommand}; @@ -58,6 +59,12 @@ enum Commands { #[command(subcommand)] command: DesktopUseCommand, }, + /// Browser Use: page control for system Chromium (cua) or Atmos embedded browser. + #[command(name = "browser-use")] + BrowserUse { + #[command(subcommand)] + command: BrowserUseCommand, + }, /// Check for or install CLI updates. Update(UpdateArgs), } @@ -80,6 +87,7 @@ async fn run() -> Result<(), String> { Commands::Runtime { command } => execute_runtime(command).await, Commands::Computer { command } => execute_computer(command).await, Commands::DesktopUse { command } => execute_desktop_use(command).await, + Commands::BrowserUse { command } => execute_browser_use(command).await, Commands::Canvas { canvas, command } => execute_canvas(cli.api, canvas, command).await, Commands::Update(args) => execute_update(args).await, } diff --git a/apps/cli/src/output.rs b/apps/cli/src/output.rs index a5832c667..daf22f0b2 100644 --- a/apps/cli/src/output.rs +++ b/apps/cli/src/output.rs @@ -53,9 +53,10 @@ impl CommandKind { } else { UpdateVerb::Install }), - Commands::Canvas { .. } | Commands::Review { .. } | Commands::DesktopUse { .. } => { - Self::Json - } + Commands::Canvas { .. } + | Commands::Review { .. } + | Commands::DesktopUse { .. } + | Commands::BrowserUse { .. } => Self::Json, } } diff --git a/apps/desktop-electron/src/app-state.ts b/apps/desktop-electron/src/app-state.ts index 0291df03f..3ef9e60ee 100644 --- a/apps/desktop-electron/src/app-state.ts +++ b/apps/desktop-electron/src/app-state.ts @@ -1,5 +1,6 @@ import type { BrowserWindow } from "electron"; import type { BrowserSurfaceManager } from "./browser/surface-manager.js"; +import type { BrowserUseControlPlane } from "./browser/browser-use-control.js"; import type { TunnelService } from "./tunnel/service.js"; export type AppState = { @@ -7,6 +8,8 @@ export type AppState = { apiPort: number | null; mainWindow: BrowserWindow | null; browser: BrowserSurfaceManager | null; + /** Loopback control plane for `atmos browser-use --backend embedded`. */ + browserUseControl: BrowserUseControlPlane | null; tunnel: TunnelService | null; /** Whether this Electron process started the Server (vs reused existing). */ startedServer: boolean; @@ -18,6 +21,7 @@ export function createAppState(): AppState { apiPort: null, mainWindow: null, browser: null, + browserUseControl: null, tunnel: null, startedServer: false, }; diff --git a/apps/desktop-electron/src/browser/browser-use-control.structural.test.ts b/apps/desktop-electron/src/browser/browser-use-control.structural.test.ts new file mode 100644 index 000000000..d6c89cf7c --- /dev/null +++ b/apps/desktop-electron/src/browser/browser-use-control.structural.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +const root = join(import.meta.dir, ".."); + +describe("browser-use-control structural", () => { + it("control plane is wired from main boot", () => { + const main = readFileSync(join(root, "main.ts"), "utf8"); + expect(main).toContain("BrowserUseControlPlane"); + expect(main).toContain("browserUseControl"); + }); + + it("exposes guest accessors on surface manager", () => { + const sm = readFileSync(join(root, "browser/surface-manager.ts"), "utf8"); + expect(sm).toContain("getGuestWebContents"); + expect(sm).toContain("listBrowserUseSessions"); + }); + + it("control plane uses loopback + control.json", () => { + const src = readFileSync(join(root, "browser/browser-use-control.ts"), "utf8"); + expect(src).toContain("127.0.0.1"); + expect(src).toContain("control.json"); + expect(src).toContain("/v1/prepare"); + expect(src).toContain("/v1/state"); + expect(src).toContain("persist:atmos-browser"); + }); +}); diff --git a/apps/desktop-electron/src/browser/browser-use-control.ts b/apps/desktop-electron/src/browser/browser-use-control.ts new file mode 100644 index 000000000..ae6581125 --- /dev/null +++ b/apps/desktop-electron/src/browser/browser-use-control.ts @@ -0,0 +1,419 @@ +/** + * Atmos Browser Use — host control plane for APP-053 embedded webviews. + * + * Exposes loopback HTTP for `atmos browser-use --backend embedded`. + * Uses guest WebContents debugger / executeJavaScript (not user-Chrome prepare). + */ + +import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import { mkdirSync, writeFileSync, unlinkSync, existsSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import type { WebContents } from "electron"; +import type { BrowserSurfaceManager } from "./surface-manager.js"; + +const CONTROL_DIR = () => + process.env.ATMOS_BROWSER_USE_HOME?.trim() || + join(homedir(), ".atmos", "browser-use"); + +type SnapshotEl = { + ref: string; + tag: string; + role: string | null; + name: string; + href?: string | null; + value?: string | null; + rect: { x: number; y: number; width: number; height: number }; +}; + +type SessionCache = { + elements: SnapshotEl[]; + url: string; + title: string; +}; + +export class BrowserUseControlPlane { + private server: Server | null = null; + private port = 0; + private readonly manager: BrowserSurfaceManager; + private readonly snapshots = new Map(); + + constructor(manager: BrowserSurfaceManager) { + this.manager = manager; + } + + start(): { baseUrl: string; port: number } { + if (this.server) { + return { baseUrl: `http://127.0.0.1:${this.port}`, port: this.port }; + } + const server = createServer((req, res) => { + void this.handle(req, res); + }); + // Bind loopback only. + server.listen(0, "127.0.0.1", () => { + const addr = server.address(); + if (addr && typeof addr === "object") { + this.port = addr.port; + this.writeMeta(); + console.log( + `[browser-use] control plane http://127.0.0.1:${this.port}`, + ); + } + }); + this.server = server; + // port may still be 0 until listen callback; meta rewritten there + return { baseUrl: `http://127.0.0.1:${this.port || 0}`, port: this.port }; + } + + stop(): void { + try { + this.server?.close(); + } catch { + /* ignore */ + } + this.server = null; + const meta = join(CONTROL_DIR(), "control.json"); + if (existsSync(meta)) { + try { + unlinkSync(meta); + } catch { + /* ignore */ + } + } + } + + private writeMeta(): void { + const dir = CONTROL_DIR(); + mkdirSync(dir, { recursive: true }); + const payload = { + base_url: `http://127.0.0.1:${this.port}`, + port: this.port, + partition: "persist:atmos-browser", + protocol: "atmos-browser-use/v1", + updated_at: new Date().toISOString(), + }; + writeFileSync(join(dir, "control.json"), JSON.stringify(payload, null, 2)); + } + + private async readBody(req: IncomingMessage): Promise> { + const chunks: Buffer[] = []; + for await (const c of req) { + chunks.push(Buffer.isBuffer(c) ? c : Buffer.from(c)); + } + const raw = Buffer.concat(chunks).toString("utf8").trim(); + if (!raw) return {}; + try { + return JSON.parse(raw) as Record; + } catch { + return {}; + } + } + + private send(res: ServerResponse, status: number, body: unknown): void { + const data = JSON.stringify(body); + res.writeHead(status, { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(data), + }); + res.end(data); + } + + private guestFor(sessionId: string): WebContents | null { + return this.manager.getGuestWebContents(sessionId); + } + + private listSessions(): Array<{ + target_id: string; + tab_id: string; + url: string; + title: string; + bound: boolean; + }> { + return this.manager.listBrowserUseSessions(); + } + + private async snapshot(sessionId: string): Promise { + const guest = this.guestFor(sessionId); + if (!guest) { + throw new Error( + `no bound webview guest for target_id=${sessionId}; open Atmos Browser tab first`, + ); + } + const script = `(() => { + const selectors = 'a,button,input,textarea,select,summary,[role="button"],[role="link"],[role="textbox"],[contenteditable="true"]'; + const nodes = Array.from(document.querySelectorAll(selectors)).slice(0, 250); + return { + url: location.href, + title: document.title || '', + elements: nodes.map((el, i) => { + const r = el.getBoundingClientRect(); + const name = + (el.getAttribute('aria-label') || + el.getAttribute('placeholder') || + el.getAttribute('name') || + (el.innerText || '').trim() || + el.getAttribute('href') || + '').slice(0, 120); + return { + ref: 'e' + i, + tag: el.tagName.toLowerCase(), + role: el.getAttribute('role'), + name, + href: el.getAttribute('href'), + value: el.value != null ? String(el.value).slice(0, 200) : null, + rect: { x: r.x, y: r.y, width: r.width, height: r.height }, + }; + }), + }; + })()`; + const raw = (await guest.executeJavaScript(script, true)) as { + url: string; + title: string; + elements: SnapshotEl[]; + }; + const cache: SessionCache = { + url: raw?.url ?? guest.getURL(), + title: raw?.title ?? guest.getTitle(), + elements: Array.isArray(raw?.elements) ? raw.elements : [], + }; + this.snapshots.set(sessionId, cache); + return cache; + } + + private async clickRef(sessionId: string, ref: string): Promise { + const guest = this.guestFor(sessionId); + if (!guest) throw new Error(`no guest for ${sessionId}`); + const cache = this.snapshots.get(sessionId) ?? (await this.snapshot(sessionId)); + const el = cache.elements.find((e) => e.ref === ref); + if (!el) throw new Error(`unknown ref ${ref}; run state snapshot first`); + const x = el.rect.x + Math.max(1, el.rect.width / 2); + const y = el.rect.y + Math.max(1, el.rect.height / 2); + // Prefer CDP via debugger when free; fall back to DOM click by index. + try { + const dbg = guest.debugger; + let attachedHere = false; + if (!dbg.isAttached()) { + dbg.attach("1.3"); + attachedHere = true; + } + await dbg.sendCommand("Input.dispatchMouseEvent", { + type: "mousePressed", + x, + y, + button: "left", + clickCount: 1, + }); + await dbg.sendCommand("Input.dispatchMouseEvent", { + type: "mouseReleased", + x, + y, + button: "left", + clickCount: 1, + }); + if (attachedHere) { + try { + dbg.detach(); + } catch { + /* ignore */ + } + } + return; + } catch { + /* fall through */ + } + const idx = Number(String(ref).replace(/^e/, "")); + await guest.executeJavaScript( + `(() => { + const selectors = 'a,button,input,textarea,select,summary,[role="button"],[role="link"],[role="textbox"],[contenteditable="true"]'; + const nodes = Array.from(document.querySelectorAll(selectors)); + const el = nodes[${Number.isFinite(idx) ? idx : -1}]; + if (!el) throw new Error('ref not found'); + el.focus?.(); + el.click?.(); + return true; + })()`, + true, + ); + } + + private async typeRef( + sessionId: string, + ref: string | null | undefined, + text: string, + ): Promise { + const guest = this.guestFor(sessionId); + if (!guest) throw new Error(`no guest for ${sessionId}`); + if (ref) { + const cache = this.snapshots.get(sessionId) ?? (await this.snapshot(sessionId)); + const el = cache.elements.find((e) => e.ref === ref); + if (!el) throw new Error(`unknown ref ${ref}; run state snapshot first`); + const idx = Number(String(ref).replace(/^e/, "")); + await guest.executeJavaScript( + `(() => { + const selectors = 'a,button,input,textarea,select,summary,[role="button"],[role="link"],[role="textbox"],[contenteditable="true"]'; + const nodes = Array.from(document.querySelectorAll(selectors)); + const el = nodes[${Number.isFinite(idx) ? idx : -1}]; + if (!el) throw new Error('ref not found'); + el.focus?.(); + if ('value' in el) { + el.value = ${JSON.stringify(text)}; + el.dispatchEvent(new Event('input', { bubbles: true })); + el.dispatchEvent(new Event('change', { bubbles: true })); + } else if (el.isContentEditable) { + el.textContent = ${JSON.stringify(text)}; + } + return true; + })()`, + true, + ); + return; + } + // Insert into active element + await guest.executeJavaScript( + `(() => { + const el = document.activeElement; + if (!el) throw new Error('no focused element'); + if ('value' in el) { + el.value = ${JSON.stringify(text)}; + el.dispatchEvent(new Event('input', { bubbles: true })); + } else if (el.isContentEditable) { + el.textContent = ${JSON.stringify(text)}; + } + return true; + })()`, + true, + ); + } + + private async handle( + req: IncomingMessage, + res: ServerResponse, + ): Promise { + try { + const url = new URL(req.url || "/", "http://127.0.0.1"); + if (req.method === "GET" && url.pathname === "/v1/health") { + this.send(res, 200, { + ok: true, + sessions: this.listSessions().length, + }); + return; + } + if (req.method !== "POST") { + this.send(res, 405, { ok: false, error: "method not allowed" }); + return; + } + const body = await this.readBody(req); + const path = url.pathname; + + if (path === "/v1/prepare") { + const sessions = this.listSessions(); + this.send(res, 200, { + ok: true, + backend: "embedded", + partition: "persist:atmos-browser", + sessions, + note: + sessions.length === 0 + ? "No open Atmos Browser webviews. Open a Browser tab in Desktop first." + : "Embedded browser sessions ready (no user-Chrome prepare).", + }); + return; + } + + if (path === "/v1/state") { + const targetId = + typeof body.target_id === "string" ? body.target_id.trim() : ""; + if (!targetId) { + const sessions = this.listSessions(); + this.send(res, 200, { + ok: true, + mode: "bind", + sessions, + // Convenience: first bound session as default target + target_id: sessions.find((s) => s.bound)?.target_id ?? null, + tab_id: "main", + }); + return; + } + const snap = await this.snapshot(targetId); + this.send(res, 200, { + ok: true, + mode: "snapshot", + target_id: targetId, + tab_id: "main", + url: snap.url, + title: snap.title, + elements: snap.elements, + element_count: snap.elements.length, + }); + return; + } + + if (path === "/v1/click") { + const targetId = String(body.target_id || "").trim(); + const ref = String(body.ref || "").trim(); + if (!targetId || !ref) { + this.send(res, 400, { + ok: false, + error: "click requires target_id and ref", + error_code: "invalid_args", + }); + return; + } + await this.clickRef(targetId, ref); + this.send(res, 200, { ok: true, target_id: targetId, ref }); + return; + } + + if (path === "/v1/type") { + const targetId = String(body.target_id || "").trim(); + const text = String(body.text ?? ""); + const ref = + typeof body.ref === "string" && body.ref.trim() + ? body.ref.trim() + : null; + if (!targetId) { + this.send(res, 400, { + ok: false, + error: "type requires target_id", + error_code: "invalid_args", + }); + return; + } + await this.typeRef(targetId, ref, text); + this.send(res, 200, { ok: true, target_id: targetId, ref, typed: text.length }); + return; + } + + if (path === "/v1/navigate") { + const targetId = String(body.target_id || "").trim(); + const navUrl = String(body.url || "").trim(); + if (!targetId || !navUrl) { + this.send(res, 400, { + ok: false, + error: "navigate requires target_id and url", + error_code: "invalid_args", + }); + return; + } + // Host-owned navigation for detached / bookkeeping; guest loadURL when bound. + const guest = this.guestFor(targetId); + if (guest) { + await guest.loadURL(navUrl); + } else { + this.manager.navigate(targetId, navUrl); + } + this.send(res, 200, { ok: true, target_id: targetId, url: navUrl }); + return; + } + + this.send(res, 404, { ok: false, error: "not found" }); + } catch (e) { + this.send(res, 500, { + ok: false, + error: e instanceof Error ? e.message : String(e), + error_code: "browser_engine_failed", + }); + } + } +} diff --git a/apps/desktop-electron/src/browser/surface-manager.ts b/apps/desktop-electron/src/browser/surface-manager.ts index d592c7cfc..4c1673fe2 100644 --- a/apps/desktop-electron/src/browser/surface-manager.ts +++ b/apps/desktop-electron/src/browser/surface-manager.ts @@ -449,6 +449,45 @@ export class BrowserSurfaceManager { return new Set(this.surfaces.keys()); } + /** Bound guest WebContents for Browser Use embedded control (APP-053). */ + getGuestWebContents(sessionId: string): WebContents | null { + const s = this.surfaces.get(sessionId); + const g = s?.guestWebContents; + if (g && !g.isDestroyed()) return g; + return null; + } + + /** + * Session summaries for `atmos browser-use --backend embedded` prepare/state bind. + */ + listBrowserUseSessions(): Array<{ + target_id: string; + tab_id: string; + url: string; + title: string; + bound: boolean; + }> { + const out: Array<{ + target_id: string; + tab_id: string; + url: string; + title: string; + bound: boolean; + }> = []; + for (const s of this.surfaces.values()) { + const g = s.guestWebContents; + const bound = Boolean(g && !g.isDestroyed()); + out.push({ + target_id: s.sessionId, + tab_id: "main", + url: bound ? g!.getURL() : s.currentUrl, + title: bound ? g!.getTitle() : "", + bound, + }); + } + return out; + } + private attachWebContentsListeners( sessionId: string, wc: WebContents, diff --git a/apps/desktop-electron/src/main.ts b/apps/desktop-electron/src/main.ts index 932b1c768..c68e0b5d3 100644 --- a/apps/desktop-electron/src/main.ts +++ b/apps/desktop-electron/src/main.ts @@ -17,6 +17,7 @@ import { createMainWindow, uiBaseUrl } from "./windows/main-window.js"; import { markAllowWindowDestroy } from "./windows/close-behavior.js"; import { ensureMacDockVisible } from "./windows/mac-dock.js"; import { BrowserSurfaceManager } from "./browser/surface-manager.js"; +import { BrowserUseControlPlane } from "./browser/browser-use-control.js"; import { ALL_PROVIDERS, TunnelService } from "./tunnel/service.js"; import { mainLog, mainLogPath } from "./main-log.js"; import { existsSync } from "node:fs"; @@ -90,6 +91,10 @@ async function boot() { if (!state.browser) { state.browser = new BrowserSurfaceManager(state); } + if (!state.browserUseControl && state.browser) { + state.browserUseControl = new BrowserUseControlPlane(state.browser); + state.browserUseControl.start(); + } if (!state.tunnel) { state.tunnel = new TunnelService(); } @@ -206,6 +211,11 @@ if (!gotLock) { isQuitting = true; void (async () => { try { + try { + state.browserUseControl?.stop(); + } catch { + /* ignore */ + } if (process.platform === "darwin") { try { const { stopTriggerListener } = await import("./appshot/trigger.js"); diff --git a/apps/web/messages/en.json b/apps/web/messages/en.json index 695d3e263..0d51a9733 100644 --- a/apps/web/messages/en.json +++ b/apps/web/messages/en.json @@ -3292,6 +3292,10 @@ "disableSkill": { "label": "Dynamic Skills", "description": "Dynamically adjust which skills Agents can see — takes effect in the current session" + }, + "browserUse": { + "label": "Browser Use", + "description": "Control Atmos in-app browser pages (DOM click / type / navigate)" } } }, @@ -8212,6 +8216,10 @@ "label": "Dynamic Skills", "description": "Dynamically adjust which skills Agents can see — takes effect in the current session" }, + "browserUseCommand": { + "label": "Browser Use", + "description": "Control Atmos in-app browser pages (DOM click / type / navigate)" + }, "skillDisable": { "chip": "Dynamic Skills", "chipTooltip": "Filter and toggle skills for Agents in this context", diff --git a/apps/web/messages/zh.json b/apps/web/messages/zh.json index 0bb8c6bfb..c4769540e 100644 --- a/apps/web/messages/zh.json +++ b/apps/web/messages/zh.json @@ -3292,6 +3292,10 @@ "disableSkill": { "label": "Dynamic Skills", "description": "动态调整 Agent 可见的技能，当前 session 即可生效" + }, + "browserUse": { + "label": "Browser Use", + "description": "控制 Atmos 内置浏览器页面（DOM 点击 / 输入 / 导航）" } } }, @@ -8212,6 +8216,10 @@ "label": "Dynamic Skills", "description": "动态调整 Agent 可见的技能，当前 session 即可生效" }, + "browserUseCommand": { + "label": "Browser Use", + "description": "控制 Atmos 内置浏览器页面（DOM 点击 / 输入 / 导航）" + }, "skillDisable": { "chip": "Dynamic Skills", "chipTooltip": "筛选并开关当前上下文中的技能", diff --git a/apps/web/src/features/terminal/components/TerminalAgentInputOverlay.tsx b/apps/web/src/features/terminal/components/TerminalAgentInputOverlay.tsx index bc4d1f4fe..c17d56386 100644 --- a/apps/web/src/features/terminal/components/TerminalAgentInputOverlay.tsx +++ b/apps/web/src/features/terminal/components/TerminalAgentInputOverlay.tsx @@ -42,6 +42,12 @@ import { type WelcomeSlashPopoverState, } from "@/features/welcome/hooks/use-welcome-slash-navigation"; import { useWelcomeSlashSearch } from "@/features/welcome/hooks/use-welcome-slash-search"; +import { + BROWSER_USE_SLASH_COMMAND_ID, + buildBrowserUseSlashCommand, + matchesBrowserUseSlashQuery, + resolveBrowserUseSkillRef, +} from "@/features/welcome/lib/slash-browser-use"; import { getAgentContextDragItems, hasAgentContextDragData, @@ -248,6 +254,14 @@ export const TerminalAgentInputOverlay = React.forwardRef< const slashCommands = React.useMemo(() => { const query = slashPopover?.query.trim().toLowerCase() ?? ""; const commands: SlashCommandOption[] = []; + if (matchesBrowserUseSlashQuery(query)) { + commands.push( + buildBrowserUseSlashCommand({ + label: t("browserUseCommand.label"), + description: t("browserUseCommand.description"), + }), + ); + } if (onStartSideChat && (!query || "side".includes(query))) { commands.push({ id: "side", @@ -441,6 +455,7 @@ export const TerminalAgentInputOverlay = React.forwardRef< }); const { + allSkills, filteredAgents, filteredProjects, filteredSkills, @@ -526,6 +541,23 @@ export const TerminalAgentInputOverlay = React.forwardRef< enterDisableSkillsView(); return; } + if (command.id === BROWSER_USE_SLASH_COMMAND_ID) { + const popover = slashPopover; + if (!popover) return; + const skill = resolveBrowserUseSkillRef(allSkills); + composerRef.current?.applySlashAtRange( + popover.slashOffset, + popover.query.length, + { + kind: "skill", + absolutePath: skill.absolutePath, + name: skill.name, + }, + ); + setSlashPopover(null); + setSlashPopoverView("menu"); + return; + } if (command.id !== "side" && command.id !== "spawn") return; const popover = slashPopover; if (!popover) return; @@ -547,7 +579,7 @@ export const TerminalAgentInputOverlay = React.forwardRef< setSlashPopoverView("menu"); setSkillDisableFilter(""); }, - [createCapturePromptContext, enterDisableSkillsView, slashPopover], + [allSkills, createCapturePromptContext, enterDisableSkillsView, slashPopover], ); const { diff --git a/apps/web/src/features/welcome/components/WelcomePage.tsx b/apps/web/src/features/welcome/components/WelcomePage.tsx index f47b43681..9f04ff1c7 100644 --- a/apps/web/src/features/welcome/components/WelcomePage.tsx +++ b/apps/web/src/features/welcome/components/WelcomePage.tsx @@ -29,6 +29,12 @@ import { useWelcomeSlashNavigation, } from "@/features/welcome/hooks/use-welcome-slash-navigation"; import type { SlashPopoverView } from "@/features/welcome/components/SlashCommandPopover"; +import { + BROWSER_USE_SLASH_COMMAND_ID, + buildBrowserUseSlashCommand, + matchesBrowserUseSlashQuery, + resolveBrowserUseSkillRef, +} from "@/features/welcome/lib/slash-browser-use"; import { useProjectStore } from "@/features/project/store/use-project-store"; import { useProjects, @@ -242,26 +248,33 @@ const WelcomePage: React.FC = ({ } = useComposerDisableSkills(skillsContext); const slashCommands = React.useMemo(() => { - if (!skillsContext) return []; const query = slashPopover?.query.trim().toLowerCase() ?? ""; + const commands: SlashCommandOption[] = []; + if (matchesBrowserUseSlashQuery(query)) { + commands.push( + buildBrowserUseSlashCommand({ + label: t("slashPopover.browserUse.label"), + description: t("slashPopover.browserUse.description"), + }), + ); + } + if (!skillsContext) return commands; if ( - query && - !"dynamic-skills".includes(query) && - !"dynamic skills".includes(query) && - !"disable-skill".includes(query) && - !"disable skill".includes(query) && - !"disable".includes(query) && - !"skill".includes(query) + !query || + "dynamic-skills".includes(query) || + "dynamic skills".includes(query) || + "disable-skill".includes(query) || + "disable skill".includes(query) || + "disable".includes(query) || + "skill".includes(query) ) { - return []; - } - return [ - { + commands.push({ id: "dynamic-skills", label: t("slashPopover.disableSkill.label"), description: t("slashPopover.disableSkill.description"), - }, - ]; + }); + } + return commands; }, [skillsContext, slashPopover?.query, t]); const { autoExtractTodos, @@ -326,6 +339,7 @@ const WelcomePage: React.FC = ({ filteredAgents, filteredProjects, filteredSkills, + allSkills, isSkillsLoading, } = useWelcomeSlashSearch({ availableAgents, @@ -500,9 +514,21 @@ const WelcomePage: React.FC = ({ (command: SlashCommandOption) => { if (command.id === "dynamic-skills") { enterDisableSkillsView(); + return; + } + if (command.id === BROWSER_USE_SLASH_COMMAND_ID) { + const popover = slashPopover; + if (!popover) return; + const skill = resolveBrowserUseSkillRef(allSkills); + composerRef.current?.applySlashAtRange( + popover.slashOffset, + popover.query.length, + { kind: "skill", absolutePath: skill.absolutePath, name: skill.name }, + ); + setSlashPopover(null); } }, - [enterDisableSkillsView], + [allSkills, enterDisableSkillsView, slashPopover], ); const { diff --git a/apps/web/src/features/welcome/hooks/use-welcome-slash-search.ts b/apps/web/src/features/welcome/hooks/use-welcome-slash-search.ts index 278fcb5a9..766109f51 100644 --- a/apps/web/src/features/welcome/hooks/use-welcome-slash-search.ts +++ b/apps/web/src/features/welcome/hooks/use-welcome-slash-search.ts @@ -34,11 +34,16 @@ export function useWelcomeSlashSearch({ const debouncedSlashQuery = useDebouncedPopoverQuery(popover, 300); const skillsQuery = useSkillsListQuery(); - const skills = React.useMemo( - () => (skillsQuery.data?.skills ?? []).filter(isSlashSurfacedSkill), + const allSkills = React.useMemo( + () => skillsQuery.data?.skills ?? [], [skillsQuery.data?.skills], ); + const skills = React.useMemo( + () => allSkills.filter(isSlashSurfacedSkill), + [allSkills], + ); + const visibleSkills = React.useMemo( () => filterSlashSkillsForProject(skills, activeProjectId ?? null), [activeProjectId, skills], @@ -96,6 +101,7 @@ export function useWelcomeSlashSearch({ }, [debouncedSlashQuery, availableAgents, agentsFuse]); return { + allSkills, filteredAgents, filteredProjects, filteredSkills, diff --git a/apps/web/src/features/welcome/lib/__tests__/slash-browser-use.test.ts b/apps/web/src/features/welcome/lib/__tests__/slash-browser-use.test.ts new file mode 100644 index 000000000..30941ce97 --- /dev/null +++ b/apps/web/src/features/welcome/lib/__tests__/slash-browser-use.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "bun:test"; +import { + BROWSER_USE_SLASH_COMMAND_ID, + BROWSER_USE_SKILL_NAME, + buildBrowserUseSlashCommand, + matchesBrowserUseSlashQuery, + resolveBrowserUseSkillRef, +} from "@/features/welcome/lib/slash-browser-use"; + +describe("slash-browser-use", () => { + it("matches common queries", () => { + expect(matchesBrowserUseSlashQuery("")).toBe(true); + expect(matchesBrowserUseSlashQuery("browser")).toBe(true); + expect(matchesBrowserUseSlashQuery("cdp")).toBe(true); + expect(matchesBrowserUseSlashQuery("desktop-only-xyz")).toBe(false); + }); + + it("builds command id", () => { + const cmd = buildBrowserUseSlashCommand({ + label: "Browser Use", + description: "Page DOM control", + }); + expect(cmd.id).toBe(BROWSER_USE_SLASH_COMMAND_ID); + expect(cmd.label).toBe("Browser Use"); + }); + + it("resolves skill path from list or fallback", () => { + const found = resolveBrowserUseSkillRef([ + { + id: BROWSER_USE_SKILL_NAME, + name: BROWSER_USE_SKILL_NAME, + path: "/tmp/atmos-browser-use", + scope: "global", + } as never, + ]); + expect(found.absolutePath).toBe("/tmp/atmos-browser-use"); + + const fallback = resolveBrowserUseSkillRef([]); + expect(fallback.name).toBe(BROWSER_USE_SKILL_NAME); + expect(fallback.absolutePath).toContain("atmos-browser-use"); + }); +}); diff --git a/apps/web/src/features/welcome/lib/slash-browser-use.ts b/apps/web/src/features/welcome/lib/slash-browser-use.ts new file mode 100644 index 000000000..8018a1335 --- /dev/null +++ b/apps/web/src/features/welcome/lib/slash-browser-use.ts @@ -0,0 +1,57 @@ +import type { SkillInfo } from "@/api/ws-api"; +import type { SlashCommandOption } from "@/features/welcome/hooks/use-welcome-slash-navigation"; + +export const BROWSER_USE_SLASH_COMMAND_ID = "browser-use"; +export const BROWSER_USE_SKILL_NAME = "atmos-browser-use"; +/** Fallback when skills_list has not synced the system skill yet. */ +export const BROWSER_USE_SKILL_FALLBACK_PATH = + "~/.atmos/skills/.system/atmos-browser-use"; + +export function matchesBrowserUseSlashQuery(query: string): boolean { + const q = query.trim().toLowerCase(); + if (!q) return true; + return ( + "browser-use".includes(q) || + "browser use".includes(q) || + "atmos-browser-use".includes(q) || + "atmos browser use".includes(q) || + "browser".includes(q) || + "webview".includes(q) || + "cdp".includes(q) || + "page".includes(q) || + "dom".includes(q) + ); +} + +export function buildBrowserUseSlashCommand(opts: { + label: string; + description: string; +}): SlashCommandOption { + return { + id: BROWSER_USE_SLASH_COMMAND_ID, + label: opts.label, + description: opts.description, + }; +} + +export function resolveBrowserUseSkillRef(skills: SkillInfo[]): { + absolutePath: string; + name: string; +} { + const found = skills.find( + (s) => + s.name === BROWSER_USE_SKILL_NAME || + s.id === BROWSER_USE_SKILL_NAME || + s.path.includes("atmos-browser-use"), + ); + if (found) { + return { + absolutePath: found.path, + name: found.name || BROWSER_USE_SKILL_NAME, + }; + } + return { + absolutePath: BROWSER_USE_SKILL_FALLBACK_PATH, + name: BROWSER_USE_SKILL_NAME, + }; +} diff --git a/crates/browser-use/Cargo.toml b/crates/browser-use/Cargo.toml new file mode 100644 index 000000000..7c3cb916f --- /dev/null +++ b/crates/browser-use/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "browser-use" +version = "0.1.0" +edition = "2021" +description = "Atmos Browser Use — page CDP control (separate from Desktop Use)" + +[dependencies] +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +dirs = "5.0" + +[dev-dependencies] +tempfile = "3.10" diff --git a/crates/browser-use/src/backends/cua.rs b/crates/browser-use/src/backends/cua.rs new file mode 100644 index 000000000..09c1fbc3b --- /dev/null +++ b/crates/browser-use/src/backends/cua.rs @@ -0,0 +1,226 @@ +//! CUA external Chromium path via managed control engine tools. + +use serde_json::{json, Value}; + +use super::BrowserBackend; +use crate::engine_client; +use crate::types::{BrowserAction, BrowserError, BrowserRequest, BrowserResult}; + +#[derive(Debug, Default)] +pub struct CuaExternalBackend; + +pub fn build_tool_call(req: &BrowserRequest) -> Result<(&'static str, Value), String> { + let session = req + .session + .clone() + .unwrap_or_else(|| "atmos-browser-use".into()); + + match req.action { + BrowserAction::Prepare => { + let pid = req + .pid + .ok_or_else(|| "prepare requires --pid (browser process)".to_string())?; + let mut a = json!({ + "pid": pid, + "session": session, + }); + if let Some(wid) = req.window_id { + a["window_id"] = json!(wid); + } + let strategy = req + .profile_strategy + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()); + match strategy { + Some(s) if s.contains("existing") => { + if req.window_id.is_none() { + return Err( + "prepare with existing_profile requires --window-id (engine 0.17)" + .into(), + ); + } + a["strategy"] = json!({ "kind": "existing_profile" }); + } + Some(s) if s.contains("isolated_named") => { + a["profile"] = json!({ "mode": "isolated_named", "name": "atmos" }); + a["allow_launch"] = json!(true); + } + Some(s) if s.contains("isolated") => { + a["profile"] = json!({ "mode": "isolated_new" }); + a["allow_launch"] = json!(true); + } + _ => {} + } + Ok(("browser_prepare", a)) + } + BrowserAction::State => { + let mut a = json!({ "session": session }); + if let Some(pid) = req.pid { + a["pid"] = json!(pid); + } + if let Some(wid) = req.window_id { + a["window_id"] = json!(wid); + } + if let Some(t) = req.target_id.as_ref() { + a["target_id"] = json!(t); + } + if let Some(t) = req.tab_id.as_ref() { + a["tab_id"] = json!(t); + } + let bind = req.pid.is_some() && req.window_id.is_some(); + let snapshot = req.target_id.is_some() && req.tab_id.is_some(); + if !bind && !snapshot { + return Err( + "state requires bind (--pid + --window-id) or snapshot (--target-id + --tab-id)" + .into(), + ); + } + Ok(("get_browser_state", a)) + } + BrowserAction::Click => { + let target = req + .target_id + .as_ref() + .ok_or_else(|| "click requires --target-id (from browser-use state)".to_string())?; + let tab = req + .tab_id + .as_ref() + .ok_or_else(|| "click requires --tab-id".to_string())?; + let r = req + .element_ref + .as_ref() + .ok_or_else(|| "click requires --ref".to_string())?; + Ok(( + "browser_click", + json!({ + "session": session, + "target_id": target, + "tab_id": tab, + "ref": r, + }), + )) + } + BrowserAction::Type => { + let target = req + .target_id + .as_ref() + .ok_or_else(|| "type requires --target-id".to_string())?; + let tab = req + .tab_id + .as_ref() + .ok_or_else(|| "type requires --tab-id".to_string())?; + let text = req + .text + .as_ref() + .ok_or_else(|| "type requires --text".to_string())?; + let r = req + .element_ref + .as_ref() + .ok_or_else(|| "type requires --ref (CUA engine 0.17)".to_string())?; + Ok(( + "browser_type", + json!({ + "session": session, + "target_id": target, + "tab_id": tab, + "text": text, + "ref": r, + }), + )) + } + BrowserAction::Navigate => { + let target = req + .target_id + .as_ref() + .ok_or_else(|| "navigate requires --target-id".to_string())?; + let tab = req + .tab_id + .as_ref() + .ok_or_else(|| "navigate requires --tab-id".to_string())?; + let url = req + .url + .as_ref() + .ok_or_else(|| "navigate requires --url".to_string())?; + Ok(( + "browser_navigate", + json!({ + "session": session, + "target_id": target, + "tab_id": tab, + "url": url, + }), + )) + } + } +} + +impl BrowserBackend for CuaExternalBackend { + fn execute(&self, req: BrowserRequest) -> BrowserResult { + let action = match req.action { + BrowserAction::Prepare => "prepare", + BrowserAction::State => "state", + BrowserAction::Click => "click", + BrowserAction::Type => "type", + BrowserAction::Navigate => "navigate", + }; + let (tool, args) = match build_tool_call(&req) { + Ok(v) => v, + Err(e) => { + let err = BrowserError::InvalidArgs(e); + return BrowserResult { + ok: false, + action: action.into(), + backend: "cua".into(), + result: None, + error: Some(err.message()), + error_code: Some(err.code().into()), + }; + } + }; + match engine_client::call_tool(tool, &args) { + Ok(v) => { + // Soft-fail engine refusals + if let Some(status) = v.get("status").and_then(|s| s.as_str()) { + let s = status.to_ascii_lowercase(); + if s == "refused" || s == "error" || s == "failed" { + return BrowserResult { + ok: false, + action: action.into(), + backend: "cua".into(), + result: Some(v), + error: Some("browser engine refused or failed".into()), + error_code: Some("browser_engine_failed".into()), + }; + } + } + if v.get("refusal").is_some() { + return BrowserResult { + ok: false, + action: action.into(), + backend: "cua".into(), + result: Some(v), + error: Some("browser engine refused".into()), + error_code: Some("browser_engine_failed".into()), + }; + } + BrowserResult { + ok: true, + action: action.into(), + backend: "cua".into(), + result: Some(v), + error: None, + error_code: None, + } + } + Err(e) => BrowserResult { + ok: false, + action: action.into(), + backend: "cua".into(), + result: None, + error: Some(e), + error_code: Some("browser_engine_failed".into()), + }, + } + } +} diff --git a/crates/browser-use/src/backends/embedded.rs b/crates/browser-use/src/backends/embedded.rs new file mode 100644 index 000000000..4a55e2327 --- /dev/null +++ b/crates/browser-use/src/backends/embedded.rs @@ -0,0 +1,324 @@ +//! Atmos embedded browser (APP-053 webview) via host-owned control plane. +//! +//! Electron Desktop writes `~/.atmos/browser-use/control.json` with a loopback +//! HTTP base URL. This backend talks to that plane — **not** user-Chrome +//! `browser_prepare`. + +use std::fs; +use std::io::{Read, Write}; +use std::net::TcpStream; +use std::path::PathBuf; +use std::time::Duration; + +use serde_json::{json, Value}; + +use super::BrowserBackend; +use crate::types::{ + BrowserAction, BrowserError, BrowserRequest, BrowserResult, ERR_EMBEDDED_HOST_UNAVAILABLE, +}; + +#[derive(Debug, Default)] +pub struct EmbeddedBackend; + +fn control_dir() -> PathBuf { + if let Ok(p) = std::env::var("ATMOS_BROWSER_USE_HOME") { + if !p.is_empty() { + return PathBuf::from(p); + } + } + dirs::home_dir() + .unwrap_or_else(|| PathBuf::from(".")) + .join(".atmos") + .join("browser-use") +} + +fn control_meta_path() -> PathBuf { + control_dir().join("control.json") +} + +/// Resolve loopback base URL from host-written control.json, e.g. `http://127.0.0.1:18765`. +pub fn read_control_base_url() -> Result { + let path = control_meta_path(); + let raw = fs::read_to_string(&path).map_err(|_| { + format!( + "Atmos Browser Use host is not running (missing {}). Open Atmos Desktop with an in-app Browser tab, then retry --backend embedded.", + path.display() + ) + })?; + let v: Value = serde_json::from_str(&raw).map_err(|e| format!("invalid control.json: {e}"))?; + let base = v + .get("base_url") + .and_then(|x| x.as_str()) + .map(str::trim) + .filter(|s| !s.is_empty()) + .ok_or_else(|| "control.json missing base_url".to_string())?; + if !(base.starts_with("http://127.0.0.1") || base.starts_with("http://localhost")) { + return Err("control base_url must be loopback HTTP".into()); + } + Ok(base.trim_end_matches('/').to_string()) +} + +/// Minimal HTTP/1.1 POST JSON (no reqwest dep). +pub fn http_post_json(base: &str, path: &str, body: &Value) -> Result { + let url = format!("{base}{path}"); + let parsed = parse_http_url(&url)?; + let payload = serde_json::to_vec(body).map_err(|e| e.to_string())?; + let req = format!( + "POST {} HTTP/1.1\r\nHost: {}:{}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + parsed.path, + parsed.host, + parsed.port, + payload.len() + ); + let mut stream = TcpStream::connect((parsed.host.as_str(), parsed.port)) + .map_err(|e| format!("connect embedded host failed: {e}"))?; + stream.set_read_timeout(Some(Duration::from_secs(30))).ok(); + stream.set_write_timeout(Some(Duration::from_secs(10))).ok(); + stream + .write_all(req.as_bytes()) + .and_then(|_| stream.write_all(&payload)) + .map_err(|e| format!("write embedded host failed: {e}"))?; + let mut buf = Vec::new(); + stream + .read_to_end(&mut buf) + .map_err(|e| format!("read embedded host failed: {e}"))?; + let text = String::from_utf8_lossy(&buf); + let body_start = text + .find("\r\n\r\n") + .map(|i| i + 4) + .ok_or_else(|| "invalid HTTP response from embedded host".to_string())?; + let status_line = text.lines().next().unwrap_or(""); + if !status_line.contains(" 200 ") && !status_line.ends_with(" 200") { + // Still try parse body for structured error + if let Ok(v) = serde_json::from_str::(text[body_start..].trim()) { + return Ok(v); + } + return Err(format!("embedded host HTTP error: {}", status_line.trim())); + } + let body_str = text[body_start..].trim(); + if body_str.is_empty() { + return Ok(json!({ "ok": true })); + } + serde_json::from_str(body_str).map_err(|e| format!("embedded host JSON: {e}")) +} + +struct ParsedUrl { + host: String, + port: u16, + path: String, +} + +fn parse_http_url(url: &str) -> Result { + let rest = url + .strip_prefix("http://") + .ok_or_else(|| "only http:// loopback is supported".to_string())?; + let (hostport, path) = match rest.split_once('/') { + Some((h, p)) => (h, format!("/{p}")), + None => (rest, "/".into()), + }; + let (host, port) = if let Some((h, p)) = hostport.split_once(':') { + ( + h.to_string(), + p.parse::() + .map_err(|_| format!("invalid port in {url}"))?, + ) + } else { + (hostport.to_string(), 80u16) + }; + Ok(ParsedUrl { host, port, path }) +} + +fn action_name(a: BrowserAction) -> &'static str { + match a { + BrowserAction::Prepare => "prepare", + BrowserAction::State => "state", + BrowserAction::Click => "click", + BrowserAction::Type => "type", + BrowserAction::Navigate => "navigate", + } +} + +fn request_path(a: BrowserAction) -> &'static str { + match a { + BrowserAction::Prepare => "/v1/prepare", + BrowserAction::State => "/v1/state", + BrowserAction::Click => "/v1/click", + BrowserAction::Type => "/v1/type", + BrowserAction::Navigate => "/v1/navigate", + } +} + +pub fn build_embedded_body(req: &BrowserRequest) -> Result { + let session = req + .session + .clone() + .unwrap_or_else(|| "atmos-browser-use".into()); + match req.action { + BrowserAction::Prepare => Ok(json!({ + "session": session, + // Optional preferred APP-053 browser session id (maps to target_id) + "target_id": req.target_id, + "url": req.url, + })), + BrowserAction::State => { + // Bind: no target → list sessions. Snapshot: target_id (+ optional tab_id) + Ok(json!({ + "session": session, + "target_id": req.target_id, + "tab_id": req.tab_id, + })) + } + BrowserAction::Click => { + let target = req + .target_id + .as_ref() + .ok_or_else(|| "click requires --target-id (embedded session id)".to_string())?; + let r = req + .element_ref + .as_ref() + .ok_or_else(|| "click requires --ref".to_string())?; + Ok(json!({ + "session": session, + "target_id": target, + "tab_id": req.tab_id.clone().unwrap_or_else(|| "main".into()), + "ref": r, + })) + } + BrowserAction::Type => { + let target = req + .target_id + .as_ref() + .ok_or_else(|| "type requires --target-id".to_string())?; + let text = req + .text + .as_ref() + .ok_or_else(|| "type requires --text".to_string())?; + Ok(json!({ + "session": session, + "target_id": target, + "tab_id": req.tab_id.clone().unwrap_or_else(|| "main".into()), + "ref": req.element_ref, + "text": text, + })) + } + BrowserAction::Navigate => { + let target = req + .target_id + .as_ref() + .ok_or_else(|| "navigate requires --target-id".to_string())?; + let url = req + .url + .as_ref() + .ok_or_else(|| "navigate requires --url".to_string())?; + Ok(json!({ + "session": session, + "target_id": target, + "tab_id": req.tab_id.clone().unwrap_or_else(|| "main".into()), + "url": url, + })) + } + } +} + +impl BrowserBackend for EmbeddedBackend { + fn execute(&self, req: BrowserRequest) -> BrowserResult { + let action = action_name(req.action); + let body = match build_embedded_body(&req) { + Ok(b) => b, + Err(e) => { + let err = BrowserError::InvalidArgs(e); + return BrowserResult { + ok: false, + action: action.into(), + backend: "embedded".into(), + result: None, + error: Some(err.message()), + error_code: Some(err.code().into()), + }; + } + }; + let base = match read_control_base_url() { + Ok(b) => b, + Err(e) => { + return BrowserResult { + ok: false, + action: action.into(), + backend: "embedded".into(), + result: Some(json!({ + "partition": "persist:atmos-browser", + "attach": "electron_debugger_host_control_plane", + "control_meta": control_meta_path().display().to_string(), + })), + error: Some(e), + error_code: Some(ERR_EMBEDDED_HOST_UNAVAILABLE.into()), + }; + } + }; + match http_post_json(&base, request_path(req.action), &body) { + Ok(v) => { + let ok = v.get("ok").and_then(|x| x.as_bool()).unwrap_or(true); + if !ok { + let msg = v + .get("error") + .and_then(|x| x.as_str()) + .unwrap_or("embedded browser action failed") + .to_string(); + let code = v + .get("error_code") + .and_then(|x| x.as_str()) + .unwrap_or("browser_engine_failed") + .to_string(); + return BrowserResult { + ok: false, + action: action.into(), + backend: "embedded".into(), + result: Some(v), + error: Some(msg), + error_code: Some(code), + }; + } + BrowserResult { + ok: true, + action: action.into(), + backend: "embedded".into(), + result: Some(v), + error: None, + error_code: None, + } + } + Err(e) => BrowserResult { + ok: false, + action: action.into(), + backend: "embedded".into(), + result: None, + error: Some(e), + error_code: Some(ERR_EMBEDDED_HOST_UNAVAILABLE.into()), + }, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::types::BrowserBackendKind; + + #[test] + fn build_click_requires_ref() { + let req = BrowserRequest { + backend: BrowserBackendKind::Embedded, + action: BrowserAction::Click, + target_id: Some("s1".into()), + ..Default::default() + }; + assert!(build_embedded_body(&req).is_err()); + } + + #[test] + fn parse_loopback_url() { + let p = parse_http_url("http://127.0.0.1:18765/v1/state").unwrap(); + assert_eq!(p.host, "127.0.0.1"); + assert_eq!(p.port, 18765); + assert_eq!(p.path, "/v1/state"); + } +} diff --git a/crates/browser-use/src/backends/mod.rs b/crates/browser-use/src/backends/mod.rs new file mode 100644 index 000000000..0ba481481 --- /dev/null +++ b/crates/browser-use/src/backends/mod.rs @@ -0,0 +1,13 @@ +pub(crate) mod cua; +pub(crate) mod embedded; + +pub use cua::CuaExternalBackend; +pub use embedded::EmbeddedBackend; + +use crate::types::{BrowserRequest, BrowserResult}; + +pub trait BrowserBackend { + fn execute(&self, req: BrowserRequest) -> BrowserResult; +} + +pub(crate) use cua::build_tool_call; diff --git a/crates/browser-use/src/engine_client.rs b/crates/browser-use/src/engine_client.rs new file mode 100644 index 000000000..a53625d22 --- /dev/null +++ b/crates/browser-use/src/engine_client.rs @@ -0,0 +1,100 @@ +//! Thin control-engine tool caller for the CUA external browser path. +//! +//! Does not depend on desktop-use host internals. Uses the managed engine binary +//! under `~/.atmos/desktop-use` when present (same socket contract as Desktop Use). + +use std::path::{Path, PathBuf}; +use std::process::Command; + +use serde_json::Value; + +fn data_dir() -> PathBuf { + if let Ok(p) = std::env::var("ATMOS_DESKTOP_USE_HOME") { + if !p.is_empty() { + return PathBuf::from(p); + } + } + dirs::home_dir() + .unwrap_or_else(|| PathBuf::from(".")) + .join(".atmos") + .join("desktop-use") +} + +pub fn engine_bin() -> PathBuf { + if let Ok(p) = std::env::var("ATMOS_DESKTOP_USE_ENGINE") { + if !p.is_empty() { + return PathBuf::from(p); + } + } + let name = if cfg!(windows) { + "atmos-desktop-control.exe" + } else { + "atmos-desktop-control" + }; + data_dir().join("bin").join(name) +} + +pub fn socket_path() -> PathBuf { + data_dir().join("engine.sock") +} + +pub fn call_tool(tool: &str, args: &Value) -> Result { + let engine = engine_bin(); + if !engine.is_file() { + return Err("Control engine is not installed. Run: atmos desktop-use driver ensure".into()); + } + let socket = socket_path(); + if !socket.exists() { + // Best-effort daemon start (direct serve; host.app path is Desktop Use concern). + let _ = Command::new(&engine) + .args([ + "serve", + "--socket", + &socket.display().to_string(), + "--no-permissions-gate", + ]) + .env("CUA_DRIVER_RS_PERMISSIONS_GATE", "0") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn(); + // Short wait for socket + for _ in 0..40 { + if socket.exists() { + break; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + } + call_tool_on(&engine, &socket, tool, args) +} + +fn call_tool_on(engine: &Path, socket: &Path, tool: &str, args: &Value) -> Result { + let args_json = serde_json::to_string(args).map_err(|e| e.to_string())?; + let output = Command::new(engine) + .arg("call") + .arg("--socket") + .arg(socket.display().to_string()) + .arg(tool) + .arg(&args_json) + .env("CUA_DRIVER_RS_PERMISSIONS_GATE", "0") + .output() + .map_err(|e| format!("control engine call failed: {e}"))?; + + let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + if !output.status.success() { + let msg = if !stderr.is_empty() { + stderr + } else if !stdout.is_empty() { + stdout + } else { + "control engine call failed".into() + }; + return Err(msg); + } + if stdout.is_empty() { + return Ok(serde_json::json!({ "ok": true })); + } + serde_json::from_str(&stdout) + .map_err(|e| format!("control engine returned non-JSON success output: {e}; raw={stdout}")) +} diff --git a/crates/browser-use/src/lib.rs b/crates/browser-use/src/lib.rs new file mode 100644 index 000000000..72fec9213 --- /dev/null +++ b/crates/browser-use/src/lib.rs @@ -0,0 +1,91 @@ +//! Atmos **Browser Use** — page-level control, orthogonal to Desktop Use. +//! +//! Backends: +//! - [`CuaExternalBackend`]: system Chromium via managed control engine tools +//! - [`EmbeddedBackend`]: Atmos in-app browser (APP-053 webview) via host control plane +//! +//! **No MCP.** + +mod backends; +mod engine_client; +mod types; + +pub use backends::{CuaExternalBackend, EmbeddedBackend}; +pub use types::{ + BrowserAction, BrowserBackendKind, BrowserError, BrowserRequest, BrowserResult, + ERR_EMBEDDED_HOST_UNAVAILABLE, ERR_EMBEDDED_NOT_IMPLEMENTED, ERR_NO_MCP, +}; + +use backends::BrowserBackend; + +/// Dispatch a browser-use request to the selected backend. +pub fn execute(req: BrowserRequest) -> BrowserResult { + match req.backend { + BrowserBackendKind::Cua | BrowserBackendKind::External => CuaExternalBackend.execute(req), + BrowserBackendKind::Embedded | BrowserBackendKind::Atmos => EmbeddedBackend.execute(req), + } +} + +/// Pure mapping of browser request → engine tool + args (CUA path). Unit-tested. +pub fn build_cua_tool_call( + req: &BrowserRequest, +) -> Result<(&'static str, serde_json::Value), String> { + backends::build_tool_call(req) +} + +/// Pure body builder for embedded host (unit-tested). +pub fn build_embedded_body(req: &BrowserRequest) -> Result { + backends::embedded::build_embedded_body(req) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn embedded_without_host_fails_closed() { + // Ensure no control meta from a previous run in test env + let dir = tempfile::tempdir().unwrap(); + // SAFETY: test isolation via env override + unsafe { + std::env::set_var("ATMOS_BROWSER_USE_HOME", dir.path()); + } + let res = execute(BrowserRequest { + backend: BrowserBackendKind::Embedded, + action: BrowserAction::Prepare, + ..Default::default() + }); + assert!(!res.ok); + assert_eq!( + res.error_code.as_deref(), + Some(ERR_EMBEDDED_HOST_UNAVAILABLE) + ); + unsafe { + std::env::remove_var("ATMOS_BROWSER_USE_HOME"); + } + } + + #[test] + fn cua_navigate_requires_ids() { + let missing = BrowserRequest { + backend: BrowserBackendKind::Cua, + action: BrowserAction::Navigate, + tab_id: Some("t1".into()), + url: Some("https://example.com".into()), + ..Default::default() + }; + assert!(build_cua_tool_call(&missing).is_err()); + + let req = BrowserRequest { + backend: BrowserBackendKind::Cua, + action: BrowserAction::Navigate, + target_id: Some("tgt".into()), + tab_id: Some("t1".into()), + url: Some("https://example.com".into()), + ..Default::default() + }; + let (tool, args) = build_cua_tool_call(&req).unwrap(); + assert_eq!(tool, "browser_navigate"); + assert_eq!(args["url"], "https://example.com"); + } +} diff --git a/crates/browser-use/src/types.rs b/crates/browser-use/src/types.rs new file mode 100644 index 000000000..a609d78a6 --- /dev/null +++ b/crates/browser-use/src/types.rs @@ -0,0 +1,107 @@ +use serde::{Deserialize, Serialize}; + +/// Structured error: Atmos embedded browser host control plane not reachable. +pub const ERR_EMBEDDED_HOST_UNAVAILABLE: &str = "embedded_browser_host_unavailable"; + +/// Legacy code still returned by older stubs. +pub const ERR_EMBEDDED_NOT_IMPLEMENTED: &str = "embedded_browser_not_implemented"; + +/// Product rule: Browser Use never exposes MCP. +pub const ERR_NO_MCP: &str = + "Browser Use has no MCP surface; use atmos browser-use CLI/skills only"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum BrowserBackendKind { + /// System Chromium via managed control engine (CUA tools). Default. + #[default] + Cua, + /// Alias for Cua. + External, + /// Atmos in-app browser (APP-053 webview + host CDP control plane). + Embedded, + /// Alias for Embedded. + Atmos, +} + +impl BrowserBackendKind { + pub fn parse(s: &str) -> Option { + match s.trim().to_ascii_lowercase().as_str() { + "cua" | "external" | "chrome" | "chromium" => Some(Self::Cua), + "embedded" | "atmos" | "webview" | "app" => Some(Self::Embedded), + _ => None, + } + } + + pub fn as_str(self) -> &'static str { + match self { + Self::Cua | Self::External => "cua", + Self::Embedded | Self::Atmos => "embedded", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum BrowserAction { + #[default] + Prepare, + State, + Click, + Type, + Navigate, +} + +#[derive(Debug, Clone, Default)] +pub struct BrowserRequest { + pub backend: BrowserBackendKind, + pub action: BrowserAction, + pub pid: Option, + /// Native CGWindowID — required for existing_profile prepare and bind-mode state (CUA). + pub window_id: Option, + /// Opaque browser target id (CUA target or embedded session id). + pub target_id: Option, + pub tab_id: Option, + pub element_ref: Option, + pub text: Option, + pub url: Option, + pub session: Option, + /// existing_profile (requires window_id) | isolated_new | isolated_named | omit + pub profile_strategy: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct BrowserResult { + pub ok: bool, + pub action: String, + pub backend: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub result: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error_code: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BrowserError { + InvalidArgs(String), + Engine(String), + EmbeddedHostUnavailable(String), +} + +impl BrowserError { + pub fn code(&self) -> &'static str { + match self { + Self::InvalidArgs(_) => "invalid_args", + Self::Engine(_) => "browser_engine_failed", + Self::EmbeddedHostUnavailable(_) => ERR_EMBEDDED_HOST_UNAVAILABLE, + } + } + + pub fn message(&self) -> String { + match self { + Self::InvalidArgs(m) | Self::Engine(m) | Self::EmbeddedHostUnavailable(m) => m.clone(), + } + } +} diff --git a/crates/infra/src/utils/system_skill_sync.rs b/crates/infra/src/utils/system_skill_sync.rs index f5c7e112c..d481ed39e 100644 --- a/crates/infra/src/utils/system_skill_sync.rs +++ b/crates/infra/src/utils/system_skill_sync.rs @@ -39,6 +39,8 @@ const ALL_SYSTEM_SKILL_NAMES: &[&str] = &[ "atmos-review-fix", // Canvas terminal-agent integration (APP-015) "atmos-canvas-agent", + // Browser Use page control (APP-052/053 embedded + CUA) + "atmos-browser-use", ]; #[derive(Clone, Debug, Deserialize)] @@ -172,6 +174,7 @@ fn repo_skill_root(skill_name: &str) -> Option<&'static str> { "git-commit" => Some("skills/git-commit"), "atmos-review-fix" => Some("skills/atmos-review-fix"), "atmos-canvas-agent" => Some("skills/atmos-canvas-agent"), + "atmos-browser-use" => Some("skills/atmos-browser-use"), _ => None, } } diff --git a/skills/atmos-browser-use/SKILL.md b/skills/atmos-browser-use/SKILL.md new file mode 100644 index 000000000..9ce3d457b --- /dev/null +++ b/skills/atmos-browser-use/SKILL.md @@ -0,0 +1,67 @@ +--- +name: atmos-browser-use +version: "1.1.0" +description: > + Control web pages via Browser Use (`atmos browser-use`), separate from Desktop Use. + Prefer for Chrome/Chromium page DOM or Atmos in-app browser (embedded). Do not use for + OS window chrome or non-browser apps (Slack/VS Code). No MCP. +--- + +# Atmos Browser Use + +**Page / tab control** — not Desktop Use. **No MCP.** + +## Backends + +| `--backend` | Meaning | +|-------------|---------| +| `cua` (default) | System Chromium via managed control engine (`browser_*` tools) | +| `embedded` | Atmos in-app browser (APP-053 ``, partition `persist:atmos-browser`) via Desktop host control plane | + +## Atmos embedded loop (in-app browser) + +Requires **Atmos Desktop** running with at least one Browser tab open (so a webview guest is bound). + +```bash +# 1) Host readiness + list sessions (target_id = APP-053 session id) +atmos browser-use --json prepare --backend embedded + +# 2) Bind list (omit target) or snapshot DOM refs for a session +atmos browser-use --json state --backend embedded +atmos browser-use --json state --backend embedded --target-id + +# 3) Act +atmos browser-use --json click --backend embedded --target-id --tab-id main --ref e0 +atmos browser-use --json type --backend embedded --target-id --tab-id main --ref e3 --text "hello" +atmos browser-use --json navigate --backend embedded --target-id --tab-id main --url https://example.com +``` + +Host writes `~/.atmos/browser-use/control.json` with a loopback `base_url`. CLI talks only to that plane — **never** user-Chrome `browser_prepare`. + +## System Chrome loop (CUA) + +```bash +# Find Chrome pid + window_id via Desktop Use +atmos desktop-use --json drive verify + +atmos browser-use --json prepare --backend cua --pid +atmos browser-use --json state --backend cua --pid --window-id +atmos browser-use --json state --backend cua --target-id … --tab-id … +atmos browser-use --json click --backend cua --target-id … --tab-id … --ref … +``` + +## Decision + +| Target | Surface | +|--------|---------| +| Atmos in-app browser page | `--backend embedded` | +| User Chrome/Chromium page | `--backend cua` | +| Window chrome / Slack / VS Code / any non-browser app | **`atmos-desktop-use`** (AX / pixel) | + +## Errors + +| Code | Meaning | +|------|---------| +| `embedded_browser_host_unavailable` | Desktop control plane not running / no control.json | +| `control_engine_not_installed` / `browser_engine_failed` | CUA path needs Desktop Use engine install | +| `invalid_args` | Missing target_id / ref / url | diff --git a/skills/system-skills-manifest.json b/skills/system-skills-manifest.json index 94706ae18..604110812 100644 --- a/skills/system-skills-manifest.json +++ b/skills/system-skills-manifest.json @@ -61,6 +61,9 @@ "skills/atmos-canvas-agent/references/document-scripts.md", "skills/atmos-canvas-agent/references/documents.md", "skills/atmos-canvas-agent/references/command-reference.md" + ], + "atmos-browser-use": [ + "skills/atmos-browser-use/SKILL.md" ] } } diff --git a/specs/APP/APP-053_desktop-browser-webview/TECH.md b/specs/APP/APP-053_desktop-browser-webview/TECH.md index 12016a5b6..8237097fc 100644 --- a/specs/APP/APP-053_desktop-browser-webview/TECH.md +++ b/specs/APP/APP-053_desktop-browser-webview/TECH.md @@ -7,6 +7,7 @@ | Area | Change | |------|--------| | Specs | APP-053 browser + webview + host selection | +| **Browser Use (embedded)** | Host control plane + `atmos browser-use --backend embedded` (see APP-052 §5.2) — CDP/DOM via guest WebContents, **not** user-Chrome `browser_prepare` | | `apps/desktop-electron/src/browser` | webviewTag, attach policy, surface manager without in-panel WebContentsView, `browser_bridge_*` | | `apps/web/src/features/browser` | mount `` for transport `desktop`; host selection UI; delete APP-029; rename symbols | | `packages/shared/browser` | `browser-runtime.js`; protocol `atmos-browser:*`; host toolbar path (`showSelectionToolbar: false` product) | From 851aeeca4a59ffe4c76c541d25b9253fde212712 Mon Sep 17 00:00:00 2001 From: AarynLu Date: Tue, 4 Aug 2026 22:10:59 +0800 Subject: [PATCH 02/43] fix(browser-use): address CodeRabbit stability and security quick-wins - Handle control-plane listen errors; detach CDP debugger on failure - Redact password field values and invalidate snapshots after navigate - Bound embedded HTTP connect/read; timeout control-engine calls - Skip /browser slash insert when the Browser Use skill is disabled --- .../src/browser/browser-use-control.ts | 74 ++++++++++++------- .../components/TerminalAgentInputOverlay.tsx | 6 ++ .../welcome/components/WelcomePage.tsx | 5 ++ .../lib/__tests__/slash-browser-use.test.ts | 14 ++++ .../features/welcome/lib/slash-browser-use.ts | 4 + crates/browser-use/src/backends/embedded.rs | 24 +++++- crates/browser-use/src/engine_client.rs | 53 +++++++++++-- 7 files changed, 144 insertions(+), 36 deletions(-) diff --git a/apps/desktop-electron/src/browser/browser-use-control.ts b/apps/desktop-electron/src/browser/browser-use-control.ts index ae6581125..79749ea84 100644 --- a/apps/desktop-electron/src/browser/browser-use-control.ts +++ b/apps/desktop-electron/src/browser/browser-use-control.ts @@ -49,6 +49,15 @@ export class BrowserUseControlPlane { const server = createServer((req, res) => { void this.handle(req, res); }); + // Handle async listen failures (e.g. EADDRINUSE) before listen() — + // without a listener Node throws and can crash the desktop process. + server.once("error", (err) => { + console.error("[browser-use] control plane listen failed:", err); + if (this.server === server) { + this.server = null; + this.port = 0; + } + }); // Bind loopback only. server.listen(0, "127.0.0.1", () => { const addr = server.address(); @@ -160,7 +169,13 @@ export class BrowserUseControlPlane { role: el.getAttribute('role'), name, href: el.getAttribute('href'), - value: el.value != null ? String(el.value).slice(0, 200) : null, + // Never surface password field values in Browser Use snapshots. + value: + el instanceof HTMLInputElement && el.type === 'password' + ? null + : el.value != null + ? String(el.value).slice(0, 200) + : null, rect: { x: r.x, y: r.y, width: r.width, height: r.height }, }; }), @@ -192,34 +207,39 @@ export class BrowserUseControlPlane { try { const dbg = guest.debugger; let attachedHere = false; - if (!dbg.isAttached()) { - dbg.attach("1.3"); - attachedHere = true; - } - await dbg.sendCommand("Input.dispatchMouseEvent", { - type: "mousePressed", - x, - y, - button: "left", - clickCount: 1, - }); - await dbg.sendCommand("Input.dispatchMouseEvent", { - type: "mouseReleased", - x, - y, - button: "left", - clickCount: 1, - }); - if (attachedHere) { - try { - dbg.detach(); - } catch { - /* ignore */ + try { + if (!dbg.isAttached()) { + dbg.attach("1.3"); + attachedHere = true; + } + await dbg.sendCommand("Input.dispatchMouseEvent", { + type: "mousePressed", + x, + y, + button: "left", + clickCount: 1, + }); + await dbg.sendCommand("Input.dispatchMouseEvent", { + type: "mouseReleased", + x, + y, + button: "left", + clickCount: 1, + }); + return; + } finally { + // Always detach if we attached — sendCommand failures must not leave + // the guest debugger stuck (blocks subsequent CDP / double-actions). + if (attachedHere) { + try { + dbg.detach(); + } catch { + /* ignore */ + } } } - return; } catch { - /* fall through */ + /* fall through to DOM click */ } const idx = Number(String(ref).replace(/^e/, "")); await guest.executeJavaScript( @@ -403,6 +423,8 @@ export class BrowserUseControlPlane { } else { this.manager.navigate(targetId, navUrl); } + // Page document changed — drop stale element refs/coords. + this.snapshots.delete(targetId); this.send(res, 200, { ok: true, target_id: targetId, url: navUrl }); return; } diff --git a/apps/web/src/features/terminal/components/TerminalAgentInputOverlay.tsx b/apps/web/src/features/terminal/components/TerminalAgentInputOverlay.tsx index c17d56386..697fa5a38 100644 --- a/apps/web/src/features/terminal/components/TerminalAgentInputOverlay.tsx +++ b/apps/web/src/features/terminal/components/TerminalAgentInputOverlay.tsx @@ -545,6 +545,12 @@ export const TerminalAgentInputOverlay = React.forwardRef< const popover = slashPopover; if (!popover) return; const skill = resolveBrowserUseSkillRef(allSkills); + // Match ordinary skill selection: do not insert disabled skills. + if (skill.status === "disabled") { + setSlashPopover(null); + setSlashPopoverView("menu"); + return; + } composerRef.current?.applySlashAtRange( popover.slashOffset, popover.query.length, diff --git a/apps/web/src/features/welcome/components/WelcomePage.tsx b/apps/web/src/features/welcome/components/WelcomePage.tsx index 9f04ff1c7..3e4dc1761 100644 --- a/apps/web/src/features/welcome/components/WelcomePage.tsx +++ b/apps/web/src/features/welcome/components/WelcomePage.tsx @@ -520,6 +520,11 @@ const WelcomePage: React.FC = ({ const popover = slashPopover; if (!popover) return; const skill = resolveBrowserUseSkillRef(allSkills); + // Match ordinary skill selection: do not insert disabled skills. + if (skill.status === "disabled") { + setSlashPopover(null); + return; + } composerRef.current?.applySlashAtRange( popover.slashOffset, popover.query.length, diff --git a/apps/web/src/features/welcome/lib/__tests__/slash-browser-use.test.ts b/apps/web/src/features/welcome/lib/__tests__/slash-browser-use.test.ts index 30941ce97..4f0927354 100644 --- a/apps/web/src/features/welcome/lib/__tests__/slash-browser-use.test.ts +++ b/apps/web/src/features/welcome/lib/__tests__/slash-browser-use.test.ts @@ -31,12 +31,26 @@ describe("slash-browser-use", () => { name: BROWSER_USE_SKILL_NAME, path: "/tmp/atmos-browser-use", scope: "global", + status: "enabled", } as never, ]); expect(found.absolutePath).toBe("/tmp/atmos-browser-use"); + expect(found.status).toBe("enabled"); + + const disabled = resolveBrowserUseSkillRef([ + { + id: BROWSER_USE_SKILL_NAME, + name: BROWSER_USE_SKILL_NAME, + path: "/tmp/atmos-browser-use", + scope: "global", + status: "disabled", + } as never, + ]); + expect(disabled.status).toBe("disabled"); const fallback = resolveBrowserUseSkillRef([]); expect(fallback.name).toBe(BROWSER_USE_SKILL_NAME); expect(fallback.absolutePath).toContain("atmos-browser-use"); + expect(fallback.status).toBe("enabled"); }); }); diff --git a/apps/web/src/features/welcome/lib/slash-browser-use.ts b/apps/web/src/features/welcome/lib/slash-browser-use.ts index 8018a1335..cfb65e9e3 100644 --- a/apps/web/src/features/welcome/lib/slash-browser-use.ts +++ b/apps/web/src/features/welcome/lib/slash-browser-use.ts @@ -37,6 +37,8 @@ export function buildBrowserUseSlashCommand(opts: { export function resolveBrowserUseSkillRef(skills: SkillInfo[]): { absolutePath: string; name: string; + /** Mirrors SkillInfo.status; fallback path is treated as enabled until list syncs. */ + status: SkillInfo["status"]; } { const found = skills.find( (s) => @@ -48,10 +50,12 @@ export function resolveBrowserUseSkillRef(skills: SkillInfo[]): { return { absolutePath: found.path, name: found.name || BROWSER_USE_SKILL_NAME, + status: found.status, }; } return { absolutePath: BROWSER_USE_SKILL_FALLBACK_PATH, name: BROWSER_USE_SKILL_NAME, + status: "enabled", }; } diff --git a/crates/browser-use/src/backends/embedded.rs b/crates/browser-use/src/backends/embedded.rs index 4a55e2327..26f21935d 100644 --- a/crates/browser-use/src/backends/embedded.rs +++ b/crates/browser-use/src/backends/embedded.rs @@ -6,10 +6,14 @@ use std::fs; use std::io::{Read, Write}; -use std::net::TcpStream; +use std::net::{SocketAddr, TcpStream, ToSocketAddrs}; use std::path::PathBuf; use std::time::Duration; +const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); +/// Cap identity-framed HTTP responses so a broken host cannot unbounded-buffer. +const MAX_RESPONSE_BYTES: u64 = 8 * 1024 * 1024; + use serde_json::{json, Value}; use super::BrowserBackend; @@ -70,7 +74,8 @@ pub fn http_post_json(base: &str, path: &str, body: &Value) -> Result Result MAX_RESPONSE_BYTES { + return Err("embedded host response too large".into()); + } let text = String::from_utf8_lossy(&buf); let body_start = text .find("\r\n\r\n") @@ -108,6 +118,14 @@ struct ParsedUrl { path: String, } +fn resolve_socket_addr(host: &str, port: u16) -> Result { + (host, port) + .to_socket_addrs() + .map_err(|e| format!("resolve embedded host failed: {e}"))? + .next() + .ok_or_else(|| format!("resolve embedded host failed: no address for {host}:{port}")) +} + fn parse_http_url(url: &str) -> Result { let rest = url .strip_prefix("http://") diff --git a/crates/browser-use/src/engine_client.rs b/crates/browser-use/src/engine_client.rs index a53625d22..1de426dfb 100644 --- a/crates/browser-use/src/engine_client.rs +++ b/crates/browser-use/src/engine_client.rs @@ -3,11 +3,16 @@ //! Does not depend on desktop-use host internals. Uses the managed engine binary //! under `~/.atmos/desktop-use` when present (same socket contract as Desktop Use). +use std::io::Read; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; use serde_json::Value; +/// Wall-clock limit for a single control-engine `call` subprocess. +const ENGINE_CALL_TIMEOUT: Duration = Duration::from_secs(30); + fn data_dir() -> PathBuf { if let Ok(p) = std::env::var("ATMOS_DESKTOP_USE_HOME") { if !p.is_empty() { @@ -62,7 +67,13 @@ pub fn call_tool(tool: &str, args: &Value) -> Result { if socket.exists() { break; } - std::thread::sleep(std::time::Duration::from_millis(50)); + std::thread::sleep(Duration::from_millis(50)); + } + if !socket.exists() { + return Err( + "control engine did not start (socket not created). Run: atmos desktop-use driver ensure" + .into(), + ); } } call_tool_on(&engine, &socket, tool, args) @@ -70,19 +81,47 @@ pub fn call_tool(tool: &str, args: &Value) -> Result { fn call_tool_on(engine: &Path, socket: &Path, tool: &str, args: &Value) -> Result { let args_json = serde_json::to_string(args).map_err(|e| e.to_string())?; - let output = Command::new(engine) + let mut child = Command::new(engine) .arg("call") .arg("--socket") .arg(socket.display().to_string()) .arg(tool) .arg(&args_json) .env("CUA_DRIVER_RS_PERMISSIONS_GATE", "0") - .output() + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() .map_err(|e| format!("control engine call failed: {e}"))?; - let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); - if !output.status.success() { + let deadline = Instant::now() + ENGINE_CALL_TIMEOUT; + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break status, + Ok(None) if Instant::now() >= deadline => { + let _ = child.kill(); + let _ = child.wait(); + return Err(format!( + "control engine call timed out after {}s", + ENGINE_CALL_TIMEOUT.as_secs() + )); + } + Ok(None) => std::thread::sleep(Duration::from_millis(50)), + Err(e) => return Err(format!("control engine wait failed: {e}")), + } + }; + + let mut stdout_buf = Vec::new(); + let mut stderr_buf = Vec::new(); + if let Some(mut out) = child.stdout.take() { + let _ = out.read_to_end(&mut stdout_buf); + } + if let Some(mut err) = child.stderr.take() { + let _ = err.read_to_end(&mut stderr_buf); + } + + let stdout = String::from_utf8_lossy(&stdout_buf).trim().to_string(); + let stderr = String::from_utf8_lossy(&stderr_buf).trim().to_string(); + if !status.success() { let msg = if !stderr.is_empty() { stderr } else if !stdout.is_empty() { From ce0653ed68e71ecd4576c57e3fc04e080c05181a Mon Sep 17 00:00:00 2001 From: AarynLu Date: Tue, 4 Aug 2026 23:54:01 +0800 Subject: [PATCH 03/43] fix(desktop): stop macOS Dock tile collapsing to zero width AppShot boot warm created a panel/always-on-top overlay and packaged builds called dock.setIcon, which left a zero-width Dock entry so Atmos looked icon-less. Skip overlay warm on darwin, drop panel type, setIcon only in dev, and recycle Dock after all-workspaces capture play. --- .../src/appshot/capture-animation.ts | 53 ++++++-- apps/desktop-electron/src/branding.ts | 7 +- apps/desktop-electron/src/main.ts | 15 +-- .../src/windows/mac-dock.test.ts | 14 ++ apps/desktop-electron/src/windows/mac-dock.ts | 125 ++++++++++++++++-- 5 files changed, 180 insertions(+), 34 deletions(-) diff --git a/apps/desktop-electron/src/appshot/capture-animation.ts b/apps/desktop-electron/src/appshot/capture-animation.ts index a5be94348..3ae3613e9 100644 --- a/apps/desktop-electron/src/appshot/capture-animation.ts +++ b/apps/desktop-electron/src/appshot/capture-animation.ts @@ -191,10 +191,14 @@ async function ensureSharedOverlay(): Promise { sharedOverlayReady = (async () => { const { BrowserWindow } = await import("electron"); + // IMPORTANT (macOS Dock): do NOT use type: "panel". + // A hidden panel BrowserWindow at boot leaves Atmos as a zero-width Dock + // tile (Accessibility still lists "Atmos", size 0×N — icon appears gone). + // Use a normal frameless window + showInactive() so we do not steal focus. const win = new BrowserWindow({ // Off-screen placeholder; real bounds applied on each play. - x: 0, - y: 0, + x: -10_000, + y: -10_000, width: 200, height: 200, show: false, @@ -208,8 +212,6 @@ async function ensureSharedOverlay(): Promise { skipTaskbar: true, hasShadow: false, focusable: false, - // panel avoids activating Atmos / stealing frontmost from the target app - type: "panel", backgroundColor: "#00000000", paintWhenInitiallyHidden: true, webPreferences: { @@ -222,8 +224,10 @@ async function ensureSharedOverlay(): Promise { win.setIgnoreMouseEvents(true, { forward: true }); win.setAlwaysOnTop(true, "screen-saver"); - // Can dismiss Dock icon on macOS — helper restores immediately (electron#26350). - await setOverlayVisibleOnAllWorkspaces(win, true); + // Intentionally do NOT call setVisibleOnAllWorkspaces here. + // Enabling it on a hidden overlay at boot dismisses / zero-sizes the Dock + // icon for the whole session (electron#26350). Apply only while the + // overlay is actually shown during playCaptureAnimation. win.on("closed", () => { if (sharedOverlay === win) { @@ -242,7 +246,6 @@ async function ensureSharedOverlay(): Promise { } sharedOverlay = win; - // Boot warm-path can still race Dock visibility; pin it again after load. await ensureMacDockVisible(); return win; })().catch((error) => { @@ -254,11 +257,16 @@ async function ensureSharedOverlay(): Promise { } /** - * Pre-create the overlay renderer so the first capture does not hitch on window spawn. - * Safe to call during app ready / first Appshot status poll. + * Pre-create the overlay renderer so the first capture does not hitch. + * + * On macOS this is intentionally a **no-op at call time**: creating any + * always-on-top overlay window during boot has been observed to leave a + * zero-width Dock tile. First capture pays the (small) window-create cost. */ export function warmCaptureAnimationOverlay(): void { - if (process.platform !== "darwin") return; + // Do not create BrowserWindows during boot on darwin — Dock tile regresses. + // Kept as a named export so call sites stay stable. + if (process.platform === "darwin") return; void ensureSharedOverlay() .then(() => ensureMacDockVisible()) .catch((error) => { @@ -280,9 +288,11 @@ export async function playCaptureAnimation( if (!frame) return; const gen = ++playGeneration; + let win: AnyBrowserWindow | null = null; + let allWorkspacesOn = false; try { - const win = await ensureSharedOverlay(); + win = await ensureSharedOverlay(); if (gen !== playGeneration || win.isDestroyed()) return; // Hide first so setBounds does not animate a visible jump across monitors. @@ -297,6 +307,11 @@ export async function playCaptureAnimation( height: frame.height, }); + // Only while visible: fullscreen workspaces so the flash works over spaces. + // Must be cleared after hide — leaving it on permanently can hide Dock. + await setOverlayVisibleOnAllWorkspaces(win, true); + allWorkspacesOn = true; + // Restart CSS animations without reloading (reload re-spawns the renderer). try { await win.webContents.executeJavaScript( @@ -315,12 +330,22 @@ export async function playCaptureAnimation( if (gen !== playGeneration || win.isDestroyed()) return; win.hide(); - // Overlay show/hide can re-trigger Dock dismiss on some Electron/macOS builds. - await ensureMacDockVisible(); } catch (error) { const msg = error instanceof Error ? error.message : String(error); mainLog(`[appshot-capture] animation overlay failed: ${msg}`, "error"); destroySharedOverlay(); - await ensureMacDockVisible(); + win = null; + allWorkspacesOn = false; + } finally { + // Always clear all-workspaces + restore Dock, including early-return races. + if (win && !win.isDestroyed() && allWorkspacesOn) { + try { + await setOverlayVisibleOnAllWorkspaces(win, false); + } catch { + await ensureMacDockVisible(); + } + } else { + await ensureMacDockVisible(); + } } } diff --git a/apps/desktop-electron/src/branding.ts b/apps/desktop-electron/src/branding.ts index 7516a2af4..dd2b2b877 100644 --- a/apps/desktop-electron/src/branding.ts +++ b/apps/desktop-electron/src/branding.ts @@ -110,8 +110,11 @@ export function applyReadyAppBranding(): void { ` dockIcon=${icons.dockIconPath ?? "(missing)"}`, ); - if (process.platform === "darwin" && app.dock) { - const dockPath = icons.dockIconPath; + // Packaged Atmos.app already has CFBundleIconFile (icon.icns). Runtime + // dock.setIcon has been observed to leave a zero-width Dock tile on + // Electron 37 / recent macOS — only setIcon for unpackaged / dev shells. + if (process.platform === "darwin" && app.dock && !app.isPackaged) { + const dockPath = icons.pngPath ?? icons.dockIconPath; if (dockPath) { try { const image = nativeImage.createFromPath(dockPath); diff --git a/apps/desktop-electron/src/main.ts b/apps/desktop-electron/src/main.ts index c68e0b5d3..8c40b4e7a 100644 --- a/apps/desktop-electron/src/main.ts +++ b/apps/desktop-electron/src/main.ts @@ -106,15 +106,10 @@ async function boot() { // Arm Appshots Left+Right Shift global gesture (macOS). Always attempt on boot. if (process.platform === "darwin") { - // Pre-create the transparent capture overlay so first dual-shift is smooth. - try { - const { warmCaptureAnimationOverlay } = await import( - "./appshot/capture-animation.js" - ); - warmCaptureAnimationOverlay(); - } catch (e) { - console.warn("[desktop-electron] AppShot overlay warm failed", e); - } + // Do NOT warm the capture overlay BrowserWindow at boot — creating that + // always-on-top surface (historically type:"panel") leaves a zero-width + // Dock tile so Atmos appears icon-less next to 豆包 / Downloads. + // First dual-shift creates the overlay lazily. try { const appshot = await import("./appshot/service.js"); const status = await appshot.appshotStatus(state); @@ -127,6 +122,8 @@ async function boot() { "error", ); } + // Pin Dock after boot work (trigger arm, etc.) in case anything dismissed it. + void ensureMacDockVisible(); } } diff --git a/apps/desktop-electron/src/windows/mac-dock.test.ts b/apps/desktop-electron/src/windows/mac-dock.test.ts index 9405f672b..6e0d5ce3f 100644 --- a/apps/desktop-electron/src/windows/mac-dock.test.ts +++ b/apps/desktop-electron/src/windows/mac-dock.test.ts @@ -36,4 +36,18 @@ describe("setOverlayVisibleOnAllWorkspaces", () => { { v: false, opts: { visibleOnFullScreen: false } }, ]); }); + + it("restores dock after both enable and disable", async () => { + if (process.platform !== "darwin") return; + + // Both paths must invoke ensureMacDockVisible (which is internal). + // We only assert setVisibleOnAllWorkspaces side effects stay correct and + // the helper does not throw when electron is available. + const win = { + setVisibleOnAllWorkspaces: mock(() => {}), + }; + await setOverlayVisibleOnAllWorkspaces(win, true); + await setOverlayVisibleOnAllWorkspaces(win, false); + expect(win.setVisibleOnAllWorkspaces).toHaveBeenCalledTimes(2); + }); }); diff --git a/apps/desktop-electron/src/windows/mac-dock.ts b/apps/desktop-electron/src/windows/mac-dock.ts index 18df77f86..07268caf7 100644 --- a/apps/desktop-electron/src/windows/mac-dock.ts +++ b/apps/desktop-electron/src/windows/mac-dock.ts @@ -1,23 +1,125 @@ /** * macOS Dock visibility helpers. * - * Electron can dismiss the Dock icon when a BrowserWindow calls - * `setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true })` - * (often used for AppShot capture overlays). See: - * https://github.com/electron/electron/issues/26350 + * Electron / macOS can leave a **zero-width Dock tile** (Accessibility still + * lists the app, but size is 0×N so the icon is invisible) when the shell: + * - creates a `type: "panel"` BrowserWindow, and/or + * - calls `setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true })` + * + * See: https://github.com/electron/electron/issues/26350 * * Call {@link ensureMacDockVisible} after any such API, and on activate / * hide-to-background paths so the user always has a Dock restore affordance. + * + * Packaged builds must rely on `CFBundleIconFile` — runtime `dock.setIcon` + * has been observed to contribute to a broken / zero-size tile on Electron 37. */ -/** Re-show the Dock icon if Electron/macOS hid it. No-op off darwin. */ +/** Debounce concurrent soft restores. */ +let softRestoreInFlight: Promise | null = null; +/** Debounce hard hide→show recycles (avoid dock blink storms). */ +let hardRestoreInFlight: Promise | null = null; +let softFollowUpTimer: ReturnType | null = null; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** + * Soft re-show: activation policy + dock.show(). Safe to call often (activate, + * ready-to-show, second-instance). Does not hide the Dock first. + */ export async function ensureMacDockVisible(): Promise { + if (process.platform !== "darwin") return; + if (softRestoreInFlight) { + await softRestoreInFlight; + return; + } + + softRestoreInFlight = restoreMacDockOnce(false).finally(() => { + softRestoreInFlight = null; + }); + await softRestoreInFlight; + + // One delayed soft pin for racey boot paths (no hide→show — that blinks). + if (softFollowUpTimer) { + clearTimeout(softFollowUpTimer); + } + softFollowUpTimer = setTimeout(() => { + softFollowUpTimer = null; + void restoreMacDockOnce(false); + }, 300); +} + +/** + * Hard recycle: dock.hide() → dock.show(). Use after APIs known to leave a + * zero-width ghost tile (all-workspaces, legacy panel overlay). Avoid calling + * from high-frequency paths (activate / every window event). + */ +export async function forceMacDockTileRefresh(): Promise { + if (process.platform !== "darwin") return; + if (hardRestoreInFlight) { + await hardRestoreInFlight; + return; + } + hardRestoreInFlight = restoreMacDockOnce(true).finally(() => { + hardRestoreInFlight = null; + }); + await hardRestoreInFlight; +} + +/** + * @param hard — when true, hide→show to force Dock to recreate the tile + * (needed after panel / all-workspaces left a 0-width ghost entry). + */ +async function restoreMacDockOnce(hard: boolean): Promise { if (process.platform !== "darwin") return; try { const { app } = await import("electron"); - // Force show even when isVisible() reports true — after - // setVisibleOnAllWorkspaces the icon can still be missing (electron#26350). - app.dock?.show(); + if (!app.dock) return; + + // Panel / all-workspaces can flip the app to accessory (no real Dock tile). + try { + app.setActivationPolicy?.("regular"); + } catch { + /* older Electron / non-mac */ + } + + if (hard) { + // Recycle the Dock tile. isVisible() often still reports true while the + // Accessibility size is 0×N (invisible). Always hide→show on hard path. + try { + app.dock.hide(); + } catch { + /* ignore */ + } + await sleep(40); + } + + const showResult = app.dock.show(); + if (showResult && typeof (showResult as Promise).then === "function") { + await showResult; + } + + // Dev only: stock Electron.app needs a runtime icon. Packaged Atmos.app + // already has CFBundleIconFile — setIcon there can zero-out the tile. + if (!app.isPackaged) { + try { + const { nativeImage } = await import("electron"); + const { getResolvedAppIcons } = await import("../branding.js"); + const icons = getResolvedAppIcons(); + // Prefer PNG for setIcon — .icns via nativeImage is flaky on some builds. + const path = icons.pngPath ?? icons.dockIconPath; + if (path) { + const image = nativeImage.createFromPath(path); + if (!image.isEmpty()) { + app.dock.setIcon(image); + } + } + } catch { + /* branding may not resolve in unit tests */ + } + } } catch { /* ignore — main process only; tests may lack electron */ } @@ -25,7 +127,10 @@ export async function ensureMacDockVisible(): Promise { /** * Apply workspace visibility for always-on-top overlays, then restore Dock. - * Order matters: setVisibleOnAllWorkspaces → dock.show (electron#26350). + * + * Prefer enabling this only while the overlay is actually shown (capture play), + * never when pre-creating a hidden warm window at boot. + * After disable, force a hard Dock tile recycle. */ export async function setOverlayVisibleOnAllWorkspaces( win: { @@ -44,5 +149,7 @@ export async function setOverlayVisibleOnAllWorkspaces( } if (visible) { await ensureMacDockVisible(); + } else { + await forceMacDockTileRefresh(); } } From 558ed4a8369d0c1938a02127b78efc11cbd52eac Mon Sep 17 00:00:00 2001 From: AarynLu Date: Tue, 4 Aug 2026 23:55:08 +0800 Subject: [PATCH 04/43] fix(browser): raise standalone traffic lights; fix element picker i18n key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a browser-only macOS traffic light y for the h-8 tab strip without changing main/agent-chat chrome, and rename desktopNative → desktop so the element picker tooltip resolves. --- apps/desktop-electron/src/windows/mac-chrome.ts | 7 ++++++- apps/desktop-electron/src/windows/secondary.ts | 9 ++++++--- apps/web/messages/en.json | 2 +- apps/web/messages/zh.json | 2 +- 4 files changed, 14 insertions(+), 6 deletions(-) diff --git a/apps/desktop-electron/src/windows/mac-chrome.ts b/apps/desktop-electron/src/windows/mac-chrome.ts index 2c28b1858..5c8c460f5 100644 --- a/apps/desktop-electron/src/windows/mac-chrome.ts +++ b/apps/desktop-electron/src/windows/mac-chrome.ts @@ -4,12 +4,17 @@ * Product header is `h-12` (48px). Lights are ~12–14px tall; y is the TOP of * the control cluster (Electron/Tauri logical coords). y≈16 centers in h-12. * Keep x aligned with Tauri main (`tauri.conf.json` trafficLightPosition.x = 14). + * + * Standalone browser tab strip is `h-8` (32px) with `h-7` tabs — use `browser` + * so lights sit higher and align with the tab row (do not reuse for main shell). */ export const MAC_TRAFFIC_LIGHTS = { - /** Main app shell + agent-chat / preview browser chrome */ + /** Main app shell + agent-chat chrome (h-12 headers) */ primary: { x: 14, y: 16 }, /** Compact title strips (permissions, short bars) */ compact: { x: 14, y: 14 }, + /** Standalone browser window — BrowserTabBar h-8 / tabs h-7 */ + browser: { x: 14, y: 10 }, } as const; export function macWindowChromeOptions( diff --git a/apps/desktop-electron/src/windows/secondary.ts b/apps/desktop-electron/src/windows/secondary.ts index fce01af5e..4f5df1c53 100644 --- a/apps/desktop-electron/src/windows/secondary.ts +++ b/apps/desktop-electron/src/windows/secondary.ts @@ -3,12 +3,14 @@ import type { AppState } from "../app-state.js"; import { appWindowBranding } from "../branding.js"; import { installBrowserWebviewHooks } from "../browser/webview-hooks.js"; import { uiBaseUrl } from "./main-window.js"; -import { macWindowChromeOptions } from "./mac-chrome.js"; +import { MAC_TRAFFIC_LIGHTS, macWindowChromeOptions } from "./mac-chrome.js"; import { wireFullscreenEvents } from "./fullscreen.js"; import { resolveAppPreloadPath } from "./preload-path.js"; const secondaryWindows = new Map(); +type MacChromeVariant = keyof typeof MAC_TRAFFIC_LIGHTS; + function openOrFocus( label: string, urlPath: string, @@ -18,7 +20,7 @@ function openOrFocus( height: number; minW: number; minH: number; - chrome?: "primary" | "compact"; + chrome?: MacChromeVariant; }, state: AppState, ): void { @@ -142,7 +144,8 @@ export function openBrowserWindow( height: 860, minW: 760, minH: 520, - chrome: "primary", + // h-8 tab strip — raise lights vs primary (h-12); leave main/agent-chat alone + chrome: "browser", }, state, ); diff --git a/apps/web/messages/en.json b/apps/web/messages/en.json index 0d51a9733..1e527abec 100644 --- a/apps/web/messages/en.json +++ b/apps/web/messages/en.json @@ -8459,7 +8459,7 @@ "tooltip": { "enterUrlFirst": "Enter a URL first.", "unavailable": "Element selection is only available for same-origin pages, local development URLs, or the desktop Browser.", - "desktopNative": "{action} element selection. Source component detection runs through the desktop native Browser and supports React, Vue, Angular, and Svelte.", + "desktop": "{action} element selection. Source component detection runs through the desktop native Browser and supports React, Vue, Angular, and Svelte.", "extension": "{action} element selection. Cross-port pages use the Atmos Inspector extension. Source component detection supports React, Vue, Angular, and Svelte.", "default": "{action} element selection. Source component detection supports React, Vue, Angular, and Svelte." } diff --git a/apps/web/messages/zh.json b/apps/web/messages/zh.json index c4769540e..2d3f0bd6a 100644 --- a/apps/web/messages/zh.json +++ b/apps/web/messages/zh.json @@ -8459,7 +8459,7 @@ "tooltip": { "enterUrlFirst": "请先输入 URL。", "unavailable": "元素选择仅适用于同源页面、本地开发 URL 或桌面浏览器。", - "desktopNative": "{action}元素选择。源码组件检测通过桌面原生浏览器运行，并支持 React、Vue、Angular 和 Svelte。", + "desktop": "{action}元素选择。源码组件检测通过桌面原生浏览器运行，并支持 React、Vue、Angular 和 Svelte。", "extension": "{action}元素选择。跨端口页面使用 Atmos Inspector 扩展。源码组件检测支持 React、Vue、Angular 和 Svelte。", "default": "{action}元素选择。源码组件检测支持 React、Vue、Angular 和 Svelte。" } From a8d13da22f88cff32d6330a6183a2ca5f9656f54 Mon Sep 17 00:00:00 2001 From: AarynLu Date: Wed, 5 Aug 2026 00:51:20 +0800 Subject: [PATCH 05/43] feat(browser-use): show Desktop Use agent chrome on page actions Wire session cursor/operation border for CUA and embedded browser-use clicks (and type-with-ref), harden chrome spawn against missing CLI, fix desktop-use skill status for typecheck, and add use-local-cli just recipe. --- apps/cli/AGENTS.md | 3 +- .../browser-use-control.structural.test.ts | 64 +++++ .../src/browser/browser-use-control.ts | 123 +++++++++ .../src/browser/surface-manager.ts | 9 + .../welcome/components/PromptComposer.tsx | 95 ++++++- .../components/SlashCommandPopover.tsx | 20 +- .../__tests__/slash-command-icons.test.ts | 58 ++++ .../features/welcome/lib/slash-desktop-use.ts | 9 +- crates/browser-use/src/backends/cua.rs | 20 ++ crates/browser-use/src/chrome.rs | 248 ++++++++++++++++++ crates/browser-use/src/lib.rs | 5 + justfile | 34 ++- .../icons/browser-use-icon-static.tsx | 45 ++++ 13 files changed, 727 insertions(+), 6 deletions(-) create mode 100644 apps/web/src/features/welcome/components/__tests__/slash-command-icons.test.ts create mode 100644 crates/browser-use/src/chrome.rs create mode 100644 packages/ui/src/components/icons/browser-use-icon-static.tsx diff --git a/apps/cli/AGENTS.md b/apps/cli/AGENTS.md index 0d6186c53..8409aab9f 100644 --- a/apps/cli/AGENTS.md +++ b/apps/cli/AGENTS.md @@ -8,7 +8,8 @@ - **Dev**: `just dev-cli` - **Build**: `just build-cli` -- **Install**: `just install-cli` +- **Install (cargo bin)**: `just install-cli` → `~/.cargo/bin/atmos` +- **Use local CLI**: `just use-local-cli` → release build + replace `~/.atmos/bin/atmos` (and `~/.cargo/bin` if present) - **Test**: `cargo test -p atmos` --- diff --git a/apps/desktop-electron/src/browser/browser-use-control.structural.test.ts b/apps/desktop-electron/src/browser/browser-use-control.structural.test.ts index d6c89cf7c..96645ef02 100644 --- a/apps/desktop-electron/src/browser/browser-use-control.structural.test.ts +++ b/apps/desktop-electron/src/browser/browser-use-control.structural.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "bun:test"; import { readFileSync } from "node:fs"; +import { homedir } from "node:os"; import { join } from "node:path"; const root = join(import.meta.dir, ".."); @@ -25,4 +26,67 @@ describe("browser-use-control structural", () => { expect(src).toContain("/v1/state"); expect(src).toContain("persist:atmos-browser"); }); + + it("wires Desktop Use agent chrome on embedded click/type", () => { + const src = readFileSync(join(root, "browser/browser-use-control.ts"), "utf8"); + expect(src).toContain("showEmbeddedBrowserChrome"); + expect(src).toContain("mapGuestRectToScreen"); + expect(src).toContain("showChromeForRef"); + expect(src).toContain("spawnDetachedQuiet"); + expect(src).toContain('child.on("error"'); + expect(src).toContain("desktop-use"); + expect(src).toContain("drive"); + expect(src).toContain("highlight"); + expect(src).toContain("Clicking page"); + expect(src).toContain("Typing in page"); + // Click path remains CDP / DOM — not drive click + expect(src).toContain("Input.dispatchMouseEvent"); + expect(src).toContain("el.click"); + }); +}); + +describe("mapGuestRectToScreen", () => { + it("maps guest rect into host content origin", async () => { + const { mapGuestRectToScreen } = await import("./browser-use-control.ts"); + const mapped = mapGuestRectToScreen( + { x: 100, y: 200, width: 800, height: 600 }, + { x: 50, y: 40, width: 20, height: 10 }, + ); + expect(mapped.bounds).toEqual({ x: 150, y: 240, width: 20, height: 10 }); + expect(mapped.cursor).toEqual({ x: 160, y: 245 }); + }); +}); + +describe("showEmbeddedBrowserChrome spawn safety", () => { + it("does not throw or raise uncaughtException when atmos CLI is missing", async () => { + const { showEmbeddedBrowserChrome } = await import("./browser-use-control.ts"); + const prev = process.env.ATMOS_CLI; + // Path that cannot exist as an executable on this machine. + process.env.ATMOS_CLI = join( + homedir(), + ".atmos-nonexistent-bin-for-chrome-test", + "no-such-atmos", + ); + let uncaught: Error | null = null; + const onUncaught = (err: Error) => { + uncaught = err; + }; + process.on("uncaughtException", onUncaught); + try { + expect(() => + showEmbeddedBrowserChrome({ + status: "Clicking page", + cursor: { x: 10, y: 20 }, + bounds: { x: 0, y: 0, width: 40, height: 30 }, + }), + ).not.toThrow(); + // Allow ChildProcess 'error' (ENOENT) to fire and be handled. + await new Promise((r) => setTimeout(r, 150)); + expect(uncaught).toBeNull(); + } finally { + process.off("uncaughtException", onUncaught); + if (prev === undefined) delete process.env.ATMOS_CLI; + else process.env.ATMOS_CLI = prev; + } + }); }); diff --git a/apps/desktop-electron/src/browser/browser-use-control.ts b/apps/desktop-electron/src/browser/browser-use-control.ts index 79749ea84..17b357bb8 100644 --- a/apps/desktop-electron/src/browser/browser-use-control.ts +++ b/apps/desktop-electron/src/browser/browser-use-control.ts @@ -6,6 +6,7 @@ */ import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import { spawn } from "node:child_process"; import { mkdirSync, writeFileSync, unlinkSync, existsSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; @@ -16,6 +17,105 @@ const CONTROL_DIR = () => process.env.ATMOS_BROWSER_USE_HOME?.trim() || join(homedir(), ".atmos", "browser-use"); +/** Session id shared with Rust browser-use chrome (Desktop Use cursor palette). */ +const BROWSER_USE_CHROME_SESSION = "atmos-browser-use"; + +/** + * Map guest-local element rect → approximate screen (logical) coords using the + * host BrowserWindow content origin. Best-effort: webview offset inside the + * page is not measured (may be slightly off; click path stays CDP/DOM). + */ +export function mapGuestRectToScreen( + hostContent: { x: number; y: number; width: number; height: number }, + rect: { x: number; y: number; width: number; height: number }, +): { + cursor: { x: number; y: number }; + bounds: { x: number; y: number; width: number; height: number }; +} { + const x = hostContent.x + rect.x; + const y = hostContent.y + rect.y; + const width = Math.max(1, rect.width); + const height = Math.max(1, rect.height); + return { + cursor: { x: x + width / 2, y: y + height / 2 }, + bounds: { x, y, width, height }, + }; +} + +/** + * Spawn detached without risking Electron main uncaughtException on ENOENT + * (missing ATMOS_CLI / ~/.atmos/bin/atmos). Mirrors control-plane listen hardening. + */ +function spawnDetachedQuiet(command: string, args: string[]): void { + try { + const child = spawn(command, args, { detached: true, stdio: "ignore" }); + child.on("error", (err) => { + // Missing binary, EACCES, etc. — never crash the main process. + console.warn( + `[browser-use] chrome spawn failed cmd=${command}: ${err instanceof Error ? err.message : String(err)}`, + ); + }); + child.unref(); + } catch (err) { + console.warn( + `[browser-use] chrome spawn threw cmd=${command}: ${err instanceof Error ? err.message : String(err)}`, + ); + } +} + +/** + * Fire-and-forget Desktop Use chrome (operation border + agent cursor). + * Reuses `atmos desktop-use drive highlight|move` — same overlay stack as Desktop Use. + * Never throws; missing CLI only logs a warning (does not crash main). + */ +export function showEmbeddedBrowserChrome(opts: { + status: string; + cursor: { x: number; y: number }; + bounds: { x: number; y: number; width: number; height: number }; + session?: string; +}): void { + const atmos = + process.env.ATMOS_CLI?.trim() || + join(homedir(), ".atmos", "bin", "atmos"); + const session = opts.session?.trim() || BROWSER_USE_CHROME_SESSION; + const b = opts.bounds; + const c = opts.cursor; + // Border around the element (or host content when bounds are large). + spawnDetachedQuiet(atmos, [ + "desktop-use", + "--json", + "drive", + "highlight", + "--mode", + "window", + "--x", + String(Math.round(b.x)), + "--y", + String(Math.round(b.y)), + "--width", + String(Math.round(b.width)), + "--height", + String(Math.round(b.height)), + "--status", + opts.status, + ]); + // Agent cursor overlay (not OS pointer steal). + spawnDetachedQuiet(atmos, [ + "desktop-use", + "--json", + "drive", + "move", + "--x", + String(Math.round(c.x)), + "--y", + String(Math.round(c.y)), + "--coord-space", + "points", + "--session", + session, + ]); +} + type SnapshotEl = { ref: string; tag: string; @@ -195,12 +295,34 @@ export class BrowserUseControlPlane { return cache; } + /** Best-effort Desktop Use chrome for embedded spatial actions. */ + private showChromeForRef( + sessionId: string, + el: SnapshotEl, + status: string, + ): void { + try { + const host = this.manager.getHostWindowForSession(sessionId); + if (!host || host.isDestroyed()) return; + const content = host.getContentBounds(); + const mapped = mapGuestRectToScreen(content, el.rect); + showEmbeddedBrowserChrome({ + status, + cursor: mapped.cursor, + bounds: mapped.bounds, + }); + } catch (e) { + console.warn("[browser-use] chrome failed", e); + } + } + private async clickRef(sessionId: string, ref: string): Promise { const guest = this.guestFor(sessionId); if (!guest) throw new Error(`no guest for ${sessionId}`); const cache = this.snapshots.get(sessionId) ?? (await this.snapshot(sessionId)); const el = cache.elements.find((e) => e.ref === ref); if (!el) throw new Error(`unknown ref ${ref}; run state snapshot first`); + this.showChromeForRef(sessionId, el, "Clicking page"); const x = el.rect.x + Math.max(1, el.rect.width / 2); const y = el.rect.y + Math.max(1, el.rect.height / 2); // Prefer CDP via debugger when free; fall back to DOM click by index. @@ -267,6 +389,7 @@ export class BrowserUseControlPlane { const cache = this.snapshots.get(sessionId) ?? (await this.snapshot(sessionId)); const el = cache.elements.find((e) => e.ref === ref); if (!el) throw new Error(`unknown ref ${ref}; run state snapshot first`); + this.showChromeForRef(sessionId, el, "Typing in page"); const idx = Number(String(ref).replace(/^e/, "")); await guest.executeJavaScript( `(() => { diff --git a/apps/desktop-electron/src/browser/surface-manager.ts b/apps/desktop-electron/src/browser/surface-manager.ts index 4c1673fe2..81523af3e 100644 --- a/apps/desktop-electron/src/browser/surface-manager.ts +++ b/apps/desktop-electron/src/browser/surface-manager.ts @@ -457,6 +457,15 @@ export class BrowserSurfaceManager { return null; } + /** Host BrowserWindow for a browser session (main or standalone). */ + getHostWindowForSession(sessionId: string): BrowserWindow | null { + const s = this.surfaces.get(sessionId); + if (s) return this.surfaceHost(s); + const main = this.state.mainWindow; + if (main && !main.isDestroyed()) return main; + return null; + } + /** * Session summaries for `atmos browser-use --backend embedded` prepare/state bind. */ diff --git a/apps/web/src/features/welcome/components/PromptComposer.tsx b/apps/web/src/features/welcome/components/PromptComposer.tsx index 941efc664..baf64927c 100644 --- a/apps/web/src/features/welcome/components/PromptComposer.tsx +++ b/apps/web/src/features/welcome/components/PromptComposer.tsx @@ -246,6 +246,85 @@ function buildMessagesSquareIcon(): SVGSVGElement { return svg; } +/** Mirrors `BrowserUseIconStatic` for contenteditable skill chips. */ +function buildBrowserUseChipIcon(): SVGSVGElement { + const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg"); + svg.setAttribute("aria-hidden", "true"); + svg.setAttribute("viewBox", "0 0 24 24"); + svg.setAttribute("width", "13"); + svg.setAttribute("height", "13"); + svg.setAttribute("fill", "none"); + svg.setAttribute("stroke", "currentColor"); + svg.setAttribute("stroke-width", "2"); + svg.setAttribute("stroke-linecap", "round"); + svg.setAttribute("stroke-linejoin", "round"); + svg.style.flexShrink = "0"; + + for (const d of [ + "M13 4H4a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V10", + "M2 8h11", + "M6 4v4", + "M10 4v4", + ]) { + const path = document.createElementNS("http://www.w3.org/2000/svg", "path"); + path.setAttribute("d", d); + svg.appendChild(path); + } + const g = document.createElementNS("http://www.w3.org/2000/svg", "g"); + g.setAttribute("transform", "translate(22.6 0.2) scale(-0.48 0.48)"); + for (const d of [ + "M14 4.1 12 6", + "m5.1 8-2.9-.8", + "m6 12-1.9 2", + "M7.2 2.2 8 5.1", + "M9.037 9.69a.498.498 0 0 1 .653-.653l11 4.5a.5.5 0 0 1-.074.949l-4.349 1.041a1 1 0 0 0-.74.739l-1.04 4.35a.5.5 0 0 1-.95.074z", + ]) { + const path = document.createElementNS("http://www.w3.org/2000/svg", "path"); + path.setAttribute("d", d); + if (!d.startsWith("M9.037")) path.setAttribute("opacity", "0.9"); + g.appendChild(path); + } + svg.appendChild(g); + return svg; +} + +/** Mirrors `DesktopUseIconStatic` for contenteditable skill chips. */ +function buildDesktopUseChipIcon(): SVGSVGElement { + const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg"); + svg.setAttribute("aria-hidden", "true"); + svg.setAttribute("viewBox", "0 0 24 24"); + svg.setAttribute("width", "13"); + svg.setAttribute("height", "13"); + svg.setAttribute("fill", "none"); + svg.setAttribute("stroke", "currentColor"); + svg.setAttribute("stroke-width", "2"); + svg.setAttribute("stroke-linecap", "round"); + svg.setAttribute("stroke-linejoin", "round"); + svg.style.flexShrink = "0"; + + for (const d of ["M13 3H4a2 2 0 0 0-2 2v10a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2v-3", "M8 21h8", "M12 17v4"]) { + const path = document.createElementNS("http://www.w3.org/2000/svg", "path"); + path.setAttribute("d", d); + svg.appendChild(path); + } + const g = document.createElementNS("http://www.w3.org/2000/svg", "g"); + g.setAttribute("transform", "translate(22.6 0.2) scale(-0.48 0.48)"); + for (const d of [ + "M14 4.1 12 6", + "m5.1 8-2.9-.8", + "m6 12-1.9 2", + "M7.2 2.2 8 5.1", + "M9.037 9.69a.498.498 0 0 1 .653-.653l11 4.5a.5.5 0 0 1-.074.949l-4.349 1.041a1 1 0 0 0-.74.739l-1.04 4.35a.5.5 0 0 1-.95.074z", + ]) { + const path = document.createElementNS("http://www.w3.org/2000/svg", "path"); + path.setAttribute("d", d); + if (!d.startsWith("M9.037")) path.setAttribute("opacity", "0.9"); + g.appendChild(path); + } + svg.appendChild(g); + return svg; +} + function buildStrokeIcon(paths: string[]): SVGSVGElement { const svg = document.createElementNS("http://www.w3.org/2000/svg", "svg"); svg.setAttribute("aria-hidden", "true"); @@ -412,8 +491,20 @@ function buildChipNode(token: string): HTMLSpanElement { const absolutePath = token.slice("/skill:".length); const filename = absolutePath.split("/").pop() || absolutePath; span.dataset.tooltip = absolutePath; - span.className += " border-yellow-500/30 bg-yellow-500/10 text-yellow-600 dark:text-yellow-400"; - span.appendChild(buildMaskIcon("/icons/puzzle.svg")); + const isBrowserUse = absolutePath.includes("atmos-browser-use"); + const isDesktopUse = absolutePath.includes("atmos-desktop-use"); + if (isBrowserUse) { + span.className += " border-sky-500/35 bg-sky-500/10 text-sky-700 dark:text-sky-300"; + span.appendChild(buildBrowserUseChipIcon()); + } else if (isDesktopUse) { + span.className += + " border-violet-500/35 bg-violet-500/10 text-violet-700 dark:text-violet-300"; + span.appendChild(buildDesktopUseChipIcon()); + } else { + span.className += + " border-yellow-500/30 bg-yellow-500/10 text-yellow-600 dark:text-yellow-400"; + span.appendChild(buildMaskIcon("/icons/puzzle.svg")); + } const label = document.createElement("span"); label.textContent = filename; span.appendChild(label); diff --git a/apps/web/src/features/welcome/components/SlashCommandPopover.tsx b/apps/web/src/features/welcome/components/SlashCommandPopover.tsx index a047fade3..b8c32044e 100644 --- a/apps/web/src/features/welcome/components/SlashCommandPopover.tsx +++ b/apps/web/src/features/welcome/components/SlashCommandPopover.tsx @@ -11,11 +11,13 @@ import { MessagesSquare, Puzzle, } from "lucide-react"; +import { BrowserUseIconStatic } from "@workspace/ui/components/icons/browser-use-icon-static"; import { DesktopUseIconStatic } from "@workspace/ui/components/icons/desktop-use-icon-static"; import type { SkillInfo } from "@/api/ws-api"; import { AgentIcon } from "@/features/agent/components/AgentIcon"; import type { SlashCommandOption } from "@/features/welcome/hooks/use-welcome-slash-navigation"; +import { BROWSER_USE_SLASH_COMMAND_ID } from "@/features/welcome/lib/slash-browser-use"; import { DESKTOP_USE_SLASH_COMMAND_ID } from "@/features/welcome/lib/slash-desktop-use"; import { scrollActiveListItemIntoView } from "@/features/welcome/lib/popover-list-scroll"; import type { AgentMenuOption } from "@/features/welcome/lib/welcome-page-helpers"; @@ -251,6 +253,8 @@ export function SlashCommandPopover({ ) : command.id === "dynamic-skills" ? ( + ) : command.id === BROWSER_USE_SLASH_COMMAND_ID ? ( + ) : command.id === DESKTOP_USE_SLASH_COMMAND_ID ? ( ) : ( @@ -284,6 +288,14 @@ export function SlashCommandPopover({ {visibleSkills.map((skill, index) => { const isDisabled = skill.status === "disabled"; const navIndex = skillsStartIndex + index; + const isBrowserUse = + skill.name === "atmos-browser-use" || + skill.id === "atmos-browser-use" || + skill.path.includes("atmos-browser-use"); + const isDesktopUse = + skill.name === "atmos-desktop-use" || + skill.id === "atmos-desktop-use" || + skill.path.includes("atmos-desktop-use"); return ( @@ -327,6 +357,8 @@ export const SelectionPopover: React.FC = ({ e.stopPropagation()} @@ -340,7 +372,6 @@ export const SelectionPopover: React.FC = ({ {detailsContent} - )} ); }; diff --git a/extension/AGENTS.md b/extension/AGENTS.md index 48f8e7a65..1ba0a86b6 100644 --- a/extension/AGENTS.md +++ b/extension/AGENTS.md @@ -10,7 +10,7 @@ Chrome/Edge extension that bridges cross-port preview element selection for Atmo | `background.js` | Service worker (minimal, install log only) | | `content.js` | Injected by Chrome into matching pages; loads `browser-runtime.js` and `injected.js` | | `injected.js` | Page-level glue — receives `host-init` from Atmos, creates runtime controller, relays messages | -| `browser-runtime.js` | Extension-specific preview runtime. Shares inspection logic with `packages/shared/browser/browser-runtime.js` but has a different overlay architecture (see below). | +| `browser-runtime.js` | **Synced copy** of `packages/shared/browser/browser-runtime.js` (run `bun run extension/scripts/sync-browser-runtime.ts`). Do not hand-edit. | ## How It Works @@ -42,20 +42,20 @@ The extension includes a version-check mechanism. When users click the element p All three values must always match. Forgetting any one will break the update detection. -## Two Runtime Variants +## Single Runtime Source -There are two `browser-runtime.js` files. They share the same inspection logic (element selection, React/Vue/Angular/Svelte source locators, event handling) and public API (`createRuntime`), but differ in overlay implementation: +**Canonical file:** `packages/shared/browser/browser-runtime.js` -| | `extension/browser-runtime.js` | `packages/shared/browser/browser-runtime.js` | -|---|---|---| -| **Used by** | Browser extension (Chrome MV3) | Desktop app (Tauri `include_str!`) | -| **Overlay** | Single root container + single-box divs | Per-segment border divs (4 thin edges per box) | -| **`setCursor()`** | Working — sets cursor on root, boxes, labels | No-op — Tauri manages cursor natively via Rust bridge | -| **UI sizing** | Larger (44px buttons, 22px icons) | Compact (34px buttons, 17px icons) | +- Desktop Electron injects that file (copied to `apps/desktop-electron/dist/browser-runtime.js` at build). +- Extension ships a copy under `extension/browser-runtime.js` — regenerate with: -The desktop variant uses per-segment borders to avoid intercepting pointer events in the cross-origin Tauri child webview, where a single overlay div would block the native cursor tracking. +```bash +bun run extension/scripts/sync-browser-runtime.ts +``` + +`injected.js` creates the runtime with `showSelectionToolbar: false` and `showHoverLabel: true` so host SelectionPopover matches desktop (guest draws hover/lock only). -When modifying inspection logic, source locators, or the public API, update **both** files. Overlay/cursor changes are variant-specific. +When changing inspection logic, edit **shared only**, then re-sync the extension. ## Downstream Consumers diff --git a/extension/browser-runtime.js b/extension/browser-runtime.js index 9065b8650..b4dc4e39a 100644 --- a/extension/browser-runtime.js +++ b/extension/browser-runtime.js @@ -1,13 +1,4 @@ -// Extension-specific preview runtime for the unpacked browser extension. -// This is NOT a mirror of packages/shared/browser/browser-runtime.js — the two -// files have intentionally diverged: -// - This version uses a single root container + single-box overlays and a -// working setCursor(), suitable for running directly in the target page. -// - packages/shared/ uses per-segment border overlays and a no-op setCursor(), -// designed for Tauri's cross-origin child webview where cursor is managed -// natively by the Rust bridge. -// Both files share the same inspection logic (locateReact, locateVue, etc.) and -// public API shape (createRuntime). Keep those in sync when changing either file. +// Canonical preview runtime shared by the desktop preview bridge and browser extension. (function () { if (window.__ATMOS_BROWSER_RUNTIME__) return; @@ -16,6 +7,10 @@ var PICKER_HOVER_BORDER_COLOR = '#15803d'; var PICKER_LOCKED_COLOR = '#fde047'; var PICKER_LOCKED_BORDER_COLOR = '#ca8a04'; + var Z_ANNOTATION_BOX = '2147483643'; + var Z_SELECTION_SEGMENT = '2147483644'; + var Z_ANNOTATION_MARKER = '2147483645'; + var Z_TOP_OVERLAY = '2147483647'; function createPreviewPickerCursor(fillColor, borderColor) { var cursorPath = @@ -166,63 +161,325 @@ cursorStyle.dataset.atmosPreviewOverlay = 'true'; doc.head.appendChild(cursorStyle); - const root = doc.createElement('div'); - root.dataset.atmosPreviewOverlay = 'true'; - root.style.position = 'fixed'; - root.style.inset = '0'; - root.style.pointerEvents = 'none'; - root.style.zIndex = '2147483646'; - root.style.cursor = 'inherit'; - doc.documentElement.appendChild(root); - function createBox(color) { - const box = doc.createElement('div'); - box.dataset.atmosPreviewOverlay = 'true'; - box.style.position = 'fixed'; - box.style.border = '2px solid ' + color; - box.style.borderRadius = '8px'; - box.style.background = color === '#2563eb' ? 'rgba(37, 99, 235, 0.08)' : 'rgba(249, 115, 22, 0.12)'; - box.style.pointerEvents = 'none'; - box.style.display = 'none'; - box.style.boxSizing = 'border-box'; - box.style.cursor = 'inherit'; - root.appendChild(box); - return box; + var segments = ['top', 'right', 'bottom', 'left'].map(function () { + var segment = doc.createElement('div'); + segment.dataset.atmosPreviewOverlay = 'true'; + segment.style.position = 'fixed'; + segment.style.background = color; + segment.style.pointerEvents = 'none'; + segment.style.display = 'none'; + segment.style.zIndex = Z_SELECTION_SEGMENT; + doc.documentElement.appendChild(segment); + return segment; + }); + return { + segments: segments, + }; } function createLabel() { const label = doc.createElement('div'); label.dataset.atmosPreviewOverlay = 'true'; label.style.position = 'fixed'; - label.style.padding = '4px 8px'; - label.style.borderRadius = '8px'; + label.style.left = '0'; + label.style.top = '0'; + label.style.padding = '6px 12px'; + label.style.borderRadius = '999px'; + label.style.border = '1px solid rgba(255, 255, 255, 0.1)'; label.style.fontSize = '12px'; - label.style.lineHeight = '16px'; + label.style.fontWeight = '500'; + label.style.lineHeight = '1'; label.style.fontFamily = 'ui-monospace, SFMono-Regular, Menlo, monospace'; - label.style.background = 'rgba(15, 23, 42, 0.92)'; - label.style.color = '#f8fafc'; + label.style.background = 'rgba(9, 9, 11, 0.88)'; + label.style.color = '#fafafa'; label.style.pointerEvents = 'none'; label.style.display = 'none'; label.style.maxWidth = '320px'; label.style.whiteSpace = 'nowrap'; label.style.overflow = 'hidden'; - label.style.textOverflow = 'ellipsis'; - label.style.cursor = 'inherit'; - root.appendChild(label); - return label; + label.style.boxShadow = '0 14px 34px rgba(0, 0, 0, 0.28)'; + label.style.backdropFilter = 'blur(18px)'; + label.style.webkitBackdropFilter = 'blur(18px)'; + label.style.transform = 'translate3d(-9999px, -9999px, 0)'; + label.style.willChange = 'transform'; + label.style.zIndex = Z_TOP_OVERLAY; + + const text = doc.createElement('span'); + text.style.position = 'relative'; + text.style.display = 'inline-flex'; + text.style.maxWidth = '100%'; + text.style.overflow = 'hidden'; + text.style.verticalAlign = 'top'; + label.appendChild(text); + doc.documentElement.appendChild(label); + + var visible = false; + var cursorKnown = false; + var cursorX = 0; + var cursorY = 0; + var fallbackX = 8; + var fallbackY = 8; + var x = 0; + var y = 0; + var velocityX = 0; + var velocityY = 0; + var animationFrame = 0; + var currentText = ''; + var prefersReducedMotion = false; + try { + prefersReducedMotion = !!(win.matchMedia && win.matchMedia('(prefers-reduced-motion: reduce)').matches); + } catch (_) {} + + var textTransition = 'opacity 160ms cubic-bezier(0.22, 1, 0.36, 1), transform 180ms cubic-bezier(0.22, 1, 0.36, 1)'; + + function truncateLabel(value) { + var normalized = (value || '').replace(/\s+/g, ' ').trim(); + if (normalized.length <= 96) return normalized; + return normalized.slice(0, 95).trimEnd() + '…'; + } + + function textCharacters(value) { + var counts = {}; + return value.split('').map(function (character) { + var lower = character.toLowerCase(); + counts[lower] = (counts[lower] || 0) + 1; + return { + id: lower + counts[lower], + label: character === ' ' ? '\u00A0' : character, + }; + }); + } + + function styleCharacter(span) { + span.style.display = 'inline-block'; + span.style.whiteSpace = 'pre'; + span.style.willChange = 'transform, opacity'; + span.style.transition = textTransition; + } + + function morphText(nextText) { + var previousSpans = {}; + var previousRects = {}; + Array.prototype.forEach.call(text.children, function (child) { + var id = child.dataset && child.dataset.morphId; + if (!id) return; + previousSpans[id] = child; + previousRects[id] = child.getBoundingClientRect(); + }); + + var characters = textCharacters(nextText); + var nextIds = {}; + var nextSpans = {}; + var fragment = doc.createDocumentFragment(); + + characters.forEach(function (character) { + nextIds[character.id] = true; + }); + + Object.keys(previousSpans).forEach(function (id) { + if (nextIds[id] || prefersReducedMotion) return; + var rect = previousRects[id]; + if (!rect) return; + var ghost = previousSpans[id].cloneNode(true); + styleCharacter(ghost); + ghost.style.position = 'fixed'; + ghost.style.left = rect.left + 'px'; + ghost.style.top = rect.top + 'px'; + ghost.style.width = rect.width + 'px'; + ghost.style.height = rect.height + 'px'; + ghost.style.margin = '0'; + ghost.style.pointerEvents = 'none'; + ghost.style.zIndex = Z_TOP_OVERLAY; + doc.documentElement.appendChild(ghost); + win.requestAnimationFrame(function () { + ghost.style.opacity = '0'; + ghost.style.transform = 'translateY(-6px)'; + }); + win.setTimeout(function () { + ghost.remove(); + }, 190); + }); + + characters.forEach(function (character) { + var span = previousSpans[character.id] || doc.createElement('span'); + span.dataset.morphId = character.id; + span.textContent = character.label; + styleCharacter(span); + if (!previousSpans[character.id] && !prefersReducedMotion) { + span.style.opacity = '0'; + span.style.transform = 'translateY(6px)'; + } else { + span.style.opacity = '1'; + span.style.transform = 'translate(0, 0)'; + } + nextSpans[character.id] = span; + fragment.appendChild(span); + }); + + text.replaceChildren(fragment); + if (prefersReducedMotion) return; + + win.requestAnimationFrame(function () { + characters.forEach(function (character) { + var span = nextSpans[character.id]; + var previousRect = previousRects[character.id]; + if (!span) return; + if (!previousRect) { + span.style.opacity = '1'; + span.style.transform = 'translateY(0)'; + return; + } + var nextRect = span.getBoundingClientRect(); + var deltaX = previousRect.left - nextRect.left; + var deltaY = previousRect.top - nextRect.top; + if (Math.abs(deltaX) < 0.5 && Math.abs(deltaY) < 0.5) return; + span.style.transition = 'none'; + span.style.transform = 'translate(' + deltaX + 'px, ' + deltaY + 'px)'; + span.getBoundingClientRect(); + span.style.transition = textTransition; + span.style.transform = 'translate(0, 0)'; + }); + }); + } + + function setText(value) { + var nextText = truncateLabel(value || ''); + if (nextText === currentText) return; + currentText = nextText; + morphText(nextText); + } + + function targetPosition() { + var width = label.offsetWidth; + var height = label.offsetHeight; + var originX = cursorKnown ? cursorX : fallbackX; + var originY = cursorKnown ? cursorY : fallbackY; + var targetX = originX + 16; + var targetY = originY + 18; + + if (targetX + width > win.innerWidth - 8) { + targetX = originX - width - 12; + } + if (targetY + height > win.innerHeight - 8) { + targetY = originY - height - 12; + } + + return { + x: clamp(targetX, 8, Math.max(8, win.innerWidth - width - 8)), + y: clamp(targetY, 8, Math.max(8, win.innerHeight - height - 8)), + }; + } + + function applyTransform() { + label.style.transform = 'translate3d(' + Math.round(x) + 'px, ' + Math.round(y) + 'px, 0)'; + } + + function animate() { + if (!visible) { + animationFrame = 0; + return; + } + + var target = targetPosition(); + if (prefersReducedMotion) { + x = target.x; + y = target.y; + applyTransform(); + animationFrame = 0; + return; + } + + var nextX = x + (target.x - x) * 0.28; + var nextY = y + (target.y - y) * 0.28; + velocityX = nextX - x; + velocityY = nextY - y; + x = nextX; + y = nextY; + applyTransform(); + + if ( + Math.abs(target.x - x) < 0.2 && + Math.abs(target.y - y) < 0.2 && + Math.abs(velocityX) < 0.2 && + Math.abs(velocityY) < 0.2 + ) { + x = target.x; + y = target.y; + velocityX = 0; + velocityY = 0; + applyTransform(); + animationFrame = 0; + return; + } + + animationFrame = win.requestAnimationFrame(animate); + } + + function startAnimation() { + if (animationFrame) return; + animationFrame = win.requestAnimationFrame(animate); + } + + return { + node: label, + updateCursor(xValue, yValue) { + if (!Number.isFinite(xValue) || !Number.isFinite(yValue)) return; + cursorKnown = true; + cursorX = xValue; + cursorY = yValue; + if (visible) startAnimation(); + }, + show(value, rect) { + if (!value) { + this.hide(); + return; + } + fallbackX = rect.x; + fallbackY = Math.max(8, rect.y - 32); + setText(value); + label.style.display = 'block'; + if (!visible) { + visible = true; + var target = targetPosition(); + x = target.x; + y = target.y; + velocityX = 0; + velocityY = 0; + applyTransform(); + } + startAnimation(); + }, + hide() { + visible = false; + label.style.display = 'none'; + velocityX = 0; + velocityY = 0; + if (animationFrame) { + win.cancelAnimationFrame(animationFrame); + animationFrame = 0; + } + }, + remove() { + this.hide(); + label.remove(); + }, + }; } const hoverBox = createBox(PICKER_HOVER_COLOR); const lockedBox = createBox(PICKER_LOCKED_COLOR); const hoverLabel = createLabel(); const lockedLabel = createLabel(); - const chromeRoot = doc.createElement('div'); - chromeRoot.dataset.atmosPreviewOverlay = 'true'; - chromeRoot.style.position = 'fixed'; - chromeRoot.style.inset = '0'; - chromeRoot.style.pointerEvents = 'none'; - chromeRoot.style.zIndex = '2147483647'; - doc.documentElement.appendChild(chromeRoot); + // Hover label is independent of host toolbar ownership. Host owns SelectionPopover + // when showSelectionToolbar is false; pick chrome (hover/locked box + labels) still + // lives in the guest. Default: show guest hover labels unless explicitly disabled. + // Desktop product sets showHoverLabel: true so pick mode matches the locked "motion.div" chip. + const showHoverLabel = options + ? (options.showHoverLabel != null + ? !!options.showHoverLabel + : !!options.showSelectionToolbar) + : false; function stopPropagation(event) { event.stopPropagation(); @@ -264,14 +521,14 @@ button.style.display = 'inline-flex'; button.style.alignItems = 'center'; button.style.justifyContent = 'center'; - button.style.gap = '10px'; - button.style.height = '44px'; + button.style.gap = '8px'; + button.style.height = '34px'; button.style.border = '0'; button.style.outline = 'none'; button.style.cursor = 'pointer'; button.style.pointerEvents = 'auto'; button.style.fontFamily = 'ui-sans-serif, -apple-system, BlinkMacSystemFont, sans-serif'; - button.style.fontSize = '16px'; + button.style.fontSize = '13px'; button.style.lineHeight = '1'; button.style.transition = 'background 140ms ease, color 140ms ease, transform 140ms ease, opacity 140ms ease'; button.addEventListener('mousedown', function (event) { @@ -286,10 +543,10 @@ function createFooterButton(label, variant, iconPath) { const button = createButtonBase(); - button.style.padding = '0 18px'; - button.style.borderRadius = '16px'; + button.style.padding = '0 15px'; + button.style.borderRadius = '12px'; button.style.fontWeight = '600'; - button.style.minWidth = label === 'Cancel' ? '116px' : label === 'Add' ? '124px' : '232px'; + button.style.minWidth = label === 'Cancel' ? '88px' : label === 'Add' || label === 'Copy' || label === 'Update' ? '96px' : '152px'; if (variant === 'primary') { button.style.background = '#f4f4f6'; button.style.color = '#1f1f24'; @@ -298,7 +555,7 @@ button.style.color = '#f5f5f7'; } if (iconPath) { - button.appendChild(createSvgIcon(iconPath, 22)); + button.appendChild(createSvgIcon(iconPath, 17)); } const text = doc.createElement('span'); text.textContent = label; @@ -314,18 +571,20 @@ const detailsCard = doc.createElement('div'); detailsCard.dataset.atmosPreviewOverlay = 'true'; + detailsCard.dataset.atmosPreviewToolbar = 'true'; detailsCard.style.position = 'fixed'; detailsCard.style.display = 'none'; detailsCard.style.pointerEvents = 'auto'; - detailsCard.style.borderRadius = '18px'; + detailsCard.style.borderRadius = '16px'; detailsCard.style.border = '1px solid rgba(255, 255, 255, 0.14)'; detailsCard.style.background = 'rgba(27, 27, 32, 0.98)'; detailsCard.style.boxShadow = '0 22px 50px rgba(0, 0, 0, 0.34)'; detailsCard.style.backdropFilter = 'blur(18px)'; detailsCard.style.webkitBackdropFilter = 'blur(18px)'; - detailsCard.style.padding = '22px 24px 24px'; + detailsCard.style.padding = '18px 20px 20px'; detailsCard.style.boxSizing = 'border-box'; - chromeRoot.appendChild(detailsCard); + detailsCard.style.zIndex = Z_TOP_OVERLAY; + doc.documentElement.appendChild(detailsCard); detailsCard.addEventListener('mousedown', stopPropagation); detailsCard.addEventListener('mouseup', stopPropagation); @@ -333,12 +592,14 @@ detailsCard.addEventListener('dblclick', stopPropagation); const addIconPath = ''; + const copyIconPath = ''; + const annotationIconPath = ''; const sourceSummary = doc.createElement('div'); sourceSummary.style.color = '#b9b9c2'; - sourceSummary.style.fontSize = '15px'; - sourceSummary.style.lineHeight = '1.45'; - sourceSummary.style.marginBottom = '18px'; + sourceSummary.style.fontSize = '13px'; + sourceSummary.style.lineHeight = '1.4'; + sourceSummary.style.marginBottom = '14px'; sourceSummary.style.whiteSpace = 'nowrap'; sourceSummary.style.overflow = 'hidden'; sourceSummary.style.textOverflow = 'ellipsis'; @@ -350,81 +611,138 @@ noteInput.spellcheck = false; noteInput.dataset.atmosPreviewOverlay = 'true'; noteInput.style.width = '100%'; - noteInput.style.minHeight = '156px'; + noteInput.style.minHeight = '104px'; noteInput.style.resize = 'none'; noteInput.style.boxSizing = 'border-box'; - noteInput.style.borderRadius = '18px'; + noteInput.style.borderRadius = '12px'; noteInput.style.border = '1px solid rgba(255, 255, 255, 0.14)'; noteInput.style.background = 'rgba(41, 41, 47, 0.98)'; noteInput.style.boxShadow = 'inset 0 0 0 1px rgba(255, 255, 255, 0.05)'; noteInput.style.color = '#f5f5f7'; - noteInput.style.padding = '20px 22px'; - noteInput.style.fontSize = '16px'; - noteInput.style.lineHeight = '1.55'; + noteInput.style.padding = '12px 14px'; + noteInput.style.fontSize = '13px'; + noteInput.style.lineHeight = '1.45'; noteInput.style.outline = 'none'; - noteInput.style.marginBottom = '22px'; - noteInput.addEventListener('mousedown', stopPropagation, true); - noteInput.addEventListener('mouseup', stopPropagation, true); - noteInput.addEventListener('click', stopPropagation, true); + noteInput.style.marginBottom = '16px'; + noteInput.addEventListener('mousedown', stopPropagation); + noteInput.addEventListener('mouseup', stopPropagation); + noteInput.addEventListener('click', stopPropagation); detailsCard.appendChild(noteInput); const confidenceSection = doc.createElement('div'); confidenceSection.style.display = 'none'; - confidenceSection.style.marginBottom = '24px'; + confidenceSection.style.marginBottom = '18px'; detailsCard.appendChild(confidenceSection); - const confidenceHeader = doc.createElement('div'); + const confidenceHeader = doc.createElement('button'); + confidenceHeader.type = 'button'; + confidenceHeader.dataset.atmosPreviewOverlay = 'true'; + confidenceHeader.setAttribute('aria-expanded', 'false'); confidenceHeader.style.display = 'flex'; confidenceHeader.style.alignItems = 'center'; confidenceHeader.style.justifyContent = 'space-between'; confidenceHeader.style.gap = '12px'; - confidenceHeader.style.marginBottom = '14px'; + confidenceHeader.style.width = '100%'; + confidenceHeader.style.margin = '0'; + confidenceHeader.style.padding = '0'; + confidenceHeader.style.border = '0'; + confidenceHeader.style.background = 'transparent'; + confidenceHeader.style.cursor = 'pointer'; + confidenceHeader.style.fontFamily = 'ui-sans-serif, -apple-system, BlinkMacSystemFont, sans-serif'; + confidenceHeader.style.textAlign = 'left'; confidenceSection.appendChild(confidenceHeader); + const confidenceTitleGroup = doc.createElement('span'); + confidenceTitleGroup.style.display = 'inline-flex'; + confidenceTitleGroup.style.alignItems = 'center'; + confidenceTitleGroup.style.gap = '6px'; + confidenceTitleGroup.style.minWidth = '0'; + confidenceHeader.appendChild(confidenceTitleGroup); + + const confidenceChevron = createSvgIcon('', 16); + confidenceChevron.style.color = '#8e8e98'; + confidenceChevron.style.flex = '0 0 auto'; + confidenceChevron.style.transition = 'transform 180ms cubic-bezier(0.22, 1, 0.36, 1)'; + confidenceTitleGroup.appendChild(confidenceChevron); + const confidenceTitle = doc.createElement('div'); confidenceTitle.textContent = 'Source Code Confidence'; confidenceTitle.style.color = '#b9b9c2'; - confidenceTitle.style.fontSize = '14px'; + confidenceTitle.style.fontSize = '12px'; confidenceTitle.style.fontWeight = '600'; - confidenceHeader.appendChild(confidenceTitle); + confidenceTitle.style.whiteSpace = 'nowrap'; + confidenceTitle.style.overflow = 'hidden'; + confidenceTitle.style.textOverflow = 'ellipsis'; + confidenceTitleGroup.appendChild(confidenceTitle); const confidenceBadge = doc.createElement('span'); confidenceBadge.style.display = 'inline-flex'; confidenceBadge.style.alignItems = 'center'; confidenceBadge.style.justifyContent = 'center'; - confidenceBadge.style.minWidth = '76px'; - confidenceBadge.style.padding = '0 16px'; - confidenceBadge.style.height = '38px'; + confidenceBadge.style.minWidth = '64px'; + confidenceBadge.style.padding = '0 12px'; + confidenceBadge.style.height = '30px'; confidenceBadge.style.borderRadius = '999px'; - confidenceBadge.style.fontSize = '14px'; + confidenceBadge.style.fontSize = '11px'; confidenceBadge.style.fontWeight = '700'; confidenceBadge.style.letterSpacing = '0.12em'; confidenceHeader.appendChild(confidenceBadge); + const confidenceSignalsWrap = doc.createElement('div'); + confidenceSignalsWrap.style.display = 'grid'; + confidenceSignalsWrap.style.gridTemplateRows = '0fr'; + confidenceSignalsWrap.style.transition = 'grid-template-rows 200ms cubic-bezier(0.22, 1, 0.36, 1)'; + confidenceSection.appendChild(confidenceSignalsWrap); + + const confidenceSignalsInner = doc.createElement('div'); + confidenceSignalsInner.style.minHeight = '0'; + confidenceSignalsInner.style.overflow = 'hidden'; + confidenceSignalsWrap.appendChild(confidenceSignalsInner); + const confidenceSignals = doc.createElement('div'); - confidenceSignals.style.borderRadius = '16px'; + confidenceSignals.style.marginTop = '10px'; + confidenceSignals.style.borderRadius = '12px'; confidenceSignals.style.border = '1px solid rgba(255, 255, 255, 0.1)'; confidenceSignals.style.background = 'rgba(35, 35, 41, 0.85)'; - confidenceSignals.style.padding = '16px 18px'; + confidenceSignals.style.padding = '12px 14px'; confidenceSignals.style.color = '#b9b9c2'; - confidenceSignals.style.fontSize = '14px'; - confidenceSignals.style.lineHeight = '1.55'; - confidenceSection.appendChild(confidenceSignals); + confidenceSignals.style.fontSize = '12px'; + confidenceSignals.style.lineHeight = '1.45'; + confidenceSignals.style.opacity = '0'; + confidenceSignals.style.transform = 'translateY(-4px)'; + confidenceSignals.style.transition = 'opacity 200ms cubic-bezier(0.22, 1, 0.36, 1), transform 200ms cubic-bezier(0.22, 1, 0.36, 1)'; + confidenceSignalsInner.appendChild(confidenceSignals); const footer = doc.createElement('div'); footer.style.display = 'flex'; footer.style.alignItems = 'center'; footer.style.justifyContent = 'space-between'; - footer.style.gap = '18px'; + footer.style.gap = '12px'; detailsCard.appendChild(footer); + const footerActions = doc.createElement('div'); + footerActions.style.display = 'flex'; + footerActions.style.alignItems = 'center'; + footerActions.style.justifyContent = 'flex-end'; + footerActions.style.gap = '10px'; + const footerCancelButton = createFooterButton('Cancel', 'ghost'); - const footerCopyButton = createFooterButton('Add', 'primary', addIconPath); + const footerAddButton = createFooterButton('Add', 'primary', addIconPath); + const footerCopyButton = createFooterButton('Copy', 'primary', copyIconPath); + const footerUpdateButton = createFooterButton('Update', 'primary'); footer.appendChild(footerCancelButton); - footer.appendChild(footerCopyButton); + footer.appendChild(footerActions); + footerActions.appendChild(footerAddButton); + footerActions.appendChild(footerCopyButton); + footerActions.appendChild(footerUpdateButton); + footerUpdateButton.style.display = 'none'; let currentMeta = null; let currentRect = null; + let confidenceExpanded = false; + let currentEditingAnnotation = null; + let nextAnnotationIndex = 1; + const annotations = []; let cancelHandler = null; let copyHandler = null; @@ -444,6 +762,252 @@ } } + function syncConfidenceDisclosure() { + confidenceHeader.setAttribute('aria-expanded', confidenceExpanded ? 'true' : 'false'); + confidenceChevron.style.transform = confidenceExpanded ? 'rotate(180deg)' : 'rotate(0deg)'; + confidenceSignalsWrap.style.gridTemplateRows = confidenceExpanded ? '1fr' : '0fr'; + confidenceSignals.style.opacity = confidenceExpanded ? '1' : '0'; + confidenceSignals.style.transform = confidenceExpanded ? 'translateY(0)' : 'translateY(-4px)'; + } + + confidenceHeader.addEventListener('mousedown', function (event) { + event.stopPropagation(); + }); + confidenceHeader.addEventListener('click', function (event) { + event.preventDefault(); + event.stopPropagation(); + confidenceExpanded = !confidenceExpanded; + syncConfidenceDisclosure(); + }); + + function createAnnotationId() { + return 'annotation-' + Date.now().toString(36) + '-' + (nextAnnotationIndex++); + } + + function createAnnotationActionButton(label) { + const button = doc.createElement('button'); + var isDelete = label === 'Delete'; + button.type = 'button'; + button.dataset.atmosPreviewOverlay = 'true'; + button.innerHTML = + '' + + '' + label + ''; + button.style.display = 'inline-flex'; + button.style.alignItems = 'center'; + button.style.justifyContent = 'center'; + button.style.flex = '1 1 0'; + button.style.gap = '4px'; + button.style.height = '100%'; + button.style.padding = '0'; + button.style.border = '0'; + button.style.borderRadius = '0'; + button.style.background = 'transparent'; + button.style.color = '#e5e7eb'; + button.style.fontSize = '11px'; + button.style.fontWeight = '700'; + button.style.lineHeight = '1'; + button.style.cursor = 'pointer'; + button.style.pointerEvents = 'auto'; + button.style.fontFamily = 'ui-sans-serif, -apple-system, BlinkMacSystemFont, sans-serif'; + button.style.transition = 'background 150ms ease, color 150ms ease'; + button.addEventListener('mousedown', function (event) { + event.stopPropagation(); + button.style.background = isDelete ? 'rgba(239, 68, 68, 0.25)' : 'rgba(255, 255, 255, 0.15)'; + }); + button.addEventListener('mouseup', function () { + button.style.background = isDelete ? 'rgba(239, 68, 68, 0.2)' : 'rgba(255, 255, 255, 0.1)'; + }); + button.addEventListener('mouseenter', function () { + button.style.background = isDelete ? 'rgba(239, 68, 68, 0.2)' : 'rgba(255, 255, 255, 0.1)'; + button.style.color = isDelete ? '#fee2e2' : '#ffffff'; + }); + button.addEventListener('mouseleave', function () { + button.style.background = 'transparent'; + button.style.color = '#e5e7eb'; + }); + return button; + } + + function setAnnotationMarkerExpanded(annotation, expanded) { + var ui = annotation && annotation.ui; + if (!ui) return; + ui.marker.style.width = expanded ? '124px' : '26px'; + ui.marker.style.background = expanded ? 'rgba(15, 23, 42, 0.96)' : 'rgba(34, 197, 94, 0.92)'; + ui.marker.style.borderColor = expanded ? 'rgba(52, 211, 153, 0.5)' : 'rgba(255, 255, 255, 0.72)'; + ui.marker.style.boxShadow = '0 10px 24px rgba(15, 23, 42, 0.28)'; + ui.compactIcon.style.opacity = expanded ? '0' : '1'; + ui.compactIcon.style.transform = expanded ? 'scale(0.84)' : 'scale(1)'; + ui.actions.style.opacity = expanded ? '1' : '0'; + ui.actions.style.transform = expanded ? 'translateX(0)' : 'translateX(-4px)'; + ui.actions.style.pointerEvents = expanded ? 'auto' : 'none'; + } + + function syncAnnotationOverlay(annotation) { + if (!annotation || !annotation.ui || !annotation.element || !doc.contains(annotation.element)) { + if (annotation && annotation.ui) { + annotation.ui.box.style.display = 'none'; + annotation.ui.marker.style.display = 'none'; + } + return; + } + var rect = getPreviewElementRect(annotation.element); + annotation.rect = rect; + var width = Math.max(2, rect.width); + var height = Math.max(2, rect.height); + annotation.ui.box.style.display = 'block'; + annotation.ui.box.style.left = rect.x + 'px'; + annotation.ui.box.style.top = rect.y + 'px'; + annotation.ui.box.style.width = width + 'px'; + annotation.ui.box.style.height = height + 'px'; + annotation.ui.marker.style.display = 'inline-flex'; + annotation.ui.marker.style.left = clamp(rect.x - 6, 6, Math.max(6, win.innerWidth - 124)) + 'px'; + annotation.ui.marker.style.top = clamp(rect.y - 14, 6, Math.max(6, win.innerHeight - 28)) + 'px'; + } + + function syncAnnotationOverlays() { + annotations.forEach(syncAnnotationOverlay); + } + + function updateFooterMode(mode) { + var isEdit = mode === 'edit'; + footerAddButton.style.display = isEdit ? 'none' : 'inline-flex'; + footerCopyButton.style.display = isEdit ? 'none' : 'inline-flex'; + footerUpdateButton.style.display = isEdit ? 'inline-flex' : 'none'; + } + + function beginEditAnnotation(annotation, event) { + if (event) { + event.preventDefault(); + event.stopPropagation(); + } + if (!annotation) return; + currentEditingAnnotation = annotation; + currentMeta = annotation.meta || {}; + noteInput.value = annotation.note || ''; + placeToolbar(annotation.rect || getPreviewElementRect(annotation.element)); + updateFooterMode('edit'); + win.setTimeout(function () { + try { noteInput.focus(); } catch (_) {} + }, 0); + } + + function removeAnnotation(annotationId, emitDelete, event) { + if (event) { + event.preventDefault(); + event.stopPropagation(); + } + var index = annotations.findIndex(function (annotation) { + return annotation.id === annotationId; + }); + if (index < 0) return; + var annotation = annotations[index]; + annotations.splice(index, 1); + if (annotation.ui) { + annotation.ui.box.remove(); + annotation.ui.marker.remove(); + } + if (currentEditingAnnotation && currentEditingAnnotation.id === annotationId) { + currentEditingAnnotation = null; + detailsCard.style.display = 'none'; + noteInput.value = ''; + updateFooterMode('select'); + } + if (emitDelete && copyHandler) { + copyHandler('delete', '', event, annotationId); + } + } + + function addAnnotationFromCurrent(note) { + var element = currentMeta && currentMeta.element; + if (!element || !doc.contains(element)) return null; + var annotation = { + id: createAnnotationId(), + element: element, + meta: currentMeta, + note: note || '', + rect: getPreviewElementRect(element), + ui: null, + }; + + const box = doc.createElement('div'); + box.dataset.atmosPreviewOverlay = 'true'; + box.dataset.atmosPreviewAnnotation = 'true'; + box.style.position = 'fixed'; + box.style.pointerEvents = 'none'; + box.style.border = '2px solid #22c55e'; + box.style.background = 'rgba(34, 197, 94, 0.08)'; + box.style.boxShadow = '0 0 0 1px rgba(34, 197, 94, 0.18)'; + box.style.borderRadius = '6px'; + box.style.zIndex = Z_ANNOTATION_BOX; + doc.documentElement.appendChild(box); + + const marker = doc.createElement('div'); + marker.dataset.atmosPreviewOverlay = 'true'; + marker.dataset.atmosPreviewAnnotation = 'true'; + marker.style.position = 'fixed'; + marker.style.display = 'inline-flex'; + marker.style.alignItems = 'center'; + marker.style.justifyContent = 'center'; + marker.style.width = '26px'; + marker.style.height = '26px'; + marker.style.overflow = 'hidden'; + marker.style.borderRadius = '6px'; + marker.style.border = '1px solid rgba(255, 255, 255, 0.72)'; + marker.style.background = 'rgba(34, 197, 94, 0.92)'; + marker.style.color = '#ffffff'; + marker.style.boxShadow = '0 10px 24px rgba(15, 23, 42, 0.28)'; + marker.style.pointerEvents = 'auto'; + marker.style.cursor = 'default'; + marker.style.zIndex = Z_ANNOTATION_MARKER; + marker.style.transition = 'width 180ms cubic-bezier(0.22, 1, 0.36, 1), background 180ms cubic-bezier(0.22, 1, 0.36, 1), border-color 180ms ease, box-shadow 180ms ease'; + doc.documentElement.appendChild(marker); + + const compactIcon = createSvgIcon(annotationIconPath, 15); + compactIcon.style.transition = 'opacity 140ms ease, transform 140ms ease'; + marker.appendChild(compactIcon); + + const actions = doc.createElement('div'); + actions.style.position = 'absolute'; + actions.style.inset = '0'; + actions.style.display = 'inline-flex'; + actions.style.alignItems = 'center'; + actions.style.justifyContent = 'center'; + actions.style.gap = '0'; + actions.style.opacity = '0'; + actions.style.transform = 'translateX(-4px)'; + actions.style.pointerEvents = 'none'; + actions.style.transition = 'opacity 140ms ease, transform 140ms ease'; + marker.appendChild(actions); + + const editButton = createAnnotationActionButton('Edit'); + const deleteButton = createAnnotationActionButton('Delete'); + actions.appendChild(editButton); + actions.appendChild(deleteButton); + + annotation.ui = { box: box, marker: marker, compactIcon: compactIcon, actions: actions }; + annotations.push(annotation); + syncAnnotationOverlay(annotation); + + marker.addEventListener('mouseenter', function () { + setAnnotationMarkerExpanded(annotation, true); + }); + marker.addEventListener('mouseleave', function () { + setAnnotationMarkerExpanded(annotation, false); + }); + editButton.addEventListener('click', function (event) { + beginEditAnnotation(annotation, event); + }); + deleteButton.addEventListener('click', function (event) { + removeAnnotation(annotation.id, true, event); + }); + + return annotation; + } + function renderSelectionMeta() { var meta = currentMeta || {}; var sourceLocation = meta.sourceLocation || {}; @@ -467,11 +1031,15 @@ if (confidence || signals.length > 0) { confidenceSection.style.display = 'block'; + confidenceExpanded = false; + syncConfidenceDisclosure(); confidenceBadge.textContent = (confidence || 'low').toUpperCase(); applyConfidence(confidence || 'low'); confidenceSignals.textContent = signals.length > 0 ? signals.join(', ') : 'No extra debug signals'; } else { confidenceSection.style.display = 'none'; + confidenceExpanded = false; + syncConfidenceDisclosure(); } } @@ -484,8 +1052,8 @@ currentRect = rect; renderSelectionMeta(); - var detailsWidth = Math.min(640, Math.max(320, win.innerWidth - 16)); - var detailsHeight = 346; + var detailsWidth = Math.min(480, Math.max(280, win.innerWidth - 16)); + var detailsHeight = 260; var centerX = rect.x + Math.min(rect.width, 220) / 2; var belowTop = rect.y + rect.height + 12; var aboveTop = rect.y - detailsHeight - 12; @@ -511,53 +1079,205 @@ event.preventDefault(); event.stopPropagation(); } + if (currentEditingAnnotation) { + currentEditingAnnotation = null; + detailsCard.style.display = 'none'; + noteInput.value = ''; + updateFooterMode('select'); + return; + } if (cancelHandler) { cancelHandler(event); } } - function copyWithNoteFromEvent(event) { + function writeClipboardText(text) { + var value = text == null ? '' : String(text); + if (!value) return false; + // Prefer the synchronous path first: it still has the click user gesture in + // WebKit/Tauri webviews. Async clipboard.writeText can race gesture expiry. + var synced = false; + try { + var helper = doc.createElement('textarea'); + helper.value = value; + helper.setAttribute('readonly', ''); + helper.style.position = 'fixed'; + helper.style.left = '-9999px'; + helper.style.top = '0'; + helper.style.opacity = '0'; + doc.documentElement.appendChild(helper); + helper.focus(); + helper.select(); + helper.setSelectionRange(0, value.length); + synced = doc.execCommand('copy'); + helper.remove(); + } catch (_) { + synced = false; + } + try { + if (navigator.clipboard && typeof navigator.clipboard.writeText === 'function') { + void navigator.clipboard.writeText(value); + } + } catch (_) {} + return synced; + } + + function formatSelectionForClipboard(meta, note) { + var source = (meta && meta.sourceLocation) || {}; + var element = meta && meta.element; + var context = element && doc.contains(element) ? inspectPreviewElement(element) : null; + // Protocol first line so PromptComposer can collapse paste into a chip while + // keeping the body (from line 2) as the exact AI prompt on send. + var lines = ['atmos://context/preview-element', '## Preview Element']; + var pageUrl = (meta && meta.pageUrl) || win.location.href; + if (pageUrl) lines.push('- **Page**: `' + pageUrl + '`'); + if (context && context.selector) lines.push('- **Selector**: `' + context.selector + '`'); + if (context && context.tagName) lines.push('- **Tag**: `' + context.tagName + '`'); + if (context && context.attributesSummary) lines.push('- **Attributes**: ' + context.attributesSummary); + if (source.framework) lines.push('- **Framework**: ' + source.framework); + if (source.componentName) lines.push('- **Source Component**: `' + source.componentName + '`'); + if (source.filePath || source.fileName) { + var sourcePath = source.filePath || source.fileName; + var sourceLoc = sourcePath; + if (source.line != null) sourceLoc += ':' + source.line; + if (source.column != null) sourceLoc += ':' + source.column; + lines.push('- **Source**: `' + sourceLoc + '`'); + } + if (source.confidence) lines.push('- **Confidence**: ' + source.confidence); + if (context && context.textPreview) { + lines.push('', '### Element Text', context.textPreview); + } + if (context && context.htmlPreview) { + lines.push('', '### Element HTML', '```html', context.htmlPreview, '```'); + } + if (note) { + lines.push('', '## Note', note); + } + return lines.join('\n').trim(); + } + + // Prevent pointerdown + click + button listener from firing the same action thrice. + var toolbarActionInFlight = false; + + function toolbarActionFromEvent(action, event) { + if (toolbarActionInFlight) return; if (event) { event.preventDefault(); event.stopPropagation(); } - if (copyHandler) { - copyHandler((noteInput.value || '').trim(), event); + toolbarActionInFlight = true; + win.setTimeout(function () { + toolbarActionInFlight = false; + }, 0); + + var annotationId = null; + var note = (noteInput.value || '').trim(); + if (action === 'copy') { + // Write clipboard in the inspector webview while the user gesture is still valid. + // Host-side clipboard after the Tauri bridge loses transient activation and fails silently. + writeClipboardText(formatSelectionForClipboard(currentMeta, note)); + detailsCard.style.display = 'none'; + noteInput.value = ''; + } else if (action === 'add') { + var annotation = addAnnotationFromCurrent(note); + if (!annotation) { + toolbarActionInFlight = false; + return; + } + annotationId = annotation.id; + detailsCard.style.display = 'none'; + currentMeta = null; + noteInput.value = ''; + } else if (action === 'update') { + if (!currentEditingAnnotation) { + toolbarActionInFlight = false; + return; + } + currentEditingAnnotation.note = note; + annotationId = currentEditingAnnotation.id; + currentEditingAnnotation = null; + detailsCard.style.display = 'none'; + currentMeta = null; + noteInput.value = ''; + updateFooterMode('select'); } - } - - function handleOverlayButtonPointerDown(event) { - if (eventTargetInside(event, footerCancelButton)) { - cancelFromEvent(event); - } else if (eventTargetInside(event, footerCopyButton)) { - copyWithNoteFromEvent(event); + if (copyHandler) { + copyHandler(action, note, event, annotationId); } } + // Use capture-phase click only (not pointerdown) so we fire once per activation. + // Button-level listeners below are the primary path; this catches clicks that + // land on child SVG/text nodes when a parent stopPropagation interferes. function handleOverlayButtonClick(event) { if (eventTargetInside(event, footerCancelButton)) { cancelFromEvent(event); + } else if (eventTargetInside(event, footerAddButton)) { + toolbarActionFromEvent('add', event); } else if (eventTargetInside(event, footerCopyButton)) { - copyWithNoteFromEvent(event); + toolbarActionFromEvent('copy', event); + } else if (eventTargetInside(event, footerUpdateButton)) { + toolbarActionFromEvent('update', event); } } - win.addEventListener('pointerdown', handleOverlayButtonPointerDown, true); win.addEventListener('click', handleOverlayButtonClick, true); footerCancelButton.addEventListener('click', cancelFromEvent); - footerCopyButton.addEventListener('click', copyWithNoteFromEvent); + footerAddButton.addEventListener('click', function (event) { + toolbarActionFromEvent('add', event); + }); + footerCopyButton.addEventListener('click', function (event) { + toolbarActionFromEvent('copy', event); + }); + footerUpdateButton.addEventListener('click', function (event) { + toolbarActionFromEvent('update', event); + }); + win.addEventListener('scroll', syncAnnotationOverlays, true); + win.addEventListener('resize', syncAnnotationOverlays, true); function place(box, label, rect, text) { - box.style.display = 'block'; - box.style.left = rect.x + 'px'; - box.style.top = rect.y + 'px'; - box.style.width = rect.width + 'px'; - box.style.height = rect.height + 'px'; - label.style.display = text ? 'block' : 'none'; - label.textContent = text || ''; - label.style.left = rect.x + 'px'; - label.style.top = Math.max(8, rect.y - 32) + 'px'; + var thickness = 2; + var width = Math.max(rect.width, thickness); + var height = Math.max(rect.height, thickness); + var top = box.segments[0]; + var right = box.segments[1]; + var bottom = box.segments[2]; + var left = box.segments[3]; + + top.style.display = 'block'; + top.style.left = rect.x + 'px'; + top.style.top = rect.y + 'px'; + top.style.width = width + 'px'; + top.style.height = thickness + 'px'; + + bottom.style.display = 'block'; + bottom.style.left = rect.x + 'px'; + bottom.style.top = (rect.y + height - thickness) + 'px'; + bottom.style.width = width + 'px'; + bottom.style.height = thickness + 'px'; + + left.style.display = 'block'; + left.style.left = rect.x + 'px'; + left.style.top = rect.y + 'px'; + left.style.width = thickness + 'px'; + left.style.height = height + 'px'; + + right.style.display = 'block'; + right.style.left = (rect.x + width - thickness) + 'px'; + right.style.top = rect.y + 'px'; + right.style.width = thickness + 'px'; + right.style.height = height + 'px'; + + if (label) { + label.show(text || '', rect); + } + } + + function clearBox(box) { + box.segments.forEach(function (segment) { + segment.style.display = 'none'; + }); } return { @@ -566,27 +1286,31 @@ win.__ATMOS_PREVIEW_PICK_CURSOR__ = nextCursor === 'default' ? '' : nextCursor; if (nextCursor === 'default') { cursorStyle.textContent = ''; - } else { - cursorStyle.textContent = - 'html, body, body * { cursor: ' + nextCursor + ' !important; }' + - '[data-atmos-browser-overlay="true"], [data-atmos-browser-overlay="true"] * { cursor: revert !important; }'; + return; } - root.style.cursor = nextCursor; - hoverBox.style.cursor = nextCursor; - lockedBox.style.cursor = nextCursor; - hoverLabel.style.cursor = nextCursor; - lockedLabel.style.cursor = nextCursor; + cursorStyle.textContent = + 'html, body, body * { cursor: ' + nextCursor + ' !important; }' + + '[data-atmos-browser-overlay="true"], [data-atmos-browser-overlay="true"] * { cursor: default !important; }' + + '[data-atmos-browser-overlay="true"] button, [data-atmos-browser-overlay="true"] button * { cursor: pointer !important; }' + + '[data-atmos-browser-overlay="true"] input, [data-atmos-browser-overlay="true"] textarea, [data-atmos-browser-overlay="true"] [contenteditable="true"] { cursor: text !important; }'; }, updateHover(rect, label) { - place(hoverBox, hoverLabel, rect, label); + place(hoverBox, showHoverLabel ? hoverLabel : null, rect, label); + }, + updateCursor(x, y) { + hoverLabel.updateCursor(x, y); + lockedLabel.updateCursor(x, y); }, clearHover() { - hoverBox.style.display = 'none'; - hoverLabel.style.display = 'none'; + clearBox(hoverBox); + hoverLabel.hide(); }, lock(rect, label, meta) { if (meta) { currentMeta = meta; + currentEditingAnnotation = null; + noteInput.value = ''; + updateFooterMode('select'); } place(lockedBox, lockedLabel, rect, label); // Always show the input card directly on selection. @@ -598,13 +1322,31 @@ } }, clearLocked() { - lockedBox.style.display = 'none'; - lockedLabel.style.display = 'none'; + clearBox(lockedBox); + lockedLabel.hide(); detailsCard.style.display = 'none'; noteInput.value = ''; currentMeta = null; currentRect = null; }, + clearAnnotations() { + annotations.forEach(function (annotation) { + if (annotation.ui) { + annotation.ui.box.remove(); + annotation.ui.marker.remove(); + } + }); + annotations.length = 0; + currentEditingAnnotation = null; + currentMeta = null; + currentRect = null; + detailsCard.style.display = 'none'; + noteInput.value = ''; + updateFooterMode('select'); + }, + syncAnnotations() { + syncAnnotationOverlays(); + }, onCancel(handler) { cancelHandler = handler; }, @@ -614,11 +1356,21 @@ destroy() { cursorStyle.remove(); win.__ATMOS_PREVIEW_PICK_CURSOR__ = ''; - win.removeEventListener('pointerdown', handleOverlayButtonPointerDown, true); + win.removeEventListener('scroll', syncAnnotationOverlays, true); + win.removeEventListener('resize', syncAnnotationOverlays, true); win.removeEventListener('click', handleOverlayButtonClick, true); detailsCard.remove(); - chromeRoot.remove(); - root.remove(); + annotations.forEach(function (annotation) { + if (annotation.ui) { + annotation.ui.box.remove(); + annotation.ui.marker.remove(); + } + }); + annotations.length = 0; + hoverBox.segments.forEach(function (segment) { segment.remove(); }); + lockedBox.segments.forEach(function (segment) { segment.remove(); }); + hoverLabel.remove(); + lockedLabel.remove(); }, }; } @@ -1096,6 +1848,9 @@ var doc = win.document; var overlay = createPreviewOverlay(win, doc, { showSelectionToolbar: !!config.showSelectionToolbar, + showHoverLabel: config.showHoverLabel != null + ? !!config.showHoverLabel + : !!config.showSelectionToolbar, }); var hoverCursor = createPreviewPickerCursor(PICKER_HOVER_COLOR, PICKER_HOVER_BORDER_COLOR); var lockedCursor = createPreviewPickerCursor(PICKER_LOCKED_COLOR, PICKER_LOCKED_BORDER_COLOR); @@ -1106,6 +1861,9 @@ locked: null, sessionId: null, }; + // When host owns the toolbar, still emit hover so iframe/extension can drive NativeFollowCursor. + // Desktop with guest hover labels may also emit; host can ignore. + var shouldEmitHover = !config.showSelectionToolbar; function emit(message) { if (!state.sessionId) return; @@ -1115,6 +1873,11 @@ }, message)); } + function emitHover(payload) { + if (!shouldEmitHover) return; + emit(Object.assign({ type: 'atmos-browser:hover' }, payload || {})); + } + function setPickerCursor(cursor) { var nextCursor = cursor || 'default'; overlay.setCursor(nextCursor); @@ -1196,6 +1959,7 @@ state.sessionId = sessionId; emit({ type: 'atmos-browser:ready', + pageUrl: win.location.href, capabilities: getCapabilities(win), extensionVersion: EXTENSION_VERSION, pageTitle: getPageTitle(), @@ -1204,22 +1968,21 @@ } function clearSelection(notifyHost) { + // Unlock only — keep pick mode so the host can dismiss SelectionPopover + // and continue hovering without a racey exit+re-enter. + // Full disable is exitPickMode (toolbar toggle / tab hide). state.locked = null; + state.hovered = null; overlay.clearLocked(); overlay.clearHover(); + emitHover(null); setPickerCursor(state.enabled ? hoverCursor : 'default'); if (notifyHost) { emit({ type: 'atmos-browser:cleared' }); - } else { - // Host-initiated clear also disables pick mode so hover - // overlays do not reappear after the selection is removed. - state.enabled = false; - state.hovered = null; - setPickerCursor('default'); } } - function selectElement(element) { + function selectElement(element, cursor) { var rect = getPreviewElementRect(element); var elementContext = inspectPreviewElement(element); var sourceLocation = locateSourceForElement(element, win); @@ -1227,30 +1990,53 @@ rect, (sourceLocation && sourceLocation.componentName) || buildElementSelector(element), { + element: element, pageUrl: win.location.href, sourceLocation: sourceLocation, label: buildElementSelector(element), } ); setPickerCursor(lockedCursor); - emit({ + var payload = { type: 'atmos-browser:selected', rect: rect, elementContext: elementContext, sourceLocation: sourceLocation, - }); + // Guest CSS viewport size — host maps rect/cursor through webview scale + // (critical when canvas CSS-transforms the widget). + viewport: { + width: win.innerWidth, + height: win.innerHeight, + }, + }; + if (cursor && Number.isFinite(cursor.x) && Number.isFinite(cursor.y)) { + payload.cursor = { x: cursor.x, y: cursor.y }; + } + emit(payload); + } + + function unlockSelectionForNextPick() { + state.locked = null; + state.hovered = null; + overlay.clearLocked(); + overlay.clearHover(); + emitHover(null); + setPickerCursor(state.enabled ? hoverCursor : 'default'); } function handleMouseMove(event) { if (!state.enabled) return; + overlay.updateCursor(event.clientX, event.clientY); if (state.locked) { overlay.clearHover(); + emitHover(null); setPickerCursor(lockedCursor); return; } var target = event.target; if (!(target instanceof Element) || isIgnoredElement(target)) { overlay.clearHover(); + emitHover(null); setPickerCursor(hoverCursor); state.hovered = null; return; @@ -1258,7 +2044,16 @@ state.hovered = target; setPickerCursor(hoverCursor); var rect = getPreviewElementRect(target); - overlay.updateHover(rect, buildElementSelector(target)); + var label = buildElementSelector(target); + overlay.updateHover(rect, label); + emitHover({ + label: label, + rect: rect, + cursor: { + x: event.clientX, + y: event.clientY, + }, + }); } function isOverlayEventTarget(target) { @@ -1276,17 +2071,35 @@ if (!state.enabled) return; var target = event.target; if (isOverlayEventTarget(target)) return; + overlay.updateCursor(event.clientX, event.clientY); event.preventDefault(); event.stopPropagation(); - if (state.locked) return; + var cursor = { x: event.clientX, y: event.clientY }; + // Second click while locked: re-pick another element, or dismiss host + // SelectionPopover (cleared) when clicking ignored/blank chrome. + if (state.locked) { + if (target instanceof Element && !isIgnoredElement(target) && target !== state.locked) { + unlockSelectionForNextPick(); + state.locked = target; + overlay.clearHover(); + emitHover(null); + selectElement(target, cursor); + return; + } + clearSelection(true); + return; + } if (!(target instanceof Element) || isIgnoredElement(target)) return; state.locked = target; overlay.clearHover(); - selectElement(target); + emitHover(null); + selectElement(target, cursor); } function handleKeyDown(event) { if (!state.enabled || event.key !== 'Escape') return; + // Esc: clear lock + notify host (popover dismiss) while pick mode stays on + // until the host exits pick mode explicitly. clearSelection(true); } @@ -1314,6 +2127,23 @@ overlay.lock(rect, buildElementSelector(state.locked)); } + // Throttled notify so host can re-query annotation/selection rects after scroll. + // Keep latency low so host annotation pins track page scroll. + var viewportChangedTimer = 0; + function emitViewportChanged() { + if (!state.sessionId) return; + if (viewportChangedTimer) return; + viewportChangedTimer = win.setTimeout(function () { + viewportChangedTimer = 0; + emit({ type: 'atmos-browser:viewport-changed' }); + }, 16); + } + + function handleViewportLayout() { + syncLockedOverlay(); + emitViewportChanged(); + } + doc.addEventListener('mousemove', handleMouseMove, true); doc.addEventListener('pointerdown', blockPagePointerEvent, true); doc.addEventListener('mousedown', blockPagePointerEvent, true); @@ -1324,8 +2154,8 @@ doc.addEventListener('dblclick', blockPagePointerEvent, true); doc.addEventListener('contextmenu', blockPagePointerEvent, true); win.addEventListener('keydown', handleKeyDown, true); - win.addEventListener('scroll', syncLockedOverlay, true); - win.addEventListener('resize', syncLockedOverlay, true); + win.addEventListener('scroll', handleViewportLayout, true); + win.addEventListener('resize', handleViewportLayout, true); overlay.onCancel(function (event) { if (event) { @@ -1334,17 +2164,41 @@ } clearSelection(true); }); - overlay.onCopy(function (note, event) { + overlay.onCopy(function (action, note, event, annotationId) { if (event) { event.preventDefault(); event.stopPropagation(); } - if (!state.locked) return; + // Copy needs a live locked element. Add may still emit when the local + // annotation was already created even if lock was cleared mid-click. + if (action === 'copy' && !state.locked) return; + if (action === 'add' && !state.locked && !annotationId) return; + // Include selection snapshot so the host can recover if the earlier + // `selected` event was dropped or cleared before the toolbar action arrived. + var lockedElement = state.locked; + var rect = lockedElement && doc.contains(lockedElement) + ? getPreviewElementRect(lockedElement) + : null; + var elementContext = lockedElement && doc.contains(lockedElement) + ? inspectPreviewElement(lockedElement) + : null; + var sourceLocation = lockedElement && doc.contains(lockedElement) + ? locateSourceForElement(lockedElement, win) + : null; emit({ type: 'atmos-browser:toolbar-action', - action: 'add', + action: action === 'copy' || action === 'update' || action === 'delete' ? action : 'add', + annotationId: annotationId || undefined, note: note || undefined, + rect: rect || undefined, + elementContext: elementContext || undefined, + sourceLocation: sourceLocation || undefined, }); + // Copy already wrote the clipboard in the overlay click path (user gesture). + // Unlock so the host does not need a successful async clipboard write to clear UI. + if (action === 'add' || action === 'copy') { + unlockSelectionForNextPick(); + } }); var lastKnownPath = win.location.pathname + win.location.hash; @@ -1432,12 +2286,20 @@ }); }, clearSelection: clearSelection, + clearAnnotations: function () { + overlay.clearAnnotations(); + }, + syncOverlays: function () { + syncLockedOverlay(); + overlay.syncAnnotations(); + }, exitPickMode: function () { state.enabled = false; state.locked = null; state.hovered = null; overlay.clearLocked(); overlay.clearHover(); + emitHover(null); setPickerCursor('default'); }, destroy: function () { diff --git a/extension/injected.js b/extension/injected.js index 507d461ef..f4706a9b3 100644 --- a/extension/injected.js +++ b/extension/injected.js @@ -16,8 +16,11 @@ function createController() { if (!window.__ATMOS_BROWSER_RUNTIME__) return null; + // Align with desktop guest: host owns SelectionPopover; guest draws hover/lock. return window.__ATMOS_BROWSER_RUNTIME__.createRuntime({ win: window, + showSelectionToolbar: false, + showHoverLabel: true, emit: function (message) { window.parent.postMessage(message, state.parentOrigin || '*'); }, diff --git a/extension/scripts/sync-browser-runtime.ts b/extension/scripts/sync-browser-runtime.ts new file mode 100644 index 000000000..fbc6b6193 --- /dev/null +++ b/extension/scripts/sync-browser-runtime.ts @@ -0,0 +1,60 @@ +/** + * Copy canonical guest runtime into the extension package. + * Source of truth: packages/shared/browser/browser-runtime.js + * + * Run: bun run extension/scripts/sync-browser-runtime.ts + * (from repo root) or via package scripts that zip the extension. + */ +import { copyFileSync, existsSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const extensionRoot = join(dirname(fileURLToPath(import.meta.url)), ".."); +const repoRoot = join(extensionRoot, ".."); +const src = join(repoRoot, "packages/shared/browser/browser-runtime.js"); +const dest = join(extensionRoot, "browser-runtime.js"); +const manifestPath = join(extensionRoot, "manifest.json"); + +if (!existsSync(src)) { + console.error(`[extension] missing source runtime: ${src}`); + process.exit(1); +} + +copyFileSync(src, dest); + +// Keep EXTENSION_VERSION in shared aligned with manifest when possible. +try { + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as { + version?: string; + }; + const version = typeof manifest.version === "string" ? manifest.version : ""; + if (version) { + let body = readFileSync(dest, "utf8"); + body = body.replace( + /var EXTENSION_VERSION = '[^']*';/, + `var EXTENSION_VERSION = '${version}';`, + ); + writeFileSync(dest, body); + // Mirror version into shared so desktop inject reports the same build id. + let shared = readFileSync(src, "utf8"); + const nextShared = shared.replace( + /var EXTENSION_VERSION = '[^']*';/, + `var EXTENSION_VERSION = '${version}';`, + ); + if (nextShared !== shared) { + writeFileSync(src, nextShared); + copyFileSync(src, dest); + // re-apply version stamp after re-copy + let body2 = readFileSync(dest, "utf8"); + body2 = body2.replace( + /var EXTENSION_VERSION = '[^']*';/, + `var EXTENSION_VERSION = '${version}';`, + ); + writeFileSync(dest, body2); + } + } +} catch (e) { + console.warn("[extension] version stamp skipped", e); +} + +console.log(`[extension] synced browser-runtime.js from packages/shared (${src})`); diff --git a/packages/shared/browser/browser-runtime.js b/packages/shared/browser/browser-runtime.js index e93667f79..b4dc4e39a 100644 --- a/packages/shared/browser/browser-runtime.js +++ b/packages/shared/browser/browser-runtime.js @@ -1968,19 +1968,17 @@ } function clearSelection(notifyHost) { + // Unlock only — keep pick mode so the host can dismiss SelectionPopover + // and continue hovering without a racey exit+re-enter. + // Full disable is exitPickMode (toolbar toggle / tab hide). state.locked = null; + state.hovered = null; overlay.clearLocked(); overlay.clearHover(); emitHover(null); setPickerCursor(state.enabled ? hoverCursor : 'default'); if (notifyHost) { emit({ type: 'atmos-browser:cleared' }); - } else { - // Host-initiated clear also disables pick mode so hover - // overlays do not reappear after the selection is removed. - state.enabled = false; - state.hovered = null; - setPickerCursor('default'); } } @@ -2076,16 +2074,32 @@ overlay.updateCursor(event.clientX, event.clientY); event.preventDefault(); event.stopPropagation(); - if (state.locked) return; + var cursor = { x: event.clientX, y: event.clientY }; + // Second click while locked: re-pick another element, or dismiss host + // SelectionPopover (cleared) when clicking ignored/blank chrome. + if (state.locked) { + if (target instanceof Element && !isIgnoredElement(target) && target !== state.locked) { + unlockSelectionForNextPick(); + state.locked = target; + overlay.clearHover(); + emitHover(null); + selectElement(target, cursor); + return; + } + clearSelection(true); + return; + } if (!(target instanceof Element) || isIgnoredElement(target)) return; state.locked = target; overlay.clearHover(); emitHover(null); - selectElement(target, { x: event.clientX, y: event.clientY }); + selectElement(target, cursor); } function handleKeyDown(event) { if (!state.enabled || event.key !== 'Escape') return; + // Esc: clear lock + notify host (popover dismiss) while pick mode stays on + // until the host exits pick mode explicitly. clearSelection(true); } @@ -2114,6 +2128,7 @@ } // Throttled notify so host can re-query annotation/selection rects after scroll. + // Keep latency low so host annotation pins track page scroll. var viewportChangedTimer = 0; function emitViewportChanged() { if (!state.sessionId) return; @@ -2121,7 +2136,7 @@ viewportChangedTimer = win.setTimeout(function () { viewportChangedTimer = 0; emit({ type: 'atmos-browser:viewport-changed' }); - }, 48); + }, 16); } function handleViewportLayout() { diff --git a/packages/ui/src/components/ui/popover.tsx b/packages/ui/src/components/ui/popover.tsx index 120567da8..c8968f88a 100644 --- a/packages/ui/src/components/ui/popover.tsx +++ b/packages/ui/src/components/ui/popover.tsx @@ -25,17 +25,24 @@ function PopoverAnchor({ const preventAutoFocus = (e: Event) => e.preventDefault() -function PopoverContent({ - className, - align = "center", - sideOffset = 4, - onOpenAutoFocus = preventAutoFocus, - onCloseAutoFocus, - ...props -}: React.ComponentProps) { +const PopoverContent = React.forwardRef< + React.ComponentRef, + React.ComponentPropsWithoutRef +>(function PopoverContent( + { + className, + align = "center", + sideOffset = 4, + onOpenAutoFocus = preventAutoFocus, + onCloseAutoFocus, + ...props + }, + ref, +) { return ( ) -} +}) +PopoverContent.displayName = "PopoverContent" export { Popover, PopoverTrigger, PopoverContent, PopoverAnchor } From 8595ca8df4a7d8bf35191a72484fc451fa55877a Mon Sep 17 00:00:00 2001 From: AarynLu Date: Wed, 5 Aug 2026 20:28:18 +0800 Subject: [PATCH 17/43] feat(web): desktop-use readiness dialog and settings polish Add readiness host/dialog flow, settings section updates, and locale copy for desktop-use permissions UX. --- apps/web/messages/en.json | 102 +++++++- apps/web/messages/zh.json | 102 +++++++- .../components/DesktopUseReadinessDialog.tsx | 114 +++++++++ .../components/DesktopUseReadinessHost.tsx | 11 + .../lib/__tests__/readiness.test.ts | 106 ++++++++ .../desktop-use/lib/readiness-modal-bus.ts | 78 ++++++ .../src/features/desktop-use/lib/readiness.ts | 242 ++++++++++++++++++ .../components/DesktopUsePermissionsPanel.tsx | 2 + .../components/DesktopUseSettingsSection.tsx | 88 ++++++- .../desktop-use-settings-section.test.ts | 18 ++ 10 files changed, 852 insertions(+), 11 deletions(-) create mode 100644 apps/web/src/features/desktop-use/components/DesktopUseReadinessDialog.tsx create mode 100644 apps/web/src/features/desktop-use/components/DesktopUseReadinessHost.tsx create mode 100644 apps/web/src/features/desktop-use/lib/__tests__/readiness.test.ts create mode 100644 apps/web/src/features/desktop-use/lib/readiness-modal-bus.ts create mode 100644 apps/web/src/features/desktop-use/lib/readiness.ts diff --git a/apps/web/messages/en.json b/apps/web/messages/en.json index 8f722ddf6..4bfa03dd4 100644 --- a/apps/web/messages/en.json +++ b/apps/web/messages/en.json @@ -5537,7 +5537,79 @@ }, "desktopUse": { "label": "Desktop Use", - "description": "Capture and control the local desktop (screenshot, click, type)" + "description": "Capture and control the local desktop (screenshot, click, type)", + "desktopOnly": "Desktop Use is only available in Atmos Desktop.", + "status": { + "webOnly": "Desktop only" + }, + "host": { + "defaultName": "Atmos Desktop Use" + }, + "engine": { + "title": "Control engine", + "hint": "Optional host engine for screen capture and UI control under a dedicated app identity.", + "installHint": "Download and install the control engine to enable Desktop Use capture and control.", + "updateHint": "Update available: v{current} → v{next}.", + "stopHint": "Stop the running engine process. You can start it again with Install / Update.", + "removeHint": "Remove the engine from this Mac. Capture and agent desktop control will fall back or become unavailable until reinstalled.", + "notInstalled": "Not installed", + "installed": "Installed", + "updateAvailable": "Update available" + }, + "actions": { + "install": "Install", + "update": "Update", + "stop": "Stop", + "uninstall": "Uninstall", + "refresh": "Refresh", + "cancel": "Cancel" + }, + "uninstallConfirm": { + "title": "Uninstall Desktop Use engine?", + "description": "This stops the engine and deletes the local host app under ~/.atmos/desktop-use.", + "consequencesTitle": "After uninstall, these will be unavailable until you install again:", + "consequences": { + "appshotHost": "AppShot dual-shift capture via the dedicated Desktop Use host identity", + "agentControl": "Agent desktop control (screenshot, click, type) through the control engine", + "cli": "CLI: atmos desktop-use drive / capture commands that need the engine", + "chrome": "Operation border and agent cursor overlays driven by the engine" + }, + "note": "macOS Accessibility and Screen Recording grants for “Atmos Desktop Use” stay in System Settings until you remove them there. Reinstall will reuse the same host identity when possible.", + "confirm": "Uninstall engine" + }, + "border": { + "title": "Operation border", + "description": "Show a border around the target window or desktop while Desktop Use is operating." + }, + "permissions": { + "title": "Permissions", + "unifiedHint": "Grant Accessibility and Screen Recording to the Desktop Use host app so capture and control work reliably.", + "installFirst": "Install the control engine first to manage permissions.", + "hostLine": "Host: {host}", + "done": "Done", + "grantAccessibility": "Open Accessibility settings", + "grantScreenRecording": "Open Screen Recording settings", + "items": { + "accessibility": { + "title": "Accessibility", + "description": "Required to inspect and control other apps." + }, + "screenRecording": { + "title": "Screen Recording", + "description": "Required to capture the screen for AppShot and Desktop Use." + } + } + }, + "errors": { + "statusFailed": "Failed to load Desktop Use status.", + "installFailed": "Failed to install the control engine.", + "updateFailed": "Failed to update the control engine.", + "stopFailed": "Failed to stop the control engine.", + "uninstallFailed": "Failed to uninstall the control engine.", + "prefsFailed": "Failed to update preferences.", + "grantFailed": "Failed to open permission settings." + }, + "title": "Desktop Use" }, "browserUse": { "label": "Browser Use", @@ -8640,6 +8712,7 @@ "browserLoadFailure": "The target page reported a browser-level load failure.", "unknownNetwork": "Unknown network error", "localHttpsHint": "The target local server may be speaking HTTP on an HTTPS URL or returning an invalid TLS response.", + "certProxyHint": "Certificate errors that appear only sometimes are often caused by a system proxy, VPN, or fake-ip DNS (for example Clash/Surge). Try disabling the proxy or use system DNS, then retry.", "networkUnreachable": "The target page could not be reached cleanly." }, "errorCard": { @@ -8886,5 +8959,32 @@ "zig_cache": "Zig build cache", "zig_out": "Zig build output" } + }, + "desktopUse": { + "readinessModal": { + "title": "Desktop Use is not ready", + "dismiss": "Not now", + "openSettings": "Open Desktop Use settings", + "hint": "Install or start the control engine and grant Accessibility + Screen Recording to Atmos Desktop Use.", + "source": { + "generic": "This feature needs the Desktop Use control engine.", + "appshot": "AppShot capture works best when Desktop Use is installed and permitted.", + "slash": "The Desktop Use slash skill needs the control engine and macOS permissions." + }, + "reason": { + "webOnly": "Desktop Use is only available in Atmos Desktop.", + "engineNotInstalled": "The control engine is not installed yet.", + "engineNotRunning": "The control engine is installed but not running.", + "permissionAccessibility": "Accessibility permission is not granted for Atmos Desktop Use.", + "permissionScreenRecording": "Screen Recording permission is not granted for Atmos Desktop Use.", + "permissionBoth": "Accessibility and Screen Recording are not granted for Atmos Desktop Use.", + "unknown": "Could not verify Desktop Use status. Check Settings to install and grant permissions." + }, + "needs": { + "engine": "Control engine installed and running", + "accessibility": "Accessibility allowed for Atmos Desktop Use", + "screenRecording": "Screen Recording allowed for Atmos Desktop Use" + } + } } } diff --git a/apps/web/messages/zh.json b/apps/web/messages/zh.json index 21d36fa9c..8db79da26 100644 --- a/apps/web/messages/zh.json +++ b/apps/web/messages/zh.json @@ -5537,7 +5537,79 @@ }, "desktopUse": { "label": "Desktop Use", - "description": "本机桌面捕获与控制（截图、点击、输入）" + "description": "本机桌面捕获与控制（截图、点击、输入）", + "desktopOnly": "Desktop Use 仅在 Atmos 桌面端可用。", + "status": { + "webOnly": "仅桌面端" + }, + "host": { + "defaultName": "Atmos Desktop Use" + }, + "engine": { + "title": "控制引擎", + "hint": "可选本机控制引擎，在独立 App 身份下完成截屏与 UI 操控。", + "installHint": "下载并安装控制引擎后，即可使用 Desktop Use 的捕获与控制能力。", + "updateHint": "有可用更新：v{current} → v{next}。", + "stopHint": "停止正在运行的引擎进程。之后可通过「安装 / 更新」再次启动。", + "removeHint": "从本机移除引擎。在重新安装前，相关捕获与 Agent 桌面控制会降级或不可用。", + "notInstalled": "未安装", + "installed": "已安装", + "updateAvailable": "有可用更新" + }, + "actions": { + "install": "安装", + "update": "更新", + "stop": "停止", + "uninstall": "卸载", + "refresh": "刷新", + "cancel": "取消" + }, + "uninstallConfirm": { + "title": "卸载 Desktop Use 引擎？", + "description": "将停止引擎并删除 ~/.atmos/desktop-use 下的本机 host 应用。", + "consequencesTitle": "卸载后，在重新安装前以下能力将不可用：", + "consequences": { + "appshotHost": "经 Desktop Use host 身份的 AppShot 双 Shift 捕获", + "agentControl": "经控制引擎的 Agent 桌面操控（截图、点击、输入）", + "cli": "依赖引擎的 CLI：atmos desktop-use drive / capture 等命令", + "chrome": "由引擎驱动的操作描边与 Agent 光标叠层" + }, + "note": "系统设置里对「Atmos Desktop Use」的辅助功能 / 屏幕录制授权不会自动撤销；若需清除请到系统设置中手动移除。重新安装时会尽量复用同一 host 身份。", + "confirm": "确认卸载" + }, + "border": { + "title": "操作描边", + "description": "Desktop Use 操作目标窗口或桌面时显示描边提示。" + }, + "permissions": { + "title": "权限", + "unifiedHint": "请为 Desktop Use host 应用授予辅助功能与屏幕录制权限，以便稳定完成捕获与控制。", + "installFirst": "请先安装控制引擎再管理权限。", + "hostLine": "Host：{host}", + "done": "完成", + "grantAccessibility": "打开辅助功能设置", + "grantScreenRecording": "打开屏幕录制设置", + "items": { + "accessibility": { + "title": "辅助功能", + "description": "用于检查并控制其他应用。" + }, + "screenRecording": { + "title": "屏幕录制", + "description": "用于 AppShot 与 Desktop Use 的屏幕捕获。" + } + } + }, + "errors": { + "statusFailed": "加载 Desktop Use 状态失败。", + "installFailed": "安装控制引擎失败。", + "updateFailed": "更新控制引擎失败。", + "stopFailed": "停止控制引擎失败。", + "uninstallFailed": "卸载控制引擎失败。", + "prefsFailed": "更新偏好设置失败。", + "grantFailed": "打开权限设置失败。" + }, + "title": "Desktop Use" }, "browserUse": { "label": "Browser Use", @@ -8640,6 +8712,7 @@ "browserLoadFailure": "目标页面报告了浏览器级别的加载失败。", "unknownNetwork": "未知网络错误", "localHttpsHint": "目标本地服务可能在 HTTPS URL 上实际提供的是 HTTP，或返回了无效的 TLS 响应。", + "certProxyHint": "证书错误若只是偶尔出现，常见原因是系统代理、VPN 或 fake-ip DNS（例如 Clash/Surge）。可先关闭代理或改用系统 DNS 后再试。", "networkUnreachable": "目标页面无法被正常访问。" }, "errorCard": { @@ -8887,5 +8960,32 @@ "zig_cache": "Zig 构建缓存", "zig_out": "Zig 构建输出" } + }, + "desktopUse": { + "readinessModal": { + "title": "Desktop Use 尚未就绪", + "dismiss": "暂不处理", + "openSettings": "打开 Desktop Use 设置", + "hint": "请安装或启动控制引擎，并为「Atmos Desktop Use」授予辅助功能与屏幕录制权限。", + "source": { + "generic": "此功能需要 Desktop Use 控制引擎。", + "appshot": "AppShot 捕获需要已安装并授权的 Desktop Use。", + "slash": "Desktop Use 斜杠技能需要控制引擎与 macOS 权限。" + }, + "reason": { + "webOnly": "Desktop Use 仅在 Atmos 桌面端可用。", + "engineNotInstalled": "控制引擎尚未安装。", + "engineNotRunning": "控制引擎已安装但未运行。", + "permissionAccessibility": "尚未为 Atmos Desktop Use 授予辅助功能权限。", + "permissionScreenRecording": "尚未为 Atmos Desktop Use 授予屏幕录制权限。", + "permissionBoth": "尚未为 Atmos Desktop Use 授予辅助功能与屏幕录制权限。", + "unknown": "无法确认 Desktop Use 状态。请到设置中安装引擎并检查权限。" + }, + "needs": { + "engine": "控制引擎已安装并运行", + "accessibility": "已为 Atmos Desktop Use 开启辅助功能", + "screenRecording": "已为 Atmos Desktop Use 开启屏幕录制" + } + } } } diff --git a/apps/web/src/features/desktop-use/components/DesktopUseReadinessDialog.tsx b/apps/web/src/features/desktop-use/components/DesktopUseReadinessDialog.tsx new file mode 100644 index 000000000..1017dc62a --- /dev/null +++ b/apps/web/src/features/desktop-use/components/DesktopUseReadinessDialog.tsx @@ -0,0 +1,114 @@ +"use client"; + +import React from "react"; +import { useTranslations } from "next-intl"; +import { + Button, + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@workspace/ui"; +import { AlertCircle, Settings2 } from "lucide-react"; +import { openDesktopUseSettingsInApp } from "@/features/appshot/lib/open-desktop-use-settings"; +import { + closeDesktopUseReadinessModal, + subscribeDesktopUseReadinessModal, + type DesktopUseReadinessModalState, +} from "../lib/readiness-modal-bus"; +import { invalidateDesktopUseReadinessCache } from "../lib/readiness"; +import type { DesktopUseReadinessReason } from "../lib/readiness"; + +function reasonMessageKey( + reason: DesktopUseReadinessReason | null | undefined, +): string { + switch (reason) { + case "web_only": + return "reason.webOnly"; + case "engine_not_installed": + return "reason.engineNotInstalled"; + case "engine_not_running": + return "reason.engineNotRunning"; + case "permission_accessibility": + return "reason.permissionAccessibility"; + case "permission_screen_recording": + return "reason.permissionScreenRecording"; + case "permission_both": + return "reason.permissionBoth"; + default: + return "reason.unknown"; + } +} + +export function DesktopUseReadinessDialog() { + const t = useTranslations("desktopUse.readinessModal"); + const [state, setState] = React.useState({ + open: false, + readiness: null, + }); + + React.useEffect(() => subscribeDesktopUseReadinessModal(setState), []); + + const reason = state.readiness?.reason ?? null; + const source = state.source ?? "generic"; + + const onOpenSettings = () => { + closeDesktopUseReadinessModal(); + // Force next entry to re-check after user may install / grant. + invalidateDesktopUseReadinessCache(); + // Non-hook path: safe in root layout prerender (no useSearchParams). + openDesktopUseSettingsInApp(); + }; + + return ( + { + if (!open) closeDesktopUseReadinessModal(); + }} + > + + +
+ +
+ {t("title")} + +
+

{t(`source.${source}`)}

+

+ {t(reasonMessageKey(reason))} +

+
    +
  • {t("needs.engine")}
    • +
  • {t("needs.accessibility")}
    • +
  • {t("needs.screenRecording")}
    • +
+

{t("hint")}

+
+ + + + + + + + + ); +} diff --git a/apps/web/src/features/desktop-use/components/DesktopUseReadinessHost.tsx b/apps/web/src/features/desktop-use/components/DesktopUseReadinessHost.tsx new file mode 100644 index 000000000..eb9a7e5dc --- /dev/null +++ b/apps/web/src/features/desktop-use/components/DesktopUseReadinessHost.tsx @@ -0,0 +1,11 @@ +"use client"; + +/** + * Mount once near app root so gateDesktopUseFeature() can open a modal + * from AppShot / slash / any entry without prop drilling. + */ +import { DesktopUseReadinessDialog } from "./DesktopUseReadinessDialog"; + +export function DesktopUseReadinessHost() { + return ; +} diff --git a/apps/web/src/features/desktop-use/lib/__tests__/readiness.test.ts b/apps/web/src/features/desktop-use/lib/__tests__/readiness.test.ts new file mode 100644 index 000000000..9b6c1dc8b --- /dev/null +++ b/apps/web/src/features/desktop-use/lib/__tests__/readiness.test.ts @@ -0,0 +1,106 @@ +import { afterEach, beforeEach, describe, expect, it, mock } from "bun:test"; + +// Minimal localStorage for bun +class MemoryStorage { + private map = new Map(); + getItem(k: string) { + return this.map.has(k) ? this.map.get(k)! : null; + } + setItem(k: string, v: string) { + this.map.set(k, v); + } + removeItem(k: string) { + this.map.delete(k); + } + clear() { + this.map.clear(); + } +} + +const storage = new MemoryStorage(); +// @ts-expect-error test polyfill +globalThis.localStorage = storage; +// @ts-expect-error test polyfill +globalThis.window = globalThis; + +const invokeMock = mock(async (_cmd: string) => ({ + engine_installed: true, + engine_ready: true, + accessibility: true, + screen_recording: true, +})); + +mock.module("@/shared/lib/desktop-bridge", () => ({ + isDesktopRuntime: () => true, + desktopInvoke: invokeMock, +})); + +describe("desktop-use readiness cache", () => { + beforeEach(() => { + storage.clear(); + invokeMock.mockClear(); + invokeMock.mockImplementation(async () => ({ + engine_installed: true, + engine_ready: true, + accessibility: true, + screen_recording: true, + })); + }); + + afterEach(() => { + storage.clear(); + }); + + it("caches ready doctor results and serves from cache", async () => { + const { + fetchDesktopUseReadiness, + peekDesktopUseReadinessCache, + invalidateDesktopUseReadinessCache, + } = await import("../readiness"); + + invalidateDesktopUseReadinessCache(); + const first = await fetchDesktopUseReadiness({ force: true }); + expect(first.ready).toBe(true); + expect(first.fromCache).toBe(false); + expect(invokeMock).toHaveBeenCalledTimes(1); + + const peek = peekDesktopUseReadinessCache(); + expect(peek?.ready).toBe(true); + expect(peek?.fromCache).toBe(true); + + const second = await fetchDesktopUseReadiness(); + expect(second.ready).toBe(true); + // Still one doctor call — served from cache + expect(invokeMock).toHaveBeenCalledTimes(1); + }); + + it("blocks when engine is not installed", async () => { + invokeMock.mockImplementation(async () => ({ + engine_installed: false, + engine_ready: false, + accessibility: null, + screen_recording: null, + })); + const { fetchDesktopUseReadiness, invalidateDesktopUseReadinessCache } = + await import("../readiness"); + invalidateDesktopUseReadinessCache(); + const r = await fetchDesktopUseReadiness({ force: true }); + expect(r.ready).toBe(false); + expect(r.reason).toBe("engine_not_installed"); + }); + + it("blocks when screen recording is explicitly denied", async () => { + invokeMock.mockImplementation(async () => ({ + engine_installed: true, + engine_ready: true, + accessibility: true, + screen_recording: false, + })); + const { fetchDesktopUseReadiness, invalidateDesktopUseReadinessCache } = + await import("../readiness"); + invalidateDesktopUseReadinessCache(); + const r = await fetchDesktopUseReadiness({ force: true }); + expect(r.ready).toBe(false); + expect(r.reason).toBe("permission_screen_recording"); + }); +}); diff --git a/apps/web/src/features/desktop-use/lib/readiness-modal-bus.ts b/apps/web/src/features/desktop-use/lib/readiness-modal-bus.ts new file mode 100644 index 000000000..a2df6887e --- /dev/null +++ b/apps/web/src/features/desktop-use/lib/readiness-modal-bus.ts @@ -0,0 +1,78 @@ +/** + * Tiny pub/sub so non-React entry points can open the Desktop Use readiness modal. + */ + +import type { DesktopUseReadiness } from "./readiness"; + +export type DesktopUseReadinessModalState = { + open: boolean; + readiness: DesktopUseReadiness | null; + /** Optional label of the feature that triggered the gate (for copy). */ + source?: "appshot" | "slash" | "generic"; +}; + +type Listener = (state: DesktopUseReadinessModalState) => void; + +let state: DesktopUseReadinessModalState = { + open: false, + readiness: null, + source: undefined, +}; +const listeners = new Set(); + +function publish() { + for (const l of listeners) { + try { + l(state); + } catch { + /* ignore subscriber errors */ + } + } +} + +export function getDesktopUseReadinessModalState(): DesktopUseReadinessModalState { + return state; +} + +export function subscribeDesktopUseReadinessModal(listener: Listener): () => void { + listeners.add(listener); + listener(state); + return () => { + listeners.delete(listener); + }; +} + +export function openDesktopUseReadinessModal( + readiness: DesktopUseReadiness, + source: DesktopUseReadinessModalState["source"] = "generic", +): void { + state = { open: true, readiness, source }; + publish(); +} + +export function closeDesktopUseReadinessModal(): void { + if (!state.open) return; + state = { open: false, readiness: state.readiness, source: state.source }; + publish(); +} + +/** + * Entry helper: background readiness check → modal when not ready. + * Does not block the caller. + */ +export function gateDesktopUseFeature( + source: DesktopUseReadinessModalState["source"] = "generic", + handlers?: { + onReady?: () => void; + }, +): void { + // Lazy import to avoid circular deps with readiness.ts call sites + void import("./readiness").then( + ({ checkDesktopUseReadinessInBackground }) => { + checkDesktopUseReadinessInBackground({ + onReady: () => handlers?.onReady?.(), + onBlocked: (r) => openDesktopUseReadinessModal(r, source), + }); + }, + ); +} diff --git a/apps/web/src/features/desktop-use/lib/readiness.ts b/apps/web/src/features/desktop-use/lib/readiness.ts new file mode 100644 index 000000000..d31aba0bf --- /dev/null +++ b/apps/web/src/features/desktop-use/lib/readiness.ts @@ -0,0 +1,242 @@ +/** + * Desktop Use readiness — async doctor check + localStorage cache. + * + * Entry points (AppShot, slash Desktop Use, …) call + * {@link checkDesktopUseReadinessInBackground} so the UI never blocks on IPC. + */ + +import { desktopInvoke, isDesktopRuntime } from "@/shared/lib/desktop-bridge"; + +export type DesktopUseReadinessReason = + | "web_only" + | "engine_not_installed" + | "engine_not_running" + | "permission_accessibility" + | "permission_screen_recording" + | "permission_both" + | "unknown"; + +export type DesktopUseReadiness = { + ready: boolean; + reason: DesktopUseReadinessReason | null; + engineInstalled: boolean; + engineReady: boolean; + accessibility: boolean | null; + screenRecording: boolean | null; + checkedAt: number; + fromCache: boolean; +}; + +type DoctorPayload = { + engine_installed?: boolean; + engine_ready?: boolean; + accessibility?: boolean | null; + screen_recording?: boolean | null; +}; + +const CACHE_KEY = "atmos:v1:desktop-use:readiness"; +/** Positive (ready) cache TTL. */ +const READY_TTL_MS = 5 * 60 * 1000; +/** Negative cache TTL — short so fixing permissions is picked up quickly. */ +const NOT_READY_TTL_MS = 45 * 1000; + +type CacheRecord = Omit; + +let memoryCache: CacheRecord | null = null; +let inflight: Promise | null = null; + +function now(): number { + return Date.now(); +} + +function readStorage(): CacheRecord | null { + if (typeof window === "undefined") return null; + try { + const raw = window.localStorage.getItem(CACHE_KEY); + if (!raw) return null; + const parsed = JSON.parse(raw) as CacheRecord; + if ( + typeof parsed?.checkedAt !== "number" || + typeof parsed?.ready !== "boolean" + ) { + return null; + } + return parsed; + } catch { + return null; + } +} + +function writeStorage(record: CacheRecord): void { + memoryCache = record; + if (typeof window === "undefined") return; + try { + window.localStorage.setItem(CACHE_KEY, JSON.stringify(record)); + } catch { + /* quota / private mode */ + } +} + +function isFresh(record: CacheRecord, at = now()): boolean { + const age = at - record.checkedAt; + if (age < 0) return false; + return age <= (record.ready ? READY_TTL_MS : NOT_READY_TTL_MS); +} + +function derive( + doctor: DoctorPayload, + checkedAt: number, + fromCache: boolean, +): DesktopUseReadiness { + const engineInstalled = Boolean(doctor.engine_installed); + const engineReady = Boolean(doctor.engine_ready); + const accessibility = + typeof doctor.accessibility === "boolean" ? doctor.accessibility : null; + const screenRecording = + typeof doctor.screen_recording === "boolean" + ? doctor.screen_recording + : null; + + let reason: DesktopUseReadinessReason | null = null; + if (!engineInstalled) { + reason = "engine_not_installed"; + } else if (!engineReady) { + reason = "engine_not_running"; + } else { + const axOk = accessibility !== false; + const scrOk = screenRecording !== false; + if (!axOk && !scrOk) reason = "permission_both"; + else if (!axOk) reason = "permission_accessibility"; + else if (!scrOk) reason = "permission_screen_recording"; + } + + // Treat null permission as unknown-but-allow when engine ready (doctor may + // not always probe). Only explicit false blocks. + const ready = reason === null; + + return { + ready, + reason, + engineInstalled, + engineReady, + accessibility, + screenRecording, + checkedAt, + fromCache, + }; +} + +function toCacheRecord(r: DesktopUseReadiness): CacheRecord { + const { fromCache: _fc, ...rest } = r; + return rest; +} + +/** Invalidate cache (e.g. after install / uninstall / grant in Settings). */ +export function invalidateDesktopUseReadinessCache(): void { + memoryCache = null; + if (typeof window === "undefined") return; + try { + window.localStorage.removeItem(CACHE_KEY); + } catch { + /* ignore */ + } +} + +/** Sync peek: memory → localStorage if still fresh. */ +export function peekDesktopUseReadinessCache(): DesktopUseReadiness | null { + const rec = memoryCache ?? readStorage(); + if (!rec) return null; + memoryCache = rec; + if (!isFresh(rec)) return { ...rec, fromCache: true }; + return { ...rec, fromCache: true }; +} + +/** + * Fetch doctor (or use fresh cache). Concurrent callers share one in-flight request. + */ +export async function fetchDesktopUseReadiness(opts?: { + force?: boolean; +}): Promise { + if (!isDesktopRuntime()) { + return { + ready: false, + reason: "web_only", + engineInstalled: false, + engineReady: false, + accessibility: null, + screenRecording: null, + checkedAt: now(), + fromCache: false, + }; + } + + if (!opts?.force) { + const cached = peekDesktopUseReadinessCache(); + if (cached && isFresh(toCacheRecord(cached))) { + return cached; + } + } + + if (inflight && !opts?.force) { + return inflight; + } + + inflight = (async () => { + try { + const doctor = await desktopInvoke("desktop_use_doctor"); + const result = derive(doctor ?? {}, now(), false); + writeStorage(toCacheRecord(result)); + return result; + } catch { + const fallback = derive( + { + engine_installed: false, + engine_ready: false, + }, + now(), + false, + ); + // Soft-fail: if we had a ready cache, keep using it briefly on transient error + const prev = memoryCache ?? readStorage(); + if (prev?.ready && now() - prev.checkedAt < READY_TTL_MS) { + return { ...prev, fromCache: true }; + } + writeStorage(toCacheRecord({ ...fallback, reason: "unknown" })); + return { ...fallback, reason: "unknown", fromCache: false }; + } finally { + inflight = null; + } + })(); + + return inflight; +} + +/** + * Non-blocking gate for feature entry points. + * + * - Fresh **ready** cache → call `onReady` immediately; soft-refresh in background. + * - Fresh **not-ready** cache → call `onBlocked` immediately; soft-refresh may update. + * - No / stale cache → refresh in background only; `onBlocked`/`onReady` when result lands. + * Never awaits from the caller's perspective. + */ +export function checkDesktopUseReadinessInBackground(handlers: { + onReady?: (r: DesktopUseReadiness) => void; + onBlocked?: (r: DesktopUseReadiness) => void; +}): void { + const emit = (r: DesktopUseReadiness) => { + if (r.ready) handlers.onReady?.(r); + else handlers.onBlocked?.(r); + }; + + const cached = peekDesktopUseReadinessCache(); + if (cached && isFresh(toCacheRecord(cached))) { + emit(cached); + // Background soft refresh without forcing UI if still same readiness class + void fetchDesktopUseReadiness({ force: true }).then((fresh) => { + if (fresh.ready !== cached.ready) emit(fresh); + }); + return; + } + + void fetchDesktopUseReadiness({ force: true }).then(emit); +} diff --git a/apps/web/src/features/settings/components/DesktopUsePermissionsPanel.tsx b/apps/web/src/features/settings/components/DesktopUsePermissionsPanel.tsx index 1e0de1b40..1a23f9f59 100644 --- a/apps/web/src/features/settings/components/DesktopUsePermissionsPanel.tsx +++ b/apps/web/src/features/settings/components/DesktopUsePermissionsPanel.tsx @@ -8,6 +8,7 @@ import React from "react"; import { useTranslations } from "next-intl"; import { Button, cn } from "@workspace/ui"; import { Check, Loader2, RefreshCw, Settings2 } from "lucide-react"; +import { invalidateDesktopUseReadinessCache } from "@/features/desktop-use/lib/readiness"; import { desktopInvoke, isDesktopRuntime } from "@/shared/lib/desktop-bridge"; type PermissionName = "accessibility" | "screen_recording"; @@ -96,6 +97,7 @@ export function DesktopUsePermissionsPanel({ setError(null); try { await desktopInvoke("desktop_use_grant_permissions", { target }); + invalidateDesktopUseReadinessCache(); stopPoll(); const started = Date.now(); pollRef.current = setInterval(() => { diff --git a/apps/web/src/features/settings/components/DesktopUseSettingsSection.tsx b/apps/web/src/features/settings/components/DesktopUseSettingsSection.tsx index 52d9993e6..814b661c7 100644 --- a/apps/web/src/features/settings/components/DesktopUseSettingsSection.tsx +++ b/apps/web/src/features/settings/components/DesktopUseSettingsSection.tsx @@ -2,9 +2,17 @@ import React, { useCallback, useEffect, useState } from "react"; import { useTranslations } from "next-intl"; -import { Button } from "@workspace/ui/components/ui/button"; -import { Skeleton } from "@workspace/ui/components/ui/skeleton"; -import { Switch } from "@workspace/ui/components/ui/switch"; +import { + Button, + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, + Skeleton, + Switch, +} from "@workspace/ui"; import { ArrowUpCircle, Download, @@ -13,6 +21,7 @@ import { Trash2, } from "lucide-react"; import { DesktopUsePermissionsPanel } from "@/features/settings/components/DesktopUsePermissionsPanel"; +import { invalidateDesktopUseReadinessCache } from "@/features/desktop-use/lib/readiness"; import { desktopInvoke, isDesktopRuntime } from "@/shared/lib/desktop-bridge"; type DriverStatus = { @@ -84,6 +93,7 @@ export function DesktopUseSettingsSection() { const [busyAction, setBusyAction] = useState(null); const [error, setError] = useState(null); const [borderBusy, setBorderBusy] = useState(false); + const [uninstallOpen, setUninstallOpen] = useState(false); const load = useCallback(async (opts?: { silent?: boolean }) => { const silent = Boolean(opts?.silent); @@ -173,6 +183,8 @@ export function DesktopUseSettingsSection() { setError(null); try { await fn(); + // Install / stop / uninstall change doctor results — drop readiness cache. + invalidateDesktopUseReadinessCache(); await load({ silent: true }); } catch (e) { const fallback = @@ -312,7 +324,7 @@ export function DesktopUseSettingsSection() { ) : null} - {/* Remove — always visible when installed */} + {/* Remove — always visible when installed; confirm before destructive uninstall */} {installed && desktop ? ( - void run("uninstall", async () => { - await desktopInvoke("desktop_use_driver_uninstall"); - }) - } + onClick={() => setUninstallOpen(true)} className="cursor-pointer text-destructive hover:text-destructive" > {busyAction === "uninstall" ? ( @@ -340,6 +348,68 @@ export function DesktopUseSettingsSection() { ) : null} + { + if (busyAction === "uninstall") return; + setUninstallOpen(open); + }} + > + + +
+ +
+ {t("uninstallConfirm.title")} + +
+

{t("uninstallConfirm.description")}

+
+

+ {t("uninstallConfirm.consequencesTitle")} +

+
    +
  • {t("uninstallConfirm.consequences.appshotHost")}
    • +
  • {t("uninstallConfirm.consequences.agentControl")}
    • +
  • {t("uninstallConfirm.consequences.cli")}
    • +
  • {t("uninstallConfirm.consequences.chrome")}
    • +
+
+

{t("uninstallConfirm.note")}

+
+ + + + + + + + + {/* Operation border chrome */} { expect(section).toContain("DesktopUsePermissionsPanel"); }); + it("uninstall requires a confirm dialog that lists feature impact", () => { + const section = readFileSync( + join( + root, + "apps/web/src/features/settings/components/DesktopUseSettingsSection.tsx", + ), + "utf8", + ); + expect(section).toContain("uninstallOpen"); + expect(section).toContain('t("uninstallConfirm.title")'); + expect(section).toContain('t("uninstallConfirm.consequences.appshotHost")'); + expect(section).toContain('t("uninstallConfirm.consequences.agentControl")'); + expect(section).toContain('t("uninstallConfirm.consequences.cli")'); + expect(section).toContain("desktop_use_driver_uninstall"); + // Must not uninstall on first click without opening confirm + expect(section).toMatch(/onClick=\{\(\)\s*=>\s*setUninstallOpen\(true\)\}/); + }); + it("exposes operation border toggle via prefs IPC", () => { const section = readFileSync( join( From 6cfe7b80423d896ef69d7807cf8ded84be58bb6b Mon Sep 17 00:00:00 2001 From: AarynLu Date: Wed, 5 Aug 2026 20:28:18 +0800 Subject: [PATCH 18/43] fix(web): desktop fullscreen bus and chrome padding polish Coordinate fullscreen state across shell chrome, traffic-light padding, welcome/appshot popover behavior, and layout hooks. --- .../src/app-shell/use-header-fullscreen.ts | 19 +--- apps/web/src/app/layout.tsx | 2 + .../components/AppshotsHistoryPopover.tsx | 6 ++ .../welcome/components/WelcomePage.tsx | 24 ++++- .../use-desktop-traffic-lights-padding.ts | 64 ++++------- .../__tests__/desktop-fullscreen-bus.test.ts | 26 +++++ .../src/shared/lib/desktop-fullscreen-bus.ts | 100 ++++++++++++++++++ 7 files changed, 175 insertions(+), 66 deletions(-) create mode 100644 apps/web/src/shared/lib/__tests__/desktop-fullscreen-bus.test.ts create mode 100644 apps/web/src/shared/lib/desktop-fullscreen-bus.ts diff --git a/apps/web/src/app-shell/use-header-fullscreen.ts b/apps/web/src/app-shell/use-header-fullscreen.ts index 7b6eb0378..f7bc7c891 100644 --- a/apps/web/src/app-shell/use-header-fullscreen.ts +++ b/apps/web/src/app-shell/use-header-fullscreen.ts @@ -2,10 +2,10 @@ import { useCallback, useEffect, useRef, useState } from "react"; import { desktopInvoke, - desktopListen, isDesktopRuntime, isTauriShell, } from "@/shared/lib/desktop-bridge"; +import { subscribeDesktopFullscreen } from "@/shared/lib/desktop-fullscreen-bus"; export function useHeaderFullscreen() { const [isDesktopFullscreen, setIsDesktopFullscreen] = useState(false); @@ -81,20 +81,9 @@ export function useHeaderFullscreen() { unlisten = off; }); } else { - void desktopListen("window-fullscreen-changed", (payload) => { - const fs = Boolean( - payload && - typeof payload === "object" && - "fullscreen" in payload && - (payload as { fullscreen?: unknown }).fullscreen, - ); - applyFullscreenState(fs); - }).then((off) => { - if (disposed) { - off(); - return; - } - unlisten = off; + // Shared bus — one IPC listener for all fullscreen consumers. + unlisten = subscribeDesktopFullscreen((fs) => { + if (!disposed) applyFullscreenState(fs); }); } diff --git a/apps/web/src/app/layout.tsx b/apps/web/src/app/layout.tsx index ffd02cb1d..3aeee59f4 100644 --- a/apps/web/src/app/layout.tsx +++ b/apps/web/src/app/layout.tsx @@ -12,6 +12,7 @@ import { DesktopExternalUrlBridge } from "@/providers/app/desktop-external-url-b import { WorkbenchIntlProvider } from "@/providers/app/workbench-intl-provider"; import UpdateNotification from "@/app-shell/UpdateNotification"; import { ServerStateEventBridge } from "@/providers/app/server-state-event-bridge"; +import { DesktopUseReadinessHost } from "@/features/desktop-use/components/DesktopUseReadinessHost"; import { AgentToastProvider, AnchoredToastProvider, @@ -83,6 +84,7 @@ export default function RootLayout({ children }: { children: React.ReactNode }) {children} + diff --git a/apps/web/src/features/appshot/components/AppshotsHistoryPopover.tsx b/apps/web/src/features/appshot/components/AppshotsHistoryPopover.tsx index 6b01ca9a9..2a3ab4eaa 100644 --- a/apps/web/src/features/appshot/components/AppshotsHistoryPopover.tsx +++ b/apps/web/src/features/appshot/components/AppshotsHistoryPopover.tsx @@ -137,6 +137,12 @@ export function AppshotsHistoryPopover({ } void refreshStatus(); void refreshHistory(); + // Background readiness gate — does not block popover open / history load. + void import("@/features/desktop-use/lib/readiness-modal-bus").then( + ({ gateDesktopUseFeature }) => { + gateDesktopUseFeature("appshot"); + }, + ); }, [open, refreshHistory, refreshStatus]); React.useEffect(() => { diff --git a/apps/web/src/features/welcome/components/WelcomePage.tsx b/apps/web/src/features/welcome/components/WelcomePage.tsx index ff9fad0da..f09347e56 100644 --- a/apps/web/src/features/welcome/components/WelcomePage.tsx +++ b/apps/web/src/features/welcome/components/WelcomePage.tsx @@ -555,12 +555,26 @@ const WelcomePage: React.FC = ({ setSlashPopover(null); return; } - composerRef.current?.applySlashAtRange( - popover.slashOffset, - popover.query.length, - { kind: "skill", absolutePath: skill.absolutePath, name: skill.name }, - ); + // Non-blocking readiness: insert skill only when engine + permissions OK; + // otherwise open Settings guide modal (cached doctor, no UI freeze). setSlashPopover(null); + void import("@/features/desktop-use/lib/readiness-modal-bus").then( + ({ gateDesktopUseFeature }) => { + gateDesktopUseFeature("slash", { + onReady: () => { + composerRef.current?.applySlashAtRange( + popover.slashOffset, + popover.query.length, + { + kind: "skill", + absolutePath: skill.absolutePath, + name: skill.name, + }, + ); + }, + }); + }, + ); return; } }, diff --git a/apps/web/src/shared/hooks/use-desktop-traffic-lights-padding.ts b/apps/web/src/shared/hooks/use-desktop-traffic-lights-padding.ts index d8b51d927..967fcfe8f 100644 --- a/apps/web/src/shared/hooks/use-desktop-traffic-lights-padding.ts +++ b/apps/web/src/shared/hooks/use-desktop-traffic-lights-padding.ts @@ -2,18 +2,18 @@ import { useEffect, useState } from "react"; -import { - desktopInvoke, - desktopListen, - isTauriShell, -} from "@/shared/lib/desktop-bridge"; +import { isTauriShell } from "@/shared/lib/desktop-bridge"; import { isDesktopRuntime } from "@/shared/lib/desktop-runtime"; +import { subscribeDesktopFullscreen } from "@/shared/lib/desktop-fullscreen-bus"; /** * True when the window shows macOS traffic lights that need content inset * (not fullscreen). Used by main Header (pl-[92px]), agent-chat, canvas, etc. * * Fullscreen: lights hide → padding off. Exit fullscreen → padding back. + * + * Electron path shares one IPC listener via desktop-fullscreen-bus (avoids + * MaxListenersExceededWarning when many components mount this hook). */ export function useDesktopTrafficLightsPadding(): boolean { const [needsPadding, setNeedsPadding] = useState(false); @@ -35,30 +35,20 @@ export function useDesktopTrafficLightsPadding(): boolean { if (!disposed) setNeedsPadding(!fullscreen); }; - const sync = async () => { - try { - if (isTauriShell()) { - const { getCurrentWindow } = await import("@tauri-apps/api/window"); - const fullscreen = await getCurrentWindow().isFullscreen(); - apply(fullscreen); - return; - } - // Electron - const fullscreen = await desktopInvoke("window_is_fullscreen"); - apply(Boolean(fullscreen)); - } catch { - // Desktop shell but query failed — still reserve lights when not browser-FS. - apply(!!document.fullscreenElement); - } - }; - - void sync(); - if (isTauriShell()) { void import("@tauri-apps/api/window").then(async ({ getCurrentWindow }) => { const currentWindow = getCurrentWindow(); - const off = await currentWindow.onResized(() => { - void sync(); + try { + apply(await currentWindow.isFullscreen()); + } catch { + apply(!!document.fullscreenElement); + } + const off = await currentWindow.onResized(async () => { + try { + apply(await currentWindow.isFullscreen()); + } catch { + apply(!!document.fullscreenElement); + } }); if (disposed) { off(); @@ -67,30 +57,12 @@ export function useDesktopTrafficLightsPadding(): boolean { unlisten = off; }); } else { - void desktopListen( - "window-fullscreen-changed", - (payload: unknown) => { - const fs = Boolean( - payload && - typeof payload === "object" && - "fullscreen" in payload && - (payload as { fullscreen?: unknown }).fullscreen, - ); - apply(fs); - }, - ).then((off) => { - if (disposed) { - off(); - return; - } - unlisten = off; - }); + unlisten = subscribeDesktopFullscreen(apply); } - // Browser Fullscreen API (web / electron document FS fallback) const onDocFs = () => { if (isDesktopRuntime() && !isTauriShell()) { - void sync(); + apply(!!document.fullscreenElement); } }; document.addEventListener("fullscreenchange", onDocFs); diff --git a/apps/web/src/shared/lib/__tests__/desktop-fullscreen-bus.test.ts b/apps/web/src/shared/lib/__tests__/desktop-fullscreen-bus.test.ts new file mode 100644 index 000000000..9919f1158 --- /dev/null +++ b/apps/web/src/shared/lib/__tests__/desktop-fullscreen-bus.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it, beforeEach } from "bun:test"; +import { + __resetDesktopFullscreenBusForTests, + subscribeDesktopFullscreen, +} from "../desktop-fullscreen-bus"; + +describe("desktop-fullscreen-bus", () => { + beforeEach(() => { + __resetDesktopFullscreenBusForTests(); + }); + + it("fans out to multiple subscribers without requiring per-hook IPC", () => { + const a: boolean[] = []; + const b: boolean[] = []; + const offA = subscribeDesktopFullscreen((fs) => { + a.push(fs); + }); + const offB = subscribeDesktopFullscreen((fs) => { + b.push(fs); + }); + expect(typeof offA).toBe("function"); + expect(typeof offB).toBe("function"); + offA(); + offB(); + }); +}); diff --git a/apps/web/src/shared/lib/desktop-fullscreen-bus.ts b/apps/web/src/shared/lib/desktop-fullscreen-bus.ts new file mode 100644 index 000000000..12752ea28 --- /dev/null +++ b/apps/web/src/shared/lib/desktop-fullscreen-bus.ts @@ -0,0 +1,100 @@ +/** + * Single-subscriber bus for `window-fullscreen-changed`. + * + * Many UI surfaces need traffic-light padding; each used to call desktopListen + * and hit MaxListenersExceededWarning (default 10). One IPC subscription fans + * out to all React hooks instead. + */ + +import { desktopInvoke, desktopListen, isDesktopRuntime, isTauriShell } from "./desktop-bridge"; + +type Listener = (fullscreen: boolean) => void; + +const listeners = new Set(); +let ipcUnlisten: (() => void) | null = null; +let subscribePromise: Promise | null = null; +let currentFullscreen = false; +let known = false; + +function emit(fullscreen: boolean) { + known = true; + currentFullscreen = fullscreen; + for (const listener of listeners) { + try { + listener(fullscreen); + } catch { + /* ignore subscriber errors */ + } + } +} + +async function ensureIpcSubscription(): Promise { + if (!isDesktopRuntime() || isTauriShell()) return; + if (ipcUnlisten || subscribePromise) { + await subscribePromise; + return; + } + + subscribePromise = (async () => { + try { + const fs = await desktopInvoke("window_is_fullscreen"); + emit(Boolean(fs)); + } catch { + emit(!!document.fullscreenElement); + } + + const off = await desktopListen("window-fullscreen-changed", (payload) => { + const fs = Boolean( + payload && + typeof payload === "object" && + "fullscreen" in payload && + (payload as { fullscreen?: unknown }).fullscreen, + ); + emit(fs); + }); + ipcUnlisten = typeof off === "function" ? off : null; + })(); + + try { + await subscribePromise; + } finally { + subscribePromise = null; + } +} + +function maybeTeardownIpc() { + if (listeners.size > 0) return; + ipcUnlisten?.(); + ipcUnlisten = null; + known = false; +} + +/** + * Subscribe to desktop window fullscreen state. Returns unsubscribe. + * Immediately invokes listener when state is already known. + */ +export function subscribeDesktopFullscreen(listener: Listener): () => void { + listeners.add(listener); + if (known) { + try { + listener(currentFullscreen); + } catch { + /* ignore */ + } + } + void ensureIpcSubscription(); + + return () => { + listeners.delete(listener); + maybeTeardownIpc(); + }; +} + +export function __resetDesktopFullscreenBusForTests(): void { + listeners.clear(); + ipcUnlisten?.(); + ipcUnlisten = null; + subscribePromise = null; + known = false; + currentFullscreen = false; +} From f8b462f2e6da148a1b6cc21cbc426d01972a203a Mon Sep 17 00:00:00 2001 From: AarynLu Date: Wed, 5 Aug 2026 20:32:08 +0800 Subject: [PATCH 19/43] fix(terminal): stop reattach from stacking TUI frames into scrollback Capture viewport-only for alt-screen and inline mouse TUIs, and clear xterm scrollback after replaying those cells so refresh no longer grows a multi-viewport junk history under Grok-class apps. --- .../features/terminal/components/Terminal.tsx | 8 ++++- crates/core-engine/src/tmux/capture.rs | 30 +++++++++++------ packages/shared/src/terminal/snapshot.test.ts | 32 +++++++++++++++++++ packages/shared/src/terminal/snapshot.ts | 6 +++- 4 files changed, 64 insertions(+), 12 deletions(-) diff --git a/apps/web/src/features/terminal/components/Terminal.tsx b/apps/web/src/features/terminal/components/Terminal.tsx index 6360bc17e..35977dea1 100644 --- a/apps/web/src/features/terminal/components/Terminal.tsx +++ b/apps/web/src/features/terminal/components/Terminal.tsx @@ -430,7 +430,13 @@ const Terminal = ({ // heuristic with the full default (includes 1003 hover). const mouseRestore = mouseTrackingRestoreSequence(snapshot); const screenMode = useAlternateScreen ? "\x1b[?1049h" : "\x1b[?1049l"; + // Always erase display; clear scrollback for non-alt. Inline mouse TUIs + // (Grok) paint in the normal buffer — after replaying cells, clear + // scrollback again so older backends that still ship long history do not + // leave multi-viewport junk under the TUI (APP-054). const clearScrollback = useAlternateScreen ? "" : "\x1b[3J"; + const postHydrateScrollbackClear = + !useAlternateScreen && mouseRestore.length > 0 ? "\x1b[3J" : ""; const clearScreen = `${screenMode}\x1b[H\x1b[2J${clearScrollback}`; const data = normalizeSnapshotData(snapshot.data); const cursorRestore = `\x1b[${snapshot.cursor_y + 1};${snapshot.cursor_x + 1}H`; @@ -448,7 +454,7 @@ const Terminal = ({ // tmux `capture-pane -N` preserves trailing spaces so background-coloured // TUI panels survive reconnect. Replay them with autowrap disabled so a // full-width captured row does not create an extra wrapped line in xterm.js. - const payload = `${clearScreen}\x1b[?7l${data}\x1b[?7h\x1b[0m${cursorRestore}${mouseRestore}`; + const payload = `${clearScreen}\x1b[?7l${data}\x1b[?7h\x1b[0m${cursorRestore}${postHydrateScrollbackClear}${mouseRestore}`; writeXtermPayload(term, payload, () => { if (!useAlternateScreen) { jumpXtermToBottom(term); diff --git a/crates/core-engine/src/tmux/capture.rs b/crates/core-engine/src/tmux/capture.rs index 4eee92911..c31f1e237 100644 --- a/crates/core-engine/src/tmux/capture.rs +++ b/crates/core-engine/src/tmux/capture.rs @@ -270,7 +270,26 @@ impl TmuxEngine { "#{cursor_x}|#{cursor_y}|#{pane_width}|#{pane_height}|#{alternate_on}", ])?; let metadata = parse_pane_metadata(&metadata); - let capture_lines = if metadata.alternate { + let current_command = self + .get_pane_current_command(session_name, window_index) + .unwrap_or_default(); + let observed = self + .get_pane_mouse_tracking(session_name, window_index) + .ok() + .flatten(); + let (restore_mouse_tracking, mouse_tracking_sequence) = + resolve_mouse_tracking_restore(observed, metadata.alternate, ¤t_command); + + // Viewport-only when the pane is a TUI that will own the wheel (alt-screen + // or inline mouse TUI). Dumping tmux history for those apps re-hydrates + // stale TUI frames into xterm scrollback on every refresh (APP-054). + let viewport_only = metadata.alternate + || restore_mouse_tracking + || super::types::should_restore_tui_mouse_tracking( + metadata.alternate, + ¤t_command, + ); + let capture_lines = if viewport_only { Some(metadata.rows.max(1) as i32) } else { lines @@ -281,15 +300,6 @@ impl TmuxEngine { capture_lines, metadata.alternate, )?; - let current_command = self - .get_pane_current_command(session_name, window_index) - .unwrap_or_default(); - let observed = self - .get_pane_mouse_tracking(session_name, window_index) - .ok() - .flatten(); - let (restore_mouse_tracking, mouse_tracking_sequence) = - resolve_mouse_tracking_restore(observed, metadata.alternate, ¤t_command); Ok(TmuxPaneSnapshot { data, diff --git a/packages/shared/src/terminal/snapshot.test.ts b/packages/shared/src/terminal/snapshot.test.ts index 45937f2c9..5baa8fb34 100644 --- a/packages/shared/src/terminal/snapshot.test.ts +++ b/packages/shared/src/terminal/snapshot.test.ts @@ -160,4 +160,36 @@ describe("buildTerminalSnapshotRestorePayload", () => { expect(restoreMouseTracking).toBe(false); expect(payload.includes(ENABLE_TUI_MOUSE_TRACKING)).toBe(false); }); + + test("clears scrollback after replaying inline mouse TUI cells", () => { + // CSI 3J after cell data so a tall capture cannot leave multi-viewport junk. + const { payload, restoreMouseTracking } = buildTerminalSnapshotRestorePayload( + baseSnapshot({ + alternate: false, + restore_mouse_tracking: true, + data: "line-a\nline-b\nline-c", + }), + ); + expect(restoreMouseTracking).toBe(true); + const dataIdx = payload.indexOf("line-a"); + const postClearIdx = payload.indexOf("\x1b[3J", dataIdx); + const mouseIdx = payload.indexOf(ENABLE_TUI_MOUSE_TRACKING); + expect(dataIdx).toBeGreaterThan(-1); + expect(postClearIdx).toBeGreaterThan(dataIdx); + expect(mouseIdx).toBeGreaterThan(postClearIdx); + }); + + test("does not post-clear scrollback for idle shell history hydrate", () => { + const { payload, restoreMouseTracking } = buildTerminalSnapshotRestorePayload( + baseSnapshot({ + alternate: false, + restore_mouse_tracking: false, + data: "hist1\nhist2", + }), + ); + expect(restoreMouseTracking).toBe(false); + // Pre-clear may still use 3J once; no second 3J after cell data. + const dataIdx = payload.indexOf("hist1"); + expect(payload.indexOf("\x1b[3J", dataIdx)).toBe(-1); + }); }); diff --git a/packages/shared/src/terminal/snapshot.ts b/packages/shared/src/terminal/snapshot.ts index 9ad2ba4c6..58c1bf44c 100644 --- a/packages/shared/src/terminal/snapshot.ts +++ b/packages/shared/src/terminal/snapshot.ts @@ -94,12 +94,16 @@ export function buildTerminalSnapshotRestorePayload(snapshot: TerminalSnapshot): const restoreMouseTracking = mouseRestore.length > 0; const screenMode = useAlternateScreen ? "\x1b[?1049h" : "\x1b[?1049l"; const clearScrollback = useAlternateScreen ? "" : "\x1b[3J"; + // After replaying cells for an inline mouse TUI, clear scrollback again so a + // tall capture of prior TUI frames does not accumulate under the viewport. + const postHydrateScrollbackClear = + !useAlternateScreen && restoreMouseTracking ? "\x1b[3J" : ""; const clearScreen = `${screenMode}\x1b[H\x1b[2J${clearScrollback}`; const data = normalizeSnapshotData(snapshot.data); const cursorRestore = `\x1b[${snapshot.cursor_y + 1};${snapshot.cursor_x + 1}H`; return { - payload: `${clearScreen}\x1b[?7l${data}\x1b[?7h\x1b[0m${cursorRestore}${mouseRestore}`, + payload: `${clearScreen}\x1b[?7l${data}\x1b[?7h\x1b[0m${cursorRestore}${postHydrateScrollbackClear}${mouseRestore}`, useAlternateScreen, restoreMouseTracking, }; From aed791c2763a8bccaaba9443977b1229d2acc62d Mon Sep 17 00:00:00 2001 From: AarynLu Date: Wed, 5 Aug 2026 20:46:57 +0800 Subject: [PATCH 20/43] fix(terminal): reduce Grok warm-switch flash of home underscore cursor Skip tmux resize when fit grid is unchanged, and hide the local xterm cursor while inline mouse TUIs own the surface so reveal/redraw does not flash the theme underline cursor at the top-left. --- .../features/terminal/components/Terminal.tsx | 58 ++++++++++++++++--- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/apps/web/src/features/terminal/components/Terminal.tsx b/apps/web/src/features/terminal/components/Terminal.tsx index 35977dea1..88f7ae4a7 100644 --- a/apps/web/src/features/terminal/components/Terminal.tsx +++ b/apps/web/src/features/terminal/components/Terminal.tsx @@ -99,7 +99,12 @@ const INITIAL_CONNECT_STABLE_FRAMES = 2; const INITIAL_CONNECT_MAX_WAIT_FRAMES = 20; const CANVAS_TERMINAL_SCALE_FIT_DEBOUNCE_MS = 300; -function fitTerminalPreservingScroll(term: XTerm, fit: FitAddon) { +function fitTerminalPreservingScroll( + term: XTerm, + fit: FitAddon, +): { cols: number; rows: number; changed: boolean } { + const prevCols = term.cols; + const prevRows = term.rows; const before = term.buffer.active; const wasAtBottom = before.viewportY >= before.baseY; const distanceFromBottom = Math.max(0, before.baseY - before.viewportY); @@ -108,11 +113,23 @@ function fitTerminalPreservingScroll(term: XTerm, fit: FitAddon) { if (wasAtBottom) { jumpXtermToBottom(term); - return; + } else { + const after = term.buffer.active; + term.scrollToLine(Math.max(0, after.baseY - distanceFromBottom)); } - const after = term.buffer.active; - term.scrollToLine(Math.max(0, after.baseY - distanceFromBottom)); + return { + cols: term.cols, + rows: term.rows, + changed: term.cols !== prevCols || term.rows !== prevRows, + }; +} + +/** Hide local xterm cursor (DEC private mode 25). Inline TUIs (Grok) paint in + * the normal buffer; a brief clear/redraw would otherwise flash the theme's + * underline cursor as `_` at home before the app repaints. */ +function hideLocalXtermCursor(term: XTerm) { + term.write("\x1b[?25l"); } const Terminal = ({ @@ -360,8 +377,13 @@ const Terminal = ({ fitAddonRef.current && isTerminalContainerVisible(containerRef.current) ) { - fitTerminalPreservingScroll(terminalRef.current, fitAddonRef.current); - sendResizeRef.current({ cols: terminalRef.current.cols, rows: terminalRef.current.rows }); + const { cols, rows, changed } = fitTerminalPreservingScroll( + terminalRef.current, + fitAddonRef.current, + ); + if (changed) { + sendResizeRef.current({ cols, rows }); + } } scheduleInputReadyFallback(); }, [resetInputReady, scheduleInputReadyFallback, sessionId]); @@ -454,7 +476,9 @@ const Terminal = ({ // tmux `capture-pane -N` preserves trailing spaces so background-coloured // TUI panels survive reconnect. Replay them with autowrap disabled so a // full-width captured row does not create an extra wrapped line in xterm.js. - const payload = `${clearScreen}\x1b[?7l${data}\x1b[?7h\x1b[0m${cursorRestore}${postHydrateScrollbackClear}${mouseRestore}`; + // Hide local cursor for inline mouse TUIs so a repaint never flashes `_` at 0,0. + const hideCursor = mouseRestore.length > 0 && !useAlternateScreen ? "\x1b[?25l" : ""; + const payload = `${clearScreen}\x1b[?7l${data}\x1b[?7h\x1b[0m${cursorRestore}${postHydrateScrollbackClear}${mouseRestore}${hideCursor}`; writeXtermPayload(term, payload, () => { if (!useAlternateScreen) { jumpXtermToBottom(term); @@ -813,6 +837,11 @@ const Terminal = ({ hostForMouseChrome = containerRef.current?.parentElement ?? containerRef.current; const syncTuiMouseChrome = (active: boolean) => { hostForMouseChrome?.classList.toggle("atmos-tui-mouse-active", active); + // Inline TUI paint runs in the normal buffer; keep the local underline + // cursor hidden so warm-reveal / redraw never flashes `_` at home. + if (active) { + hideLocalXtermCursor(terminal); + } }; attachTuiMouseWheelMultiplier(terminal, { onMouseTrackingActiveChange: syncTuiMouseChrome, @@ -1362,6 +1391,8 @@ const Terminal = ({ }, [sessionId, workspaceId, cwd, projectRootPath]); // eslint-disable-line react-hooks/exhaustive-deps // Warm/inactive: disconnect RO so sibling hops do not thrash. Reveal: re-observe + one fit. + // Only pin tmux when the grid actually changes — unnecessary SIGWINCH makes + // inline Grok-class TUIs clear+repaint (flash of xterm `_` cursor at home). useEffect(() => { const ro = resizeObserverRef.current; const el = containerRef.current; @@ -1389,8 +1420,17 @@ const Terminal = ({ const fit = fitAddonRef.current; if (!term || !fit) return; if (!isTerminalContainerVisible(containerRef.current)) return; - fitTerminalPreservingScroll(term, fit); - sendResizeRef.current({ cols: term.cols, rows: term.rows }); + // Inline mouse TUI: hide local cursor before any possible repaint flash. + if ( + tuiMouseDesiredRef.current || + isTerminalMouseTrackingActive(term) + ) { + hideLocalXtermCursor(term); + } + const { cols, rows, changed } = fitTerminalPreservingScroll(term, fit); + if (changed) { + sendResizeRef.current({ cols, rows }); + } }); }); return () => { From 956048212268c008a9a59594b7e9161fd832332c Mon Sep 17 00:00:00 2001 From: AarynLu Date: Wed, 5 Aug 2026 20:50:34 +0800 Subject: [PATCH 21/43] fix(terminal): clear history when inline TUI disables mouse Grok paints into the normal buffer, so quit left full TUI frames in tmux scrollback and reattach rehydrated a ghost UI above the shell. On mouse mode off, clear-history; client also CSI 3J when shell CMD_END drops mouse. --- .../features/terminal/components/Terminal.tsx | 4 ++- crates/core-engine/src/tmux/mod.rs | 11 +++++++ .../src/service/terminal/mouse_mode_watch.rs | 20 +++++++++--- .../src/service/terminal/runtime.rs | 31 ++++++++++++++----- 4 files changed, 52 insertions(+), 14 deletions(-) diff --git a/apps/web/src/features/terminal/components/Terminal.tsx b/apps/web/src/features/terminal/components/Terminal.tsx index 88f7ae4a7..9ba7f7883 100644 --- a/apps/web/src/features/terminal/components/Terminal.tsx +++ b/apps/web/src/features/terminal/components/Terminal.tsx @@ -1073,7 +1073,9 @@ const Terminal = ({ if (shouldDisableTuiMouseOnCmdEnd(terminal.buffer.active.type)) { tuiMouseDesiredRef.current = false; lastMouseRestoreSequenceRef.current = ""; - terminal.write(DISABLE_TUI_MOUSE_TRACKING); + // Disable mouse + clear local scrollback so inline TUI (Grok) frames + // painted into the normal buffer do not linger above the shell. + terminal.write(`${DISABLE_TUI_MOUSE_TRACKING}\x1b[3J`); syncTuiMouseChrome(false); } } diff --git a/crates/core-engine/src/tmux/mod.rs b/crates/core-engine/src/tmux/mod.rs index 03ea672cd..c271ca9d9 100644 --- a/crates/core-engine/src/tmux/mod.rs +++ b/crates/core-engine/src/tmux/mod.rs @@ -428,6 +428,17 @@ impl TmuxEngine { ])?; Ok(MouseModeState::decode_persist(raw.trim())) } + + /// Drop scrollback for a pane (`clear-history`). + /// + /// Inline mouse TUIs (e.g. Grok) paint into the **normal** buffer, so every + /// full-frame redraw pollutes tmux history. After the app disables mouse + /// reporting (typically on exit), clear that junk so reattach does not + /// rehydrate a ghost TUI above the shell (APP-054). + pub fn clear_pane_history_by_id(&self, pane_id: &str) -> Result<()> { + self.run_tmux(&["clear-history", "-t", pane_id])?; + Ok(()) + } } #[cfg(test)] diff --git a/crates/core-service/src/service/terminal/mouse_mode_watch.rs b/crates/core-service/src/service/terminal/mouse_mode_watch.rs index 874b13338..49b0c9c2b 100644 --- a/crates/core-service/src/service/terminal/mouse_mode_watch.rs +++ b/crates/core-service/src/service/terminal/mouse_mode_watch.rs @@ -308,11 +308,21 @@ fn run_mouse_mode_watch( pane_id: pid, data, .. }) if pid == pane_id => { let data = passthrough.push(&data); - if !data.is_empty() && mouse_modes.observe_bytes(&data) { - if let Err(error) = - tmux.set_pane_mouse_tracking_by_id(&pane_id, &mouse_modes) - { - debug!("mouse-mode watch persist failed for {key}: {error}"); + if !data.is_empty() { + let mouse_was_active = mouse_modes.is_active(); + if mouse_modes.observe_bytes(&data) { + if let Err(error) = + tmux.set_pane_mouse_tracking_by_id(&pane_id, &mouse_modes) + { + debug!("mouse-mode watch persist failed for {key}: {error}"); + } + if mouse_was_active && !mouse_modes.is_active() { + if let Err(error) = tmux.clear_pane_history_by_id(&pane_id) { + debug!( + "mouse-mode watch clear-history failed for {key}: {error}" + ); + } + } } } } diff --git a/crates/core-service/src/service/terminal/runtime.rs b/crates/core-service/src/service/terminal/runtime.rs index 22e6ce965..9ed0bc6de 100644 --- a/crates/core-service/src/service/terminal/runtime.rs +++ b/crates/core-service/src/service/terminal/runtime.rs @@ -261,14 +261,29 @@ pub(super) fn run_control_mode_tmux_session( // unwrapper so its state machine stays in sync // with the live stream even while we drop them. let data = passthrough_unwrapper.push(&data); - if !data.is_empty() && mouse_modes.observe_bytes(&data) { - if let Err(error) = - tmux.set_pane_mouse_tracking_by_id(&reader_pane_id, &mouse_modes) - { - debug!( - "Failed to persist mouse tracking for {}: {}", - reader_session_id, error - ); + if !data.is_empty() { + let mouse_was_active = mouse_modes.is_active(); + if mouse_modes.observe_bytes(&data) { + if let Err(error) = tmux + .set_pane_mouse_tracking_by_id(&reader_pane_id, &mouse_modes) + { + debug!( + "Failed to persist mouse tracking for {}: {}", + reader_session_id, error + ); + } + // Inline TUI exit: drop normal-buffer paint history so + // refresh does not resurrect the TUI frame above the shell. + if mouse_was_active && !mouse_modes.is_active() { + if let Err(error) = + tmux.clear_pane_history_by_id(&reader_pane_id) + { + debug!( + "Failed to clear pane history after mouse off for {}: {}", + reader_session_id, error + ); + } + } } } if suppress_pane_output { From a6cad265d2c6d77f9367b1e4f8bdd1afdd0c285c Mon Sep 17 00:00:00 2001 From: AarynLu Date: Wed, 5 Aug 2026 20:58:48 +0800 Subject: [PATCH 22/43] fix(terminal,web): keep pre-TUI history; polish composer menu UX Revert pane clear-history on mouse-off so shell history before an inline TUI is preserved. Tighten Browser Use glyph spacing/color vs side, and add min gap between @/slash list items so selection/hover no longer fuse. --- .../welcome/components/PromptComposer.tsx | 8 ++--- .../components/SlashCommandPopover.tsx | 20 ++++++------ .../components/WelcomeMentionPopover.tsx | 8 ++--- .../__tests__/slash-command-icons.test.ts | 8 ++--- crates/core-engine/src/tmux/mod.rs | 11 ------- .../src/service/terminal/mouse_mode_watch.rs | 20 +++--------- .../src/service/terminal/runtime.rs | 31 +++++-------------- .../icons/browser-use-icon-static.tsx | 9 +++--- 8 files changed, 40 insertions(+), 75 deletions(-) diff --git a/apps/web/src/features/welcome/components/PromptComposer.tsx b/apps/web/src/features/welcome/components/PromptComposer.tsx index baf64927c..185666375 100644 --- a/apps/web/src/features/welcome/components/PromptComposer.tsx +++ b/apps/web/src/features/welcome/components/PromptComposer.tsx @@ -261,8 +261,8 @@ function buildBrowserUseChipIcon(): SVGSVGElement { svg.style.flexShrink = "0"; for (const d of [ - "M13 4H4a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V10", - "M2 8h11", + "M12 4H4a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2h16a2 2 0 0 0 2-2V9", + "M2 8h10", "M6 4v4", "M10 4v4", ]) { @@ -271,7 +271,7 @@ function buildBrowserUseChipIcon(): SVGSVGElement { svg.appendChild(path); } const g = document.createElementNS("http://www.w3.org/2000/svg", "g"); - g.setAttribute("transform", "translate(22.6 0.2) scale(-0.48 0.48)"); + g.setAttribute("transform", "translate(23.4 2.0) scale(-0.45 0.45)"); for (const d of [ "M14 4.1 12 6", "m5.1 8-2.9-.8", @@ -494,7 +494,7 @@ function buildChipNode(token: string): HTMLSpanElement { const isBrowserUse = absolutePath.includes("atmos-browser-use"); const isDesktopUse = absolutePath.includes("atmos-desktop-use"); if (isBrowserUse) { - span.className += " border-sky-500/35 bg-sky-500/10 text-sky-700 dark:text-sky-300"; + span.className += " border-amber-500/35 bg-amber-500/10 text-amber-700 dark:text-amber-300"; span.appendChild(buildBrowserUseChipIcon()); } else if (isDesktopUse) { span.className += diff --git a/apps/web/src/features/welcome/components/SlashCommandPopover.tsx b/apps/web/src/features/welcome/components/SlashCommandPopover.tsx index b8c32044e..aa2f275df 100644 --- a/apps/web/src/features/welcome/components/SlashCommandPopover.tsx +++ b/apps/web/src/features/welcome/components/SlashCommandPopover.tsx @@ -227,7 +227,7 @@ export function SlashCommandPopover({ }; const menuContent = ( -
+
{showCommands && visibleCommands.length > 0 ? ( <>
@@ -241,7 +241,7 @@ export function SlashCommandPopover({ setItemRef(index, el); }} className={cn( - "flex w-full items-center gap-2 rounded-md px-2.5 py-1 text-left hover:bg-muted", + "flex w-full items-center gap-2 rounded-md px-2.5 py-1.5 text-left hover:bg-muted", index === activeIndex && "bg-muted", )} onMouseDown={(event) => { @@ -254,7 +254,7 @@ export function SlashCommandPopover({ ) : command.id === "dynamic-skills" ? ( ) : command.id === BROWSER_USE_SLASH_COMMAND_ID ? ( - + ) : command.id === DESKTOP_USE_SLASH_COMMAND_ID ? ( ) : ( @@ -306,7 +306,7 @@ export function SlashCommandPopover({ disabled={isDisabled} aria-disabled={isDisabled} className={cn( - "flex w-full items-center gap-2 rounded-md px-2.5 py-1 text-left", + "flex w-full items-center gap-2 rounded-md px-2.5 py-1.5 text-left", isDisabled ? "cursor-default text-muted-foreground opacity-80" : "hover:bg-muted", @@ -319,7 +319,7 @@ export function SlashCommandPopover({ }} > {isBrowserUse ? ( - + ) : isDesktopUse ? ( ) : ( @@ -344,7 +344,7 @@ export function SlashCommandPopover({ setItemRef(skillsStartIndex + skillsCount, el); }} className={cn( - "flex w-full items-center gap-2 rounded-md px-2.5 py-1 text-left text-[11px] text-muted-foreground hover:bg-muted", + "flex w-full items-center gap-2 rounded-md px-2.5 py-1.5 text-left text-[11px] text-muted-foreground hover:bg-muted", skillsStartIndex + skillsCount === activeIndex && "bg-muted", )} onMouseDown={(event) => { @@ -381,7 +381,7 @@ export function SlashCommandPopover({ setItemRef(navIndex, el); }} className={cn( - "flex w-full items-center gap-2 rounded-md px-2.5 py-1 text-left hover:bg-muted", + "flex w-full items-center gap-2 rounded-md px-2.5 py-1.5 text-left hover:bg-muted", navIndex === activeIndex && "bg-muted", )} onMouseDown={(event) => { @@ -401,7 +401,7 @@ export function SlashCommandPopover({ setItemRef(projectsStartIndex + projectsCount, el); }} className={cn( - "flex w-full items-center gap-2 rounded-md px-2.5 py-1 text-left text-[11px] text-muted-foreground hover:bg-muted", + "flex w-full items-center gap-2 rounded-md px-2.5 py-1.5 text-left text-[11px] text-muted-foreground hover:bg-muted", projectsStartIndex + projectsCount === activeIndex && "bg-muted", )} onMouseDown={(event) => { @@ -438,7 +438,7 @@ export function SlashCommandPopover({ setItemRef(navIndex, el); }} className={cn( - "flex w-full items-center gap-2 rounded-md px-2.5 py-1 text-left hover:bg-muted", + "flex w-full items-center gap-2 rounded-md px-2.5 py-1.5 text-left hover:bg-muted", navIndex === activeIndex && "bg-muted", )} onMouseDown={(event) => { @@ -464,7 +464,7 @@ export function SlashCommandPopover({ setItemRef(agentsStartIndex + Math.min(filteredAgents.length, 3), el); }} className={cn( - "flex w-full items-center gap-2 rounded-md px-2.5 py-1 text-left text-[11px] text-muted-foreground hover:bg-muted", + "flex w-full items-center gap-2 rounded-md px-2.5 py-1.5 text-left text-[11px] text-muted-foreground hover:bg-muted", agentsStartIndex + Math.min(filteredAgents.length, 3) === activeIndex && "bg-muted", )} diff --git a/apps/web/src/features/welcome/components/WelcomeMentionPopover.tsx b/apps/web/src/features/welcome/components/WelcomeMentionPopover.tsx index 39de9127a..08b730dd7 100644 --- a/apps/web/src/features/welcome/components/WelcomeMentionPopover.tsx +++ b/apps/web/src/features/welcome/components/WelcomeMentionPopover.tsx @@ -111,7 +111,7 @@ export function WelcomeMentionPopover({ />
{ @@ -166,7 +166,7 @@ export function WelcomeMentionPopover({ onSetItemRef(prIndex, element); }} className={cn( - "flex w-full items-center gap-2 rounded-md px-2.5 py-1 text-left hover:bg-muted", + "flex w-full items-center gap-2 rounded-md px-2.5 py-1.5 text-left hover:bg-muted", prIndex === activeIndex && "bg-muted", )} onMouseDown={(event) => { @@ -218,7 +218,7 @@ export function WelcomeMentionPopover({ onSetItemRef(navIndex, element); }} className={cn( - "flex w-full items-center gap-2 rounded-md px-2.5 py-1 text-left hover:bg-muted", + "flex w-full items-center gap-2 rounded-md px-2.5 py-1.5 text-left hover:bg-muted", item.isHidden && "text-muted-foreground", navIndex === activeIndex && "bg-muted", )} diff --git a/apps/web/src/features/welcome/components/__tests__/slash-command-icons.test.ts b/apps/web/src/features/welcome/components/__tests__/slash-command-icons.test.ts index cddb8eff8..96ddf2c61 100644 --- a/apps/web/src/features/welcome/components/__tests__/slash-command-icons.test.ts +++ b/apps/web/src/features/welcome/components/__tests__/slash-command-icons.test.ts @@ -35,11 +35,11 @@ describe("composer slash command icons", () => { ); expect(icon).toContain("BrowserUseIconStatic"); // Title-bar chrome (browser window), not desktop stand - expect(icon).toContain("M13 4H4a2 2 0 0 0-2 2v12"); - expect(icon).toContain("M2 8h11"); + expect(icon).toContain("M12 4H4a2 2 0 0 0-2 2v12"); + expect(icon).toContain("M2 8h10"); expect(icon).not.toContain("M8 21h8"); - // Shared pointer composition with Desktop Use - expect(icon).toContain("translate(22.6 0.2) scale(-0.48 0.48)"); + // Pointer inset from window top edge (gap vs desktop-style spacing) + expect(icon).toContain("translate(23.4 2.0) scale(-0.45 0.45)"); }); it("Desktop Use static icon remains monitor + pointer", () => { diff --git a/crates/core-engine/src/tmux/mod.rs b/crates/core-engine/src/tmux/mod.rs index c271ca9d9..03ea672cd 100644 --- a/crates/core-engine/src/tmux/mod.rs +++ b/crates/core-engine/src/tmux/mod.rs @@ -428,17 +428,6 @@ impl TmuxEngine { ])?; Ok(MouseModeState::decode_persist(raw.trim())) } - - /// Drop scrollback for a pane (`clear-history`). - /// - /// Inline mouse TUIs (e.g. Grok) paint into the **normal** buffer, so every - /// full-frame redraw pollutes tmux history. After the app disables mouse - /// reporting (typically on exit), clear that junk so reattach does not - /// rehydrate a ghost TUI above the shell (APP-054). - pub fn clear_pane_history_by_id(&self, pane_id: &str) -> Result<()> { - self.run_tmux(&["clear-history", "-t", pane_id])?; - Ok(()) - } } #[cfg(test)] diff --git a/crates/core-service/src/service/terminal/mouse_mode_watch.rs b/crates/core-service/src/service/terminal/mouse_mode_watch.rs index 49b0c9c2b..874b13338 100644 --- a/crates/core-service/src/service/terminal/mouse_mode_watch.rs +++ b/crates/core-service/src/service/terminal/mouse_mode_watch.rs @@ -308,21 +308,11 @@ fn run_mouse_mode_watch( pane_id: pid, data, .. }) if pid == pane_id => { let data = passthrough.push(&data); - if !data.is_empty() { - let mouse_was_active = mouse_modes.is_active(); - if mouse_modes.observe_bytes(&data) { - if let Err(error) = - tmux.set_pane_mouse_tracking_by_id(&pane_id, &mouse_modes) - { - debug!("mouse-mode watch persist failed for {key}: {error}"); - } - if mouse_was_active && !mouse_modes.is_active() { - if let Err(error) = tmux.clear_pane_history_by_id(&pane_id) { - debug!( - "mouse-mode watch clear-history failed for {key}: {error}" - ); - } - } + if !data.is_empty() && mouse_modes.observe_bytes(&data) { + if let Err(error) = + tmux.set_pane_mouse_tracking_by_id(&pane_id, &mouse_modes) + { + debug!("mouse-mode watch persist failed for {key}: {error}"); } } } diff --git a/crates/core-service/src/service/terminal/runtime.rs b/crates/core-service/src/service/terminal/runtime.rs index 9ed0bc6de..22e6ce965 100644 --- a/crates/core-service/src/service/terminal/runtime.rs +++ b/crates/core-service/src/service/terminal/runtime.rs @@ -261,29 +261,14 @@ pub(super) fn run_control_mode_tmux_session( // unwrapper so its state machine stays in sync // with the live stream even while we drop them. let data = passthrough_unwrapper.push(&data); - if !data.is_empty() { - let mouse_was_active = mouse_modes.is_active(); - if mouse_modes.observe_bytes(&data) { - if let Err(error) = tmux - .set_pane_mouse_tracking_by_id(&reader_pane_id, &mouse_modes) - { - debug!( - "Failed to persist mouse tracking for {}: {}", - reader_session_id, error - ); - } - // Inline TUI exit: drop normal-buffer paint history so - // refresh does not resurrect the TUI frame above the shell. - if mouse_was_active && !mouse_modes.is_active() { - if let Err(error) = - tmux.clear_pane_history_by_id(&reader_pane_id) - { - debug!( - "Failed to clear pane history after mouse off for {}: {}", - reader_session_id, error - ); - } - } + if !data.is_empty() && mouse_modes.observe_bytes(&data) { + if let Err(error) = + tmux.set_pane_mouse_tracking_by_id(&reader_pane_id, &mouse_modes) + { + debug!( + "Failed to persist mouse tracking for {}: {}", + reader_session_id, error + ); } } if suppress_pane_output { diff --git a/packages/ui/src/components/icons/browser-use-icon-static.tsx b/packages/ui/src/components/icons/browser-use-icon-static.tsx index 9c0a3c3d4..1b70017aa 100644 --- a/packages/ui/src/components/icons/browser-use-icon-static.tsx +++ b/packages/ui/src/components/icons/browser-use-icon-static.tsx @@ -25,13 +25,14 @@ export function BrowserUseIconStatic({ aria-hidden > {/* Window frame open at top-right (notch for the control cursor). */} - + {/* Title-bar chrome (only under the closed portion of the top edge). */} - + - {/* Click pointer on the top-right notch, tip pointing ↗. */} - + {/* Pointer sits in the notch with a clear gap from the window top edge + (matches Desktop Use spacing feel). */} + From 60cceab74401f23cc2e25c78bd762c5dd13a3681 Mon Sep 17 00:00:00 2001 From: AarynLu Date: Wed, 5 Aug 2026 21:04:22 +0800 Subject: [PATCH 23/43] feat(web): regroup Desktop Use settings into collapsible cards Nest Stop/Uninstall under Control engine, put Permissions second, and move Operation border into a Visual feedback group. Each group uses SettingsGroupCard with an icon and smart default expand/collapse. --- .../components/DesktopUseSettingsSection.tsx | 369 ++++++++++-------- 1 file changed, 200 insertions(+), 169 deletions(-) diff --git a/apps/web/src/features/settings/components/DesktopUseSettingsSection.tsx b/apps/web/src/features/settings/components/DesktopUseSettingsSection.tsx index 814b661c7..ca7625747 100644 --- a/apps/web/src/features/settings/components/DesktopUseSettingsSection.tsx +++ b/apps/web/src/features/settings/components/DesktopUseSettingsSection.tsx @@ -1,6 +1,6 @@ "use client"; -import React, { useCallback, useEffect, useState } from "react"; +import React, { useCallback, useEffect, useRef, useState } from "react"; import { useTranslations } from "next-intl"; import { Button, @@ -15,12 +15,19 @@ import { } from "@workspace/ui"; import { ArrowUpCircle, + Cpu, Download, Loader2, + Scan, + Shield, Square, Trash2, } from "lucide-react"; import { DesktopUsePermissionsPanel } from "@/features/settings/components/DesktopUsePermissionsPanel"; +import { + SettingsGroupCard, + SettingsGroupRow, +} from "@/features/settings/components/settings/SettingsGroupCard"; import { invalidateDesktopUseReadinessCache } from "@/features/desktop-use/lib/readiness"; import { desktopInvoke, isDesktopRuntime } from "@/shared/lib/desktop-bridge"; @@ -51,40 +58,11 @@ type DesktopUseStatus = { prefs?: DesktopUsePrefs; }; -/** Same row chrome as Canvas / Terminal settings cards. */ -function SettingsItemCard({ - title, - description, - children, - wide = true, -}: { - title: string; - description?: string; - children: React.ReactNode; - wide?: boolean; -}) { - return ( -
-
-
-

{title}

- {description ? ( -

- {description} -

- ) : null} -
-
{children}
-
-
- ); -} +type DoctorLite = { + engine_installed?: boolean; + accessibility?: boolean | null; + screen_recording?: boolean | null; +}; export function DesktopUseSettingsSection() { const t = useTranslations("settings.desktopUse"); @@ -95,6 +73,12 @@ export function DesktopUseSettingsSection() { const [borderBusy, setBorderBusy] = useState(false); const [uninstallOpen, setUninstallOpen] = useState(false); + // Collapsible groups — defaults applied once after first status/doctor load. + const [engineOpen, setEngineOpen] = useState(true); + const [permissionsOpen, setPermissionsOpen] = useState(true); + const [visibilityOpen, setVisibilityOpen] = useState(true); + const defaultsAppliedRef = useRef(false); + const load = useCallback(async (opts?: { silent?: boolean }) => { const silent = Boolean(opts?.silent); if (!silent) setLoading(true); @@ -116,10 +100,38 @@ export function DesktopUseSettingsSection() { highlight_idle_ms: 8000, }, }); + if (!defaultsAppliedRef.current) { + defaultsAppliedRef.current = true; + // Not installed / not desktop → keep engine + permissions expanded. + setEngineOpen(true); + setPermissionsOpen(true); + } return; } const res = await desktopInvoke("desktop_use_status"); setStatus(res); + + // One-shot default collapse: installed engine ready → collapse; + // all permissions granted → collapse. User can always re-expand. + if (!defaultsAppliedRef.current) { + defaultsAppliedRef.current = true; + const installed = Boolean(res?.driver?.installed); + const updateAvailable = Boolean(res?.update_available); + // Collapse when installed and no pending update (actions are secondary). + setEngineOpen(!(installed && !updateAvailable)); + + let allGranted = false; + try { + const doctor = await desktopInvoke("desktop_use_doctor"); + allGranted = + Boolean(doctor?.engine_installed) && + doctor?.accessibility === true && + doctor?.screen_recording === true; + } catch { + allGranted = false; + } + setPermissionsOpen(!allGranted); + } } catch (e) { setError(e instanceof Error ? e.message : t("errors.statusFailed")); } finally { @@ -222,131 +234,145 @@ export function DesktopUseSettingsSection() { return ver ? `${t("engine.installed")} · ${ver}` : t("engine.installed"); })(); + const engineGroupDescription = (() => { + if (!desktop) return t("desktopOnly"); + if (!installed) return t("engine.installHint"); + if (updateAvailable) { + return t("engine.updateHint", { + current: installedVersion ?? "—", + next: pinnedVersion ?? "—", + }); + } + return t("groups.engine.description"); + })(); + return (
- {/* Control engine */} - - {loading ? ( - - ) : !desktop ? ( - {engineStatusLabel} - ) : !installed ? ( - - ) : updateAvailable ? ( -
+ + {loading ? ( + + ) : !desktop ? ( {engineStatusLabel} + ) : !installed ? ( -
- ) : ( - {engineStatusLabel} - )} - + ) : updateAvailable ? ( +
+ + {engineStatusLabel} + + +
+ ) : ( + + {engineStatusLabel} + + )} + - {/* Stop — always visible when installed */} - {installed && desktop ? ( - - - - ) : null} + {installed && desktop ? ( + <> + + + - {/* Remove — always visible when installed; confirm before destructive uninstall */} - {installed && desktop ? ( - - - - ) : null} + + + + + ) : null} + - {/* Operation border chrome */} - - {loading ? ( - - ) : !desktop ? ( - {t("status.webOnly")} - ) : ( - void setOperationBorder(!!checked)} - aria-label={t("border.title")} - /> - )} - + + - {/* Permissions card — same outer chrome */} -
-
-

- {t("permissions.title")} -

-

- {t("permissions.unifiedHint")} -

-
-
- -
-
+ {/* 3. Visual feedback — operation border (+ future under-cursor cues) */} + + + {loading ? ( + + ) : !desktop ? ( + + {t("status.webOnly")} + + ) : ( + void setOperationBorder(!!checked)} + aria-label={t("border.title")} + /> + )} + + {(error || status?.driver?.error) && (

From 940dd89f38f9902f7f8b14d7964e7083859ddb37 Mon Sep 17 00:00:00 2001 From: AarynLu Date: Wed, 5 Aug 2026 21:04:30 +0800 Subject: [PATCH 24/43] feat(web): add Desktop Use settings group i18n (en/zh) Add group titles/descriptions for engine, permissions, and visibility, plus engine status row labels and settings-search border entry. --- apps/web/messages/en.json | 24 ++++++++++++++++++++++-- apps/web/messages/zh.json | 24 ++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/apps/web/messages/en.json b/apps/web/messages/en.json index 4bfa03dd4..91de0be9e 100644 --- a/apps/web/messages/en.json +++ b/apps/web/messages/en.json @@ -4632,11 +4632,15 @@ "desktopUse": { "engine": { "label": "Control engine", - "description": "Install or remove the Desktop Use control engine" + "description": "Install, stop, or remove the Desktop Use control engine" }, "permissions": { "label": "Desktop permissions", "description": "Screen Recording and Accessibility for capture" + }, + "border": { + "label": "Operation border", + "description": "Visual feedback while Desktop Use operates" } } } @@ -5545,6 +5549,20 @@ "host": { "defaultName": "Atmos Desktop Use" }, + "groups": { + "engine": { + "title": "Control engine", + "description": "Install, update, stop, or remove the host engine used for capture and UI control." + }, + "permissions": { + "title": "Permissions", + "description": "Grant Accessibility and Screen Recording to the Desktop Use host app so capture and control work reliably." + }, + "visibility": { + "title": "Visual feedback", + "description": "On-screen cues while Desktop Use operates — borders now, under-cursor details later." + } + }, "engine": { "title": "Control engine", "hint": "Optional host engine for screen capture and UI control under a dedicated app identity.", @@ -5554,7 +5572,9 @@ "removeHint": "Remove the engine from this Mac. Capture and agent desktop control will fall back or become unavailable until reinstalled.", "notInstalled": "Not installed", "installed": "Installed", - "updateAvailable": "Update available" + "updateAvailable": "Update available", + "statusTitle": "Engine", + "statusDescription": "Download state and version of the Desktop Use control engine." }, "actions": { "install": "Install", diff --git a/apps/web/messages/zh.json b/apps/web/messages/zh.json index 8db79da26..9776cc60a 100644 --- a/apps/web/messages/zh.json +++ b/apps/web/messages/zh.json @@ -4632,11 +4632,15 @@ "desktopUse": { "engine": { "label": "控制引擎", - "description": "安装或移除 Desktop Use 控制引擎" + "description": "安装、停止或移除 Desktop Use 控制引擎" }, "permissions": { "label": "桌面权限", "description": "用于捕获的屏幕录制与辅助功能权限" + }, + "border": { + "label": "操作描边", + "description": "Desktop Use 操作时的视觉反馈" } } } @@ -5545,6 +5549,20 @@ "host": { "defaultName": "Atmos Desktop Use" }, + "groups": { + "engine": { + "title": "控制引擎", + "description": "安装、更新、停止或移除用于捕获与 UI 操控的 host 引擎。" + }, + "permissions": { + "title": "权限", + "description": "为 Desktop Use host 应用授予辅助功能与屏幕录制权限，以便稳定完成捕获与控制。" + }, + "visibility": { + "title": "视觉反馈", + "description": "Desktop Use 操作时的屏幕提示——当前支持描边，后续可扩展光标下方内容等。" + } + }, "engine": { "title": "控制引擎", "hint": "可选本机控制引擎，在独立 App 身份下完成截屏与 UI 操控。", @@ -5554,7 +5572,9 @@ "removeHint": "从本机移除引擎。在重新安装前，相关捕获与 Agent 桌面控制会降级或不可用。", "notInstalled": "未安装", "installed": "已安装", - "updateAvailable": "有可用更新" + "updateAvailable": "有可用更新", + "statusTitle": "引擎", + "statusDescription": "Desktop Use 控制引擎的下载状态与版本。" }, "actions": { "install": "安装", From d97f8c5fddc31eed29cec7d8b07d51b8dff2d7b9 Mon Sep 17 00:00:00 2001 From: AarynLu Date: Wed, 5 Aug 2026 21:04:30 +0800 Subject: [PATCH 25/43] test(web): cover Desktop Use settings groups and search keywords Assert collapsible group order/icons and extend settings search for stop/uninstall plus operation border under visual feedback. --- .../desktop-use-settings-section.test.ts | 36 ++++++++++++++++++- .../components/settings-modal-data.ts | 17 ++++++++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/apps/web/src/features/settings/components/__tests__/desktop-use-settings-section.test.ts b/apps/web/src/features/settings/components/__tests__/desktop-use-settings-section.test.ts index f16cd00df..8822a1be6 100644 --- a/apps/web/src/features/settings/components/__tests__/desktop-use-settings-section.test.ts +++ b/apps/web/src/features/settings/components/__tests__/desktop-use-settings-section.test.ts @@ -62,6 +62,39 @@ describe("Desktop Use settings wiring", () => { expect(section).toContain("DesktopUsePermissionsPanel"); }); + it("groups engine (stop/uninstall), permissions, and visibility as collapsible cards", () => { + const section = readFileSync( + join( + root, + "apps/web/src/features/settings/components/DesktopUseSettingsSection.tsx", + ), + "utf8", + ); + expect(section).toContain("SettingsGroupCard"); + expect(section).toContain("SettingsGroupRow"); + // Icons per group + expect(section).toContain("Cpu"); + expect(section).toContain("Shield"); + expect(section).toContain("Scan"); + // Group order: engine → permissions → visibility + const engineIdx = section.indexOf('t("groups.engine.title")'); + const permsIdx = section.indexOf('t("groups.permissions.title")'); + const visIdx = section.indexOf('t("groups.visibility.title")'); + expect(engineIdx).toBeGreaterThan(-1); + expect(permsIdx).toBeGreaterThan(engineIdx); + expect(visIdx).toBeGreaterThan(permsIdx); + // Stop / Uninstall live under the engine group (not standalone cards) + expect(section).toContain('t("actions.stop")'); + expect(section).toContain('t("actions.uninstall")'); + expect(section).toContain('t("engine.stopHint")'); + expect(section).toContain('t("engine.removeHint")'); + // Default collapse: installed engine (no update) → collapse; all perms → collapse + expect(section).toContain("defaultsAppliedRef"); + expect(section).toContain("setEngineOpen"); + expect(section).toContain("setPermissionsOpen"); + expect(section).toContain("desktop_use_doctor"); + }); + it("uninstall requires a confirm dialog that lists feature impact", () => { const section = readFileSync( join( @@ -80,7 +113,7 @@ describe("Desktop Use settings wiring", () => { expect(section).toMatch(/onClick=\{\(\)\s*=>\s*setUninstallOpen\(true\)\}/); }); - it("exposes operation border toggle via prefs IPC", () => { + it("exposes operation border toggle via prefs IPC inside visibility group", () => { const section = readFileSync( join( root, @@ -89,6 +122,7 @@ describe("Desktop Use settings wiring", () => { "utf8", ); expect(section).toContain('t("border.title")'); + expect(section).toContain('t("groups.visibility.title")'); expect(section).toContain("desktop_use_prefs_set"); expect(section).toContain("operationBorder"); expect(section).toContain("operation_border_enabled"); diff --git a/apps/web/src/features/settings/components/settings-modal-data.ts b/apps/web/src/features/settings/components/settings-modal-data.ts index 245212611..d181d2a38 100644 --- a/apps/web/src/features/settings/components/settings-modal-data.ts +++ b/apps/web/src/features/settings/components/settings-modal-data.ts @@ -872,11 +872,26 @@ const SETTINGS_SETTING_ITEMS: Record Date: Wed, 5 Aug 2026 21:44:48 +0800 Subject: [PATCH 26/43] fix(desktop-use): usable chrome, window coords, and session recovery Make agent status "{Agent} - {operation}" under the pointer only, hide window borders when covered, convert screen points for window-scoped actions, support type --x/--y, and auto-reset after desktop escalate so window tools are not stuck on window_scope_disabled. --- apps/cli/src/commands/desktop_use.rs | 18 +- .../assets/highlight_overlay.swift | 77 +++- crates/desktop-use/src/control.rs | 373 +++++++++++++++--- crates/desktop-use/src/drive_tools.rs | 44 +++ crates/desktop-use/src/highlight.rs | 92 +++-- skills/atmos-desktop-use/SKILL.md | 17 +- skills/atmos-desktop-use/references/cli.md | 26 +- 7 files changed, 548 insertions(+), 99 deletions(-) diff --git a/apps/cli/src/commands/desktop_use.rs b/apps/cli/src/commands/desktop_use.rs index 39ec9d00f..bda8a324a 100644 --- a/apps/cli/src/commands/desktop_use.rs +++ b/apps/cli/src/commands/desktop_use.rs @@ -398,10 +398,10 @@ pub struct ClickArgs { /// Border highlight: auto (default) | desktop | clear | off #[arg(long, default_value = "auto")] pub highlight: String, - /// Under-arrow status text (what the agent is doing). Overrides auto copy. + /// Under-arrow operation text. Combined as "{agent} - {status}". Also used alone when already prefixed. #[arg(long)] pub status: Option, - /// Agent display name for fallback "{name} Operating". Also reads ATMOS_DESKTOP_USE_AGENT_NAME. + /// Agent display name for "{name} - …". Also reads ATMOS_DESKTOP_USE_AGENT_NAME / AGENT_NAME. #[arg(long)] pub agent_name: Option, } @@ -410,6 +410,12 @@ pub struct ClickArgs { pub struct TypeArgs { #[arg(long)] pub text: String, + /// Optional pixel focus before type (engine 0.17). Use after a click fails to keep focus, + /// or for empty-AX apps: same coordinate rules as `drive click`. + #[arg(long)] + pub x: Option, + #[arg(long)] + pub y: Option, #[arg(long)] pub pid: Option, #[arg(long)] @@ -417,6 +423,9 @@ pub struct TypeArgs { /// background (default) or foreground. #[arg(long, default_value = "background")] pub delivery_mode: String, + /// Desktop/window coord space for --x/--y: `png` (default) or `points` (screen/AX logical). + #[arg(long, default_value = "png")] + pub coord_space: String, /// AX element token from `drive window-state` (preferred for focused fields). #[arg(long)] pub element_token: Option, @@ -740,12 +749,17 @@ fn drive_cmd(command: DriveCommand) -> Result { a.highlight ) })?; + let coord_space = CoordSpace::parse(&a.coord_space) + .ok_or_else(|| format!("invalid --coord-space {:?}", a.coord_space))?; DriveRequest { action: DriveAction::Type, text: Some(a.text), + x: a.x, + y: a.y, pid: a.pid, window_id: a.window_id, delivery_mode: Some(a.delivery_mode), + coord_space, element_token: a.element_token, element_index: a.element_index, snapshot_id: a.snapshot_id, diff --git a/crates/desktop-use/assets/highlight_overlay.swift b/crates/desktop-use/assets/highlight_overlay.swift index b4af1e8e7..0de150942 100644 --- a/crates/desktop-use/assets/highlight_overlay.swift +++ b/crates/desktop-use/assets/highlight_overlay.swift @@ -166,6 +166,63 @@ func makeOverlayWindow(frame: NSRect) -> NSWindow { return win } +func cgNumber(_ value: Any?) -> Int? { + if let i = value as? Int { return i } + if let n = value as? NSNumber { return n.intValue } + return nil +} + +func cgFloatVal(_ value: Any?) -> CGFloat? { + if let d = value as? Double { return CGFloat(d) } + if let f = value as? CGFloat { return f } + if let i = value as? Int { return CGFloat(i) } + if let n = value as? NSNumber { return CGFloat(n.doubleValue) } + return nil +} + +func cgRect(from bounds: [String: Any]?) -> CGRect? { + guard let bounds = bounds, + let x = cgFloatVal(bounds["X"]), + let y = cgFloatVal(bounds["Y"]), + let w = cgFloatVal(bounds["Width"]), + let h = cgFloatVal(bounds["Height"]) + else { return nil } + return CGRect(x: x, y: y, width: w, height: h) +} + +/// On-screen window list (front → back). +func onScreenWindows() -> [[String: Any]] { + (CGWindowListCopyWindowInfo( + [.optionOnScreenOnly, .excludeDesktopElements], + kCGNullWindowID + ) as? [[String: Any]]) ?? [] +} + +/// True when another on-screen window intersects the target (user is working on top). +/// When the target is covered we hide chrome so border/capsule never paint over other apps. +func targetIsCovered(windowId: Int) -> Bool { + let list = onScreenWindows() + guard let target = list.first(where: { cgNumber($0[kCGWindowNumber as String]) == windowId }), + let targetRect = cgRect(from: target[kCGWindowBounds as String] as? [String: Any]) + else { + // Unknown / off-screen target — hide rather than paint over the user's work. + return true + } + for info in list { + let num = cgNumber(info[kCGWindowNumber as String]) + if num == windowId { break } + guard let r = cgRect(from: info[kCGWindowBounds as String] as? [String: Any]) else { continue } + // Ignore tiny system chrome / menubar-ish strips. + if r.height < 24 || r.width < 24 { continue } + // Skip fully transparent / zero-alpha if reported. + if let alpha = cgFloatVal(info[kCGWindowAlpha as String]), alpha < 0.05 { continue } + if r.intersects(targetRect) { + return true + } + } + return false +} + /// Place our window just above the target CGWindow so covering windows also cover us. func orderWithTarget(_ win: NSWindow, aboveWindowId: Int?) { guard let wid = aboveWindowId, wid > 0 else { @@ -173,6 +230,12 @@ func orderWithTarget(_ win: NSWindow, aboveWindowId: Int?) { win.orderFront(nil) return } + // If the user has another app covering the target, hide chrome entirely so we + // never flash a border/capsule on top of their work. + if targetIsCovered(windowId: wid) { + win.orderOut(nil) + return + } // CGWindowID matches Cocoa windowNumber for standard windows on modern macOS. win.order(.above, relativeTo: wid) } @@ -249,6 +312,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate { NSApp.terminate(nil) return } + // Caption mode is always anchored at the provided x/y (agent pointer). let win = makeCaptionWindow( label: label, accent: accent, @@ -282,9 +346,9 @@ final class AppDelegate: NSObject, NSApplicationDelegate { orderWithTarget(win, aboveWindowId: args.aboveWindowId) windows.append(win) - if !label.isEmpty { - let cx = args.cursorX ?? (x + min(48, w * 0.08)) - let cy = args.cursorY ?? (y + 8) + // Status capsule ONLY under the agent pointer (cursor-x/y). Never float a + // free-standing pill near the window chrome when there is no cursor. + if !label.isEmpty, let cx = args.cursorX, let cy = args.cursorY { let cap = makeCaptionWindow( label: label, accent: accent, @@ -299,10 +363,15 @@ final class AppDelegate: NSObject, NSApplicationDelegate { startBreathing(periodMs: args.blinkPeriodMs) } - // Keep sitting just above the target if the window stack reshuffles. + // Keep sitting just above the target if the window stack reshuffles; + // hide entirely while the user covers the target with another app. if let wid = args.aboveWindowId, wid > 0 { reorderTimer = Timer.scheduledTimer(withTimeInterval: 0.35, repeats: true) { [weak self] _ in guard let self = self else { return } + if targetIsCovered(windowId: wid) { + for w in self.windows { w.orderOut(nil) } + return + } for w in self.windows { w.order(.above, relativeTo: wid) } diff --git a/crates/desktop-use/src/control.rs b/crates/desktop-use/src/control.rs index 9a96658ed..1821978c0 100644 --- a/crates/desktop-use/src/control.rs +++ b/crates/desktop-use/src/control.rs @@ -396,17 +396,41 @@ fn status_for_request(req: &DriveRequest, target_app: Option<&str>, action_name: } fn cursor_points_for_request(req: &DriveRequest) -> Option<(f64, f64)> { - // Prefer explicit screen points (logical). For png desktop clicks the engine - // uses 2× pixels; caption is best-effort near the pointer after move_cursor. - match (req.x, req.y) { - (Some(x), Some(y)) if matches!(req.coord_space, CoordSpace::Points) => { - Some((x as f64, y as f64)) + cursor_points_for_request_with_window(req, None) +} + +/// Logical screen points for the under-arrow caption. +/// +/// When `window_bounds` is known and coords are window-local PNG/points, convert +/// to screen space so the capsule sits under the agent pointer — not at a free +/// window corner. +fn cursor_points_for_request_with_window( + req: &DriveRequest, + window_bounds: Option<(f64, f64, f64, f64)>, +) -> Option<(f64, f64)> { + let (x, y) = match (req.x, req.y) { + (Some(x), Some(y)) => (x as f64, y as f64), + _ => return None, + }; + match (req.coord_space, window_bounds, req.window_id) { + (CoordSpace::Points, Some((bx, by, bw, bh)), Some(_)) => { + // Agents often pass screen-absolute points with --window-id. If the + // point lies inside the window, use it as-is; otherwise treat as local. + if x >= bx && x <= bx + bw && y >= by && y <= by + bh { + Some((x, y)) + } else { + Some((bx + x, by + y)) + } + } + (CoordSpace::Points, _, _) => Some((x, y)), + (CoordSpace::Png, Some((bx, by, _, _)), Some(_)) => { + // Window-local PNG pixels → approximate screen points (Retina 2×). + Some((bx + x / 2.0, by + y / 2.0)) } - (Some(x), Some(y)) => { - // Approximate: many Retina displays are 2×; caption still near target. - Some((x as f64 / 2.0, y as f64 / 2.0)) + (CoordSpace::Png, _, _) => { + // Desktop PNG → logical points (best-effort 2× Retina). + Some((x / 2.0, y / 2.0)) } - _ => None, } } @@ -417,9 +441,22 @@ fn run_highlight(req: &DriveRequest, action_name: &str) -> DriveResult { }; let label = status_for_request(req, None, action_name); + // Explicit `drive highlight --mode window --x --y --width --height` uses x/y as + // **window bounds**, not pointer position. Do not treat those as cursor anchors + // (that recreated the free-floating capsule at the window top-left). + let bounds_mode = matches!( + (req.width, req.height), + (Some(w), Some(h)) if w > 0 && h > 0 + ); + let cursor = if bounds_mode { + None + } else { + cursor_points_for_request(req) + }; + // Capsule only when we have a real pointer anchor. let style = HighlightStyle { - label: Some(label), - cursor: cursor_points_for_request(req), + label: cursor.as_ref().map(|_| label), + cursor, blink: true, idle_ms: None, above_window_id: req.window_id, @@ -461,6 +498,12 @@ fn run_highlight(req: &DriveRequest, action_name: &str) -> DriveResult { /// Apply border chrome + under-arrow status for an in-progress drive action /// (best-effort; never fails the action). +/// +/// Rules: +/// - Window border only when `window_id` resolves (no full-desktop fallback that +/// paints over the user's other apps). +/// - Status capsule only when a cursor position is known (under the agent arrow). +/// - Explicit `HighlightMode::Desktop` still draws desktop chrome. fn apply_action_highlight( engine: &Path, socket: &Path, @@ -491,10 +534,15 @@ fn apply_action_highlight( None } }); + let window_bounds = list + .as_ref() + .and_then(|l| req.window_id.and_then(|wid| bounds_for_window_id(l, wid))); + let cursor = cursor_points_for_request_with_window(req, window_bounds); let label = status_for_request(req, target_app.as_deref(), action_name); - let cursor = cursor_points_for_request(req); + // Capsule only under the agent pointer — never a free-standing status pill. + let caption = cursor.map(|_| label.clone()); let style = HighlightStyle { - label: Some(label.clone()), + label: caption, cursor, blink: true, idle_ms: None, @@ -513,27 +561,35 @@ fn apply_action_highlight( } if let Some(wid) = req.window_id { - if let Some(list) = list.as_ref() { - if let Some((x, y, w, h)) = bounds_for_window_id(list, wid) { - let hl = show_window_highlight_styled( - x, - y, - w, - h, - HighlightStyle { - above_window_id: Some(wid), - ..style - }, - ); - return serde_json::to_value(hl).ok(); - } + if let Some((x, y, w, h)) = window_bounds { + let hl = show_window_highlight_styled( + x, + y, + w, + h, + HighlightStyle { + above_window_id: Some(wid), + ..style + }, + ); + return serde_json::to_value(hl).ok(); } + // Window requested but bounds unknown — do NOT fall back to full-desktop + // chrome (that paints over whatever app the user is working in). + return None; } - // Desktop-scope / fallback chrome so the user always sees what is under control. + // True desktop-scope actions (no window_id): border on the active display. + // Still only attach a caption when we know a cursor point. if matches!( req.action, - DriveAction::Click | DriveAction::Type | DriveAction::WindowState + DriveAction::Click + | DriveAction::DoubleClick + | DriveAction::RightClick + | DriveAction::MoveCursor + | DriveAction::Drag + | DriveAction::Scroll + | DriveAction::Type ) { let hl = show_desktop_highlight_styled(style); return serde_json::to_value(hl).ok(); @@ -753,6 +809,8 @@ fn ensure_drive_session(engine: &Path, socket: &Path, session: &str) { }), ); // Keep the agent cursor visible for long runs (default engine idle hide ~20s). + // Note: the engine cursor is a session HUD; our border/caption stay window-scoped + // and hide when the target is covered so they do not paint over the user's work. let _ = host::call_tool( engine, socket, @@ -770,6 +828,62 @@ fn ensure_drive_session(engine: &Path, socket: &Path, session: &str) { ); } +/// Engine 0.17: `escalate_session` is one-way. After a desktop-scope click the +/// live session is locked to desktop and window-scoped tools return +/// `window_scope_disabled`. End + restart so the next window action works. +fn reset_drive_session(engine: &Path, socket: &Path, session: &str) { + let _ = host::call_tool( + engine, + socket, + "end_session", + &json!({ "session": session }), + ); + ensure_drive_session(engine, socket, session); +} + +fn session_effective_scope(engine: &Path, socket: &Path, session: &str) -> Option { + let v = host::call_tool( + engine, + socket, + "get_session_state", + &json!({ "session": session }), + ) + .ok()?; + v.get("effective_scope") + .or_else(|| v.get("capture_scope")) + .or_else(|| v.pointer("/session/effective_scope")) + .and_then(|x| x.as_str()) + .map(|s| s.to_ascii_lowercase()) +} + +fn needs_window_scope(req: &DriveRequest, args: &serde_json::Value) -> bool { + if req.element_token.is_some() || req.element_index.is_some() { + return true; + } + if req.window_id.is_some() { + return true; + } + // Engine args may carry window_id even when request was filled via auto-pid. + if args.get("window_id").is_some() + && args.get("scope").and_then(|s| s.as_str()) != Some("desktop") + { + return true; + } + matches!( + req.action, + DriveAction::WindowState + | DriveAction::BringToFront + | DriveAction::SetWindowFrame + | DriveAction::InvokeMenu + | DriveAction::VerifyState + ) +} + +fn is_window_scope_disabled(fail: &str) -> bool { + let f = fail.to_ascii_lowercase(); + f.contains("window_scope_disabled") || f.contains("window scope disabled") +} + /// Desktop-scope scale: PNG pixels / logical screen points (often 2.0 on Retina). /// Source of truth is get_desktop_state metadata, not get_screen_size.scale_factor /// (which can report 1.0 while the PNG is still 2×). @@ -868,6 +982,57 @@ fn to_engine_desktop_xy( } } +/// Convert logical points → window-local PNG pixels for engine 0.17. +/// +/// Screen-absolute points that fall inside the window bounds are converted to +/// window-local; otherwise the values are treated as already window-local points. +fn to_engine_window_xy( + engine: &Path, + socket: &Path, + window_id: i64, + x: i32, + y: i32, +) -> Result<(i32, i32, Option), String> { + let list = host::call_tool(engine, socket, "list_windows", &json!({})) + .map_err(|e| format!("list_windows for window coord conversion failed: {e}"))?; + let (bx, by, bw, bh) = crate::highlight::bounds_for_window_id(&list, window_id) + .ok_or_else(|| format!("window_id {window_id} not found in list_windows"))?; + + let xf = x as f64; + let yf = y as f64; + let (local_x, local_y, input_kind) = if xf >= bx && xf <= bx + bw && yf >= by && yf <= by + bh { + (xf - bx, yf - by, "screen_points") + } else { + (xf, yf, "window_local_points") + }; + + let (sx, sy, mut meta) = desktop_png_scale(engine, socket)?; + let px = (local_x * sx).round() as i32; + let py = (local_y * sy).round() as i32; + if let Some(obj) = meta.as_object_mut() { + obj.insert("window_id".into(), json!(window_id)); + obj.insert( + "window_local_points".into(), + json!({ "x": local_x, "y": local_y }), + ); + obj.insert( + "window_bounds_points".into(), + json!({ "x": bx, "y": by, "width": bw, "height": bh }), + ); + obj.insert("input_kind".into(), json!(input_kind)); + obj.insert("output_space".into(), json!("window_png")); + } else { + meta = json!({ + "window_id": window_id, + "window_local_points": { "x": local_x, "y": local_y }, + "input_kind": input_kind, + "scale_x": sx, + "scale_y": sy, + }); + } + Ok((px, py, Some(meta))) +} + fn inject_session(args: &mut serde_json::Value, session: Option<&str>) { if let (Some(s), Some(obj)) = (session, args.as_object_mut()) { obj.insert("session".into(), json!(s)); @@ -929,40 +1094,73 @@ fn run_engine( } else { None }; - if matches!( - req.action, - DriveAction::Click - | DriveAction::DoubleClick - | DriveAction::RightClick - | DriveAction::MoveCursor - ) && req.window_id.is_none() - && req.pid.is_none() - && req.element_token.is_none() - && req.x.is_some() - && req.y.is_some() - && matches!(req.coord_space, CoordSpace::Points) + + // Convert coordinates for the engine when agents pass logical points: + // - desktop-scope + points → desktop PNG pixels + // - window-scope + points → window-local PNG pixels + // (screen-absolute points inside the window are auto-normalized — common + // agent mistake when combining --window-id with list_windows / AX coords) + if req_for_call.element_token.is_none() + && req_for_call.element_index.is_none() + && req_for_call.x.is_some() + && req_for_call.y.is_some() + && matches!(req_for_call.coord_space, CoordSpace::Points) + && matches!( + req_for_call.action, + DriveAction::Click + | DriveAction::DoubleClick + | DriveAction::RightClick + | DriveAction::MoveCursor + | DriveAction::Type + | DriveAction::Scroll + ) { - if let (Some(x), Some(y)) = (req.x, req.y) { - match to_engine_desktop_xy(engine, &socket, x, y, CoordSpace::Points) { - Ok((ex, ey, meta)) => { - req_for_call.x = Some(ex); - req_for_call.y = Some(ey); - req_for_call.coord_space = CoordSpace::Png; - scale_meta = meta; + if let (Some(x), Some(y)) = (req_for_call.x, req_for_call.y) { + if let Some(wid) = req_for_call.window_id { + match to_engine_window_xy(engine, &socket, wid, x, y) { + Ok((ex, ey, meta)) => { + req_for_call.x = Some(ex); + req_for_call.y = Some(ey); + req_for_call.coord_space = CoordSpace::Png; + scale_meta = meta; + } + Err(e) => { + return DriveResult { + ok: false, + action: action_name.into(), + detail: None, + capture: None, + result: None, + error: Some(scrub_vendor(&format!( + "{}: {e}", + strings::ERR_ENGINE_FAILED + ))), + error_code: Some("control_engine_failed".into()), + }; + } } - Err(e) => { - return DriveResult { - ok: false, - action: action_name.into(), - detail: None, - capture: None, - result: None, - error: Some(scrub_vendor(&format!( - "{}: {e}", - strings::ERR_ENGINE_FAILED - ))), - error_code: Some("control_engine_failed".into()), - }; + } else if req_for_call.pid.is_none() { + match to_engine_desktop_xy(engine, &socket, x, y, CoordSpace::Points) { + Ok((ex, ey, meta)) => { + req_for_call.x = Some(ex); + req_for_call.y = Some(ey); + req_for_call.coord_space = CoordSpace::Png; + scale_meta = meta; + } + Err(e) => { + return DriveResult { + ok: false, + action: action_name.into(), + detail: None, + capture: None, + result: None, + error: Some(scrub_vendor(&format!( + "{}: {e}", + strings::ERR_ENGINE_FAILED + ))), + error_code: Some("control_engine_failed".into()), + }; + } } } } @@ -1021,6 +1219,21 @@ fn run_engine( inject_session(&mut args, session.as_deref()); + // If a prior desktop escalate locked this session, restore window capability + // before the next window-scoped tool (engine escalate is one-way). + let window_scoped = needs_window_scope(&req_for_call, &args); + if window_scoped { + if let Some(ref s) = session { + if session_effective_scope(engine, &socket, s) + .as_deref() + .is_some_and(|sc| sc.contains("desktop")) + { + reset_drive_session(engine, &socket, s); + inject_session(&mut args, Some(s)); + } + } + } + // Desktop-scope actions need an escalated auto session (engine 0.17). if args.get("scope").and_then(|s| s.as_str()) == Some("desktop") { if let Some(ref s) = session { @@ -1037,7 +1250,21 @@ fn run_engine( } } - match host::call_tool(engine, &socket, tool, &args) { + let mut engine_result = host::call_tool(engine, &socket, tool, &args); + // One automatic recovery if escalate left us desktop-locked mid-run. + if window_scoped { + if let (Some(ref s), Ok(ref v)) = (session.as_ref(), &engine_result) { + if let Some(fail) = crate::engine_protocol::engine_payload_is_failure(v) { + if is_window_scope_disabled(&fail) { + reset_drive_session(engine, &socket, s); + inject_session(&mut args, Some(s)); + engine_result = host::call_tool(engine, &socket, tool, &args); + } + } + } + } + + match engine_result { Ok(v) => { if let Some(fail) = crate::engine_protocol::engine_payload_is_failure(&v) { let fail_l = fail.to_ascii_lowercase(); @@ -1119,6 +1346,28 @@ atmos desktop-use driver grant-permissions --target screen_recording" } } + // Type without element/pixel focus often lands in the wrong app on + // empty-AX surfaces — surface a short recovery ladder. + if matches!(req.action, DriveAction::Type) + && req.element_token.is_none() + && req.element_index.is_none() + && req.x.is_none() + { + if let Some(obj) = result.as_object_mut() { + obj.entry("atmos_type_hint").or_insert_with(|| { + json!({ + "note": "type without --element-token or --x/--y uses AX focus in the target pid (or frontmost). Empty-AX / custom UI apps often drop text.", + "next_steps": [ + "Prefer: window-state → type --element-token … (true background when AX exists)", + "Empty AX: click the field (PNG or points), then type --text … --x --y (same coords) --pid --window-id", + "If still no input: --delivery-mode foreground once (brief front→type→restore)", + "Do not drive front every turn" + ] + }) + }); + } + } + DriveResult { ok: true, action: action_name.into(), diff --git a/crates/desktop-use/src/drive_tools.rs b/crates/desktop-use/src/drive_tools.rs index 5e43664bd..a03b62540 100644 --- a/crates/desktop-use/src/drive_tools.rs +++ b/crates/desktop-use/src/drive_tools.rs @@ -259,6 +259,12 @@ pub fn build_engine_call(req: &DriveRequest) -> Result<(&'static str, Value), St let mut a = json!({ "text": text, "delivery_mode": delivery }); inject_target(&mut a, req); inject_element(&mut a, req); + // Engine 0.17: pass either element_token/index (AX) OR x,y (pixel-focus + // the field then type). Pixel path is required for empty-AX apps + // (Electron / custom UI like QQ Music) after a focus click. + if req.element_token.is_none() && req.element_index.is_none() { + inject_xy_if_present(&mut a, req); + } Ok(("type_text", a)) } DriveAction::SetValue => { @@ -455,6 +461,44 @@ mod tests { assert!(args.get("scope").is_none()); } + #[test] + fn type_with_pixel_focus_xy() { + let req = DriveRequest { + action: DriveAction::Type, + text: Some("hello".into()), + x: Some(120), + y: Some(40), + pid: Some(9), + window_id: Some(99), + ..Default::default() + }; + let (tool, args) = build_engine_call(&req).unwrap(); + assert_eq!(tool, "type_text"); + assert_eq!(args["text"], "hello"); + assert_eq!(args["x"], 120); + assert_eq!(args["y"], 40); + assert_eq!(args["pid"], 9); + assert_eq!(args["window_id"], 99); + } + + #[test] + fn type_element_token_skips_xy() { + let req = DriveRequest { + action: DriveAction::Type, + text: Some("hello".into()), + x: Some(1), + y: Some(2), + element_token: Some("tok".into()), + pid: Some(1), + ..Default::default() + }; + let (tool, args) = build_engine_call(&req).unwrap(); + assert_eq!(tool, "type_text"); + assert_eq!(args["element_token"], "tok"); + assert!(args.get("x").is_none()); + assert!(args.get("y").is_none()); + } + #[test] fn click_element_index_requires_snapshot_and_window() { let missing_snap = DriveRequest { diff --git a/crates/desktop-use/src/highlight.rs b/crates/desktop-use/src/highlight.rs index c9a3cfb5b..239c84cdd 100644 --- a/crates/desktop-use/src/highlight.rs +++ b/crates/desktop-use/src/highlight.rs @@ -657,32 +657,47 @@ pub fn resolve_agent_name(explicit: Option<&str>) -> String { "Agent".into() } -/// Build dynamic status text under the agent arrow. +/// Build under-arrow status text as `{Agent} - {operation}`. /// -/// Prefer an explicit status; otherwise describe the live action + target app; -/// final fallback: `"{AgentName} Operating"`. +/// Prefer an explicit `--status` as the operation; otherwise describe the live +/// action + target app; final fallback operation: `Operating`. +/// +/// If the explicit status already starts with `{Agent} - `, it is returned as-is +/// (agents may pass a fully formatted string). pub fn build_status_label( explicit_status: Option<&str>, agent_name: Option<&str>, action: &str, target_app: Option<&str>, ) -> String { - if let Some(s) = explicit_status.map(str::trim).filter(|s| !s.is_empty()) { - return s.to_string(); - } let agent = resolve_agent_name(agent_name); - let app = target_app.map(str::trim).filter(|s| !s.is_empty()); - match (action, app) { - ("click", Some(app)) => format!("Clicking {app}"), - ("type", Some(app)) => format!("Typing in {app}"), - ("window_state", Some(app)) => format!("Inspecting {app}"), - ("screenshot", Some(app)) => format!("Capturing {app}"), - ("highlight", Some(app)) => format!("{agent} · {app}"), - (_, Some(app)) => format!("{agent} · {app}"), - ("click", None) => format!("{agent} clicking"), - ("type", None) => format!("{agent} typing"), - _ => format!("{agent} Operating"), - } + let operation = if let Some(s) = explicit_status.map(str::trim).filter(|s| !s.is_empty()) { + // Already fully formatted by the agent. + let prefix_dash = format!("{agent} - "); + let prefix_dot = format!("{agent} · "); + if s.starts_with(&prefix_dash) || s.starts_with(&prefix_dot) { + return s.to_string(); + } + s.to_string() + } else { + let app = target_app.map(str::trim).filter(|s| !s.is_empty()); + match (action, app) { + ("click", Some(app)) => format!("Clicking {app}"), + ("type", Some(app)) => format!("Typing in {app}"), + ("double_click", Some(app)) => format!("Double-clicking {app}"), + ("right_click", Some(app)) => format!("Right-clicking {app}"), + ("window_state", Some(app)) => format!("Inspecting {app}"), + ("screenshot", Some(app)) => format!("Capturing {app}"), + ("highlight", Some(app)) => app.to_string(), + (_, Some(app)) => format!("Operating {app}"), + ("click", None) => "Clicking".into(), + ("type", None) => "Typing".into(), + ("double_click", None) => "Double-clicking".into(), + ("right_click", None) => "Right-clicking".into(), + _ => "Operating".into(), + } + }; + format!("{agent} - {operation}") } #[cfg(test)] @@ -711,30 +726,61 @@ mod tests { "click", Some("Orca") ), - "Open dashboard" + "Claude - Open dashboard" ); assert_eq!( build_status_label(None, Some("Claude"), "click", Some("Orca")), - "Clicking Orca" + "Claude - Clicking Orca" ); assert_eq!( build_status_label(None, Some("Claude"), "type", Some("Notes")), - "Typing in Notes" + "Claude - Typing in Notes" ); assert_eq!( build_status_label(None, Some("Grok"), "click", None), - "Grok clicking" + "Grok - Clicking" ); assert_eq!( build_status_label(None, Some("Claude"), "verify", None), - "Claude Operating" + "Claude - Operating" + ); + // Already formatted — do not double-prefix. + assert_eq!( + build_status_label( + Some("Claude - Opening QQ Music"), + Some("Claude"), + "click", + None + ), + "Claude - Opening QQ Music" ); } #[test] fn resolve_agent_defaults() { assert_eq!(resolve_agent_name(Some(" Claude ")), "Claude"); + // Explicit empty falls through to env / default — isolate from developer env. + let keys = [ + "ATMOS_DESKTOP_USE_AGENT_NAME", + "AGENT_NAME", + "ATMOS_AGENT_NAME", + ]; + let saved: Vec<(String, Option)> = keys + .iter() + .map(|k| ((*k).to_string(), std::env::var(k).ok())) + .collect(); + for k in &keys { + // SAFETY: test-only; single-threaded unit test process. + unsafe { std::env::remove_var(k) }; + } assert_eq!(resolve_agent_name(Some("")), "Agent"); + assert_eq!(resolve_agent_name(None), "Agent"); + for (k, v) in saved { + match v { + Some(val) => unsafe { std::env::set_var(&k, val) }, + None => unsafe { std::env::remove_var(&k) }, + } + } } #[test] diff --git a/skills/atmos-desktop-use/SKILL.md b/skills/atmos-desktop-use/SKILL.md index 17278e9cf..9566cf8b7 100644 --- a/skills/atmos-desktop-use/SKILL.md +++ b/skills/atmos-desktop-use/SKILL.md @@ -1,6 +1,6 @@ --- name: atmos-desktop-use -version: "1.6.0" +version: "1.6.1" description: > Capture and control the local macOS desktop via `atmos desktop-use` (screenshot, click, type, window list) without MCP. Use whenever the user or task needs local @@ -155,8 +155,9 @@ atmos desktop-use --json drive click --x --y \ |-------------|---------| | See where agent points | Session + `move_cursor`; idle hide extended to ~1h for drive session | | App / desktop **border highlight** | **Yes, blinking** — blue window / green desktop. Auto-clears after idle (~8s) or `drive session-end`. Toggle in Settings → Desktop Use → Operation border | -| **Under-arrow status text** | **Dynamic** — what is being operated now (e.g. `Clicking Orca`). Explicit: `--status "…"`. Fallback: `{agent} Operating` via `--agent-name` or env `ATMOS_DESKTOP_USE_AGENT_NAME` / `AGENT_NAME` | -| Must activate target app | **No** for default background path | +| **Under-arrow status text** | **`{Agent} - {operation}`** under the pointer only. Explicit: `--status "…"`. Auto: `Clicking Orca` / `Operating`. No free-floating capsule without a cursor position. | +| **Border z-order** | Window border sits just above the target; **hides** when another app covers the target (does not paint over your work). No full-desktop fallback for window-scoped actions. | +| Must activate target app | **No** for default background path — avoid `drive front` every turn | ```bash # End a drive run (clears border + engine session cursor) @@ -171,7 +172,15 @@ atmos desktop-use --json prefs set --operation-border false # Agent should pass its own identity + live status when known: export ATMOS_DESKTOP_USE_AGENT_NAME="Claude" atmos desktop-use --json drive click --x … --y … --agent-name "Claude" --status "Opening 智能体仪表盘" -# Auto: "Clicking Orca" when window_id resolves; else "Claude Operating" +# Caption: "Claude - Opening 智能体仪表盘" (under pointer only) +# Auto without --status: "Claude - Clicking Orca" when window_id resolves + +# Empty-AX type (QQ Music / custom UI): pixel-focus then type at same coords +atmos desktop-use --json drive click --x --y +atmos desktop-use --json drive type --text "周杰伦" --x --y +# logical screen points + window (auto-normalized to window-local PNG): +atmos desktop-use --json drive type --text "周杰伦" --x 780 --y 78 \ + --coord-space points --window-id --pid ``` ## Desktop vs page CDP diff --git a/skills/atmos-desktop-use/references/cli.md b/skills/atmos-desktop-use/references/cli.md index 8eb701eb5..0632651d1 100644 --- a/skills/atmos-desktop-use/references/cli.md +++ b/skills/atmos-desktop-use/references/cli.md @@ -70,9 +70,15 @@ atmos desktop-use --json drive click --element-token '' [--pid ] [-- atmos desktop-use --json drive click --element-index --snapshot-id --window-id [--pid ] # Logical points (list_windows bounds / AX positions): atmos desktop-use --json drive click --x --y --coord-space points -# Window-local pixels: +# Window-local PNG pixels (default when --window-id without --coord-space points): atmos desktop-use --json drive click --x --y --window-id [--pid ] -atmos desktop-use --json drive type --text "..." [--element-token ] [--pid ] [--window-id ] +# Screen/AX points + window_id: CLI normalizes to window-local PNG automatically +atmos desktop-use --json drive click --x --y --coord-space points --window-id [--pid ] +# Type: AX token (preferred) or pixel-focus with --x/--y after a click +atmos desktop-use --json drive type --text "..." --element-token [--pid ] [--window-id ] +atmos desktop-use --json drive type --text "..." --x --y [--coord-space png|points] [--window-id ] [--pid ] +# Status under pointer: "{agent} - {operation}" +atmos desktop-use --json drive click --x … --y … --agent-name "Grok" --status "Searching Jay Chou" # Border highlight (auto on click/type; or explicit): atmos desktop-use --json drive highlight --mode desktop atmos desktop-use --json drive highlight --mode window --x N --y N --width N --height N @@ -114,7 +120,15 @@ Every successful `drive window-state` JSON includes `result.atmos_surface`: When `kind=ax_empty`, stop retrying tokens; use screenshot + pixel path. -Pixel-path clicks also set `result.atmos_addressing` with the same ladder reminder. +Pixel-path clicks also set `result.atmos_addressing` with the same ladder reminder. +`type` without token/xy sets `result.atmos_type_hint` with the focus ladder. + +### Visibility chrome + +- Status format: **`{Agent} - {operation}`** (from `--agent-name` / env + `--status` or auto action text) +- Capsule appears **only under the agent pointer** (never a free-standing pill without cursor) +- Window border tracks the target window z-order and **hides while another app covers it** +- Window-scoped actions do **not** fall back to a full-desktop border ### Click coordinate contract (engine 0.17) @@ -125,12 +139,16 @@ Pixel-path clicks also set `result.atmos_addressing` with the same ladder remind | `--x --y` only | **PNG pixels** (`--coord-space png`, default) | From `drive screenshot` image. CLI sends `scope=desktop`. | | `--x --y --coord-space points` | **Logical points** | Same as window bounds / AX; CLI scales to PNG before the engine. | | `--x --y --window-id` | **Window-local** image pixels | Optional `--pid`. | +| `--x --y --coord-space points --window-id` | Screen or local **points** | CLI converts to window-local PNG (screen-absolute points inside the window are auto-normalized). | | `--x --y --pid` (no window_id) | **Ignored pid** | CLI strips bare `--pid` so desktop coords stay valid. | +| `type --x --y` | Same as click | Pixel-focus then type (needed for empty-AX / custom UI). | `delivery_mode`: - `background` (default): no focus steal when possible -- `foreground`: briefly front target (`window_id` recommended) for stubborn Electron UIs +- `foreground`: briefly front target (`window_id` recommended) for stubborn Electron UIs + +Avoid `drive front` every turn — only escalate after background pixel fails. `session`: defaults to `atmos-desktop-use` so the engine agent cursor stays active. Pass `--session ''` to disable. From 8842c9654c2ab89d3f47fb46304345ab27ad714f Mon Sep 17 00:00:00 2001 From: AarynLu Date: Thu, 6 Aug 2026 00:49:13 +0800 Subject: [PATCH 27/43] feat(desktop-use): native highlight binary and Electron grant overlay Move the macOS highlight overlay into a native prebuilt helper, add the Accessibility grant overlay preload, and wire readiness/permissions UI for Desktop Use control flows. --- apps/cli/src/commands/desktop_use.rs | 32 +- apps/desktop-electron/scripts/build.ts | 9 +- .../grant-overlay.structural.test.ts | 37 ++ .../src/desktop-use/grant-overlay.ts | 347 ++++++++++++++++++ .../src/desktop-use/grant-preload.ts | 18 + apps/desktop-electron/src/ipc/handlers.ts | 47 ++- .../components/DesktopUseReadinessDialog.tsx | 83 ++++- .../readiness-gate-wiring.structural.test.ts | 70 ++++ .../desktop-use/lib/readiness-modal-bus.ts | 72 +++- .../components/DesktopUsePermissionsPanel.tsx | 31 +- crates/desktop-use/native/highlight/README.md | 20 + .../highlight}/highlight_overlay.swift | 0 .../prebuilt/atmos-desktop-highlight | Bin 0 -> 280296 bytes crates/desktop-use/src/doctor.rs | 2 +- crates/desktop-use/src/highlight.rs | 125 +++++-- crates/desktop-use/src/host.rs | 52 ++- crates/desktop-use/src/lib.rs | 2 +- crates/desktop-use/src/manager.rs | 4 +- 18 files changed, 877 insertions(+), 74 deletions(-) create mode 100644 apps/desktop-electron/src/desktop-use/grant-overlay.structural.test.ts create mode 100644 apps/desktop-electron/src/desktop-use/grant-overlay.ts create mode 100644 apps/desktop-electron/src/desktop-use/grant-preload.ts create mode 100644 apps/web/src/features/desktop-use/lib/__tests__/readiness-gate-wiring.structural.test.ts create mode 100644 crates/desktop-use/native/highlight/README.md rename crates/desktop-use/{assets => native/highlight}/highlight_overlay.swift (100%) create mode 100755 crates/desktop-use/native/highlight/prebuilt/atmos-desktop-highlight diff --git a/apps/cli/src/commands/desktop_use.rs b/apps/cli/src/commands/desktop_use.rs index bda8a324a..dc7492002 100644 --- a/apps/cli/src/commands/desktop_use.rs +++ b/apps/cli/src/commands/desktop_use.rs @@ -553,20 +553,36 @@ fn driver_grant(args: GrantPermissionsArgs) -> Result { ) })?; let mgr = DesktopUseManager::new(); - mgr.open_permission_grant_target(target)?; - let target_label = match target { - PermissionGrantTarget::All => "all", - PermissionGrantTarget::Accessibility => "accessibility", - PermissionGrantTarget::ScreenRecording => "screen_recording", + let outcome = mgr.open_permission_grant_target(target)?; + let target_label = outcome.target.as_str(); + let host_name = outcome + .host_app_name + .as_deref() + .unwrap_or("Atmos Desktop Use"); + + // Do not open Finder / Reveal — Settings is enough; Desktop shows a grant panel. + + let hint = if outcome.accessibility_pane { + format!( + "System Settings → Accessibility opened. Enable the toggle for {host_name}. If it is not listed, add the host app with + (path in host_app_path), then Refresh. Atmos Desktop shows a drag chip for the same path." + ) + } else { + format!( + "System Settings → Screen Recording opened for {host_name}. Enable the toggle there, then return and Refresh." + ) }; + Ok(json!({ "ok": true, "action": "grant_permissions", "target": target_label, - "host": "Atmos Desktop Use", + "host": host_name, + "host_app_path": outcome.host_app_path, + "host_app_name": outcome.host_app_name, + "accessibility_pane": outcome.accessibility_pane, "applied": true, - "opened_settings": true, - "hint": "System Settings → Privacy & Security opened for Atmos Desktop Use. Enable the toggle there, then return and Refresh. No extra system alert is shown — Settings is the grant surface.", + "opened_settings": outcome.opened_settings, + "hint": hint, })) } diff --git a/apps/desktop-electron/scripts/build.ts b/apps/desktop-electron/scripts/build.ts index 608ad99b3..37e9cef95 100644 --- a/apps/desktop-electron/scripts/build.ts +++ b/apps/desktop-electron/scripts/build.ts @@ -62,6 +62,13 @@ await esbuild.build({ entryPoints: [join(root, "src/appshot/shift-helper-main.ts")], outfile: join(dist, "shift-helper-main.js"), }); +// Desktop Use Accessibility grant panel (drag host .app into System Settings). +await esbuild.build({ + ...shared, + format: "cjs", + entryPoints: [join(root, "src/desktop-use/grant-preload.ts")], + outfile: join(dist, "grant-preload.cjs"), +}); // Guest element-select runtime is read at runtime via executeJavaScript inject. // Ship next to main.js so packaged apps resolve dist/browser-runtime.js (not monorepo). @@ -78,5 +85,5 @@ if (!existsSync(browserRuntimeSrc)) { copyFileSync(browserRuntimeSrc, browserRuntimeDest); console.log( - "[build] dist/main.js, dist/preload.cjs, dist/browser-preload.cjs, dist/browser-runtime.js, dist/shift-helper-main.js", + "[build] dist/main.js, dist/preload.cjs, dist/browser-preload.cjs, dist/browser-runtime.js, dist/shift-helper-main.js, dist/grant-preload.cjs", ); diff --git a/apps/desktop-electron/src/desktop-use/grant-overlay.structural.test.ts b/apps/desktop-electron/src/desktop-use/grant-overlay.structural.test.ts new file mode 100644 index 000000000..3b5f6493d --- /dev/null +++ b/apps/desktop-electron/src/desktop-use/grant-overlay.structural.test.ts @@ -0,0 +1,37 @@ +import { describe, expect, it } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +describe("desktop-use grant overlay", () => { + it("uses Electron startDrag and never embeds third-party grant brands", () => { + const src = readFileSync( + join(import.meta.dir, "grant-overlay.ts"), + "utf8", + ); + expect(src).toContain("startDrag"); + expect(src).toContain("hostAppPath"); + expect(src.toLowerCase()).not.toContain("chatgpt"); + expect(src.toLowerCase()).not.toContain("codex"); + expect(src.toLowerCase()).not.toContain("trycua"); + }); + + it("grant preload only exposes drag/close bridge", () => { + const src = readFileSync( + join(import.meta.dir, "grant-preload.ts"), + "utf8", + ); + expect(src).toContain("startDrag"); + expect(src).toContain("sendSync"); + expect(src).toContain("desktop-use-grant-drag-start"); + expect(src).toContain("desktop-use-grant-close"); + }); + + it("uses dragstart not mousedown for file drag", () => { + const src = readFileSync( + join(import.meta.dir, "grant-overlay.ts"), + "utf8", + ); + expect(src).toContain("dragstart"); + expect(src).not.toContain("addEventListener('mousedown'"); + }); +}); diff --git a/apps/desktop-electron/src/desktop-use/grant-overlay.ts b/apps/desktop-electron/src/desktop-use/grant-overlay.ts new file mode 100644 index 000000000..4006b7bfe --- /dev/null +++ b/apps/desktop-electron/src/desktop-use/grant-overlay.ts @@ -0,0 +1,347 @@ +/** + * Accessibility grant overlay (desktop shell). + * + * Opens after System Settings → Accessibility. Primary action is enable the + * host toggle when the app is already listed; optional drag adds the host + * `.app` when it is missing from the list. + * + * Drag uses Electron `webContents.startDrag` from a **dragstart** handler + * (mousedown + async IPC does not start a macOS file drag). + */ + +import { BrowserWindow, ipcMain, nativeImage, screen } from "electron"; +import { existsSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +export type GrantOverlayOptions = { + hostAppPath: string; + hostAppName?: string; + /** BCP-47-ish language tag; used for in-panel copy. */ + locale?: string; +}; + +type GrantState = { + hostAppPath: string; + hostAppName: string; + title: string; + secondary: string; + chipLabel: string; +}; + +let grantWindow: BrowserWindow | null = null; +let grantState: GrantState | null = null; +let ipcWired = false; + +function grantPreloadPath(): string { + const besideMain = join( + dirname(fileURLToPath(import.meta.url)), + "grant-preload.cjs", + ); + if (existsSync(besideMain)) return besideMain; + return join( + dirname(fileURLToPath(import.meta.url)), + "..", + "..", + "dist", + "grant-preload.cjs", + ); +} + +function isZh(locale: string | undefined): boolean { + return (locale ?? "").toLowerCase().startsWith("zh"); +} + +function buildCopy( + hostAppName: string, + locale?: string, +): Pick { + if (isZh(locale)) { + return { + title: `在列表中找到「${hostAppName}」并打开右侧开关`, + secondary: "列表里没有？按住下方芯片拖入列表添加", + chipLabel: hostAppName, + }; + } + return { + title: `Find “${hostAppName}” in the list and turn the toggle on`, + secondary: "Not listed? Drag the chip below into the list to add it", + chipLabel: hostAppName, + }; +} + +function resolveDragIcon(hostAppPath: string): Electron.NativeImage { + const candidates = [ + join(hostAppPath, "Contents", "Resources", "AppIcon.icns"), + join(hostAppPath, "Contents", "Resources", "icon.icns"), + ]; + for (const p of candidates) { + if (!existsSync(p)) continue; + const img = nativeImage.createFromPath(p); + if (!img.isEmpty()) { + return img.resize({ width: 32, height: 32 }); + } + } + // Minimal non-empty 32×32 blue PNG (required on some platforms). + const png = Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAAaElEQVRYR+2WMQ4AIAgD7f8f" + + "zcbBxMHB2kBJuAZSWigAZgYz8z0zO+8dERExM7M+gPkA5gOYD2A+gPkA5gOYD2A+gPkA5gOY" + + "D2A+gPkA5gOYD2A+gPkA5gOYD2A+gPkA5gOYD2A+gPkA5gOYD/gA1dYBvQ1vS5QAAAAASUVO" + + "RK5CYII=", + "base64", + ); + const fallback = nativeImage.createFromBuffer(png); + return fallback.isEmpty() ? nativeImage.createEmpty() : fallback; +} + +function panelHtml(state: GrantState, locale?: string): string { + const title = escapeHtml(state.title); + const secondary = escapeHtml(state.secondary); + const chip = escapeHtml(state.chipLabel); + const closeLabel = isZh(locale) ? "关闭" : "Close"; + return ` + + + + + Desktop Use permissions + + + +

+
+
${title}
+ +
+
${secondary}
+
+
+
${chip}
+
+
+ + +`; +} + +function escapeHtml(s: string): string { + return s + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +} + +function wireIpcOnce(): void { + if (ipcWired) return; + ipcWired = true; + + // sendSync so startDrag runs inside the dragstart stack (required on macOS). + ipcMain.on("desktop-use-grant-drag-start", (event) => { + const path = grantState?.hostAppPath; + if (!path || !existsSync(path)) { + event.returnValue = { ok: false, error: "missing_host" }; + return; + } + try { + const icon = resolveDragIcon(path); + event.sender.startDrag({ + file: path, + icon, + }); + event.returnValue = { ok: true }; + } catch (err) { + console.warn("[desktop-use] startDrag failed", err); + event.returnValue = { + ok: false, + error: err instanceof Error ? err.message : String(err), + }; + } + }); + + ipcMain.on("desktop-use-grant-close", () => { + closeAccessibilityGrantOverlay(); + }); +} + +function positionBottomCenter(win: BrowserWindow): void { + const display = screen.getPrimaryDisplay(); + const { width: sw, height: sh, x: sx, y: sy } = display.workArea; + const size = win.getSize(); + const ww = size[0] ?? 400; + const wh = size[1] ?? 150; + const x = Math.round(sx + (sw - ww) / 2); + const y = Math.round(sy + sh - wh - 36); + win.setPosition(x, y); +} + +/** Show (or focus) the Accessibility grant panel for the host app. */ +export function showAccessibilityGrantOverlay( + opts: GrantOverlayOptions, +): { ok: boolean; error?: string } { + if (process.platform !== "darwin") { + return { ok: false, error: "Grant overlay is only supported on macOS" }; + } + if (!opts.hostAppPath || !existsSync(opts.hostAppPath)) { + return { + ok: false, + error: "Host app path missing; install the control engine first", + }; + } + + wireIpcOnce(); + + const hostAppName = + opts.hostAppName?.trim() || + opts.hostAppPath.split("/").pop()?.replace(/\.app$/i, "") || + "Atmos Desktop Use"; + const copy = buildCopy(hostAppName, opts.locale); + grantState = { + hostAppPath: opts.hostAppPath, + hostAppName, + ...copy, + }; + + if (grantWindow && !grantWindow.isDestroyed()) { + void grantWindow.loadURL( + `data:text/html;charset=utf-8,${encodeURIComponent(panelHtml(grantState, opts.locale))}`, + ); + positionBottomCenter(grantWindow); + grantWindow.showInactive(); + return { ok: true }; + } + + const preload = grantPreloadPath(); + // Prefer a normal always-on-top window over type:"panel" — panel windows + // often break HTML5/Electron file drag on macOS. + const win = new BrowserWindow({ + width: 400, + height: 150, + frame: false, + transparent: true, + resizable: false, + maximizable: false, + minimizable: false, + fullscreenable: false, + skipTaskbar: true, + alwaysOnTop: true, + hasShadow: false, + show: false, + focusable: true, + webPreferences: { + preload: existsSync(preload) ? preload : undefined, + contextIsolation: true, + nodeIntegration: false, + sandbox: false, + }, + }); + + win.setAlwaysOnTop(true, "floating"); + win.setVisibleOnAllWorkspaces(true, { visibleOnFullScreen: true }); + + grantWindow = win; + win.on("closed", () => { + if (grantWindow === win) grantWindow = null; + }); + + void win + .loadURL( + `data:text/html;charset=utf-8,${encodeURIComponent(panelHtml(grantState, opts.locale))}`, + ) + .then(() => { + positionBottomCenter(win); + win.showInactive(); + }); + + return { ok: true }; +} + +export function closeAccessibilityGrantOverlay(): void { + if (grantWindow && !grantWindow.isDestroyed()) { + grantWindow.close(); + } + grantWindow = null; +} diff --git a/apps/desktop-electron/src/desktop-use/grant-preload.ts b/apps/desktop-electron/src/desktop-use/grant-preload.ts new file mode 100644 index 000000000..b104322d2 --- /dev/null +++ b/apps/desktop-electron/src/desktop-use/grant-preload.ts @@ -0,0 +1,18 @@ +/** + * Preload for the Accessibility grant panel. + * startDrag must run synchronously during the renderer dragstart event. + */ +import { contextBridge, ipcRenderer } from "electron"; + +contextBridge.exposeInMainWorld("atmosGrant", { + /** Sync IPC so main can call startDrag before dragstart returns. */ + startDrag() { + return ipcRenderer.sendSync("desktop-use-grant-drag-start") as { + ok?: boolean; + error?: string; + }; + }, + close() { + ipcRenderer.send("desktop-use-grant-close"); + }, +}); diff --git a/apps/desktop-electron/src/ipc/handlers.ts b/apps/desktop-electron/src/ipc/handlers.ts index 804d23900..1952c5bce 100644 --- a/apps/desktop-electron/src/ipc/handlers.ts +++ b/apps/desktop-electron/src/ipc/handlers.ts @@ -617,7 +617,45 @@ export function createAllHandlers( raw === "accessibility" || raw === "screen_recording" || raw === "all" ? raw : "all"; - return client.desktopUseGrantPermissions(target); + const result = (await client.desktopUseGrantPermissions(target)) as { + ok?: boolean; + host_app_path?: string | null; + host_app_name?: string | null; + accessibility_pane?: boolean; + [key: string]: unknown; + }; + + // Desktop shell: Accessibility gets a drag-to-list panel (host .app path). + const wantsDrag = + result?.accessibility_pane === true || + target === "accessibility" || + target === "all"; + const hostPath = + typeof result?.host_app_path === "string" ? result.host_app_path : ""; + if (wantsDrag && hostPath && process.platform === "darwin") { + const { showAccessibilityGrantOverlay } = await import( + "../desktop-use/grant-overlay.js" + ); + const locale = + typeof args?.locale === "string" + ? args.locale + : typeof args?.lang === "string" + ? args.lang + : undefined; + const overlay = showAccessibilityGrantOverlay({ + hostAppPath: hostPath, + hostAppName: + typeof result?.host_app_name === "string" + ? result.host_app_name + : undefined, + locale, + }); + return { + ...result, + drag_overlay: overlay, + }; + } + return result; }, async desktop_use_drive_verify() { const client = await import("../desktop-use/client.js"); @@ -649,6 +687,13 @@ export function createAllHandlers( const client = await import("../desktop-use/client.js"); return client.desktopUseDriveSessionEnd(); }, + async desktop_use_close_grant_overlay() { + const { closeAccessibilityGrantOverlay } = await import( + "../desktop-use/grant-overlay.js" + ); + closeAccessibilityGrantOverlay(); + return { ok: true }; + }, // --- tunnel --- async tunnel_connector_detect() { diff --git a/apps/web/src/features/desktop-use/components/DesktopUseReadinessDialog.tsx b/apps/web/src/features/desktop-use/components/DesktopUseReadinessDialog.tsx index 1017dc62a..99089ea92 100644 --- a/apps/web/src/features/desktop-use/components/DesktopUseReadinessDialog.tsx +++ b/apps/web/src/features/desktop-use/components/DesktopUseReadinessDialog.tsx @@ -10,9 +10,10 @@ import { DialogFooter, DialogHeader, DialogTitle, + cn, } from "@workspace/ui"; import { AlertCircle, Settings2 } from "lucide-react"; -import { openDesktopUseSettingsInApp } from "@/features/appshot/lib/open-desktop-use-settings"; +import { useOpenDesktopUseSettings } from "@/features/appshot/lib/open-desktop-use-settings"; import { closeDesktopUseReadinessModal, subscribeDesktopUseReadinessModal, @@ -21,6 +22,9 @@ import { import { invalidateDesktopUseReadinessCache } from "../lib/readiness"; import type { DesktopUseReadinessReason } from "../lib/readiness"; +/** Ignore dismiss / clicks for this long after open (Popover→Dialog handoff race). */ +const OPEN_SETTLE_MS = 400; + function reasonMessageKey( reason: DesktopUseReadinessReason | null | undefined, ): string { @@ -44,32 +48,94 @@ function reasonMessageKey( export function DesktopUseReadinessDialog() { const t = useTranslations("desktopUse.readinessModal"); + const openDesktopUseSettings = useOpenDesktopUseSettings(); const [state, setState] = React.useState({ open: false, readiness: null, }); + /** After open, block outside-dismiss and button clicks until the open gesture settles. */ + const [settled, setSettled] = React.useState(false); + const openedAtRef = React.useRef(0); React.useEffect(() => subscribeDesktopUseReadinessModal(setState), []); + React.useEffect(() => { + if (!state.open) { + setSettled(false); + return; + } + openedAtRef.current = Date.now(); + setSettled(false); + const id = window.setTimeout(() => setSettled(true), OPEN_SETTLE_MS); + return () => window.clearTimeout(id); + }, [state.open, state.readiness?.checkedAt, state.source]); + const reason = state.readiness?.reason ?? null; const source = state.source ?? "generic"; + const isWithinSettleWindow = () => + Date.now() - openedAtRef.current < OPEN_SETTLE_MS; + const onOpenSettings = () => { + if (!settled || isWithinSettleWindow()) return; closeDesktopUseReadinessModal(); // Force next entry to re-check after user may install / grant. invalidateDesktopUseReadinessCache(); - // Non-hook path: safe in root layout prerender (no useSearchParams). - openDesktopUseSettingsInApp(); + // nuqs path — reliable under NuqsAdapter (unlike bare pushState + popstate). + openDesktopUseSettings(); }; return ( { - if (!open) closeDesktopUseReadinessModal(); + // Swallow the synthetic dismiss that fires when a Popover closes in the + // same gesture that opened this dialog (Appshots / slash popovers). + if (!open) { + if (!settled || isWithinSettleWindow()) return; + closeDesktopUseReadinessModal(); + } }} + modal > - + {/* + High z-index: Appshots history uses an extreme ceiling. Settle window + disables pointer events so the opening click cannot ghost-activate + "Open settings" (which wrote URL params and made Settings appear later). + */} + { + // Do not focus the primary CTA — avoids Enter/ghost activation. + event.preventDefault(); + }} + onCloseAutoFocus={(event) => { + event.preventDefault(); + }} + onPointerDownOutside={(event) => { + if (!settled || isWithinSettleWindow()) { + event.preventDefault(); + } + }} + onInteractOutside={(event) => { + if (!settled || isWithinSettleWindow()) { + event.preventDefault(); + } + }} + onEscapeKeyDown={(event) => { + if (!settled || isWithinSettleWindow()) { + event.preventDefault(); + } + }} + >
@@ -94,13 +160,18 @@ export function DesktopUseReadinessDialog() {
- + {project.name} - {projectAgentState !== AGENT_STATE.IDLE ? ( + {/* In attention filter mode, prefer the sticky bell over a live running spinner. */} + {attentionFilterMode && projectAttentionReason ? ( + + ) : projectAgentState !== AGENT_STATE.IDLE ? ( ) : projectAttentionReason ? ( @@ -775,7 +808,7 @@ export const ProjectItem = React.memo(function ProjectItem({ onShowLess={showLess} className="ml-4" /> - {project.workspaces.length === 0 && ( + {project.workspaces.length === 0 && !attentionFilterMode && (
{t("leftSidebarControls.noWorkspaces")}
)}
diff --git a/apps/web/src/app-shell/sidebar/WorkspaceContent.tsx b/apps/web/src/app-shell/sidebar/WorkspaceContent.tsx index e125d2099..9aa150380 100644 --- a/apps/web/src/app-shell/sidebar/WorkspaceContent.tsx +++ b/apps/web/src/app-shell/sidebar/WorkspaceContent.tsx @@ -36,7 +36,10 @@ import { gitApi } from "@/api/ws-api"; import { AGENT_STATE, useAgentHooksStore } from "@/features/agent/store/agent-hooks-store"; import { AgentAttentionIndicator } from "@/features/agent/components/AgentAttentionIndicator"; import { AgentHookStatusIndicator } from "@/features/agent/components/AgentHookStatusIndicator"; -import { useAgentAttentionStore } from "@/features/agent/store/agent-attention-store"; +import { + selectAttentionFilterMode, + useAgentAttentionStore, +} from "@/features/agent/store/agent-attention-store"; import { WorkspaceGroupSelect, WorkspaceLabelBadges, @@ -455,9 +458,10 @@ export const WorkspaceContent = React.memo(function Works const workspaceAgentState = useAgentHooksStore((s) => s.getAgentStateForContextId(workspace.id), ); + const attentionFilterMode = useAgentAttentionStore(selectAttentionFilterMode); const workspaceAttentionReason = useAgentAttentionStore( (s) => s.getContextReason(workspace.id), - );; + ); const handleSaveName = React.useCallback(async () => { const nextName = editableName.trim(); @@ -573,10 +577,18 @@ export const WorkspaceContent = React.memo(function Works )} - {workspaceAgentState !== AGENT_STATE.IDLE ? ( + {/* In attention filter mode, prefer the sticky bell over a live running spinner. */} + {attentionFilterMode && workspaceAttentionReason ? ( + + ) : workspaceAgentState !== AGENT_STATE.IDLE ? ( ) : workspaceAttentionReason ? ( diff --git a/apps/web/src/features/agent/components/AgentHookStatusIndicator.tsx b/apps/web/src/features/agent/components/AgentHookStatusIndicator.tsx index a3f19eff4..c64bea8ad 100644 --- a/apps/web/src/features/agent/components/AgentHookStatusIndicator.tsx +++ b/apps/web/src/features/agent/components/AgentHookStatusIndicator.tsx @@ -1,61 +1,17 @@ "use client"; -import React, { useState, useEffect, useRef } from "react"; +import React, { useEffect, useRef } from "react"; import { useTranslations } from "next-intl"; import { TextShimmer, FilledBellIcon, cn } from "@workspace/ui"; import type { AnimatedIconHandle } from "@workspace/ui"; import { AGENT_STATE, type AgentHookState } from "@/features/agent/store/agent-hooks-store"; - -const SPINNER_NAMES = [ - "braille", "helix", "scan", "cascade", "orbit", - "snake", "breathe", "pulse", "dna", "rain", -] as const; - -const BRAILLE_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"]; -const BRAILLE_INTERVAL = 80; - -function useBrailleSpinner() { - const [frame, setFrame] = useState(0); - - useEffect(() => { - const timer = setInterval(() => { - setFrame((f) => (f + 1) % BRAILLE_FRAMES.length); - }, BRAILLE_INTERVAL); - return () => clearInterval(timer); - }, []); - - return BRAILLE_FRAMES[frame]; -} - -function useFullSpinner() { - const [frame, setFrame] = useState(0); - const [spinner, setSpinner] = useState<{ frames: readonly string[]; interval: number } | null>(null); - - useEffect(() => { - let cancelled = false; - const name = SPINNER_NAMES[Math.floor(Math.random() * SPINNER_NAMES.length)]; - - import("unicode-animations").then((mod) => { - if (cancelled) return; - const spinners = mod.default ?? mod; - const s = spinners[name as keyof typeof spinners]; - if (s) setSpinner(s); - }); - - return () => { cancelled = true; }; - }, []); - - useEffect(() => { - if (!spinner) return; - const timer = setInterval(() => { - setFrame((f) => f + 1); - }, spinner.interval); - return () => clearInterval(timer); - }, [spinner]); - - if (!spinner) return "⠋"; - return spinner.frames[frame % spinner.frames.length]; -} +import { AgentRunningGlyph } from "@/features/agent/components/AgentRunningGlyph"; +import { + DEFAULT_INDICATOR_BY_PLACEMENT, + type AgentActivityIndicatorId, + type AgentIndicatorPlacement, +} from "@/features/agent/lib/agent-activity-indicator-styles"; +import { useAgentActivityIndicatorSettingsStore } from "@/features/settings/store/agent-activity-indicator-settings-store"; export type AgentHookIndicatorVariant = "compact" | "full"; @@ -64,6 +20,16 @@ interface AgentHookStatusIndicatorProps { variant?: AgentHookIndicatorVariant; className?: string; tool?: string; + /** + * Where this indicator is shown. Selects the user-configured running glyph. + * Omit when forcing `styleId` (e.g. settings previews). + */ + placement?: AgentIndicatorPlacement; + /** + * Force a specific running glyph (settings previews / tests). + * Wins over `placement` when both are set. + */ + styleId?: AgentActivityIndicatorId; } const STATE_DOT_COLORS: Record = { @@ -87,7 +53,10 @@ function PermissionBellCompact() { const bellRef = useRef(null); useLoopingBell(bellRef); return ( - + ); @@ -109,9 +78,26 @@ function PermissionBellFull({ tool }: { tool?: string }) { ); } -function CompactIndicator({ state }: { state: AgentHookState }) { +function useResolvedStyleId( + placement: AgentIndicatorPlacement | undefined, + styleId: AgentActivityIndicatorId | undefined, +): AgentActivityIndicatorId { + const fromStore = useAgentActivityIndicatorSettingsStore((s) => + placement ? s[placement] : DEFAULT_INDICATOR_BY_PLACEMENT.left_sidebar, + ); + if (styleId) return styleId; + if (placement) return fromStore; + return DEFAULT_INDICATOR_BY_PLACEMENT.left_sidebar; +} + +function CompactIndicator({ + state, + styleId, +}: { + state: AgentHookState; + styleId: AgentActivityIndicatorId; +}) { const t = useTranslations("Agent.components.hookStatus"); - const spinnerChar = useBrailleSpinner(); if (state === AGENT_STATE.IDLE) { return null; @@ -121,21 +107,20 @@ function CompactIndicator({ state }: { state: AgentHookState }) { return ; } - return ( - - {spinnerChar} - - ); + return ; } -function RunningFullSpinner({ tool }: { tool?: string }) { +function RunningFullSpinner({ + tool, + styleId, +}: { + tool?: string; + styleId: AgentActivityIndicatorId; +}) { const t = useTranslations("Agent.components.hookStatus"); - const spinnerChar = useFullSpinner(); return (
- - {spinnerChar} - + {tool ? t("runningWithTool", { tool }) : t("agentRunning")} @@ -143,7 +128,15 @@ function RunningFullSpinner({ tool }: { tool?: string }) { ); } -function FullIndicator({ state, tool }: { state: AgentHookState; tool?: string }) { +function FullIndicator({ + state, + tool, + styleId, +}: { + state: AgentHookState; + tool?: string; + styleId: AgentActivityIndicatorId; +}) { const t = useTranslations("Agent.components.hookStatus"); if (state === AGENT_STATE.IDLE) { return ( @@ -160,7 +153,7 @@ function FullIndicator({ state, tool }: { state: AgentHookState; tool?: string } return ; } - return ; + return ; } export function AgentHookStatusIndicator({ @@ -168,13 +161,17 @@ export function AgentHookStatusIndicator({ variant = "compact", className, tool, + placement, + styleId, }: AgentHookStatusIndicatorProps) { + const resolvedStyleId = useResolvedStyleId(placement, styleId); + return (
{variant === "compact" ? ( - + ) : ( - + )}
); diff --git a/apps/web/src/features/agent/components/AgentRunningGlyph.tsx b/apps/web/src/features/agent/components/AgentRunningGlyph.tsx new file mode 100644 index 000000000..29a89e119 --- /dev/null +++ b/apps/web/src/features/agent/components/AgentRunningGlyph.tsx @@ -0,0 +1,139 @@ +"use client"; + +import React, { useEffect, useMemo, useState } from "react"; +import { cn } from "@workspace/ui"; +import { Orb, type OrbVariant } from "@/features/agent/components/orbs/Orb"; +import { + isOrbIndicatorId, + RANDOM_UNICODE_ID, + UNICODE_SPINNER_IDS, + type AgentActivityIndicatorId, + type UnicodeSpinnerId, +} from "@/features/agent/lib/agent-activity-indicator-styles"; + +const BRAILLE_FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"] as const; +const BRAILLE_INTERVAL = 80; + +type SpinnerData = { frames: readonly string[]; interval: number }; + +function pickUnicodeName(id: AgentActivityIndicatorId): UnicodeSpinnerId { + if (id === RANDOM_UNICODE_ID) { + return UNICODE_SPINNER_IDS[Math.floor(Math.random() * UNICODE_SPINNER_IDS.length)]; + } + if ((UNICODE_SPINNER_IDS as readonly string[]).includes(id)) { + return id as UnicodeSpinnerId; + } + return "braille"; +} + +function useUnicodeSpinner(id: AgentActivityIndicatorId): string { + const [frame, setFrame] = useState(0); + const [spinner, setSpinner] = useState(null); + + // Stable while `id` is unchanged; re-roll only when id becomes "random" again. + const resolvedName = useMemo(() => pickUnicodeName(id), [id]); + + useEffect(() => { + // Fast path for the classic braille compact indicator. + if (resolvedName === "braille") { + setSpinner({ frames: BRAILLE_FRAMES, interval: BRAILLE_INTERVAL }); + return; + } + + let cancelled = false; + import("unicode-animations").then((mod) => { + if (cancelled) return; + const spinners = (mod.default ?? mod) as Record; + const s = spinners[resolvedName]; + if (s?.frames?.length) setSpinner(s); + else setSpinner({ frames: BRAILLE_FRAMES, interval: BRAILLE_INTERVAL }); + }); + return () => { + cancelled = true; + }; + }, [resolvedName]); + + useEffect(() => { + if (!spinner) return; + const timer = setInterval(() => { + setFrame((f) => f + 1); + }, spinner.interval); + return () => clearInterval(timer); + }, [spinner]); + + if (!spinner) return BRAILLE_FRAMES[0]; + return spinner.frames[frame % spinner.frames.length] ?? BRAILLE_FRAMES[0]; +} + +export interface AgentRunningGlyphProps { + styleId: AgentActivityIndicatorId; + /** + * `compact` — sidebar / tab / footer icon slot (~20px). + * `full` — terminal pane status next to shimmer text (~14px glyph). + */ + density?: "compact" | "full"; + className?: string; + title?: string; +} + +/** + * Renders only the animated running glyph (unicode spinner or AIcss Orb). + * Permission / idle states stay in AgentHookStatusIndicator. + */ +export function AgentRunningGlyph({ + styleId, + density = "compact", + className, + title, +}: AgentRunningGlyphProps) { + if (isOrbIndicatorId(styleId)) { + const size = density === "full" ? 14 : 18; + return ( + + + + ); + } + + return ( + + ); +} + +function UnicodeRunningGlyph({ + styleId, + density, + className, + title, +}: { + styleId: AgentActivityIndicatorId; + density: "compact" | "full"; + className?: string; + title?: string; +}) { + const spinnerChar = useUnicodeSpinner(styleId); + return ( + + {spinnerChar} + + ); +} diff --git a/apps/web/src/features/agent/components/orbs/Orb.module.css b/apps/web/src/features/agent/components/orbs/Orb.module.css new file mode 100644 index 000000000..63229f4c1 --- /dev/null +++ b/apps/web/src/features/agent/components/orbs/Orb.module.css @@ -0,0 +1,875 @@ +/* Orbs — agent activity indicators (vendored from AIcss). + * Source: https://www.aicss.dev/components/orbs + * Adapted for Atmos: dark theme via html.dark (next-themes class strategy). + */ + +/* Orbs — two families of agent activity indicator. + * + * The geometry is authored at a 28px stage and scaled with --orb-k, so + * the hand-tuned dot sizes, pitch and blur radii hold at any size. + * + * Per-segment easings inside @keyframes are written as literals: an + * `animation-timing-function` declaration inside a keyframe block is + * read by the animation engine, not resolved against the element, so a + * var() there would not resolve. The numbers mirror the three custom + * properties below exactly. */ + +.root { + --orb-ease-smooth: cubic-bezier(0.22, 1, 0.36, 1); + --orb-ease-out: cubic-bezier(0.17, 1, 0.32, 1); + --orb-ease-in-out: cubic-bezier(0.66, 0, 0.34, 1); + + display: inline-flex; + align-items: center; + vertical-align: middle; + color: currentColor; +} + +/* The inline pill form — same component, wrapped. */ +.root[data-pill] { + gap: 7px; + height: 30px; + padding: 0 11px 0 5px; + border-radius: 999px; + background: #ffffff; + /* A hairline ring rather than a border, so it can't affect layout, plus + a tight contact shadow and a wider ambient one. */ + box-shadow: + 0 0 0 0.5px rgba(0, 0, 0, 0.08), + 0 1px 2px rgba(0, 0, 0, 0.05), + 0 2px 4px rgba(0, 0, 0, 0.02); +} + +.pillLabel { + font-family: "Inter", system-ui, sans-serif; + font-size: 11.5px; + font-weight: 425; + line-height: 1; + color: #a1a1a1; + white-space: nowrap; +} + +.glyph { + position: relative; + display: block; + flex: none; + width: 20px; + height: 20px; + overflow: hidden; + contain: strict; +} + +/* Root keeps color: currentColor so host text tokens (e.g. muted-foreground) apply. */ + +:global(html.dark) .root[data-pill] { + background: #1a1a1a; + box-shadow: + 0 0 0 0.5px rgba(255, 255, 255, 0.12), + 0 1px 2px rgba(0, 0, 0, 0.4), + 0 2px 4px rgba(0, 0, 0, 0.3); +} + +:global(html.dark) .pillLabel { + color: #a3a3a3; +} + +/* --- Lattice: discrete dots on a fixed 3×3 grid ------------------- */ + +.lattice { + position: absolute; + left: 0; + top: 0; + width: 28px; + height: 28px; + transform-origin: 0 0; + /* Three 3px dots on a 6px pitch measure 15px, so the grid is offset to sit + centred on the 28px stage. It deliberately does not fill the stage: that + is what keeps its visual weight level with the Lens circles. */ + transform: scale(var(--orb-k, 1)) translate(6.5px, 6.5px); + /* Resting ink of an unlit cell — the grid stays legible between beats. + --orb-dim is for cells sitting a choreography out entirely. */ + --orb-rest: 0.14; + --orb-dim: 0.07; +} + +/* Light ink on a dark surface reads dimmer at the same alpha. */ +:global(html.dark) .lattice { + --orb-rest: 0.2; + --orb-dim: 0.1; +} + + +.cell { + position: absolute; + width: 3px; + height: 3px; + border-radius: 50%; + background: currentColor; + opacity: var(--orb-rest); +} + +/* One wave shape drives all three sweeps. What separates them is the pair of + duration and per-cell stagger: the stagger sets how fast the wavefront + travels, the duration how many cells it holds lit at once — which is to + say, how wide the band reads. */ +.lattice[data-variant="S1"] .cell { + animation: orb-wave 1.7s var(--orb-ease-in-out) infinite both; +} + +.lattice[data-variant="S2"] .cell { + animation: orb-wave 1.7s var(--orb-ease-in-out) infinite both; +} + +.lattice[data-variant="S4"] .cell { + animation: orb-wave 1.6s var(--orb-ease-in-out) infinite both; +} + +.lattice[data-variant="S3"] .cell { + animation: orb-comet 1.7s var(--orb-ease-smooth) infinite both; +} + +/* Interior cells sit out `orbit` and drop back, so the ring reads as a + ring and the travelling head has something to stand out against. */ +.lattice[data-variant="S3"] .cell[data-still] { + animation: none; + opacity: var(--orb-dim); +} + +.lattice[data-variant="S5"] .cell { + animation: orb-comet 1.7s var(--orb-ease-smooth) infinite both; +} + +.lattice[data-variant="S5"] .cell[data-still] { + animation: none; + opacity: var(--orb-dim); +} + +/* Swells and subsides on the same symmetric curve, so there is no flash and + no hard edge — the cell rises out of its resting ink and sinks back into + it. The long tail after 56% is the gap between beats. */ +@keyframes orb-wave { + 0% { + opacity: var(--orb-rest); + transform: scale(1); + animation-timing-function: cubic-bezier(0.66, 0, 0.34, 1); + } + 28% { + opacity: 1; + transform: scale(1.18); + animation-timing-function: cubic-bezier(0.66, 0, 0.34, 1); + } + 56% { + opacity: var(--orb-rest); + transform: scale(1); + } + 100% { + opacity: var(--orb-rest); + transform: scale(1); + } +} + +/* Starts lit and decays, so staggered ring cells form a head and tail. + The decay spans ~3.5 of the 8 ring positions — enough to read as a comet. */ +@keyframes orb-comet { + 0% { + opacity: 1; + transform: scale(1.2); + /* Linear, so the cells behind the head form an even gradient instead + of collapsing to rest within the first two. */ + animation-timing-function: linear; + } + 45% { + opacity: var(--orb-rest); + transform: scale(1); + } + 100% { + opacity: var(--orb-rest); + transform: scale(1); + } +} + +/* --- Lens: three circles at depth, blur reads as distance --------- */ + +.lens { + position: absolute; + left: 0; + top: 0; + width: 28px; + height: 28px; + transform-origin: 0 0; + transform: scale(var(--orb-k, 1)); +} + +.shape { + position: absolute; + left: 50%; + top: 50%; + width: var(--orb-d, 7px); + height: var(--orb-d, 7px); + /* Pulled back by its own half-size, so --orb-d is the only knob a variant + has to touch to resize the cast and it stays centred on the stage. */ + margin: calc(var(--orb-d, 7px) / -2) 0 0 calc(var(--orb-d, 7px) / -2); + border-radius: 50%; + background: currentColor; +} + +/* focus — attention travels the cast: each circle pulls into focus in turn. + Four circles on the corners of a square, one size for all of them, so the + only thing separating them is which one is sharp. + + A second longer than the three-circle version it grew out of: the square + has four stations to visit and each one keeps the same unhurried second. + + The delays count down rather than up because a more negative delay seeds a + circle further into its cycle: -3s of a 4s cycle runs three quarters ahead, + which is what sends focus round the square clockwise. */ +.lens[data-variant="B1"] .shape { + --orb-d: 6px; + animation: orb-focus 4s var(--orb-ease-smooth) infinite both; +} + +.lens[data-variant="B1"] .shapeA { + --orb-ox: -4.5px; + --orb-oy: -4.5px; + animation-delay: 0s; +} + +.lens[data-variant="B1"] .shapeB { + --orb-ox: 4.5px; + --orb-oy: -4.5px; + animation-delay: -3s; +} + +.lens[data-variant="B1"] .shapeC { + --orb-ox: 4.5px; + --orb-oy: 4.5px; + animation-delay: -2s; +} + +.lens[data-variant="B1"] .shapeD { + --orb-ox: -4.5px; + --orb-oy: 4.5px; + animation-delay: -1s; +} + +/* Opacity gradient around the square: active = 1.0, next neighbour = 0.30, + diagonal = 0.10, far = 0.05. Two circles are always clearly visible, the + rest are ghost hints. */ +@keyframes orb-focus { + 0%, + 100% { + opacity: 0.05; + filter: blur(2px); + transform: translate(var(--orb-ox), var(--orb-oy)) scale(1.12); + animation-timing-function: cubic-bezier(0.4, 0, 0.2, 1); + } + 12% { + opacity: 1; + filter: blur(0); + transform: translate(var(--orb-ox), var(--orb-oy)) scale(1); + animation-timing-function: linear; + } + 22% { + opacity: 1; + filter: blur(0); + transform: translate(var(--orb-ox), var(--orb-oy)) scale(1); + animation-timing-function: cubic-bezier(0.4, 0, 0.2, 1); + } + /* Next neighbour — one quarter away: clearly visible */ + 38% { + opacity: 0.3; + filter: blur(1.2px); + transform: translate(var(--orb-ox), var(--orb-oy)) scale(1.06); + animation-timing-function: cubic-bezier(0.4, 0, 0.2, 1); + } + /* Diagonal — half a cycle away: ghost */ + 58% { + opacity: 0.1; + filter: blur(1.8px); + transform: translate(var(--orb-ox), var(--orb-oy)) scale(1.1); + animation-timing-function: linear; + } + /* Far neighbour — three quarters away: barely there */ + 82% { + opacity: 0.05; + filter: blur(2px); + transform: translate(var(--orb-ox), var(--orb-oy)) scale(1.12); + } +} + +/* drift — the cast circles the stage on one track, sharp at the front and + blurred away at the back, so the orb reads as looking around. Uniform + size: the depth cue is doing the work, a size ladder would fight it. */ +.lens[data-variant="B2"] .shape { + animation: orb-revolve 3.3s linear infinite both; +} + +/* Evenly spaced around the track, so one is always at the front. */ +.lens[data-variant="B2"] .shapeA { + animation-delay: 0s; +} + +.lens[data-variant="B2"] .shapeB { + animation-delay: -1.1s; +} + +.lens[data-variant="B2"] .shapeC { + animation-delay: -2.2s; +} + +/* rotate() then translateY() walks a circle. Linear all the way: an eased + rotation on a circular path reads as a wobble, not as travel. */ +@keyframes orb-revolve { + 0% { + opacity: 1; + filter: blur(0); + transform: rotate(0deg) translateY(6.5px) scale(1); + } + 25% { + opacity: 0.55; + filter: blur(1.3px); + transform: rotate(90deg) translateY(6.5px) scale(0.82); + } + 50% { + opacity: 0.28; + filter: blur(2.4px); + transform: rotate(180deg) translateY(6.5px) scale(0.66); + } + 75% { + opacity: 0.55; + filter: blur(1.3px); + transform: rotate(270deg) translateY(6.5px) scale(0.82); + } + 100% { + opacity: 1; + filter: blur(0); + transform: rotate(360deg) translateY(6.5px) scale(1); + } +} + +/* bloom — shapes emanate from the centre, blurring out as they grow. + Linear keeps the total ink even; on a front-loaded curve the shapes + jump to their large, blurred end state and the orb alternates between + a heavy blot and an empty haze. */ +.lens[data-variant="B3"] .shape { + animation: orb-bloom 4.2s linear infinite both; +} + +.lens[data-variant="B3"] .shapeA { + animation-delay: 0s; +} + +.lens[data-variant="B3"] .shapeB { + animation-delay: -1.4s; +} + +.lens[data-variant="B3"] .shapeC { + animation-delay: -2.8s; +} + +/* Each ripple dies at 62% and waits out the rest, so the three overlapping + blooms leave gaps. Without the gap the aggregate is a constant haze and + the outward motion stops reading at all. Sharp circle appears, holds + briefly, then dissolves outward — blur only kicks in once opacity starts + dropping, so the circle stays crisp while it's visible and the blur reads + as the ripple dissipating. */ +@keyframes orb-bloom { + 0% { + opacity: 0; + filter: blur(0); + transform: scale(0.35); + animation-timing-function: cubic-bezier(0, 0, 0.2, 1); + } + 8% { + opacity: 1; + filter: blur(0); + transform: scale(0.55); + animation-timing-function: linear; + } + 24% { + opacity: 1; + filter: blur(0); + transform: scale(0.72); + animation-timing-function: cubic-bezier(0.16, 1, 0.3, 1); + } + 42% { + opacity: 0.1; + filter: blur(1.8px); + transform: scale(1.5); + } + 62% { + opacity: 0; + filter: blur(2.8px); + transform: scale(2.4); + } + 100% { + opacity: 0; + filter: blur(2.8px); + transform: scale(2.4); + } +} + +/* converge — a single circle traces an equilateral triangle (top → bottom-right + → bottom-left → top) with handoff-style easing: full size and sharp at each + vertex, smaller and slightly blurred in transit. orbB breathes at the + centroid as a subtle depth cue; orbC is hidden. */ +.lens[data-variant="B4"] .shapeA { + animation: orb-converge 3.6s linear infinite both; +} +.lens[data-variant="B4"] .shapeB { + animation: orb-breathe 3.6s ease-in-out infinite both; +} +.lens[data-variant="B4"] .shapeC { + display: none; +} + +@keyframes orb-converge { + 0% { + transform: translate(0px, -5px) scale(1); + filter: blur(0); + animation-timing-function: linear; + } + 10% { + transform: translate(0px, -5px) scale(1); + filter: blur(0); + animation-timing-function: cubic-bezier(0.55, 0, 1, 0.45); + } + 22% { + transform: translate(2.15px, -1.25px) scale(0.72); + filter: blur(0.8px); + animation-timing-function: cubic-bezier(0.33, 1, 0.68, 1); + } + 33% { + transform: translate(4.3px, 2.5px) scale(1); + filter: blur(0); + animation-timing-function: linear; + } + 43% { + transform: translate(4.3px, 2.5px) scale(1); + filter: blur(0); + animation-timing-function: cubic-bezier(0.55, 0, 1, 0.45); + } + 55% { + transform: translate(0px, 2.5px) scale(0.72); + filter: blur(0.8px); + animation-timing-function: cubic-bezier(0.33, 1, 0.68, 1); + } + 66% { + transform: translate(-4.3px, 2.5px) scale(1); + filter: blur(0); + animation-timing-function: linear; + } + 77% { + transform: translate(-4.3px, 2.5px) scale(1); + filter: blur(0); + animation-timing-function: cubic-bezier(0.55, 0, 1, 0.45); + } + 88% { + transform: translate(-2.15px, -1.25px) scale(0.72); + filter: blur(0.8px); + animation-timing-function: cubic-bezier(0.33, 1, 0.68, 1); + } + 100% { + transform: translate(0px, -5px) scale(1); + filter: blur(0); + } +} + +/* handoff — the cast crosses the focal plane one after another, always left + to right, like work being passed on. The shorthand curve is only a + fallback; every segment below sets its own. */ +.lens[data-variant="B5"] .shape { + animation: orb-handoff 2.8s linear infinite both; +} + +/* Half a cycle apart, so one is always at the focal plane while the other is + invisible at an end and the loop point cannot be seen. */ +.lens[data-variant="B5"] .shapeA { + animation-delay: 0s; +} + +.lens[data-variant="B5"] .shapeC { + animation-delay: -1.4s; +} + +/* The third holds the centre and breathes — a soft depth cue behind the + traffic rather than another traveller. */ +.lens[data-variant="B5"] .shapeB { + animation-name: orb-breathe; + animation-duration: 3.6s; +} + +/* Enters small from the left, reaches standard size at the focal plane, then + shrinks and fades out to the right. At the dwell (centre) the circle is + exactly 1× — no pulsing, no bounce, just a clean handoff. */ +@keyframes orb-handoff { + 0% { + opacity: 0; + filter: blur(2.4px); + transform: translateX(-11px) scale(0.55); + animation-timing-function: cubic-bezier(0.33, 1, 0.68, 1); + } + 22% { + opacity: 1; + filter: blur(0); + transform: translateX(-1px) scale(1); + animation-timing-function: linear; + } + 37% { + opacity: 1; + filter: blur(0); + transform: translateX(0) scale(1); + animation-timing-function: linear; + } + 52% { + opacity: 1; + filter: blur(0); + transform: translateX(1px) scale(1); + animation-timing-function: cubic-bezier(0.33, 1, 0.68, 1); + } + 70% { + opacity: 0; + filter: blur(2.4px); + transform: translateX(11px) scale(0.55); + } + 100% { + opacity: 0; + filter: blur(2.4px); + transform: translateX(11px) scale(0.55); + } +} + +@keyframes orb-breathe { + 0%, + 100% { + opacity: 0.16; + filter: blur(2.4px); + transform: scale(1.2); + } + 50% { + opacity: 0.32; + filter: blur(1.6px); + transform: scale(0.98); + } +} + +/* --- Ring: eight circles on a fixed ring ----------------------------- */ + +.ring { + position: absolute; + inset: 0; + transform: scale(var(--orb-k, 1)); + --orb-ring-rest: 0.22; +} + +:global(html.dark) .ring { + --orb-ring-rest: 0.3; +} + + +.ringDot { + position: absolute; + left: 50%; + top: 50%; + width: 3px; + height: 3px; + margin: -1.5px 0 0 -1.5px; + border-radius: 50%; + background: currentColor; + transform: translate(var(--orb-rx), var(--orb-ry)); +} + +.ring[data-variant="C1"] .ringDot { + opacity: var(--orb-ring-rest); + animation: orb-ring-chase 1.6s linear infinite both; +} + +@keyframes orb-ring-chase { + 0%, 11% { + opacity: 1; + } + 12.5%, 100% { + opacity: var(--orb-ring-rest); + } +} + +.ring[data-variant="C2"] .ringDot { + animation: orb-ring-pulse 2s ease-in-out infinite both; +} + +@keyframes orb-ring-pulse { + 0%, 100% { + opacity: 0.18; + transform: translate(var(--orb-rx), var(--orb-ry)) scale(0.7); + } + 50% { + opacity: 1; + transform: translate(var(--orb-rx), var(--orb-ry)) scale(1.15); + } +} + +.ring[data-variant="C3"] .ringDot { + animation: orb-ring-comet 1.8s ease-in-out infinite both; +} + +@keyframes orb-ring-comet { + 0%, 100% { + opacity: 0.08; + transform: translate(var(--orb-rx), var(--orb-ry)); + } + 12% { + opacity: 1; + transform: translate(var(--orb-rx), var(--orb-ry)); + animation-timing-function: cubic-bezier(0.33, 1, 0.68, 1); + } + 35% { + opacity: 0.5; + transform: translate(var(--orb-rx), var(--orb-ry)); + } + 60% { + opacity: 0.12; + transform: translate(var(--orb-rx), var(--orb-ry)); + } +} + +.ring[data-variant="C4"] .ringDot { + animation: orb-ring-stagger 1.6s ease-in-out infinite both; +} + +@keyframes orb-ring-stagger { + 0%, 100% { + opacity: 1; + transform: translate(var(--orb-rx), var(--orb-ry)); + } + 50% { + opacity: 0.15; + transform: translate(var(--orb-rx), var(--orb-ry)); + } +} + +.ring[data-variant="C5"] .ringDot { + animation: orb-ring-comet 1.8s ease-in-out infinite both; +} + +/* ---- Globe (Helix family) ---- */ +.helix { + position: absolute; + inset: 0; + transform: scale(var(--orb-k, 1)); +} + +.helixDot { + position: absolute; + left: 50%; + top: 50%; + width: 2px; + height: 2px; + margin: -1px 0 0 -1px; + border-radius: 50%; + background: currentColor; + will-change: transform, opacity; +} + +.helix[data-variant="G1"] .helixDot { + animation: orb-globe-spin 4.5s linear infinite both; +} +.helix[data-variant="G2"] .helixDot { + animation: orb-globe-spin 3.6s linear infinite both; +} +.helix[data-variant="G3"] .helixDot { + animation: orb-globe-ringturn 2.8s linear infinite both; +} +.helix[data-variant="G4"] .helixDot { + animation: orb-globe-ringturn 2.8s linear infinite both; +} +.helix[data-variant="G5"] .helixDot { + animation: orb-globe-breathe 3.6s linear infinite both; +} + +@keyframes orb-globe-spin { + 0%, 100% { + transform: translate(var(--g0x), var(--g0y)); + opacity: var(--g0o); + } + 12.5% { + transform: translate(var(--g1x), var(--g1y)); + opacity: var(--g1o); + } + 25% { + transform: translate(var(--g2x), var(--g2y)); + opacity: var(--g2o); + } + 37.5% { + transform: translate(var(--g3x), var(--g3y)); + opacity: var(--g3o); + } + 50% { + transform: translate(var(--g4x), var(--g4y)); + opacity: var(--g4o); + } + 62.5% { + transform: translate(var(--g5x), var(--g5y)); + opacity: var(--g5o); + } + 75% { + transform: translate(var(--g6x), var(--g6y)); + opacity: var(--g6o); + } + 87.5% { + transform: translate(var(--g7x), var(--g7y)); + opacity: var(--g7o); + } +} + +@keyframes orb-globe-ringturn { + 0% { transform: translate(var(--g0x), var(--g0y)); opacity: var(--g0o); } + 2.5% { transform: translate(var(--g1x), var(--g1y)); opacity: var(--g1o); } + 5% { transform: translate(var(--g2x), var(--g2y)); opacity: var(--g2o); } + 7.5%, 10% { transform: translate(var(--g3x), var(--g3y)); opacity: var(--g3o); } + 12.5% { transform: translate(var(--g4x), var(--g4y)); opacity: var(--g4o); } + 15% { transform: translate(var(--g5x), var(--g5y)); opacity: var(--g5o); } + 17.5%, 20% { transform: translate(var(--g6x), var(--g6y)); opacity: var(--g6o); } + 22.5% { transform: translate(var(--g7x), var(--g7y)); opacity: var(--g7o); } + 25% { transform: translate(var(--g8x), var(--g8y)); opacity: var(--g8o); } + 27.5%, 30% { transform: translate(var(--g9x), var(--g9y)); opacity: var(--g9o); } + 32.5% { transform: translate(var(--g10x), var(--g10y)); opacity: var(--g10o); } + 35% { transform: translate(var(--g11x), var(--g11y)); opacity: var(--g11o); } + 37.5%, 40% { transform: translate(var(--g12x), var(--g12y)); opacity: var(--g12o); } + 42.5% { transform: translate(var(--g13x), var(--g13y)); opacity: var(--g13o); } + 45% { transform: translate(var(--g14x), var(--g14y)); opacity: var(--g14o); } + 47.5%, 50% { transform: translate(var(--g15x), var(--g15y)); opacity: var(--g15o); } + 52.5% { transform: translate(var(--g16x), var(--g16y)); opacity: var(--g16o); } + 55% { transform: translate(var(--g17x), var(--g17y)); opacity: var(--g17o); } + 57.5%, 60% { transform: translate(var(--g18x), var(--g18y)); opacity: var(--g18o); } + 62.5% { transform: translate(var(--g19x), var(--g19y)); opacity: var(--g19o); } + 65% { transform: translate(var(--g20x), var(--g20y)); opacity: var(--g20o); } + 67.5%, 70% { transform: translate(var(--g21x), var(--g21y)); opacity: var(--g21o); } + 72.5% { transform: translate(var(--g22x), var(--g22y)); opacity: var(--g22o); } + 75% { transform: translate(var(--g23x), var(--g23y)); opacity: var(--g23o); } + 77.5%, 80% { transform: translate(var(--g24x), var(--g24y)); opacity: var(--g24o); } + 82.5% { transform: translate(var(--g25x), var(--g25y)); opacity: var(--g25o); } + 85% { transform: translate(var(--g26x), var(--g26y)); opacity: var(--g26o); } + 87.5%, 90% { transform: translate(var(--g27x), var(--g27y)); opacity: var(--g27o); } + 92.5% { transform: translate(var(--g28x), var(--g28y)); opacity: var(--g28o); } + 95% { transform: translate(var(--g29x), var(--g29y)); opacity: var(--g29o); } + 97.5%, 100% { transform: translate(var(--g30x), var(--g30y)); opacity: var(--g30o); } +} + +@keyframes orb-globe-breathe { + 0% { + transform: translate(var(--g0x), var(--g0y)); + opacity: var(--g0o); + } + 19% { + transform: translate(var(--g1x), var(--g1y)); + opacity: var(--g1o); + } + 25% { + transform: translate(var(--g2x), var(--g2y)); + opacity: var(--g2o); + } + 44% { + transform: translate(var(--g3x), var(--g3y)); + opacity: var(--g3o); + } + 50% { + transform: translate(var(--g4x), var(--g4y)); + opacity: var(--g4o); + } + 69% { + transform: translate(var(--g5x), var(--g5y)); + opacity: var(--g5o); + } + 75% { + transform: translate(var(--g6x), var(--g6y)); + opacity: var(--g6o); + } + 94% { + transform: translate(var(--g7x), var(--g7y)); + opacity: var(--g7o); + } + 100% { + transform: translate(var(--g8x), var(--g8y)); + opacity: var(--g8o); + } +} + +/* ---- Morph ---- */ +.morph { + position: absolute; + inset: 0; + transform: scale(var(--orb-k, 1)); +} + +.morphDot { + position: absolute; + left: 50%; + top: 50%; + width: 3px; + height: 3px; + margin: -1.5px 0 0 -1.5px; + border-radius: 50%; + background: currentColor; + animation: orb-morph 4.8s cubic-bezier(0.4, 0, 0.2, 1) infinite both; +} + +@keyframes orb-morph { + 0%, 5% { transform: translate(var(--m-1)); } + 25%, 30% { transform: translate(var(--m-2)); } + 50%, 55% { transform: translate(var(--m-3)); } + 75%, 80% { transform: translate(var(--m-4)); } + 100% { transform: translate(var(--m-1)); } +} + +.morph[data-variant="M2"] { + animation: orb-morph-twist 9.6s linear infinite; +} + +.morph[data-variant="M4"] { + animation: orb-morph-twist 9.6s linear infinite; +} + +.morph[data-variant="M5"] .morphDot { + animation: orb-morph-scatter 2.8s cubic-bezier(0.4, 0, 0.2, 1) infinite both; +} + +@keyframes orb-morph-scatter { + 0%, 12% { transform: translate(var(--m-1)); opacity: 1; } + 38%, 62% { transform: translate(var(--m-2)); opacity: calc(1 - 0.6 * var(--m-depth, 0)); } + 88%, 100% { transform: translate(var(--m-1)); opacity: 1; } +} + +@keyframes orb-morph-twist { + from { transform: scale(var(--orb-k, 1)) rotate(0deg); } + to { transform: scale(var(--orb-k, 1)) rotate(360deg); } +} + +@media (prefers-reduced-motion: reduce) { + .cell, + .shape, + .ringDot, + .helixDot, + .morphDot { + animation: none !important; + } + .cell[data-mid] { + opacity: 1 !important; + } + .shape { + opacity: 0.3 !important; + filter: blur(1.4px) !important; + transform: none !important; + } + .shapeA { + opacity: 1 !important; + filter: blur(0) !important; + } + .ringDot { + opacity: 0.7 !important; + } + .ring, + .helix, + .morph { + animation: none !important; + } +} \ No newline at end of file diff --git a/apps/web/src/features/agent/components/orbs/Orb.tsx b/apps/web/src/features/agent/components/orbs/Orb.tsx new file mode 100644 index 000000000..334f6ff21 --- /dev/null +++ b/apps/web/src/features/agent/components/orbs/Orb.tsx @@ -0,0 +1,684 @@ +/** + * Orbs — compact agent activity indicators. + * Vendored from AIcss: https://www.aicss.dev/components/orbs + * Attribution: @kvnkld / AIcss (copy-paste free component). + */ + +import type { CSSProperties } from "react"; +import styles from "./Orb.module.css"; + +/** The stage the geometry is tuned on; --orb-k scales it to `size`. */ +const STAGE = 28; + +/** Default rendered size — 20×20 indicator box. */ +const SIZE = 20; + +export type LatticeVariant = "S1" | "S2" | "S3" | "S4" | "S5"; +export type LensVariant = "B1" | "B2" | "B3" | "B4" | "B5"; +export type RingVariant = "C1" | "C2" | "C3" | "C4" | "C5"; +export type HelixVariant = "G1" | "G2" | "G3" | "G4" | "G5"; +export type MorphVariant = "M1" | "M2" | "M3" | "M4" | "M5"; +export type OrbVariant = LatticeVariant | LensVariant | RingVariant | HelixVariant | MorphVariant; + +export const LATTICE_VARIANTS: LatticeVariant[] = ["S1", "S2", "S3", "S4", "S5"]; + +export const LENS_VARIANTS: LensVariant[] = [ + "B1", + "B2", + "B3", + "B4", + "B5", +]; + +export const RING_VARIANTS: RingVariant[] = ["C1", "C2", "C3", "C4", "C5"]; + +export const HELIX_VARIANTS: HelixVariant[] = ["G1", "G2", "G3", "G4", "G5"]; + +export const MORPH_VARIANTS: MorphVariant[] = ["M1", "M2", "M3", "M4", "M5"]; + +export const ORB_TASKS: Record = { + S1: "Thinking", + S2: "Processing", + S3: "Working", + S4: "Searching", + S5: "Finalizing", + B1: "Thinking", + B2: "Searching", + B3: "Generating", + B4: "Solving", + B5: "Routing", + C1: "Loading", + C2: "Listening", + C3: "Streaming", + C4: "Analyzing", + C5: "Compiling", + G1: "Processing", + G2: "Sequencing", + G3: "Uploading", + G4: "Syncing", + G5: "Idling", + M1: "Shaping", + M2: "Expanding", + M3: "Unfolding", + M4: "Transforming", + M5: "Dispersing", +}; + +function isLattice(v: OrbVariant): v is LatticeVariant { + return (LATTICE_VARIANTS as OrbVariant[]).includes(v); +} + +function isRing(v: OrbVariant): v is RingVariant { + return (RING_VARIANTS as OrbVariant[]).includes(v); +} + +function isHelix(v: OrbVariant): v is HelixVariant { + return (HELIX_VARIANTS as OrbVariant[]).includes(v); +} + +function isMorph(v: OrbVariant): v is MorphVariant { + return (MORPH_VARIANTS as OrbVariant[]).includes(v); +} + +const N = 3; // lattice is N×N +const PITCH = 6; // centre-to-centre spacing in stage px; the dot size is CSS +const MID = (N - 1) / 2; + +/** Clockwise walk of the lattice perimeter — the track `orbit` runs on. */ +const RING: [number, number][] = (() => { + const ring: [number, number][] = []; + for (let x = 0; x < N; x++) ring.push([x, 0]); + for (let y = 1; y < N; y++) ring.push([N - 1, y]); + for (let x = N - 2; x >= 0; x--) ring.push([x, N - 1]); + for (let y = N - 2; y >= 1; y--) ring.push([0, y]); + return ring; +})(); + +const RING_INDEX = new Map(RING.map(([x, y], i) => [x + "," + y, i])); + +/** + * Per-cell `animation-delay` in ms. Negative values seed a cell partway + * into its cycle, which is what turns 8 identical animations into one + * comet travelling the ring. + */ +function cellDelay(v: LatticeVariant, x: number, y: number): number { + const dx = x - MID; + const dy = y - MID; + const ring = Math.max(Math.abs(dx), Math.abs(dy)); + switch (v) { + // Radiates from the centre on a round wavefront. Centre leads a beat + // early so the next swell doesn't sit behind the outer fade. + case "S1": + return Math.hypot(dx, dy) * 700 - (dx === 0 && dy === 0 ? 180 : 0); + // A broad band crosses the grid on the diagonal. The spread is close to + // the wave duration, which both widens the band and makes the sweep + // continuous — the far corner restarts as the near one does. + case "S2": + return ((x + y) / (2 * (N - 1))) * 1500; + // One head with a decaying tail, running the perimeter clockwise. + case "S3": { + const i = RING_INDEX.get(x + "," + y); + if (i === undefined) return 0; + return -(((RING.length - i) % RING.length) / RING.length) * 1700; + } + // A soft column travels left to right. + case "S4": + return (x / (N - 1)) * 1100; + // Like S3 but scrambled order — the pulse jumps pseudo-randomly. + case "S5": { + const i = RING_INDEX.get(x + "," + y); + if (i === undefined) return 0; + const scrambled = (i * 3) % RING.length; + return -(scrambled / RING.length) * 1700; + } + } +} + +/** + * `settle` gathers each cell from a position rotated one way around the + * centre and releases it to the mirror rotation, so the cycle keeps swirling + * the same way instead of rewinding to where it came from. + */ +const SWIRL = 1.05; // radians of rotation at each end, ~60° +const SPREAD = 1.6; // outward push, on top of the rotation + +/** Offset from a cell's own grid slot to its swirled position, in stage px. */ +function swirl(x: number, y: number, angle: number): [number, number] { + const dx = x - MID; + const dy = y - MID; + const cos = Math.cos(angle); + const sin = Math.sin(angle); + return [ + ((dx * cos - dy * sin) * SPREAD - dx) * PITCH, + ((dx * sin + dy * cos) * SPREAD - dy) * PITCH, + ]; +} + +interface Cell { + key: string; + left: number; + top: number; + delay: number; + /** Where `settle` gathers this cell from, and releases it to. */ + ax: number; + ay: number; + bx: number; + by: number; + /** Sits out the choreography (interior cells during `orbit`). */ + still: boolean; + /** Centre cell — the static frame under reduced motion. */ + mid: boolean; +} + +/** The 9 lattice cells, with position, phase and swirl vectors. */ +function latticeCells(v: LatticeVariant): Cell[] { + const cells: Cell[] = []; + for (let y = 0; y < N; y++) { + for (let x = 0; x < N; x++) { + const [ax, ay] = swirl(x, y, -SWIRL); + const [bx, by] = swirl(x, y, SWIRL); + cells.push({ + key: x + "," + y, + left: x * PITCH, + top: y * PITCH, + delay: cellDelay(v, x, y), + ax, + ay, + bx, + by, + still: (v === "S3" || v === "S5") && !RING_INDEX.has(x + "," + y), + mid: x === MID && y === MID, + }); + } + } + return cells; +} + +const RING_N = 8; +const RING_R = 8; + +interface RingDot { + key: number; + rx: number; + ry: number; + delay: number; +} + +function ringDuration(v: RingVariant): number { + switch (v) { + case "C1": return 1600; + case "C2": return 2000; + case "C3": return 1800; + case "C4": return 1600; + case "C5": return 2200; + } +} + +function ringDelay(v: RingVariant, i: number): number { + const dur = ringDuration(v); + switch (v) { + case "C1": + return -((RING_N - 1 - i) / RING_N) * dur; + case "C2": + case "C3": + return -((RING_N - 1 - i) / RING_N) * dur; + case "C4": + return i % 2 === 0 ? 0 : -(dur / 2); + case "C5": { + const scrambled = (i * 3) % RING_N; + return -(scrambled / RING_N) * dur; + } + default: + return -(i / RING_N) * dur; + } +} + +function ringDots(v: RingVariant): RingDot[] { + const dots: RingDot[] = []; + for (let i = 0; i < RING_N; i++) { + const angle = (i / RING_N) * Math.PI * 2 - Math.PI / 2; + dots.push({ + key: i, + rx: Math.cos(angle) * RING_R, + ry: Math.sin(angle) * RING_R, + delay: ringDelay(v, i), + }); + } + return dots; +} + +const GLOBE_R = 8.5; +const GLOBE_TILT = (14 * Math.PI) / 180; +const GLOBE_STEPS = 8; + +const GLOBE_RINGS: { lat: number; count: number }[] = [ + { lat: 52, count: 8 }, + { lat: 26, count: 8 }, + { lat: 0, count: 8 }, + { lat: -26, count: 8 }, + { lat: -52, count: 8 }, +]; + +interface GlobeDot { + key: number; + style: Record; + css: string; +} + +function projectGlobe(x: number, y: number, z: number, spin: number) { + const cs = Math.cos(spin); + const ss = Math.sin(spin); + const x1 = x * cs - z * ss; + const z1 = x * ss + z * cs; + const y1 = y; + const ct = Math.cos(GLOBE_TILT); + const st = Math.sin(GLOBE_TILT); + return { + x: x1, + y: y1 * ct - z1 * st, + z: y1 * st + z1 * ct, + }; +} + +function globeOpacity(z: number) { + const t = Math.max(0, Math.min(1, (z / GLOBE_R + 0.15) / 1.15)); + return 0.12 + 0.88 * t * t; +} + +type RingMove = { ring: number; angle: number }; +const RING_HALF = Math.PI; +const RING_ARC = 3; + +function ringDir(ring: number) { + return ring % 2 === 0 ? -1 : 1; +} + +const G3_MOVES: RingMove[] = (() => { + const moves: RingMove[] = []; + for (let pass = 0; pass < 2; pass++) { + for (let r = 0; r < GLOBE_RINGS.length; r++) { + moves.push({ ring: r, angle: ringDir(r) * RING_HALF }); + } + } + return moves; +})(); + +const G4_MOVES: RingMove[] = [2, 1, 3, 0, 4, 2, 1, 3, 0, 4].map((ring) => ({ + ring, + angle: ringDir(ring) * RING_HALF, +})); + +function ringTurnPoses( + x0: number, + y0: number, + z0: number, + ringIndex: number, + moves: RingMove[], +): [number, number, number][] { + let x = x0; + let y = y0; + let z = z0; + const poses: [number, number, number][] = [[x, y, z]]; + for (let m = 0; m < moves.length; m++) { + const move = moves[m]; + const xS = x; + const yS = y; + const zS = z; + for (let s = 1; s <= RING_ARC; s++) { + if (ringIndex === move.ring) { + const a = move.angle * (s / RING_ARC); + const c = Math.cos(a); + const sn = Math.sin(a); + x = xS * c - zS * sn; + y = yS; + z = xS * sn + zS * c; + } + poses.push([x, y, z]); + } + } + return poses; +} + +const G5_SLOW = 0.4; +const G5_BURST = (Math.PI * 2 - G5_SLOW * 4) / 4; +const G5_POSES: { s: number; spin: number }[] = (() => { + const poses: { s: number; spin: number }[] = [{ s: 1.0, spin: 0 }]; + let spin = 0; + const steps: { s: number; kind: "slow" | "burst" }[] = [ + { s: 1.0, kind: "slow" }, + { s: 0.9, kind: "burst" }, + { s: 0.9, kind: "slow" }, + { s: 0.8, kind: "burst" }, + { s: 0.8, kind: "slow" }, + { s: 0.9, kind: "burst" }, + { s: 0.9, kind: "slow" }, + { s: 1.0, kind: "burst" }, + ]; + for (const step of steps) { + spin += step.kind === "slow" ? G5_SLOW : G5_BURST; + poses.push({ s: step.s, spin }); + } + return poses; +})(); + +function globeKeyframeStyle( + x0: number, + y0: number, + z0: number, + variant: HelixVariant, + ringIndex: number, + j = 0, +): Record { + const style: Record = {}; + + if (variant === "G5") { + for (let k = 0; k < G5_POSES.length; k++) { + const sc = G5_POSES[k].s; + const spin = G5_POSES[k].spin; + const p = projectGlobe(x0 * sc, y0 * sc, z0 * sc, spin); + style["--g" + k + "x"] = p.x.toFixed(2) + "px"; + style["--g" + k + "y"] = (-p.y).toFixed(2) + "px"; + style["--g" + k + "o"] = globeOpacity(p.z).toFixed(3); + } + return style; + } + + if (variant === "G3" || variant === "G4") { + const poses = ringTurnPoses( + x0, + y0, + z0, + ringIndex, + variant === "G3" ? G3_MOVES : G4_MOVES, + ); + for (let k = 0; k < poses.length; k++) { + const pos = poses[k]; + const p = projectGlobe(pos[0], pos[1], pos[2], 0); + style["--g" + k + "x"] = p.x.toFixed(2) + "px"; + style["--g" + k + "y"] = (-p.y).toFixed(2) + "px"; + style["--g" + k + "o"] = globeOpacity(p.z).toFixed(3); + } + return style; + } + + const dir = variant === "G2" && ringIndex % 2 === 1 ? -1 : 1; + + for (let k = 0; k < GLOBE_STEPS; k++) { + const phase = k / GLOBE_STEPS; + const spin = dir * phase * Math.PI * 2; + const p = projectGlobe(x0, y0, z0, spin); + style["--g" + k + "x"] = p.x.toFixed(2) + "px"; + style["--g" + k + "y"] = (-p.y).toFixed(2) + "px"; + style["--g" + k + "o"] = globeOpacity(p.z).toFixed(3); + } + return style; +} + +function globeDots(v: HelixVariant): GlobeDot[] { + const dots: GlobeDot[] = []; + let idx = 0; + for (let ringIndex = 0; ringIndex < GLOBE_RINGS.length; ringIndex++) { + const ring = GLOBE_RINGS[ringIndex]; + const latRad = (ring.lat * Math.PI) / 180; + const y0 = Math.sin(latRad) * GLOBE_R; + const ringR = Math.cos(latRad) * GLOBE_R; + for (let j = 0; j < ring.count; j++) { + const lon = (j / ring.count) * Math.PI * 2; + const style = globeKeyframeStyle( + Math.cos(lon) * ringR, + y0, + Math.sin(lon) * ringR, + v, + ringIndex, + j, + ); + dots.push({ + key: idx, + style, + css: Object.keys(style) + .map((k) => k + ":" + style[k]) + .join(";"), + }); + idx++; + } + } + return dots; +} + +const MORPH_N = 8; +const MORPH_R = 7; + +type ShapeFn = (i: number) => [number, number]; + +const shapeCircle: ShapeFn = (i) => { + const a = (i / MORPH_N) * Math.PI * 2 - Math.PI / 2; + return [Math.cos(a) * MORPH_R, Math.sin(a) * MORPH_R]; +}; + +const shapeOctagon: ShapeFn = (i) => { + const a = (i / MORPH_N) * Math.PI * 2 - Math.PI / 2; + const r = MORPH_R * 0.92; + const sector = Math.round(a / (Math.PI / 4)) * (Math.PI / 4); + return [Math.cos(sector) * r, Math.sin(sector) * r]; +}; + +const shapeSquare: ShapeFn = (i) => { + const h = MORPH_R * 0.85; + const corners: [number, number][] = [[-h, -h], [h, -h], [h, h], [-h, h]]; + const t = ((i / MORPH_N) * 4 + 0.5) % 4; + const side = Math.floor(t) % 4; + const frac = t - Math.floor(t); + const from = corners[side]; + const to = corners[(side + 1) % 4]; + return [from[0] + (to[0] - from[0]) * frac, from[1] + (to[1] - from[1]) * frac]; +}; + +const shapeCircleAt = + (turn: number): ShapeFn => + (i) => { + const a = (i / MORPH_N) * Math.PI * 2 - Math.PI / 2 + turn; + return [Math.cos(a) * MORPH_R, Math.sin(a) * MORPH_R]; + }; + +const SCATTER_TRAIL = 0.12; + +const shapeScatterA: ShapeFn = (i) => { + const a = (i / MORPH_N) * Math.PI * 2 - Math.PI / 2; + return [-Math.cos(a) * MORPH_R, Math.sin(a) * MORPH_R]; +}; + +const shapeScatterB: ShapeFn = shapeCircle; +const shapeScatterC: ShapeFn = shapeScatterA; + +const shapeDiamond: ShapeFn = (i) => { + const corners: [number, number][] = [[0, -MORPH_R], [MORPH_R, 0], [0, MORPH_R], [-MORPH_R, 0]]; + const t = (i / MORPH_N) * 4; + const side = Math.floor(t) % 4; + const frac = t - Math.floor(t); + const from = corners[side]; + const to = corners[(side + 1) % 4]; + return [from[0] + (to[0] - from[0]) * frac, from[1] + (to[1] - from[1]) * frac]; +}; + +const shapeCenter: ShapeFn = (i) => { + const a = (i / MORPH_N) * Math.PI * 2 - Math.PI / 2; + return [Math.cos(a) * 1.5, Math.sin(a) * 1.5]; +}; + +function morphShapes(v: MorphVariant): [ShapeFn, ShapeFn, ShapeFn, ShapeFn] { + switch (v) { + case "M1": return [shapeCircle, shapeSquare, shapeDiamond, shapeSquare]; + case "M2": return [shapeCenter, shapeCircle, shapeCenter, shapeCircle]; + case "M3": + return [ + shapeCircleAt(0), + shapeCircleAt(Math.PI / 2), + shapeCircleAt(Math.PI), + shapeCircleAt(Math.PI * 1.5), + ]; + case "M4": return [shapeCircle, shapeDiamond, shapeCircle, shapeDiamond]; + case "M5": return [shapeCircle, shapeScatterA, shapeScatterB, shapeScatterC]; + } +} + +interface MorphDot { + key: number; + m1: string; + m2: string; + m3: string; + m4: string; + delay?: string; + depth?: string; +} + +function morphDots(v: MorphVariant): MorphDot[] { + const [s1, s2, s3, s4] = morphShapes(v); + const dots: MorphDot[] = []; + for (let i = 0; i < MORPH_N; i++) { + const [x1, y1] = s1(i); + const [x2, y2] = s2(i); + const [x3, y3] = s3(i); + const [x4, y4] = s4(i); + dots.push({ + key: i, + m1: x1.toFixed(1) + "px, " + y1.toFixed(1) + "px", + m2: x2.toFixed(1) + "px, " + y2.toFixed(1) + "px", + m3: x3.toFixed(1) + "px, " + y3.toFixed(1) + "px", + m4: x4.toFixed(1) + "px, " + y4.toFixed(1) + "px", + delay: v === "M5" ? -i * 10 + "ms" : undefined, + depth: v === "M5" ? Math.abs(Math.cos((i / MORPH_N) * Math.PI * 2 - Math.PI / 2)).toFixed(2) : undefined, + }); + } + return dots; +} + +export interface OrbProps { + variant?: OrbVariant; + /** Rendered edge length in px. The 28px geometry scales to fit. */ + size?: number; + /** Accessible label, and the status text when `pill` is set. */ + label?: string; + /** Wraps the orb and its label in a status pill. */ + pill?: boolean; + className?: string; + style?: CSSProperties; +} + +export function Orb({ + variant = "S1", + size = SIZE, + label, + pill, + className, + style, +}: OrbProps) { + const text = label ?? ORB_TASKS[variant] + "…"; + return ( + + + {isLattice(variant) ? ( + + {latticeCells(variant).map((c) => ( + + ))} + + ) : isRing(variant) ? ( + + {ringDots(variant).map((d) => ( + + ))} + + ) : isHelix(variant) ? ( + + {globeDots(variant).map((d) => ( + + ))} + + ) : isMorph(variant) ? ( + + {morphDots(variant).map((d) => ( + + ))} + + ) : ( + + + + + {/* focus is the one variant that needs a fourth circle: its cast + sits on the corners of a square, and three corners do not + make a square. */} + {variant === "B1" && ( + + )} + + )} + + {pill && {text}} + + ); +} + +/* Usage: + + + + + */ \ No newline at end of file diff --git a/apps/web/src/features/agent/lib/__tests__/agent-activity-indicator-styles.test.ts b/apps/web/src/features/agent/lib/__tests__/agent-activity-indicator-styles.test.ts new file mode 100644 index 000000000..0c6f3bdb7 --- /dev/null +++ b/apps/web/src/features/agent/lib/__tests__/agent-activity-indicator-styles.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, test } from "bun:test"; +import { + ALL_INDICATOR_STYLE_OPTIONS, + DEFAULT_INDICATOR_BY_PLACEMENT, + INDICATOR_STYLE_GROUPS, + isAgentActivityIndicatorId, + isOrbIndicatorId, + isUnicodeIndicatorId, + ORB_VARIANT_IDS, + resolveIndicatorId, + UNICODE_SPINNER_IDS, +} from "../agent-activity-indicator-styles"; + +describe("agent-activity-indicator-styles", () => { + test("includes project unicode spinners and all aicss orb variants", () => { + expect(UNICODE_SPINNER_IDS).toContain("braille"); + expect(UNICODE_SPINNER_IDS).toContain("helix"); + expect(ORB_VARIANT_IDS).toHaveLength(25); + expect(ORB_VARIANT_IDS).toContain("S1"); + expect(ORB_VARIANT_IDS).toContain("M5"); + }); + + test("catalog is unique and covers every group", () => { + const ids = ALL_INDICATOR_STYLE_OPTIONS.map((o) => o.id); + expect(new Set(ids).size).toBe(ids.length); + expect(INDICATOR_STYLE_GROUPS).toHaveLength(6); + for (const group of INDICATOR_STYLE_GROUPS) { + expect(group.options.length).toBeGreaterThan(0); + } + }); + + test("resolveIndicatorId falls back to placement defaults", () => { + expect(resolveIndicatorId(undefined, "left_sidebar")).toBe( + DEFAULT_INDICATOR_BY_PLACEMENT.left_sidebar, + ); + expect(resolveIndicatorId("not-a-style", "terminal_panel")).toBe( + DEFAULT_INDICATOR_BY_PLACEMENT.terminal_panel, + ); + expect(resolveIndicatorId("S4", "footer")).toBe("S4"); + expect(resolveIndicatorId("random", "terminal_panel")).toBe("random"); + }); + + test("type guards", () => { + expect(isAgentActivityIndicatorId("braille")).toBe(true); + expect(isAgentActivityIndicatorId("B2")).toBe(true); + expect(isAgentActivityIndicatorId("nope")).toBe(false); + expect(isOrbIndicatorId("G1")).toBe(true); + expect(isOrbIndicatorId("braille")).toBe(false); + expect(isUnicodeIndicatorId("random")).toBe(true); + expect(isUnicodeIndicatorId("orbit")).toBe(true); + expect(isUnicodeIndicatorId("S1")).toBe(false); + }); +}); diff --git a/apps/web/src/features/agent/lib/agent-activity-indicator-styles.ts b/apps/web/src/features/agent/lib/agent-activity-indicator-styles.ts new file mode 100644 index 000000000..16c3571d8 --- /dev/null +++ b/apps/web/src/features/agent/lib/agent-activity-indicator-styles.ts @@ -0,0 +1,189 @@ +/** + * Agent activity indicator style catalog. + * + * Combines: + * - Unicode spinners already used in Atmos (unicode-animations) + * - AIcss Orbs (https://www.aicss.dev/components/orbs) + */ + +import { + HELIX_VARIANTS, + LATTICE_VARIANTS, + LENS_VARIANTS, + MORPH_VARIANTS, + RING_VARIANTS, + type OrbVariant, +} from "@/features/agent/components/orbs/Orb"; + +/** Placements that can pick their own running indicator. */ +export type AgentIndicatorPlacement = + | "left_sidebar" + | "center_terminal" + | "terminal_panel" + | "footer"; + +/** Unicode spinner names historically used in Atmos agent status UI. */ +export const UNICODE_SPINNER_IDS = [ + "braille", + "helix", + "scan", + "cascade", + "orbit", + "snake", + "breathe", + "pulse", + "dna", + "rain", +] as const; + +export type UnicodeSpinnerId = (typeof UNICODE_SPINNER_IDS)[number]; + +/** Special id: pick a random unicode spinner per mount (legacy full-indicator behavior). */ +export const RANDOM_UNICODE_ID = "random" as const; + +export type AgentActivityIndicatorId = + | UnicodeSpinnerId + | typeof RANDOM_UNICODE_ID + | OrbVariant; + +export type IndicatorFamily = + | "unicode" + | "lattice" + | "lens" + | "ring" + | "helix" + | "morph"; + +export interface IndicatorStyleOption { + id: AgentActivityIndicatorId; + family: IndicatorFamily; + /** Short label for picker cards (variant code or spinner name). */ + label: string; +} + +export const ORB_VARIANT_IDS: readonly OrbVariant[] = [ + ...LATTICE_VARIANTS, + ...LENS_VARIANTS, + ...RING_VARIANTS, + ...HELIX_VARIANTS, + ...MORPH_VARIANTS, +]; + +export const UNICODE_STYLE_OPTIONS: readonly IndicatorStyleOption[] = [ + { id: RANDOM_UNICODE_ID, family: "unicode", label: "Random" }, + ...UNICODE_SPINNER_IDS.map( + (id): IndicatorStyleOption => ({ + id, + family: "unicode", + label: id, + }), + ), +]; + +function orbFamily(id: OrbVariant): IndicatorFamily { + if ((LATTICE_VARIANTS as readonly string[]).includes(id)) return "lattice"; + if ((LENS_VARIANTS as readonly string[]).includes(id)) return "lens"; + if ((RING_VARIANTS as readonly string[]).includes(id)) return "ring"; + if ((HELIX_VARIANTS as readonly string[]).includes(id)) return "helix"; + return "morph"; +} + +export const ORB_STYLE_OPTIONS: readonly IndicatorStyleOption[] = ORB_VARIANT_IDS.map( + (id) => ({ + id, + family: orbFamily(id), + label: id, + }), +); + +/** All selectable styles, unicode first then orbs by family order. */ +export const ALL_INDICATOR_STYLE_OPTIONS: readonly IndicatorStyleOption[] = [ + ...UNICODE_STYLE_OPTIONS, + ...ORB_STYLE_OPTIONS, +]; + +export const INDICATOR_STYLE_GROUPS: ReadonlyArray<{ + family: IndicatorFamily; + options: readonly IndicatorStyleOption[]; +}> = [ + { family: "unicode", options: UNICODE_STYLE_OPTIONS }, + { + family: "lattice", + options: ORB_STYLE_OPTIONS.filter((o) => o.family === "lattice"), + }, + { + family: "lens", + options: ORB_STYLE_OPTIONS.filter((o) => o.family === "lens"), + }, + { + family: "ring", + options: ORB_STYLE_OPTIONS.filter((o) => o.family === "ring"), + }, + { + family: "helix", + options: ORB_STYLE_OPTIONS.filter((o) => o.family === "helix"), + }, + { + family: "morph", + options: ORB_STYLE_OPTIONS.filter((o) => o.family === "morph"), + }, +]; + +const VALID_IDS = new Set( + ALL_INDICATOR_STYLE_OPTIONS.map((o) => o.id), +); + +export function isAgentActivityIndicatorId( + value: unknown, +): value is AgentActivityIndicatorId { + return typeof value === "string" && VALID_IDS.has(value); +} + +export function isOrbIndicatorId(id: AgentActivityIndicatorId): id is OrbVariant { + return (ORB_VARIANT_IDS as readonly string[]).includes(id); +} + +export function isUnicodeIndicatorId( + id: AgentActivityIndicatorId, +): id is UnicodeSpinnerId | typeof RANDOM_UNICODE_ID { + return id === RANDOM_UNICODE_ID || (UNICODE_SPINNER_IDS as readonly string[]).includes(id); +} + +/** Defaults match the previous compact braille spinner everywhere. */ +export const DEFAULT_INDICATOR_BY_PLACEMENT: Record< + AgentIndicatorPlacement, + AgentActivityIndicatorId +> = { + left_sidebar: "braille", + center_terminal: "braille", + /** Full pane status previously used a random unicode spinner each mount. */ + terminal_panel: "random", + footer: "braille", +}; + +export const INDICATOR_PLACEMENTS: readonly AgentIndicatorPlacement[] = [ + "left_sidebar", + "center_terminal", + "terminal_panel", + "footer", +] as const; + +/** function_settings.json keys under `agent_cli`. */ +export const INDICATOR_SETTING_KEYS: Record< + AgentIndicatorPlacement, + `activity_indicator_${AgentIndicatorPlacement}` +> = { + left_sidebar: "activity_indicator_left_sidebar", + center_terminal: "activity_indicator_center_terminal", + terminal_panel: "activity_indicator_terminal_panel", + footer: "activity_indicator_footer", +}; + +export function resolveIndicatorId( + value: unknown, + placement: AgentIndicatorPlacement, +): AgentActivityIndicatorId { + return isAgentActivityIndicatorId(value) + ? value + : DEFAULT_INDICATOR_BY_PLACEMENT[placement]; +} diff --git a/apps/web/src/features/settings/components/AgentActivityIndicatorsSettingsSection.tsx b/apps/web/src/features/settings/components/AgentActivityIndicatorsSettingsSection.tsx new file mode 100644 index 000000000..36941e906 --- /dev/null +++ b/apps/web/src/features/settings/components/AgentActivityIndicatorsSettingsSection.tsx @@ -0,0 +1,285 @@ +"use client"; + +import React, { useEffect, useState } from "react"; +import { useTranslations } from "next-intl"; +import { + Collapsible, + CollapsibleContent, + CollapsibleTrigger, + cn, +} from "@workspace/ui"; +import { Activity, Bot, ChevronDown, Terminal as TerminalIcon } from "lucide-react"; +import { AgentHookStatusIndicator } from "@/features/agent/components/AgentHookStatusIndicator"; +import { AgentRunningGlyph } from "@/features/agent/components/AgentRunningGlyph"; +import { AGENT_STATE } from "@/features/agent/store/agent-hooks-store"; +import { + INDICATOR_PLACEMENTS, + INDICATOR_STYLE_GROUPS, + type AgentActivityIndicatorId, + type AgentIndicatorPlacement, + type IndicatorFamily, +} from "@/features/agent/lib/agent-activity-indicator-styles"; +import { useAgentActivityIndicatorSettingsStore } from "@/features/settings/store/agent-activity-indicator-settings-store"; + +function familyLabelKey(family: IndicatorFamily): string { + return `families.${family}`; +} + +function placementTitleKey(placement: AgentIndicatorPlacement): string { + return `placements.${placement}.title`; +} + +function placementDescriptionKey(placement: AgentIndicatorPlacement): string { + return `placements.${placement}.description`; +} + +/** 1:1 mock of left-sidebar project/workspace row trailing indicator. */ +function LeftSidebarMockPreview({ styleId }: { styleId: AgentActivityIndicatorId }) { + return ( +
+ + atmos + + +
+ ); +} + +/** 1:1 mock of center-stage terminal tab content (icon + title + compact indicator). */ +function CenterTerminalMockPreview({ styleId }: { styleId: AgentActivityIndicatorId }) { + return ( +
+ + + claude + + +
+ ); +} + +/** 1:1 mock of terminal mosaic pane toolbar full agent status. */ +function TerminalPanelMockPreview({ styleId }: { styleId: AgentActivityIndicatorId }) { + return ( +
+ + pane + +
+ ); +} + +/** 1:1 mock of footer agent status ticker button. */ +function FooterMockPreview({ styleId }: { styleId: AgentActivityIndicatorId }) { + return ( +
+ + + atmos + - + main + +
+ ); +} + +function PlacementMockPreview({ + placement, + styleId, +}: { + placement: AgentIndicatorPlacement; + styleId: AgentActivityIndicatorId; +}) { + switch (placement) { + case "left_sidebar": + return ; + case "center_terminal": + return ; + case "terminal_panel": + return ; + case "footer": + return ; + } +} + +function IndicatorStylePicker({ + value, + onChange, + disabled, +}: { + value: AgentActivityIndicatorId; + onChange: (id: AgentActivityIndicatorId) => void; + disabled?: boolean; +}) { + const t = useTranslations("settings.codeAgentSection.activityIndicators"); + + return ( +
+ {INDICATOR_STYLE_GROUPS.map((group) => ( +
+

+ {t(familyLabelKey(group.family) as never)} +

+
+ {group.options.map((option) => { + const selected = option.id === value; + return ( + + ); + })} +
+
+ ))} +
+ ); +} + +function PlacementRow({ + placement, + open, + onOpenChange, +}: { + placement: AgentIndicatorPlacement; + open: boolean; + onOpenChange: (open: boolean) => void; +}) { + const t = useTranslations("settings.codeAgentSection.activityIndicators"); + const styleId = useAgentActivityIndicatorSettingsStore((s) => s[placement]); + const syncingPlacement = useAgentActivityIndicatorSettingsStore((s) => s.syncingPlacement); + const setIndicator = useAgentActivityIndicatorSettingsStore((s) => s.setIndicator); + const disabled = syncingPlacement === placement; + + return ( + +
+ + + + + +
+

+ {t(placementTitleKey(placement) as never)} +

+

+ {t(placementDescriptionKey(placement) as never)} +

+
+ + +
+ +
+
+ + + { + void setIndicator(placement, id).catch(() => { + // store rolls back; no toast per product preference for local settings + }); + }} + /> + + + ); +} + +export function AgentActivityIndicatorsSettingsSection() { + const t = useTranslations("settings.codeAgentSection.activityIndicators"); + const loadSettings = useAgentActivityIndicatorSettingsStore((s) => s.loadSettings); + const [openPlacement, setOpenPlacement] = useState(null); + const [sectionOpen, setSectionOpen] = useState(false); + + useEffect(() => { + void loadSettings(); + }, [loadSettings]); + + return ( + +
+ +
+ + + + +
+

{t("title")}

+

{t("description")}

+
+
+ +
+ + +
+ {INDICATOR_PLACEMENTS.map((placement) => ( + setOpenPlacement(open ? placement : null)} + /> + ))} +
+ + + ); +} diff --git a/apps/web/src/features/settings/components/CodeAgentSettingsSection.tsx b/apps/web/src/features/settings/components/CodeAgentSettingsSection.tsx index e5528b988..f76157f9a 100644 --- a/apps/web/src/features/settings/components/CodeAgentSettingsSection.tsx +++ b/apps/web/src/features/settings/components/CodeAgentSettingsSection.tsx @@ -16,6 +16,7 @@ import { AGENT_OPTIONS, getInteractiveAgentParams } from "@/features/wiki/compon import { resolveAgentLaunchFlags } from "@/features/agent/lib/terminal-agent-yolo"; import { AgentIcon } from "@/features/agent/components/AgentIcon"; import type { CodeAgentCustomEntry } from "@/api/ws-api"; +import { AgentActivityIndicatorsSettingsSection } from "@/features/settings/components/AgentActivityIndicatorsSettingsSection"; import { AgentHookStatusCard } from "@/features/settings/components/AgentHookStatusCard"; import { CodeAgentRunConfigSettingsSection } from "@/features/settings/components/CodeAgentRunConfigSettingsSection"; import { SaveActionButton } from "@/features/settings/components/settings/SaveActionButton"; @@ -51,6 +52,9 @@ interface CodeAgentSettingsSectionProps { yoloModeRestoring: boolean; onYoloModeChange: (enabled: boolean) => void; onRestoreAllYoloMode: () => void; + showAgentNameInTerminalTitles: boolean; + showAgentNameInTerminalTitlesSyncing: boolean; + onShowAgentNameInTerminalTitlesChange: (enabled: boolean) => void; onAddCustomAgent: () => void; onAgentSettingChange: (agentId: string, field: "cmd" | "flags" | "interactiveFlags" | "enabled", value: string | boolean) => void; onBuiltInEnabledChange: (agentId: string, enabled: boolean) => void; @@ -95,6 +99,9 @@ export function CodeAgentSettingsSection({ yoloModeRestoring, onYoloModeChange, onRestoreAllYoloMode, + showAgentNameInTerminalTitles, + showAgentNameInTerminalTitlesSyncing, + onShowAgentNameInTerminalTitlesChange, onAddCustomAgent, onAgentSettingChange, onBuiltInEnabledChange, @@ -150,6 +157,29 @@ export function CodeAgentSettingsSection({
+
+
+
+ +
+

+ {t("showAgentName.title")} +

+

+ {t("showAgentName.description")} +

+
+
+ onShowAgentNameInTerminalTitlesChange(!!checked)} + /> +
+
+ + + = "uninstall hooks", "behaviour", "idle session cleanup", + "agent activity indicators", + "activity indicator", + "status indicator", + "orbs", + "unicode spinner", + "left sidebar indicator", + "terminal panel indicator", + "footer agent status", ], terminal: [ "terminal", @@ -930,7 +938,15 @@ const SETTINGS_SETTING_ITEMS: Record; + +interface AgentActivityIndicatorSettingsState extends AgentActivityIndicatorSettings { + loaded: boolean; + loading: boolean; + syncingPlacement: AgentIndicatorPlacement | null; + loadSettings: (force?: boolean) => Promise; + setIndicator: ( + placement: AgentIndicatorPlacement, + id: AgentActivityIndicatorId, + ) => Promise; +} + +function readAll( + settings: { + agent_cli?: Partial>; + } | null | undefined, +): AgentActivityIndicatorSettings { + const agentCli = settings?.agent_cli ?? {}; + return { + left_sidebar: resolveIndicatorId( + agentCli[INDICATOR_SETTING_KEYS.left_sidebar], + "left_sidebar", + ), + center_terminal: resolveIndicatorId( + agentCli[INDICATOR_SETTING_KEYS.center_terminal], + "center_terminal", + ), + terminal_panel: resolveIndicatorId( + agentCli[INDICATOR_SETTING_KEYS.terminal_panel], + "terminal_panel", + ), + footer: resolveIndicatorId(agentCli[INDICATOR_SETTING_KEYS.footer], "footer"), + }; +} + +export const useAgentActivityIndicatorSettingsStore = create( + (set, get) => ({ + ...DEFAULT_INDICATOR_BY_PLACEMENT, + loaded: false, + loading: false, + syncingPlacement: null, + + loadSettings: async (force = false) => { + if (!force && (get().loaded || get().loading)) return; + set({ loading: true }); + try { + const settings = await useFunctionSettingsStore.getState().load(); + set({ + ...readAll(settings), + loaded: true, + loading: false, + }); + } catch { + set({ loading: false }); + } + }, + + setIndicator: async (placement, id) => { + const previous = get()[placement]; + if (previous === id) return; + set({ [placement]: id, syncingPlacement: placement }); + try { + await functionSettingsApi.update( + "agent_cli", + INDICATOR_SETTING_KEYS[placement], + id, + ); + useFunctionSettingsStore.getState().invalidate(); + await useFunctionSettingsStore.getState().load(); + set({ + ...readAll(useFunctionSettingsStore.getState().settings), + loaded: true, + syncingPlacement: null, + }); + } catch { + set({ [placement]: previous, syncingPlacement: null }); + throw new Error("Failed to update agent activity indicator setting"); + } + }, + }), +); + +export function useIndicatorIdForPlacement( + placement: AgentIndicatorPlacement, +): AgentActivityIndicatorId { + return useAgentActivityIndicatorSettingsStore((s) => s[placement]); +} diff --git a/apps/web/src/providers/app/websocket-provider.tsx b/apps/web/src/providers/app/websocket-provider.tsx index c1db340aa..ac1923b46 100644 --- a/apps/web/src/providers/app/websocket-provider.tsx +++ b/apps/web/src/providers/app/websocket-provider.tsx @@ -5,6 +5,8 @@ import { useWebSocketStore } from '@/features/connection/hooks/use-websocket'; import { useAgentHooksStore } from '@/features/agent/store/agent-hooks-store'; import { useAgentNotifications } from '@/features/agent/hooks/use-agent-notifications'; import { useLayoutSettingsStore } from '@/features/settings/store/layout-settings-store'; +import { useAgentTitleSettingsStore } from '@/features/settings/store/agent-title-settings-store'; +import { useAgentActivityIndicatorSettingsStore } from '@/features/settings/store/agent-activity-indicator-settings-store'; import { useExperimentSettingsStore } from '@/features/settings/store/experiment-settings-store'; import { useHostedConnectionStore } from '@/features/connection/store/hosted-connection-store'; import { isHostedAtmosOrigin } from '@/shared/lib/desktop-runtime'; @@ -96,6 +98,8 @@ export function WebSocketProvider({ children }: WebSocketProviderProps) { useAgentHooksStore.getState().init(); useLayoutSettingsStore.getState().loadSettings(); void useExperimentSettingsStore.getState().loadSettings(); + void useAgentTitleSettingsStore.getState().loadSettings(); + void useAgentActivityIndicatorSettingsStore.getState().loadSettings(); // Project bootstrap is now Query-owned; reconnect invalidation handles refresh. void ensureProjectBootstrap().catch(() => undefined); } From 96f1e595831f7edd5bc30393393197cef927fa21 Mon Sep 17 00:00:00 2001 From: AarynLu Date: Thu, 6 Aug 2026 00:49:19 +0800 Subject: [PATCH 31/43] refactor(ui): use border-beam package and dialog polish Delegate BorderBeam to the published package and tighten dialog defaults. --- bun.lock | 3 + packages/ui/package.json | 1 + packages/ui/src/components/ui/border-beam.tsx | 121 +++--------------- packages/ui/src/components/ui/dialog.tsx | 5 +- 4 files changed, 25 insertions(+), 105 deletions(-) diff --git a/bun.lock b/bun.lock index be2392d8c..eeb8e8710 100644 --- a/bun.lock +++ b/bun.lock @@ -385,6 +385,7 @@ "@streamdown/mermaid": "^1.0.2", "ai": "^6.0.86", "ansi-to-react": "^6.2.6", + "border-beam": "^1.3.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", @@ -1959,6 +1960,8 @@ "boolean": ["boolean@3.2.0", "", {}, "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw=="], + "border-beam": ["border-beam@1.3.0", "", { "peerDependencies": { "react": ">=18.0.0", "react-dom": ">=18.0.0" } }, "sha512-oZSISp3aQau4ASTmzm5pzxeBinO6Fh2HW0gyEixJgtpUZvRiU0vNpP+IwGRrNpIyqPjA7Rf5EBnVqDSJ49cm5w=="], + "bplist-creator": ["bplist-creator@0.1.0", "", { "dependencies": { "stream-buffers": "2.2.x" } }, "sha512-sXaHZicyEEmY86WyueLTQesbeoH/mquvarJaQNbjuOQO+7gbFcDEWqKmcWA4cOTLzFlfgvkiVxolk1k5bBIpmg=="], "bplist-parser": ["bplist-parser@0.3.2", "", { "dependencies": { "big-integer": "1.6.x" } }, "sha512-apC2+fspHGI3mMKj+dGevkGo/tCqVB8jMb6i+OX+E29p0Iposz07fABkRIfVUPNd5A5VbuOz1bZbnmkKLYF+wQ=="], diff --git a/packages/ui/package.json b/packages/ui/package.json index fdecc51f9..03715f4a9 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -43,6 +43,7 @@ "@streamdown/mermaid": "^1.0.2", "ai": "^6.0.86", "ansi-to-react": "^6.2.6", + "border-beam": "^1.3.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "^1.1.1", diff --git a/packages/ui/src/components/ui/border-beam.tsx b/packages/ui/src/components/ui/border-beam.tsx index 1f594d1ec..d74b7e1b7 100644 --- a/packages/ui/src/components/ui/border-beam.tsx +++ b/packages/ui/src/components/ui/border-beam.tsx @@ -1,107 +1,20 @@ "use client" -import { motion, MotionStyle, Transition } from "motion/react" +/** + * Design-system re-export of `border-beam`. + * + * @example + * ```tsx + * + * Content + * + * ``` + */ +export { BorderBeam } from "border-beam" -import { cn } from "@/lib/utils" - -interface BorderBeamProps { - /** - * The size of the border beam. - */ - size?: number - /** - * The duration of the border beam. - */ - duration?: number - /** - * The delay of the border beam. - */ - delay?: number - /** - * The color of the border beam from. - */ - colorFrom?: string - /** - * The color of the border beam to. - */ - colorTo?: string - /** - * The motion transition of the border beam. - */ - transition?: Transition - /** - * The class name of the border beam. - */ - className?: string - /** - * The style of the border beam. - */ - style?: React.CSSProperties - /** - * Whether to reverse the animation direction. - */ - reverse?: boolean - /** - * The initial offset position (0-100). - */ - initialOffset?: number - /** - * The border width of the beam. - */ - borderWidth?: number -} - -export const BorderBeam = ({ - className, - size = 50, - delay = 0, - duration = 6, - colorFrom = "#ffaa40", - colorTo = "#9c40ff", - transition, - style, - reverse = false, - initialOffset = 0, - borderWidth = 1, -}: BorderBeamProps) => { - return ( -
- -
- ) -} +export type { + BorderBeamProps, + BorderBeamSize, + BorderBeamColorVariant, + BorderBeamTheme, +} from "border-beam" diff --git a/packages/ui/src/components/ui/dialog.tsx b/packages/ui/src/components/ui/dialog.tsx index 74d7909c1..fbcd034e6 100644 --- a/packages/ui/src/components/ui/dialog.tsx +++ b/packages/ui/src/components/ui/dialog.tsx @@ -51,13 +51,16 @@ function DialogContent({ className, children, showCloseButton = true, + overlayClassName, ...props }: React.ComponentProps & { showCloseButton?: boolean + /** Optional class for the dimmed backdrop (e.g. raise z-index above popovers). */ + overlayClassName?: string }) { return ( - + Date: Thu, 6 Aug 2026 00:49:19 +0800 Subject: [PATCH 32/43] fix(web): shell chrome, surface policies, and i18n polish Update workspace surface policies/hotkeys, attention filter shortcuts, appshot/welcome/desktop-use copy, and related e2e coverage. --- apps/web/messages/en.json | 49 +++++++++-- apps/web/messages/zh.json | 49 +++++++++-- .../web/src/app-shell/HeaderAttentionBell.tsx | 11 ++- apps/web/src/app-shell/LeftSidebar.tsx | 7 +- .../workspace-surface-policies.test.ts | 7 ++ .../workspace-surface-switch.test.ts | 14 ++- .../src/app-shell/left-sidebar-controls.tsx | 32 ++++++- .../src/app-shell/sidebar/CommitActions.tsx | 86 +++++++++++-------- .../sidebar/UserGroupSidebarContent.tsx | 35 ++++++-- apps/web/src/app-shell/use-header-hotkeys.ts | 15 ++++ .../app-shell/workspace-surface-policies.ts | 18 +++- .../src/app-shell/workspace-surface-switch.ts | 25 ++++-- apps/web/src/app/globals.css | 42 +++++++++ apps/web/src/app/layout.tsx | 9 +- .../components/AppshotsHeaderButton.tsx | 8 +- .../components/AppshotsHistoryPopover.tsx | 13 ++- .../components/ShortcutsSettingsSection.tsx | 1 + .../welcome/components/WelcomePage.tsx | 23 +++-- .../APP-043_workspace-surface-cache.e2e.ts | 17 ++-- 19 files changed, 365 insertions(+), 96 deletions(-) diff --git a/apps/web/messages/en.json b/apps/web/messages/en.json index 91de0be9e..1e334e898 100644 --- a/apps/web/messages/en.json +++ b/apps/web/messages/en.json @@ -267,6 +267,7 @@ "toggleLeftSidebar": "Toggle left sidebar", "refreshPage": "Refresh page", "toggleQuotaUsage": "Toggle Quota Usage", + "toggleNeedAttention": "Toggle need attention filter", "toggleMenu": "Toggle menu", "toggleRightSidebar": "Toggle right sidebar" }, @@ -4164,6 +4165,7 @@ "errors": { "saveBuiltInAgentTitle": "Failed to save built-in agent", "updateYoloModeTitle": "Failed to update YOLO mode", + "updateShowAgentNameTitle": "Failed to update agent name display", "restoreYoloModeTitle": "Failed to restore YOLO mode", "updateAgentVisibilityTitle": "Failed to update agent visibility", "removeCustomAgentTitle": "Failed to remove custom agent", @@ -4811,7 +4813,8 @@ "navigateBack": "Navigate back", "navigateForward": "Navigate forward", "toggleActionMenu": "Toggle action menu", - "toggleQuotaPopover": "Toggle quota usage popover" + "toggleQuotaPopover": "Toggle quota usage popover", + "toggleNeedAttention": "Toggle need attention filter" } }, "workspace": { @@ -5058,6 +5061,40 @@ "restoreHint": "Re-enable YOLO and clear custom flag overrides on all built-in agents so they use the built-in YOLO defaults again.", "restoreAll": "Restore YOLO for all agents" }, + "showAgentName": { + "title": "Show agent name in terminal titles", + "description": "When a terminal or terminal tab detects an agent, always show its icon. Turn this off to hide the agent brand name and save space — the session topic (OSC) then sits next to the icon with a space instead of a pipe." + }, + "activityIndicators": { + "title": "Agent Activity Indicators", + "description": "Choose the running-state icon shown before agent status in each area of the app. Expand a row to preview and pick a unicode spinner or an AIcss Orb.", + "families": { + "unicode": "Unicode spinners", + "lattice": "Orbs · Lattice", + "lens": "Orbs · Lens", + "ring": "Orbs · Ring", + "helix": "Orbs · Helix", + "morph": "Orbs · Morph" + }, + "placements": { + "left_sidebar": { + "title": "Left sidebar", + "description": "Project and workspace list rows." + }, + "center_terminal": { + "title": "Center terminal tabs", + "description": "Terminal tab labels in the center stage." + }, + "terminal_panel": { + "title": "Terminal panel", + "description": "Pane toolbar status next to the terminal title." + }, + "footer": { + "title": "Footer", + "description": "Agent status ticker and session list in the footer." + } + } + }, "custom": { "title": "Custom Agents", "description": "Add your own agents with custom commands and parameters.", @@ -5607,18 +5644,19 @@ "installFirst": "Install the control engine first to manage permissions.", "hostLine": "Host: {host}", "done": "Done", - "grantAccessibility": "Open Accessibility settings", + "grantAccessibility": "Grant Accessibility", "grantScreenRecording": "Open Screen Recording settings", "items": { "accessibility": { "title": "Accessibility", - "description": "Required to inspect and control other apps." + "description": "Required to inspect and control other apps. Opens System Settings so you can enable Atmos Desktop Use (drag the chip only if it is not already listed)." }, "screenRecording": { "title": "Screen Recording", "description": "Required to capture the screen for AppShot and Desktop Use." } - } + }, + "dragHint": "Turn on Atmos Desktop Use in the list. If it is missing, drag the floating chip into the list." }, "errors": { "statusFailed": "Failed to load Desktop Use status.", @@ -8989,7 +9027,8 @@ "source": { "generic": "This feature needs the Desktop Use control engine.", "appshot": "AppShot capture works best when Desktop Use is installed and permitted.", - "slash": "The Desktop Use slash skill needs the control engine and macOS permissions." + "slash": "The Desktop Use slash skill needs the control engine and macOS permissions.", + "browser": "Browser Use needs the Desktop Use control engine and macOS permissions for agent chrome and capture." }, "reason": { "webOnly": "Desktop Use is only available in Atmos Desktop.", diff --git a/apps/web/messages/zh.json b/apps/web/messages/zh.json index 9776cc60a..5a16fc847 100644 --- a/apps/web/messages/zh.json +++ b/apps/web/messages/zh.json @@ -267,6 +267,7 @@ "toggleLeftSidebar": "切换左侧边栏", "refreshPage": "刷新页面", "toggleQuotaUsage": "切换配额用量", + "toggleNeedAttention": "切换需要关注筛选", "toggleMenu": "切换菜单", "toggleRightSidebar": "切换右侧边栏" }, @@ -4164,6 +4165,7 @@ "errors": { "saveBuiltInAgentTitle": "保存内置 Agent 失败", "updateYoloModeTitle": "更新 YOLO 模式失败", + "updateShowAgentNameTitle": "更新 Agent 名称显示失败", "restoreYoloModeTitle": "恢复 YOLO 模式失败", "updateAgentVisibilityTitle": "更新 Agent 可见性失败", "removeCustomAgentTitle": "移除自定义 Agent 失败", @@ -4811,7 +4813,8 @@ "navigateBack": "返回上一步", "navigateForward": "前进", "toggleActionMenu": "切换操作菜单", - "toggleQuotaPopover": "切换配额用量弹层" + "toggleQuotaPopover": "切换配额用量弹层", + "toggleNeedAttention": "切换需要关注筛选" } }, "workspace": { @@ -5058,6 +5061,40 @@ "restoreHint": "重新开启 YOLO，并清除所有内置 Agent 的自定义参数覆盖，使其恢复内置 YOLO 默认参数。", "restoreAll": "一键恢复全部 Agent 的 YOLO" }, + "showAgentName": { + "title": "在终端标题中显示 Agent 名称", + "description": "终端或 Terminal tabs 检测到 Agent 时始终显示图标。关闭后可隐藏 Agent 品牌名称以节省空间——此时会话主题（OSC）会直接跟在图标后，用空格分隔，不再使用竖线。" + }, + "activityIndicators": { + "title": "Agent 活动指示器", + "description": "分别为应用中不同区域选择 Agent 运行状态前的图标。展开某一行即可预览并选择 Unicode 旋转指示器或 AIcss Orb。", + "families": { + "unicode": "Unicode 旋转指示器", + "lattice": "Orbs · 点阵 Lattice", + "lens": "Orbs · 透镜 Lens", + "ring": "Orbs · 圆环 Ring", + "helix": "Orbs · 螺旋 Helix", + "morph": "Orbs · 形变 Morph" + }, + "placements": { + "left_sidebar": { + "title": "左侧边栏", + "description": "项目与工作区列表行。" + }, + "center_terminal": { + "title": "中间 Terminal 标签", + "description": "中心区域 Terminal 标签标题旁。" + }, + "terminal_panel": { + "title": "Terminal 面板", + "description": "终端窗格工具栏中、标题旁的状态。" + }, + "footer": { + "title": "页脚", + "description": "页脚 Agent 状态 ticker 与会话列表。" + } + } + }, "custom": { "title": "自定义 Agents", "description": "添加你自己的 Agent，并配置自定义命令和参数。", @@ -5607,18 +5644,19 @@ "installFirst": "请先安装控制引擎再管理权限。", "hostLine": "Host：{host}", "done": "完成", - "grantAccessibility": "打开辅助功能设置", + "grantAccessibility": "授予辅助功能权限", "grantScreenRecording": "打开屏幕录制设置", "items": { "accessibility": { "title": "辅助功能", - "description": "用于检查并控制其他应用。" + "description": "用于检查并控制其他应用。会打开系统设置以便开启 Atmos Desktop Use（仅在列表中没有时才需拖入悬浮芯片）。" }, "screenRecording": { "title": "屏幕录制", "description": "用于 AppShot 与 Desktop Use 的屏幕捕获。" } - } + }, + "dragHint": "在列表中打开 Atmos Desktop Use 开关；若没有该项，再将悬浮芯片拖入列表。" }, "errors": { "statusFailed": "加载 Desktop Use 状态失败。", @@ -8990,7 +9028,8 @@ "source": { "generic": "此功能需要 Desktop Use 控制引擎。", "appshot": "AppShot 捕获需要已安装并授权的 Desktop Use。", - "slash": "Desktop Use 斜杠技能需要控制引擎与 macOS 权限。" + "slash": "Desktop Use 斜杠技能需要控制引擎与 macOS 权限。", + "browser": "Browser Use 需要 Desktop Use 控制引擎与 macOS 权限，用于 Agent 光标与捕获。" }, "reason": { "webOnly": "Desktop Use 仅在 Atmos 桌面端可用。", diff --git a/apps/web/src/app-shell/HeaderAttentionBell.tsx b/apps/web/src/app-shell/HeaderAttentionBell.tsx index 6430718f6..0faa26c23 100644 --- a/apps/web/src/app-shell/HeaderAttentionBell.tsx +++ b/apps/web/src/app-shell/HeaderAttentionBell.tsx @@ -3,7 +3,7 @@ import React from "react"; import { useTranslations } from "next-intl"; import { AnimatePresence, motion } from "motion/react"; -import { Bell } from "lucide-react"; +import { ArrowBigUp, Bell, Command } from "lucide-react"; import { cn, Tooltip, TooltipContent, TooltipTrigger } from "@workspace/ui"; import { selectAttentionCount, @@ -79,7 +79,14 @@ export function HeaderAttentionBell() { - {filterMode ? t("tooltipActive") : t("tooltip")} +
+ {filterMode ? t("tooltipActive") : t("tooltip")} + + + + U + +
diff --git a/apps/web/src/app-shell/LeftSidebar.tsx b/apps/web/src/app-shell/LeftSidebar.tsx index 7ea913545..567dec708 100644 --- a/apps/web/src/app-shell/LeftSidebar.tsx +++ b/apps/web/src/app-shell/LeftSidebar.tsx @@ -125,12 +125,15 @@ const LeftSidebar: React.FC = () => { return filterProjectsByAttention(projects, ids); }, [attentionContextKey, attentionFilterMode, projects]); - // When filtering to attention items, expand projects so matching workspaces are visible. + // When filtering to attention items, expand projects that still have workspaces + // to show. Project-only attention keeps the row collapsed (children are hidden). useEffect(() => { if (!attentionFilterMode) return; setExpandedProjects((prev) => { const next = new Set(prev); - for (const project of listProjects) next.add(project.id); + for (const project of listProjects) { + if (project.workspaces.length > 0) next.add(project.id); + } const merged = Array.from(next); if (merged.length === prev.length && merged.every((id, i) => id === prev[i])) { return prev; diff --git a/apps/web/src/app-shell/__tests__/workspace-surface-policies.test.ts b/apps/web/src/app-shell/__tests__/workspace-surface-policies.test.ts index 9357013ad..8bb5764bc 100644 --- a/apps/web/src/app-shell/__tests__/workspace-surface-policies.test.ts +++ b/apps/web/src/app-shell/__tests__/workspace-surface-policies.test.ts @@ -12,6 +12,7 @@ import { resolveFrameActiveTab, selectEditorMountSet, sweepWarmByTtl, + terminalKeepAlivePanelClass, terminalMountKey, namedTerminalMountKey, isProtected, @@ -78,6 +79,12 @@ describe("resolveFrameActiveTab / panel visibility", () => { }), ).toBe(false); }); + + it("terminal keep-alive panels avoid display:none class names", () => { + expect(terminalKeepAlivePanelClass(true)).toBe("atmos-terminal-panel-active"); + expect(terminalKeepAlivePanelClass(false)).toBe("atmos-terminal-panel-keepalive"); + expect(terminalKeepAlivePanelClass(false)).not.toContain("hidden"); + }); }); describe("applyWarmTouch LRU + protect", () => { diff --git a/apps/web/src/app-shell/__tests__/workspace-surface-switch.test.ts b/apps/web/src/app-shell/__tests__/workspace-surface-switch.test.ts index adedad92a..89060c8c9 100644 --- a/apps/web/src/app-shell/__tests__/workspace-surface-switch.test.ts +++ b/apps/web/src/app-shell/__tests__/workspace-surface-switch.test.ts @@ -110,21 +110,29 @@ describe("promoteWorkspaceSurfaceSwitch + prepareWorkspaceContextNavigation", () expect(s.warm.map((w) => w.contextId)).toContain("ws-a"); }); - it("applyWorkspaceFrameVisualDom toggles shell hidden without unmount markers", () => { + it("applyWorkspaceFrameVisualDom toggles data-tier without display:none", () => { // jsdom may not exist in bun unit tests; skip when document is unavailable. if (typeof document === "undefined") return; document.body.innerHTML = `
- + `; applyWorkspaceFrameVisualDom("ws-b"); const a = document.querySelector('[data-workspace-frame="ws-a"]') as HTMLElement; const b = document.querySelector('[data-workspace-frame="ws-b"]') as HTMLElement; - expect(a.hidden).toBe(true); + // Warm uses visibility stacking (data-tier), not HTML hidden / display:none. + expect(a.hidden).toBe(false); + expect(a.hasAttribute("hidden")).toBe(false); + expect(a.classList.contains("hidden")).toBe(false); expect(a.getAttribute("data-tier")).toBe("warm"); + expect(a.getAttribute("aria-hidden")).toBe("true"); + expect(a.hasAttribute("inert")).toBe(true); expect(b.hidden).toBe(false); expect(b.getAttribute("data-tier")).toBe("active"); + expect(b.getAttribute("aria-hidden")).toBe("false"); + expect(b.hasAttribute("inert")).toBe(false); expect(b.classList.contains("hidden")).toBe(false); + expect(b.style.contentVisibility).toBe(""); }); it("prime does not claim cold targets and clears stale visual lead", () => { diff --git a/apps/web/src/app-shell/left-sidebar-controls.tsx b/apps/web/src/app-shell/left-sidebar-controls.tsx index a23522a6e..ed17bbaf4 100644 --- a/apps/web/src/app-shell/left-sidebar-controls.tsx +++ b/apps/web/src/app-shell/left-sidebar-controls.tsx @@ -44,6 +44,10 @@ import { type FlattenedWorkspaceEntry, type WorkspaceGroup, } from "@/app-shell/sidebar/workspace-grouping"; +import { + selectAttentionFilterMode, + useAgentAttentionStore, +} from "@/features/agent/store/agent-attention-store"; export { LeftSidebarFooter, LeftSidebarTabsHeader } from "./left-sidebar-tab-footer-controls"; type DndSensors = React.ComponentProps["sensors"]; @@ -434,6 +438,8 @@ function SortableWorkspaceGroupSection({ showLess, } = useWorkspaceListVisibleCount(group.items.length, group.key); const visibleItems = group.items.slice(0, visibleCount); + // Attention filter: dim group chrome so latched workspaces stay the focus. + const attentionFilterMode = useAgentAttentionStore(selectAttentionFilterMode); return (
-
+