trycua · f-trycua · May 27, 2026 · May 27, 2026 · May 27, 2026 · May 27, 2026
diff --git a/docs/content/docs/cua-driver/guide/getting-started/meta.json b/docs/content/docs/cua-driver/guide/getting-started/meta.json
@@ -3,5 +3,5 @@
   "description": "Get up and running with Cua Driver",
   "icon": "Rocket",
   "defaultOpen": true,
-  "pages": ["introduction", "installation", "quickstart", "windows-ssh", "linux", "autostart", "integrations", "swift-integration", "process-model", "comparison", "faq"]
+  "pages": ["introduction", "installation", "quickstart", "windows-ssh", "linux", "autostart", "pip-preview", "integrations", "swift-integration", "process-model", "comparison", "faq"]
 }
diff --git a/docs/content/docs/cua-driver/guide/getting-started/pip-preview.mdx b/docs/content/docs/cua-driver/guide/getting-started/pip-preview.mdx
@@ -0,0 +1,94 @@
+---
+title: PiP Preview (Experimental)
+description: Always-on-top picture-in-picture window showing the agent's per-action screenshots and a one-line action label.
+---
+
+import { Callout } from 'fumadocs-ui/components/callout';
+
+<Callout type="warn">
+**Experimental — opt-in, default OFF.** macOS only today; Windows and Linux ship as compile-clean stubs that print a "not yet implemented" notice when `--experimental-pip` is on argv. The flag, geometry, and frame schema may change before the feature is promoted out of `experimental`. Don't build production tooling against it yet.
+</Callout>
+
+`--experimental-pip` opens a small always-on-top window next to your work that shows what the cua-driver agent is doing in real time: the post-action screenshot of the target window, plus a one-line label describing the tool call that produced it (`click element_index=2`, `type_text "hello world"`, etc.).
+
+It's intended as a live "agent's-eye view" — a passive observation surface, not a debugger. The window updates on every action tool call; it doesn't continuously capture the desktop.
+
+## Enabling it
+
+Two ways. The persistent path uses the same `~/.cua-driver/config.json`
+file that the `set_config` MCP tool writes to, so PiP survives across
+daemon restarts without re-running `claude mcp add` with the flag in
+the args list.
+
+### A. Persistent — edit `~/.cua-driver/config.json` (recommended)
+
+```json
+{
+  "experimental_pip": true,
+  "experimental_pip_geometry": "320x200+24+24"
+}
+```
+
+Both keys are optional; `experimental_pip_geometry` defaults to
+`320x200` in the top-right corner. Restart your MCP client (or kill
+the running `cua-driver` daemon) for the new config to take effect.
+
+### B. One-off — CLI flag
+
+```bash
+# MCP server (stdio)
+cua-driver mcp --experimental-pip
+
+# HTTP / Unix-socket serve daemon
+cua-driver serve --experimental-pip
+
+# Override geometry
+cua-driver serve --experimental-pip --experimental-pip-geometry 640x400
+cua-driver serve --experimental-pip --experimental-pip-geometry 480x300+24+24
+```
+
+The geometry string is the standard X11 `WxH+X+Y` form. `+X+Y` is the
+top-left origin of the window in screen points (AppKit's bottom-left
+convention is hidden inside the backend). **CLI flags override
+`config.json`** — they're not additive.
+
+On startup you'll see:
+
+```
+⚗️  PiP preview enabled (experimental — macOS only today; see https://github.com/trycua/cua/issues for follow-up)
+```
+
+## What gets pushed
+
+PiP receives a frame for the same set of tool calls the recording pipeline writes a `turn-NNNNN/screenshot.png` for — every non-read-only, non-meta call: `click`, `double_click`, `right_click`, `type_text`, `press_key`, `hotkey`, `scroll`, `drag`, `set_value`, `launch_app`, plus the get/refresh AX tools that take a fresh screenshot anyway.
+
+The PNG bytes themselves come from the same `SCREENSHOT_FN` callback the recorder uses, so the live view always matches what a replay would show for that turn.
+
+## Window properties (macOS)
+
+- **Always-on-top:** `NSFloatingWindowLevel` — above your normal apps, below menus / accessibility overlays.
+- **Never key:** `setBecomesKeyOnlyIfNeeded(true)` plus a transient / no-cycle collection behavior — the window never steals keyboard focus from your frontmost app.
+- **Visible across all spaces:** `CanJoinAllSpaces | FullScreenAuxiliary | Stationary` — survives space switches and full-screen apps.
+- **Closeable:** the red traffic-light button dismisses the window and decouples it from the session. Re-enabling means restarting the daemon with the flag.
+
+## Platform support
+
+| Platform | Status | Notes |
+|---|---|---|
+| macOS | Working | NSWindow + NSImageView, frame updates via `dispatch_async` to the main queue |
+| Windows | Stub | `--experimental-pip` is accepted; backend logs "not yet implemented" and the daemon continues without a window |
+| Linux | Stub | Same as Windows |
+
+Track Win + Linux follow-up work via the [trycua/cua issue tracker](https://github.com/trycua/cua/issues).
+
+## Non-goals (today)
+
+- **Continuous capture.** PiP follows tool calls, not a frame rate. If you need a video, use `start_recording`.
+- **Click-to-drag repositioning.** Window position is static for the session; use `--experimental-pip-geometry` if you need to move it.
+- **Audio / recording-to-disk from PiP.** The recording feature already handles those; PiP is a live view only.
+
+## Troubleshooting
+
+- **Window never appears.** Make sure the cursor overlay is enabled (the AppKit event loop is shared) — if you passed `--no-overlay`, the main thread is parked and AppKit doesn't run. Workaround: drop `--no-overlay`.
+- **Window appears but no frames.** Confirm a real tool call is landing — bare reads (`get_window_state`, `list_windows`) skip the push path. Try `cua-driver call launch_app '{"bundle_id":"com.apple.calculator"}'` against a running daemon.
+- **Label is truncated.** Increase the width via `--experimental-pip-geometry 720x400`.
@@ -9,6 +9,7 @@ members = [
     "crates/platform-linux",
     "crates/cursor-overlay",
     "crates/focus-monitor-win",
+    "crates/pip-preview",
 ]
 
 [workspace.package]

@@ -14,6 +14,7 @@ pub mod cdp;
 pub mod element_cache;
 pub mod image_utils;
 pub mod page;
+pub mod pip_hook;
 pub mod protocol;
 pub mod cursor_sampler;
 pub mod recording;

@@ -0,0 +1,49 @@
+//! PiP frame-push hook — registered once by `main.rs` when the
+//! `--experimental-pip` flag is on argv.
+//!
+//! The trait + factory live in the `pip-preview` crate so the platform
+//! backends can implement them without depending on `cua-driver-core`.
+//! What lives here is just the per-process callback that the tool
+//! dispatcher uses to push frames after each successful tool call —
+//! a thin shim so `tool.rs` doesn't need to know about `pip-preview`
+//! directly and we keep the dependency graph one-directional.
+//!
+//! The PNG bytes pushed through here come from the existing
+//! `SCREENSHOT_FN` callback (the same source `screenshot.png` uses in
+//! the recording pipeline), so PiP shows exactly what the recorder
+//! captures.
+
+use std::sync::OnceLock;
+
+/// Synthesized per-call frame payload. Kept structurally identical
+/// to `pip_preview::PipFrame` — duplicated here to keep `cua-driver-core`
+/// from importing `pip-preview` (the dependency would be circular once
+/// platform backends pull both crates in).
+pub struct PipHookFrame {
+    pub png_bytes: Vec<u8>,
+    pub action_label: String,
+    pub timestamp_ms: u64,
+}
+
+type PipPushFnBox = Box<dyn Fn(PipHookFrame) + Send + Sync>;
+static PIP_PUSH_FN: OnceLock<PipPushFnBox> = OnceLock::new();
+
+/// Register the platform-side push callback. `main.rs` calls this
+/// once after starting the PiP backend.
+pub fn set_pip_push_fn(f: impl Fn(PipHookFrame) + Send + Sync + 'static) {
+    let _ = PIP_PUSH_FN.set(Box::new(f));
+}
+
+/// True when a PiP backend is wired up. Tool dispatcher uses this to
+/// skip the screenshot-bytes path when nothing would consume the
+/// frame (avoiding wasted capture work in the common --pip-off case).
+pub fn pip_enabled() -> bool {
+    PIP_PUSH_FN.get().is_some()
+}
+
+/// Push a frame to the PiP window. No-op when no backend is registered.
+pub fn push_pip_frame(frame: PipHookFrame) {
+    if let Some(f) = PIP_PUSH_FN.get() {
+        f(frame);
+    }
+}
@@ -35,6 +35,14 @@ pub fn set_screenshot_fn(f: impl Fn(Option<u64>, Option<i64>) -> Option<Vec<u8>>
     let _ = SCREENSHOT_FN.set(Box::new(f));
 }
 
+/// Invoke the registered screenshot callback. Returns `None` when no
+/// callback was registered or when the platform capture failed. Used
+/// by the PiP push hook (and by anything else that wants to share the
+/// per-turn screenshot pipeline without duplicating the platform glue).
+pub fn screenshot_for(window_id: Option<u64>, pid: Option<i64>) -> Option<Vec<u8>> {
+    SCREENSHOT_FN.get().and_then(|f| f(window_id, pid))
+}
+
 // ── Platform click-marker callback ───────────────────────────────────────────
 //
 // Takes (png_bytes, cx, cy) and returns modified PNG bytes with a red crosshair

@@ -7,13 +7,15 @@ use async_trait::async_trait;
 use serde_json::Value;
 
 use crate::{
+    pip_hook,
     protocol::{Content, ToolResult},
-    recording::{now_ms, RecordingSession},
+    recording::{now_ms, screenshot_for, RecordingSession},
     recording_tools::{
         GetRecordingStateTool, ReplayTrajectoryTool, StartRecordingTool,
         StopRecordingTool,
         init_replay_registry,
     },
+    tool_args::ArgsExt,
 };
 
 /// Metadata for a single tool.
@@ -163,6 +165,25 @@ impl ToolRegistry {
             self.recording.record(name, &args, result_text, start_ms);
         }
 
+        // Experimental PiP push — only when --experimental-pip is on argv
+        // (otherwise `pip_enabled()` is false and we skip the screenshot
+        // entirely to avoid wasted capture work). We push for the same set
+        // of action tools the recording pipeline cares about (non-read-only,
+        // not the recording-control meta-tools) so the live view matches
+        // what the recorder would have captured for the turn.
+        if pip_hook::pip_enabled() && should_record {
+            let window_id = args.opt_u64("window_id");
+            let pid = args.opt_i64("pid");
+            if let Some(png_bytes) = screenshot_for(window_id, pid) {
+                let label = synthesize_action_label(name, &args);
+                pip_hook::push_pip_frame(pip_hook::PipHookFrame {
+                    png_bytes,
+                    action_label: label,
+                    timestamp_ms: now_ms(),
+                });
+            }
+        }
+
         result
     }
 }
@@ -172,3 +193,50 @@ impl Default for ToolRegistry {
         Self::new()
     }
 }
+
+/// Build a short, human-friendly label for the PiP overlay from the
+/// tool name + raw args. Kept under ~60 chars so the macOS NSTextField
+/// has room without truncation at default geometry.
+fn synthesize_action_label(tool_name: &str, args: &Value) -> String {
+    let arg = |k: &str| -> Option<String> {
+        args.get(k).map(|v| match v {
+            Value::String(s) => s.clone(),
+            other => other.to_string(),
+        })
+    };
+    let summary = match tool_name {
+        "click" | "double_click" | "right_click" => {
+            if let Some(idx) = args.opt_u64("element_index") {
+                format!("element_index={idx}")
+            } else if let (Some(x), Some(y)) = (args.opt_f64("x"), args.opt_f64("y")) {
+                format!("({x:.0}, {y:.0})")
+            } else {
+                "".into()
+            }
+        }
+        "type_text" => {
+            let text = arg("text").unwrap_or_default();
+            let trimmed: String = text.chars().take(40).collect();
+            if text.chars().count() > 40 {
+                format!("\"{trimmed}…\"")
+            } else {
+                format!("\"{trimmed}\"")
+            }
+        }
+        "press_key" | "hotkey" => arg("key").or_else(|| arg("keys")).unwrap_or_default(),
+        "scroll" => format!(
+            "dx={} dy={}",
+            arg("dx").unwrap_or_else(|| "0".into()),
+            arg("dy").unwrap_or_else(|| "0".into())
+        ),
+        "drag" => "drag".into(),
+        "set_value" => arg("value").unwrap_or_default(),
+        "launch_app" => arg("bundle_id").or_else(|| arg("name")).unwrap_or_default(),
+        _ => String::new(),
+    };
+    if summary.is_empty() {
+        tool_name.to_owned()
+    } else {
+        format!("{tool_name}: {summary}")
+    }
+}
@@ -18,6 +18,7 @@ tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
 cua-driver-core = { path = "../cua-driver-core" }
 cursor-overlay = { path = "../cursor-overlay" }
+pip-preview = { path = "../pip-preview" }
 async-trait = "0.1"
 base64 = { workspace = true }
 uuid = { workspace = true }

@@ -110,6 +110,10 @@ const VALUE_FLAGS: &[&str] = &[
     "--cursor-icon", "--cursor-id", "--cursor-palette",
     "--glide-ms", "--dwell-ms", "--idle-hide-ms",
     "--screenshot-out-file", "--client", "--socket", "--pid-file", "--type",
+    // Experimental PiP preview — value flag for the optional geometry
+    // override (--experimental-pip itself is a bare flag and doesn't
+    // need to be listed here).
+    "--experimental-pip-geometry",
 ];
 
 /// Parse the first non-flag positional argument from argv to determine which
@@ -152,6 +156,14 @@ pub fn parse_command() -> Command {
         println!();
         println!("doctor options:");
         println!("  --json                  Emit the probe report as JSON for scripting.");
+        println!();
+        println!("experimental options (default: off):");
+        println!("  --experimental-pip          Show a small always-on-top window with the latest");
+        println!("                              post-action screenshot + a 1-line label. macOS only");
+        println!("                              today; Win/Linux print a not-yet-implemented notice.");
+        println!("  --experimental-pip-geometry WxH[+X+Y]   Override window size (and optional top-left");
+        println!("                                          origin). Defaults to 480x360 in the top-right");
+        println!("                                          corner of the main display.");
         std::process::exit(0);
     }