diff --git a/libs/cua-driver/rust/Cargo.lock b/libs/cua-driver/rust/Cargo.lock index 751f8f6e3d..5b2a2d2ce7 100644 --- a/libs/cua-driver/rust/Cargo.lock +++ b/libs/cua-driver/rust/Cargo.lock @@ -474,7 +474,7 @@ dependencies = [ [[package]] name = "cua-driver" -version = "0.5.1" +version = "0.5.2" dependencies = [ "anyhow", "async-trait", @@ -505,7 +505,7 @@ dependencies = [ [[package]] name = "cua-driver-core" -version = "0.5.1" +version = "0.5.2" dependencies = [ "anyhow", "async-trait", @@ -520,7 +520,7 @@ dependencies = [ [[package]] name = "cua-driver-uia" -version = "0.5.1" +version = "0.5.2" dependencies = [ "anyhow", "cua-driver-core", @@ -535,7 +535,7 @@ dependencies = [ [[package]] name = "cursor-overlay" -version = "0.5.1" +version = "0.5.2" dependencies = [ "anyhow", "image", @@ -608,6 +608,12 @@ dependencies = [ "futures-util", ] +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + [[package]] name = "embed-manifest" version = "1.5.0" @@ -750,7 +756,7 @@ checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" [[package]] name = "focus-monitor-win" -version = "0.5.1" +version = "0.5.2" dependencies = [ "windows 0.58.0", ] @@ -1510,7 +1516,7 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pip-preview" -version = "0.5.1" +version = "0.5.2" dependencies = [ "anyhow", "serde_json", @@ -1536,12 +1542,13 @@ checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "platform-linux" -version = "0.5.1" +version = "0.5.2" dependencies = [ "anyhow", "async-trait", "atspi", "base64", + "bitflags 2.11.1", "cua-driver-core", "cursor-overlay", "image", @@ -1553,12 +1560,16 @@ dependencies = [ "tiny-skia", "tokio", "tracing", + "wayland-backend", + "wayland-client", + "wayland-protocols-wlr", + "wayland-scanner", "x11rb", ] [[package]] name = "platform-macos" -version = "0.5.1" +version = "0.5.2" dependencies = [ "anyhow", "async-trait", @@ -1591,7 +1602,7 @@ dependencies = [ [[package]] name = "platform-windows" -version = "0.5.1" +version = "0.5.2" dependencies = [ "anyhow", "async-trait", @@ -2846,6 +2857,76 @@ dependencies = [ "semver", ] +[[package]] +name = "wayland-backend" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2857dd20b54e916ec7253b3d6b4d5c4d7d4ca2c33c2e11c6c76a99bd8744755d" +dependencies = [ + "cc", + "downcast-rs", + "rustix", + "smallvec", + "wayland-sys", +] + +[[package]] +name = "wayland-client" +version = "0.31.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c7c96bb74690c3189b5c9cb4ca1627062bb23693a4fad9d8c3de958260144" +dependencies = [ + "bitflags 2.11.1", + "rustix", + "wayland-backend", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols" +version = "0.32.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "563a85523cade2429938e790815fd7319062103b9f4a2dc806e9b53b95982d8f" +dependencies = [ + "bitflags 2.11.1", + "wayland-backend", + "wayland-client", + "wayland-scanner", +] + +[[package]] +name = "wayland-protocols-wlr" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb04e52f7836d7c7976c78ca0250d61e33873c34156a2a1fc9474828ec268234" +dependencies = [ + "bitflags 2.11.1", + "wayland-backend", + "wayland-client", + "wayland-protocols", + "wayland-scanner", +] + +[[package]] +name = "wayland-scanner" +version = "0.31.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c324a910fd86ebdc364a3e61ec1f11737d3b1d6c273c0239ee8ff4bc0d24b4a" +dependencies = [ + "proc-macro2", + "quick-xml", + "quote", +] + +[[package]] +name = "wayland-sys" +version = "0.31.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8eab23fefc9e41f8e841df4a9c707e8a8c4ed26e944ef69297184de2785e3be" +dependencies = [ + "pkg-config", +] + [[package]] name = "webpki-roots" version = "1.0.7" diff --git a/libs/cua-driver/rust/Cargo.toml b/libs/cua-driver/rust/Cargo.toml index e111672560..d4456f3063 100644 --- a/libs/cua-driver/rust/Cargo.toml +++ b/libs/cua-driver/rust/Cargo.toml @@ -13,7 +13,7 @@ members = [ ] [workspace.package] -version = "0.5.1" +version = "0.5.2" edition = "2021" authors = ["trycua"] license = "MIT" @@ -30,3 +30,12 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } base64 = "0.22" uuid = { version = "1", features = ["v4"] } image = { version = "0.25", default-features = false, features = ["png", "jpeg"] } +# Wayland capture (Linux): hyprland-toplevel-export per-window capture + +# wlr-screencopy video frames. wayland-backend and bitflags must be direct +# deps of any crate running wayland-scanner macros (generated code references +# both by crate-root path). +wayland-client = "0.31" +wayland-backend = "0.3" +wayland-scanner = "0.31" +wayland-protocols-wlr = { version = "0.3", features = ["client"] } +bitflags = "2" diff --git a/libs/cua-driver/rust/Skills/cua-driver/LINUX.md b/libs/cua-driver/rust/Skills/cua-driver/LINUX.md index 20d6bf9dca..51b1059b55 100644 --- a/libs/cua-driver/rust/Skills/cua-driver/LINUX.md +++ b/libs/cua-driver/rust/Skills/cua-driver/LINUX.md @@ -26,14 +26,18 @@ behaviors that the macOS / Windows skills consider table-stakes are KDE-KWin with `org.freedesktop.portal.RemoteDesktop` enabled, some click and key paths work. Under most other compositors, input synthesis is denied by the security model and the tool surface - degrades to "passive" (snapshot, screenshot) only. + degrades to "passive" (snapshot, screenshot) only. Hyprland is the + exception: it is fully supported for background element-index + workflows — see the Hyprland section below. - **UIA / AX-tree equivalent**: AT-SPI when available, otherwise empty. Many GTK4 / Qt6 apps populate AT-SPI lazily; agents should expect partial trees and re-snapshot. - **launch_app**: backed by `xdg-open` / `gtk-launch` / `dbus-send` with display-environment scrubbing to avoid stealing the user's workspace. Not yet equivalent to macOS `FocusRestoreGuard`. -- **Recording**: not supported. +- **Recording**: supported. Per-turn screenshots + `app_state.json` + (AT-SPI tree) + video. Video uses wlr-screencopy on Wayland and + `x11grab` on X11; requires ffmpeg on PATH. See `SKILL.md` (macOS) and `WINDOWS.md` (Windows) for the full patterns. This file will grow as the Linux backend reaches GA. For @@ -55,6 +59,36 @@ If you're agent-driving on Linux and a tool call surprises you: /org/freedesktop/portal/desktop`). Without it, input synthesis is denied. +## Hyprland + +Hyprland is the exception to the "passive-only under Wayland" rule — +background element-index workflows are fully supported. What's +specific to it: + +- **Window ids**: `window_id` values come from hyprctl window + addresses and exceed `u32::MAX`. That's expected — pass them + through verbatim, don't truncate. +- **Per-window screenshots**: captured via the + `hyprland-toplevel-export-v1` protocol — true surface capture, so + the screenshot shows the correct content even for occluded / + background windows and windows on other workspaces. This is what + makes background computer use verifiable on Hyprland. grim + region-crop is the fallback when the protocol is unavailable. +- **Input**: native-Wayland windows accept `element_index` actions + (AT-SPI) but not pixel input. +- **launch_app**: if a newly launched window steals focus, the driver + restores the previously active window. Best-effort, watches for + ~2 s after launch. +- **Recording**: video captures the focused monitor via + wlr-screencopy frames piped to ffmpeg. `cursor.jsonl` sampling + works via the Hyprland IPC `cursorpos` query (global logical + coords; empty on other Linux sessions). +- **Permission caveat**: if `ecosystem:enforce_permissions` is + enabled in the Hyprland config and screencopy is denied, captures + silently return black "permission denied" frames — no error is + raised. Add an allow rule for the cua-driver binary to the + Hyprland permission config. + ## Forbidden vectors Same idea as macOS / Windows — don't shell out to anything that @@ -78,10 +112,10 @@ ask the user. | Element-indexed click | ⚠️ AT-SPI `accDoDefaultAction` when supported | | Type text | ⚠️ XTest, focus-sensitive | | Hotkey | ⚠️ XTest, focus-sensitive | -| Screenshot full-display | ✅ X11 (xshm); ⚠️ Wayland (portal-gated) | -| Screenshot per-window | ⚠️ X11 with composite extension; Wayland TBD | -| launch_app | ⚠️ xdg-open / gtk-launch; no FocusRestoreGuard yet | -| Recording | ❌ not implemented | +| Screenshot full-display | ✅ X11 (xshm); ✅ Wayland via grim (no portal) | +| Screenshot per-window | ✅ X11; ✅ Hyprland via toplevel-export (correct even when occluded); other Wayland TBD | +| launch_app | ✅ direct exec / xdg-open; focus-restore guard on Hyprland (see Hyprland section) | +| Recording | ✅ wlr-screencopy on Wayland / `x11grab` on X11; ffmpeg required | Until Linux reaches GA, treat this doc as a planning placeholder rather than a contract. diff --git a/libs/cua-driver/rust/Skills/cua-driver/README.md b/libs/cua-driver/rust/Skills/cua-driver/README.md index 07bd363fac..737f779506 100644 --- a/libs/cua-driver/rust/Skills/cua-driver/README.md +++ b/libs/cua-driver/rust/Skills/cua-driver/README.md @@ -36,7 +36,8 @@ platform: no focus steal, no cursor warp. `set_value` workaround). Windows web-apps coverage lives in `WINDOWS.md`'s "Web apps on Windows" section. - Trajectory recording (`RECORDING.md`) — optional per-session - recording + replay for demos and regressions. macOS-only today. + recording + replay for demos and regressions. Cross-platform + (macOS / Windows / Linux). - Canvas/viewport apps (Blender, Unity, GHOST, Qt, wxWidgets) — fallback paths when the AX/UIA/AT-SPI tree is empty. @@ -169,8 +170,8 @@ Use MCP for this Claude Code vision/computer-use-style path. CLI screenshots sti Electron, Tauri, minimized-Chrome keyboard-commit caveat). Loaded on demand from `SKILL.md`. **Note**: Windows web-apps coverage lives in `WINDOWS.md`'s "Web apps on Windows" section. -- `RECORDING.md` — trajectory recording / replay (macOS-only - today; Windows / Linux not yet supported). +- `RECORDING.md` — trajectory recording / replay (cross-platform: + macOS / Windows / Linux). - `TESTS.md` — manual test scripts for end-to-end skill verification. ## Troubleshooting diff --git a/libs/cua-driver/rust/Skills/cua-driver/RECORDING.md b/libs/cua-driver/rust/Skills/cua-driver/RECORDING.md index a7b2e55a98..28047c55e3 100644 --- a/libs/cua-driver/rust/Skills/cua-driver/RECORDING.md +++ b/libs/cua-driver/rust/Skills/cua-driver/RECORDING.md @@ -1,9 +1,9 @@ # Recording & replaying trajectories > **Cross-platform.** Recording is available on macOS (native -> ScreenCaptureKit), Windows (ffmpeg + `gdigrab`), and Linux (ffmpeg + -> `x11grab`). Replay is cross-platform as long as the recorded artifacts -> are present. +> ScreenCaptureKit), Windows (ffmpeg + `gdigrab`), and Linux (ffmpeg; +> wlr-screencopy frames on Wayland, `x11grab` on X11). Replay is +> cross-platform as long as the recorded artifacts are present. Session-scoped capture of action sequences + pre/post state, suitable for demos, regression diffs, and training data. Invoked only when the @@ -17,10 +17,10 @@ turn folder under a caller-chosen output directory. Read-only tools permission probes, agent-cursor getters / setters, and the recording controls themselves) are not recorded. -**Video on by default.** `start_recording` also captures the main -display to `/recording.mp4` (H.264 / 30 fps) for the -lifetime of the session. The mp4 is finalized on `stop_recording`. Opt -out with `record_video: false` when you don't want video. +**Video off by default.** Pass `record_video: true` to also capture +the main display to `/recording.mp4` (H.264 / 30 fps) for +the lifetime of the session. The mp4 is finalized on +`stop_recording`. **macOS — native ScreenCaptureKit, zero-config.** On macOS the recorder uses an in-process `SCStream` + `SCRecordingOutput`, so it @@ -29,7 +29,8 @@ subprocess prompt, no fast-fail, no second TCC dance. Requires macOS 15.0+ (SCRecordingOutput introduced in macOS 15). No ffmpeg needed. **Windows / Linux — ffmpeg subprocess.** Outside macOS the recorder -shells to ffmpeg with `gdigrab` (Windows) or `x11grab` (Linux). The +shells to ffmpeg with `gdigrab` (Windows), wlr-screencopy frames +piped to ffmpeg (Linux Wayland), or `x11grab` (Linux X11). The binary needs to be on PATH (`winget install Gyan.FFmpeg` / `apt install ffmpeg`); when missing, the per-turn capture continues without video and `last_error` carries the install hint. ffmpeg @@ -73,8 +74,10 @@ Each action writes to `turn-NNNNN/` (five-digit zero-padded counter): screenshot fields — those live in `screenshot.png`). On macOS the recorder resolves a frontmost window internally when the action's args don't carry one; on Windows it uses the first window of the - target pid. **Omitted on Linux** — ATSPI doesn't expose a cheap - whole-tree snapshot, and the file is left out rather than faked. + target pid. On Linux the snapshot is the AT-SPI tree (same shape as + macOS); editable / text widgets surface their Text-interface + content as `value="…"` even when the widget has a name, so typed / + set text is verifiable from the tree. - `screenshot.png` — post-action capture of the target window. Omitted when the pid has no visible window. - `action.json` — the tool name, full input arguments, result @@ -102,8 +105,8 @@ Each action writes to `turn-NNNNN/` (five-digit zero-padded counter): This skill does **not** auto-enable recording. The client invokes `start_recording` explicitly when the user asks to capture a session. If the user says "record this session" or similar, call -`start_recording({output_dir:…})` before the first action (video on -by default; pass `record_video: false` to opt out), and +`start_recording({output_dir:…})` before the first action (video off +by default; pass `record_video: true` to capture it), and `stop_recording({})` when done. ## Replaying a recorded trajectory diff --git a/libs/cua-driver/rust/Skills/cua-driver/WINDOWS.md b/libs/cua-driver/rust/Skills/cua-driver/WINDOWS.md index 6255aeb435..b1d364fdd8 100644 --- a/libs/cua-driver/rust/Skills/cua-driver/WINDOWS.md +++ b/libs/cua-driver/rust/Skills/cua-driver/WINDOWS.md @@ -449,8 +449,8 @@ subcommands: registers it (idempotent — replaces existing). `kick` runs it immediately without waiting for a fresh logon. - **`cua-driver recording start|stop|status`** — see `RECORDING.md`. - **Note: recording is currently macOS-only on the Rust port. The - command is registered but returns "not yet supported" on Windows.** + Recording works on Windows via ffmpeg + `gdigrab`; ffmpeg must be + on PATH (`winget install Gyan.FFmpeg`). Canonical multi-step workflow: @@ -895,11 +895,8 @@ registered to auto-start at logon AND whether it's currently running: ## Recording -Screen recording is **not yet supported on Windows** in -cua-driver-rs. The `recording start|stop|status` subcommands are -registered but return "Recording is currently macOS-only" on -Windows. Tracking: see the cua-driver-rs roadmap in the main repo. - -For now, capture state via `screenshot` (per-window or full-desktop) -or `get_window_state` (returns a screenshot embedded alongside the -UIA tree). +Screen recording **works on Windows** in cua-driver-rs via an ffmpeg +subprocess (`gdigrab`). ffmpeg must be on PATH — install with +`winget install Gyan.FFmpeg`. When ffmpeg is missing, per-turn +capture continues without video and `last_error` carries the install +hint. See `RECORDING.md` for the full recording / replay surface. diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/cursor_sampler.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/cursor_sampler.rs index 19cc4c2534..b53c7f6e61 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/cursor_sampler.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/cursor_sampler.rs @@ -8,8 +8,9 @@ //! Per-platform polling: //! - **Windows:** `GetCursorPos` (returns physical screen coords) //! - **macOS:** `CGEventCreate` + `CGEventGetLocation` -//! - **Linux X11:** `XQueryPointer` against the root window -//! - **Linux Wayland:** no portable API exists; sampler runs but logs +//! - **Linux (Hyprland):** `cursorpos` over the Hyprland IPC socket +//! (global logical coordinates) +//! - **Linux (other):** no portable API exists; sampler runs but logs //! no samples — the resulting cursor.jsonl is empty and the zoom //! renderer falls back to the click-point-only path. @@ -132,15 +133,92 @@ fn sample_cursor() -> Option<(f64, f64)> { #[cfg(target_os = "linux")] fn sample_cursor() -> Option<(f64, f64)> { - // Wayland has no equivalent portable poll; on X11 use XQueryPointer. - // We try the X11 path via the `x11` crate if available; otherwise - // return None and the sampler writes an empty cursor.jsonl. + // Wayland has no portable cursor poll, but Hyprland exposes one over + // its IPC socket (`cursorpos` — the same query `hyprctl cursorpos` + // runs). One short-lived unix-socket connect per sample is the + // protocol's request model and is cheap at 30 Hz. // - // The X11 dep isn't always present in cua-driver's Linux build - // (Wayland-only hosts), so this fallback is "no-op when X11 isn't - // wired up" — the renderer copes by falling back to click-point- - // only zoom (no cursor-follow between actions). - None + // cursorpos is global LOGICAL layout coordinates, but the screencopy + // video records the focused-at-start monitor in PHYSICAL pixels — so + // samples are translated by that monitor's origin and multiplied by + // its scale, and samples while the cursor is on another monitor are + // dropped (the recorded screen doesn't show the cursor then anyway). + // The monitor snapshot is per sampler thread, i.e. per recording + // session — the same focused-monitor choice the video backend makes. + // + // Non-Hyprland sessions return None and the sampler writes an empty + // cursor.jsonl — the renderer falls back to click-point-only zoom. + thread_local! { + static MONITOR: std::cell::OnceCell> = + const { std::cell::OnceCell::new() }; + } + MONITOR.with(|m| { + let mon = (*m.get_or_init(hyprland_focused_monitor))?; + let (cx, cy) = hyprland_cursorpos()?; + let px = (cx - mon.x) * mon.scale; + let py = (cy - mon.y) * mon.scale; + if px < 0.0 || py < 0.0 || px > mon.width_px || py > mon.height_px { + return None; + } + Some((px, py)) + }) +} + +#[cfg(target_os = "linux")] +#[derive(Clone, Copy)] +struct FocusedMonitor { + /// Logical layout origin. + x: f64, + y: f64, + scale: f64, + /// Mode size in physical pixels. + width_px: f64, + height_px: f64, +} + +#[cfg(target_os = "linux")] +fn hyprland_focused_monitor() -> Option { + let raw = hyprland_query("j/monitors")?; + let monitors: serde_json::Value = serde_json::from_str(&raw).ok()?; + let mon = monitors + .as_array()? + .iter() + .find(|m| m.get("focused").and_then(|f| f.as_bool()).unwrap_or(false))?; + let num = |k: &str| mon.get(k).and_then(|v| v.as_f64()); + let scale = num("scale").filter(|s| *s > 0.0).unwrap_or(1.0); + Some(FocusedMonitor { + x: num("x")?, + y: num("y")?, + scale, + width_px: num("width")?, + height_px: num("height")?, + }) +} + +#[cfg(target_os = "linux")] +fn hyprland_query(command: &str) -> Option { + use std::io::{Read, Write}; + use std::os::unix::net::UnixStream; + + let sig = std::env::var("HYPRLAND_INSTANCE_SIGNATURE").ok()?; + let runtime = std::env::var("XDG_RUNTIME_DIR").ok()?; + let path = PathBuf::from(runtime).join("hypr").join(sig).join(".socket.sock"); + + let mut stream = UnixStream::connect(path).ok()?; + stream.set_read_timeout(Some(Duration::from_millis(200))).ok()?; + stream.set_write_timeout(Some(Duration::from_millis(200))).ok()?; + stream.write_all(command.as_bytes()).ok()?; + let mut buf = String::new(); + stream.read_to_string(&mut buf).ok()?; + Some(buf) +} + +#[cfg(target_os = "linux")] +fn hyprland_cursorpos() -> Option<(f64, f64)> { + // Response shape: "1234, 567" + let buf = hyprland_query("cursorpos")?; + let (x, y) = buf.trim().split_once(',')?; + Some((x.trim().parse().ok()?, y.trim().parse().ok()?)) } #[cfg(not(any(target_os = "windows", target_os = "macos", target_os = "linux")))] diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/recording_loader.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/recording_loader.rs index 59ada94f73..0b0286c5b6 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/recording_loader.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/recording_loader.rs @@ -193,20 +193,21 @@ fn parse_click_turn(action_path: &Path) -> Option { // Coordinate recovery, in priority order: // 1. `arguments.{x,y}` — pixel-addressed clicks pass coords directly. - // 2. `click_point.{x,y}` — recorded by the session for some click paths. - // 3. `result_summary` text scan — for element-indexed clicks the - // recorder doesn't yet write click_point but the tool result - // surfaces the screen coords like "Performed UIA Invoke on [28] - // (screen (745,679))". Parsing that out keeps zoom working for - // the element_index workflow until the session is updated to - // record click_point for that path too. + // 2. `result_summary` text scan — element-indexed clicks whose tool + // result surfaces screen coords like "Performed UIA Invoke on [28] + // (screen (745,679))". + // + // `click_point` is deliberately NOT a zoom source: the element-bounds + // hook records it in window-local screenshot pixels (the click.png + // marker space), while zoompan x/y are full-video-frame coordinates — + // treating one as the other zooms onto the wrong region for any + // non-maximized window. Translating would require per-turn window + // bounds, which the recorder doesn't write yet. let (x, y) = if let Some(args) = v.get("arguments") { let ax = double_value(args.get("x")); let ay = double_value(args.get("y")); if let (Some(x), Some(y)) = (ax, ay) { (x, y) - } else if let Some(cp) = v.get("click_point") { - (double_value(cp.get("x"))?, double_value(cp.get("y"))?) } else if let Some(screen) = parse_screen_from_summary( v.get("result_summary").and_then(|s| s.as_str()).unwrap_or("") ) { @@ -214,8 +215,6 @@ fn parse_click_turn(action_path: &Path) -> Option { } else { return None; } - } else if let Some(cp) = v.get("click_point") { - (double_value(cp.get("x"))?, double_value(cp.get("y"))?) } else { return None; }; diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/video_ffmpeg.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/video_ffmpeg.rs index 4ab065d3db..c0cc002913 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/video_ffmpeg.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/video_ffmpeg.rs @@ -198,7 +198,10 @@ pub fn find_ffprobe() -> Option { if p.exists() { Some(p) } else { None } } -pub(crate) fn find_ffmpeg() -> Option { +/// Locate the ffmpeg binary (PATH first, then well-known install dirs). +/// Public so platform video backends (e.g. the Linux Wayland screencopy +/// pipeline) can reuse the same resolution logic. +pub fn find_ffmpeg() -> Option { if Command::new("ffmpeg").arg("-version") .stdout(Stdio::null()).stderr(Stdio::null()) .status().map(|s| s.success()).unwrap_or(false) diff --git a/libs/cua-driver/rust/crates/cua-driver/src/main.rs b/libs/cua-driver/rust/crates/cua-driver/src/main.rs index 81a6770202..b2ca80ad47 100644 --- a/libs/cua-driver/rust/crates/cua-driver/src/main.rs +++ b/libs/cua-driver/rust/crates/cua-driver/src/main.rs @@ -768,8 +768,16 @@ fn build_registry(cursor_cfg: cursor_overlay::CursorConfig) -> cua_driver_core:: cua_driver_core::recording::set_click_marker_fn(|png_bytes, cx, cy| { platform_linux::capture::crosshair_png_bytes(png_bytes, cx, cy).ok() }); + cua_driver_core::recording::set_ax_snapshot_fn(|window_id, pid| { + platform_linux::recording_hooks::app_state_json_for(window_id, pid) + }); + cua_driver_core::recording::set_element_bounds_fn(|wid, pid, idx| { + platform_linux::recording_hooks::element_window_local_xy(wid, pid, idx) + }); + // Wayland sessions record via wlr-screencopy (x11grab only sees the + // XWayland root); pure-X11 sessions keep the core ffmpeg backend. cua_driver_core::video::set_video_backend_factory( - Box::new(cua_driver_core::video_ffmpeg::FfmpegVideoBackendFactory), + Box::new(platform_linux::video::LinuxVideoBackendFactory), ); { let mut r = platform_linux::register_tools_with_cursor(cursor_cfg, compat); check_update_tool::register_into(&mut r); r } } @@ -840,8 +848,16 @@ fn build_registry_no_cursor() -> cua_driver_core::tool::ToolRegistry { cua_driver_core::recording::set_click_marker_fn(|png_bytes, cx, cy| { platform_linux::capture::crosshair_png_bytes(png_bytes, cx, cy).ok() }); + cua_driver_core::recording::set_ax_snapshot_fn(|window_id, pid| { + platform_linux::recording_hooks::app_state_json_for(window_id, pid) + }); + cua_driver_core::recording::set_element_bounds_fn(|wid, pid, idx| { + platform_linux::recording_hooks::element_window_local_xy(wid, pid, idx) + }); + // Wayland sessions record via wlr-screencopy (x11grab only sees the + // XWayland root); pure-X11 sessions keep the core ffmpeg backend. cua_driver_core::video::set_video_backend_factory( - Box::new(cua_driver_core::video_ffmpeg::FfmpegVideoBackendFactory), + Box::new(platform_linux::video::LinuxVideoBackendFactory), ); { let mut r = platform_linux::register_tools_with_cursor( diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/mcp_protocol_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/mcp_protocol_test.rs index 2962a1a9e5..aa50617efb 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/mcp_protocol_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/mcp_protocol_test.rs @@ -817,6 +817,71 @@ fn test_recording_session() { let _ = std::fs::remove_dir_all(&tmp_dir); } +#[test] +#[cfg(target_os = "linux")] +fn test_recording_session_linux() { + //! Linux recording parity: enable recording, invoke launch_app (non-read-only, + //! headless-safe with a trivial command), verify next_turn advances and + //! turn-00001/action.json lands on disk. Video off (no compositor in CI). + let binary = binary_path(); + if !binary.exists() { return; } + + let tmp_dir = std::env::temp_dir().join(format!("cua-driver-rs-rec-test-linux-{}", std::process::id())); + let tmp_str = tmp_dir.to_string_lossy().to_string(); + + let mut child = Command::new(&binary) + .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null()) + .spawn().expect("spawn"); + let stdin = child.stdin.as_mut().unwrap(); + let mut stdout = BufReader::new(child.stdout.as_mut().unwrap()); + + send_request(stdin, &serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}})); + read_response(&mut stdout); + + send_request(stdin, &serde_json::json!({ + "jsonrpc":"2.0","id":2,"method":"tools/call", + "params":{"name":"start_recording","arguments":{"output_dir":tmp_str,"record_video":false}} + })); + let resp = read_response(&mut stdout); + assert!(!resp["result"]["isError"].as_bool().unwrap_or(false), "start_recording failed: {resp:?}"); + assert!(resp["result"]["structuredContent"]["enabled"].as_bool().unwrap_or(false)); + + // launch_app with a trivial command — recordable and headless-safe. + send_request(stdin, &serde_json::json!({ + "jsonrpc":"2.0","id":3,"method":"tools/call", + "params":{"name":"launch_app","arguments":{"name":"true"}} + })); + read_response(&mut stdout); + + send_request(stdin, &serde_json::json!({ + "jsonrpc":"2.0","id":4,"method":"tools/call", + "params":{"name":"get_recording_state","arguments":{}} + })); + let resp = read_response(&mut stdout); + let next_turn = resp["result"]["structuredContent"]["next_turn"].as_u64().unwrap_or(0); + assert!(next_turn >= 2, "expected next_turn >= 2, got {next_turn}"); + + send_request(stdin, &serde_json::json!({ + "jsonrpc":"2.0","id":5,"method":"tools/call", + "params":{"name":"stop_recording","arguments":{}} + })); + let resp = read_response(&mut stdout); + assert!(!resp["result"]["structuredContent"]["enabled"].as_bool().unwrap_or(true)); + child.kill().ok(); + + std::thread::sleep(Duration::from_millis(50)); + + let action_path = tmp_dir.join("turn-00001").join("action.json"); + assert!(action_path.exists(), "Expected {action_path:?} to exist"); + let content: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(&action_path).unwrap() + ).unwrap(); + assert_eq!(content["tool"].as_str().unwrap_or(""), "launch_app"); + assert_eq!(content["arguments"]["name"].as_str().unwrap_or(""), "true"); + + let _ = std::fs::remove_dir_all(&tmp_dir); +} + #[test] #[cfg(target_os = "macos")] fn test_recording_screenshot_capture() { diff --git a/libs/cua-driver/rust/crates/platform-linux/Cargo.toml b/libs/cua-driver/rust/crates/platform-linux/Cargo.toml index 1d7cdfa48f..31305cc5f4 100644 --- a/libs/cua-driver/rust/crates/platform-linux/Cargo.toml +++ b/libs/cua-driver/rust/crates/platform-linux/Cargo.toml @@ -29,3 +29,13 @@ libc = "0.2" # typelibs at runtime. `tokio` matches the driver's async runtime; `zbus` # re-exports the bus types we need (fdo::DBusProxy for pid resolution). atspi = { version = "0.30", features = ["tokio", "zbus"] } +# Wayland capture: hyprland-toplevel-export-v1 (vendored XML, per-window +# capture that works for occluded/background windows) + wlr-screencopy +# (full-output frames for video recording). wayland-backend and bitflags +# are direct deps because wayland-scanner's generated code references them +# by crate-root path. +wayland-client = { workspace = true } +wayland-backend = { workspace = true } +wayland-scanner = { workspace = true } +wayland-protocols-wlr = { workspace = true } +bitflags = { workspace = true } diff --git a/libs/cua-driver/rust/crates/platform-linux/protocols/hyprland-toplevel-export-v1.xml b/libs/cua-driver/rust/crates/platform-linux/protocols/hyprland-toplevel-export-v1.xml new file mode 100644 index 0000000000..b1185aa54f --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-linux/protocols/hyprland-toplevel-export-v1.xml @@ -0,0 +1,228 @@ + + + + Copyright © 2022 Vaxry + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + 3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + + This protocol allows clients to ask for exporting another toplevel's + surface(s) to a buffer. + + Particularly useful for sharing a single window. + + + + + This object is a manager which offers requests to start capturing from a + source. + + + + + Capture the next frame of a toplevel. (window) + + The captured frame will not contain any server-side decorations and will + ignore the compositor-set geometry, like e.g. rounded corners. + + It will contain all the subsurfaces and popups, however the latter will be clipped + to the geometry of the base surface. + + The handle parameter refers to the address of the window as seen in `hyprctl clients`. + For example, for d161e7b0 it would be 3512854448. + + + + + + + + + All objects created by the manager will still remain valid, until their + appropriate destroy request has been called. + + + + + + + Same as capture_toplevel, but with a zwlr_foreign_toplevel_handle_v1 handle. + + + + + + + + + + + This object represents a single frame. + + When created, a series of buffer events will be sent, each representing a + supported buffer type. The "buffer_done" event is sent afterwards to + indicate that all supported buffer types have been enumerated. The client + will then be able to send a "copy" request. If the capture is successful, + the compositor will send a "flags" followed by a "ready" event. + + wl_shm buffers are always supported, ie. the "buffer" event is guaranteed to be sent. + + If the capture failed, the "failed" event is sent. This can happen anytime + before the "ready" event. + + Once either a "ready" or a "failed" event is received, the client should + destroy the frame. + + + + + Provides information about wl_shm buffer parameters that need to be + used for this frame. This event is sent once after the frame is created + if wl_shm buffers are supported. + + + + + + + + + + Copy the frame to the supplied buffer. The buffer must have the + correct size, see hyprland_toplevel_export_frame_v1.buffer and + hyprland_toplevel_export_frame_v1.linux_dmabuf. The buffer needs to have a + supported format. + + If the frame is successfully copied, a "flags" and a "ready" event is + sent. Otherwise, a "failed" event is sent. + + This event will wait for appropriate damage to be copied, unless the ignore_damage + arg is set to a non-zero value. + + + + + + + + This event is sent right before the ready event when ignore_damage was + not set. It may be generated multiple times for each copy + request. + + The arguments describe a box around an area that has changed since the + last copy request that was derived from the current screencopy manager + instance. + + The union of all regions received between the call to copy + and a ready event is the total damage since the prior ready event. + + + + + + + + + + + + + + + + + + + Provides flags about the frame. This event is sent once before the + "ready" event. + + + + + + + Called as soon as the frame is copied, indicating it is available + for reading. This event includes the time at which presentation happened + at. + + The timestamp is expressed as tv_sec_hi, tv_sec_lo, tv_nsec triples, + each component being an unsigned 32-bit value. Whole seconds are in + tv_sec which is a 64-bit value combined from tv_sec_hi and tv_sec_lo, + and the additional fractional part in tv_nsec as nanoseconds. Hence, + for valid timestamps tv_nsec must be in [0, 999999999]. The seconds part + may have an arbitrary offset at start. + + After receiving this event, the client should destroy the object. + + + + + + + + + This event indicates that the attempted frame copy has failed. + + After receiving this event, the client should destroy the object. + + + + + + Destroys the frame. This request can be sent at any time by the client. + + + + + + Provides information about linux-dmabuf buffer parameters that need to + be used for this frame. This event is sent once after the frame is + created if linux-dmabuf buffers are supported. + + + + + + + + + This event is sent once after all buffer events have been sent. + + The client should proceed to create a buffer of one of the supported + types, and send a "copy" request. + + + + diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs index a67edb413d..49e0765fa1 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs @@ -17,6 +17,7 @@ use std::time::Duration; use anyhow::{anyhow, Result}; use atspi::connection::AccessibilityConnection; use atspi::proxy::accessible::AccessibleProxy; +use atspi::proxy::component::ComponentProxy; use atspi::proxy::proxy_ext::ProxyExt; use atspi::{CoordType, Interface, State}; @@ -266,9 +267,21 @@ async fn collect_visited<'a>( } } - // Surface Text content as the display name when the widget has no name. + // Surface Text content as the display name when the widget has no + // name. When an EDITABLE widget has a name (GTK entries name + // themselves after their label/prompt), surface the Text content + // as the value instead — otherwise the typed text is invisible in + // snapshots and a background set_value can't be verified from the + // tree. Restricted to editables so read-only text nodes (labels, + // documents) keep their pre-existing snapshot shape. if name.trim().is_empty() && !text_content.trim().is_empty() { name = text_content; + } else if value.is_none() + && has_editable + && !text_content.trim().is_empty() + && text_content != name + { + value = Some(text_content); } // Children inherit web-document context, plus this node's own role. @@ -312,8 +325,14 @@ fn render(visited: &[Visited<'_>]) -> (String, Vec) { let indent = " ".repeat(v.depth); if !v.actions.is_empty() { let act_str = v.actions.join(","); + // Escape newlines/quotes: entry text is user-controlled and the + // tree markdown is line-oriented — a literal newline or quote + // would corrupt the node line for downstream parsers. let val_part = match &v.value { - Some(val) if !val.is_empty() => format!(" value=\"{val}\""), + Some(val) if !val.is_empty() => { + let escaped = val.replace('\\', "\\\\").replace('"', "\\\"").replace('\n', "\\n").replace('\r', ""); + format!(" value=\"{escaped}\"") + } _ => String::new(), }; md.push_str(&format!( @@ -349,6 +368,59 @@ fn format_value(v: f64) -> String { format!("{v:?}") } +fn first_window_origin_for_pid(pid: u32) -> Option<(i32, i32)> { + let window = crate::x11::list_windows(Some(pid)).into_iter().next()?; + Some((window.x, window.y)) +} + +async fn component_extents_for_pid( + comp: &ComponentProxy<'_>, + pid: u32, +) -> Option<(i32, i32, u32, u32)> { + let screen_extents = call(comp.get_extents(CoordType::Screen)) + .await + .and_then(|r| r.ok()); + + if let Some((x, y, w, h)) = screen_extents { + if plausible_extents(x, y, w, h) && (x != 0 || y != 0) { + return Some((x, y, w as u32, h as u32)); + } + + // GTK-on-XWayland can report every component at (0,0) in Screen + // coordinates while still returning correct sizes. In that case try + // the Window coordinate frame and translate by the X11 toplevel origin. + if let Some((wx, wy, ww, wh)) = call(comp.get_extents(CoordType::Window)) + .await + .and_then(|r| r.ok()) + { + if plausible_extents(wx, wy, ww, wh) && (wx != 0 || wy != 0) { + if let Some((origin_x, origin_y)) = first_window_origin_for_pid(pid) { + return Some((origin_x + wx, origin_y + wy, ww as u32, wh as u32)); + } + } + } + + if plausible_extents(x, y, w, h) { + return Some((x, y, w as u32, h as u32)); + } + } + + call(comp.get_extents(CoordType::Window)) + .await + .and_then(|r| r.ok()) + .and_then(|(wx, wy, ww, wh)| { + if !plausible_extents(wx, wy, ww, wh) { + return None; + } + let (origin_x, origin_y) = first_window_origin_for_pid(pid)?; + Some((origin_x + wx, origin_y + wy, ww as u32, wh as u32)) + }) +} + +fn plausible_extents(x: i32, y: i32, w: i32, h: i32) -> bool { + x != i32::MIN && y != i32::MIN && x >= -16384 && y >= -16384 && w > 1 && h > 1 +} + // ── Public (sync) entry points ─────────────────────────────────────────────── pub fn walk_tree(pid: u32) -> Result)>> { @@ -663,11 +735,9 @@ pub fn get_element_bounds(pid: u32, idx: usize) -> Result<(i32, i32, u32, u32)> .component() .await .map_err(|e| anyhow!("Component unavailable: {e}"))?; - let (x, y, w, h) = comp - .get_extents(CoordType::Screen) + component_extents_for_pid(&comp, pid) .await - .map_err(|e| anyhow!("getExtents failed: {e}"))?; - Ok((x, y, w.max(0) as u32, h.max(0) as u32)) + .ok_or_else(|| anyhow!("getExtents returned no usable bounds for element {idx}")) }) } @@ -718,16 +788,8 @@ pub fn get_all_element_bounds(pid: u32) -> Result c, _ => continue, }; - if let Some(Ok((x, y, w, h))) = call(comp.get_extents(CoordType::Screen)).await { - // Unrealized widgets (e.g. items inside closed menus/popovers) - // report GetExtents as the i32::MIN sentinel and/or a degenerate - // 0x0 / 1x1 size. Emitting those poisons downstream consumers - // (overlay renderers, click targeting), so keep only elements - // with plausible on-screen geometry. - if x == i32::MIN || y == i32::MIN || x < -16384 || y < -16384 || w <= 1 || h <= 1 { - continue; - } - out.push((idx, x, y, w as u32, h as u32)); + if let Some((x, y, w, h)) = component_extents_for_pid(&comp, pid).await { + out.push((idx, x, y, w, h)); } } Ok(out) diff --git a/libs/cua-driver/rust/crates/platform-linux/src/capture.rs b/libs/cua-driver/rust/crates/platform-linux/src/capture.rs index 831ae61bb9..617ce75f09 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/capture.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/capture.rs @@ -12,6 +12,9 @@ use std::process::Command; /// Capture a window by X11 XID. Returns raw PNG bytes. pub fn screenshot_window_bytes(xid: u64) -> Result> { + if xid > u32::MAX as u64 { + return crate::hyprland::screenshot_window_bytes(xid); + } // Try `import -window png:-` (ImageMagick). if let Ok(bytes) = capture_via_import(xid) { return Ok(bytes); @@ -25,6 +28,11 @@ pub fn screenshot_window_bytes(xid: u64) -> Result> { /// Capture a window by X11 XID. Returns (base64_png, width, height). pub fn screenshot_window(xid: u64) -> Result<(String, u32, u32)> { + if xid > u32::MAX as u64 { + let bytes = crate::hyprland::screenshot_window_bytes(xid)?; + let (w, h) = cua_driver_core::image_utils::png_dimensions(&bytes)?; + return Ok((BASE64.encode(&bytes), w, h)); + } // Try `import -window png:-` (ImageMagick). if let Ok(bytes) = capture_via_import(xid) { let (w, h) = cua_driver_core::image_utils::png_dimensions(&bytes)?; @@ -98,8 +106,17 @@ pub fn png_dimensions_pub(data: &[u8]) -> Result<(u32, u32)> { // `CUA_DRIVER_RS_DEDUP_AUDIT.md`. RGBA-encoding callers below now go // through `cua_driver_core::image_utils::encode_rgba_to_png`. -/// Capture the primary display (root window) as raw PNG bytes. +/// Capture the primary display as raw PNG bytes. +/// +/// Under Wayland the X11 root only shows XWayland content, so grim (all +/// outputs composited) is tried first; X11 paths remain the fallback and +/// the only path on pure-X11 sessions. pub fn screenshot_display_bytes() -> Result> { + if std::env::var_os("WAYLAND_DISPLAY").is_some() { + if let Ok(png) = crate::hyprland::screenshot_display_bytes_grim() { + return Ok(png); + } + } // Try `import -window root png:-` (ImageMagick). let out = Command::new("import") .args(["-window", "root", "png:-"]) @@ -160,4 +177,3 @@ pub fn resize_png_if_needed(png_bytes: &[u8], max_dim: u32) -> Result> { pub fn crosshair_png_bytes(png_bytes: &[u8], cx: f64, cy: f64) -> Result> { cua_driver_core::image_utils::crosshair_png_bytes(png_bytes, cx, cy) } - diff --git a/libs/cua-driver/rust/crates/platform-linux/src/hyprland.rs b/libs/cua-driver/rust/crates/platform-linux/src/hyprland.rs new file mode 100644 index 0000000000..47a54559ad --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-linux/src/hyprland.rs @@ -0,0 +1,293 @@ +//! Hyprland window discovery and capture helpers. +//! +//! The primary Linux backend is X11/XWayland. On Hyprland, native Wayland +//! clients never appear in _NET_CLIENT_LIST, but hyprctl clients -j exposes +//! enough read-only metadata for list_windows, and per-window screenshots +//! go through the hyprland-toplevel-export-v1 protocol +//! (`crate::wayland_capture`) — which copies the toplevel's own buffer, so +//! occluded/background windows capture their real content. grim region +//! cropping remains only as a fallback when the protocol path fails. + +use anyhow::{bail, Result}; +use serde::Deserialize; +use std::process::Command; +use std::time::{Duration, Instant}; + +use crate::x11::WindowInfo; + +#[derive(Debug, Deserialize)] +struct HyprClient { + address: String, + mapped: bool, + hidden: bool, + pid: i64, + title: String, + class: String, + at: [i32; 2], + size: [i32; 2], + #[serde(default)] + monitor: Option, +} + +#[derive(Debug, Deserialize)] +struct HyprMonitor { + id: i64, + name: String, + #[serde(default)] + scale: f64, + #[serde(default)] + focused: bool, +} + +#[derive(Debug, Deserialize)] +struct HyprActiveWindow { + #[serde(default)] + address: Option, +} + +pub fn list_windows(filter_pid: Option) -> Vec { + list_windows_inner(filter_pid).unwrap_or_default() +} + +/// Per-window screenshot. Tries hyprland-toplevel-export first (true +/// surface capture: correct content for occluded/background windows and +/// windows on other workspaces), falling back to a grim screen-region crop +/// of the client geometry when the protocol path is unavailable. +/// +/// The protocol capture runs on a bounded scratch thread: the frame +/// dispatch loops are deadline-bounded internally, but the initial +/// connect/registry handshake is not, and this function is called +/// synchronously from the recording write path — a wedged compositor must +/// cost at most the timeout, not a hang. +pub fn screenshot_window_bytes(window_id: u64) -> Result> { + let (tx, rx) = std::sync::mpsc::sync_channel(1); + let spawn = std::thread::Builder::new().name("wl-shot".into()).spawn(move || { + let _ = tx.send(crate::wayland_capture::capture_toplevel_png(window_id)); + }); + let result = match spawn { + Ok(_) => rx + .recv_timeout(Duration::from_secs(6)) + .map_err(|_| anyhow::anyhow!("toplevel-export capture timed out")), + Err(e) => Err(anyhow::anyhow!("capture thread spawn failed: {e}")), + }; + match result { + Ok(Ok(png)) => return Ok(png), + Ok(Err(e)) => { + tracing::debug!( + "toplevel-export capture failed for 0x{window_id:x} ({e:#}); \ + falling back to grim region crop" + ); + } + Err(e) => { + tracing::debug!( + "toplevel-export capture for 0x{window_id:x} did not complete ({e:#}); \ + falling back to grim region crop" + ); + } + } + screenshot_window_bytes_grim(window_id) +} + +fn screenshot_window_bytes_grim(window_id: u64) -> Result> { + let client = clients()? + .into_iter() + .find(|c| parse_address(&c.address) == Some(window_id)) + .ok_or_else(|| anyhow::anyhow!("Hyprland client 0x{window_id:x} not found"))?; + if client.size[0] <= 1 || client.size[1] <= 1 { + bail!("Hyprland client 0x{window_id:x} has invalid geometry"); + } + + let geometry = format!( + "{},{} {}x{}", + client.at[0], client.at[1], client.size[0], client.size[1] + ); + let out = Command::new("grim") + .args(["-g", &geometry, "-t", "png", "-"]) + .output()?; + if !out.status.success() || out.stdout.is_empty() { + bail!("grim failed for Hyprland geometry {geometry}"); + } + Ok(out.stdout) +} + +/// Full-desktop screenshot via grim (all outputs composited). Used by the +/// display capture path when running under Wayland, where the X11 root +/// only shows XWayland content. +pub fn screenshot_display_bytes_grim() -> Result> { + let out = Command::new("grim").args(["-t", "png", "-"]).output()?; + if !out.status.success() || out.stdout.is_empty() { + bail!("grim full-output capture failed"); + } + Ok(out.stdout) +} + +/// True when running inside a Hyprland session (hyprctl reachable). +pub fn is_hyprland_session() -> bool { + std::env::var_os("HYPRLAND_INSTANCE_SIGNATURE").is_some() +} + +/// Name of the currently focused monitor (e.g. "DP-1"), for selecting the +/// wl_output to record. +pub fn focused_monitor_name() -> Option { + monitors() + .ok()? + .into_iter() + .find(|m| m.focused) + .map(|m| m.name) +} + +/// Render scale of the monitor a window currently sits on (e.g. 1.5 for +/// fractional scaling). toplevel-export buffers are physical pixels at +/// this scale while hyprctl/AT-SPI geometry is logical, so element +/// coordinates must be multiplied by it to land in screenshot pixels. +pub fn monitor_scale_for_window(window_id: u64) -> Option { + let client = clients() + .ok()? + .into_iter() + .find(|c| parse_address(&c.address) == Some(window_id))?; + let monitor_id = client.monitor?; + monitors() + .ok()? + .into_iter() + .find(|m| m.id == monitor_id) + .map(|m| if m.scale > 0.0 { m.scale } else { 1.0 }) +} + +/// Address of the currently active (focused) Hyprland window, if any. +pub fn active_window_address() -> Option { + if !is_hyprland_session() { + return None; + } + let out = Command::new("hyprctl").args(["activewindow", "-j"]).output().ok()?; + if !out.status.success() || out.stdout.is_empty() { + return None; + } + let active: HyprActiveWindow = serde_json::from_slice(&out.stdout).ok()?; + parse_address(&active.address?) +} + +/// Refocus a window by address (best effort). +/// +/// Hyprland ≥0.55 replaced the hyprlang dispatch grammar with Lua +/// (`hl.dsp.focus({ window = 'address:0x...' })`); older releases use the +/// legacy `focuswindow address:0x...` form. Try modern first, then legacy. +pub fn focus_window(address: u64) { + let modern = format!("hl.dsp.focus({{ window = 'address:0x{address:x}' }})"); + if hyprctl_dispatch(&modern) { + return; + } + let legacy = format!("focuswindow address:0x{address:x}"); + let _ = hyprctl_dispatch(&legacy); +} + +/// Run `hyprctl dispatch `; true only when the compositor answered +/// "ok" (hyprctl can exit 0 while printing an error). +fn hyprctl_dispatch(arg: &str) -> bool { + Command::new("hyprctl") + .args(["dispatch", arg]) + .output() + .map(|o| { + o.status.success() + && String::from_utf8_lossy(&o.stdout).trim_start().starts_with("ok") + }) + .unwrap_or(false) +} + +/// Preserve the active window across an app launch: snapshot the focused +/// window now and watch — for the next ~2 s — for focus moving to a +/// window that did not exist before the launch, putting it back. +/// +/// Hyprland focuses newly mapped windows by default; the driver's contract +/// is that the user's frontmost window must not change, so launch_app +/// restores it. Only focus grabs by NEW windows are reverted — a user +/// alt-tabbing to a pre-existing window during the watch window is left +/// alone. Detached and best-effort: if the previous window closed or +/// hyprctl fails, focus is simply left alone. +pub fn spawn_focus_restore_guard() { + if !is_hyprland_session() { + return; + } + let Some(previous) = active_window_address() else { + return; + }; + let preexisting: std::collections::HashSet = clients() + .unwrap_or_default() + .iter() + .filter_map(|c| parse_address(&c.address)) + .collect(); + std::thread::spawn(move || { + let deadline = Instant::now() + Duration::from_secs(2); + while Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(100)); + match active_window_address() { + Some(current) if current != previous && !preexisting.contains(¤t) => { + focus_window(previous); + return; + } + _ => {} + } + } + }); +} + +fn list_windows_inner(filter_pid: Option) -> Result> { + let mut out = Vec::new(); + for client in clients()? { + if !client.mapped || client.hidden || client.size[0] <= 1 || client.size[1] <= 1 { + continue; + } + let pid = u32::try_from(client.pid).ok(); + if let Some(filter_pid) = filter_pid { + if pid != Some(filter_pid) { + continue; + } + } + let Some(window_id) = parse_address(&client.address) else { + continue; + }; + let title = if client.title.trim().is_empty() { + client.class + } else { + client.title + }; + if title.trim().is_empty() { + continue; + } + out.push(WindowInfo { + xid: window_id, + pid, + title, + x: client.at[0], + y: client.at[1], + width: client.size[0] as u32, + height: client.size[1] as u32, + }); + } + Ok(out) +} + +fn clients() -> Result> { + if !is_hyprland_session() { + return Ok(Vec::new()); + } + let out = Command::new("hyprctl").args(["clients", "-j"]).output()?; + if !out.status.success() || out.stdout.is_empty() { + bail!("hyprctl clients -j failed"); + } + Ok(serde_json::from_slice(&out.stdout)?) +} + +fn monitors() -> Result> { + if !is_hyprland_session() { + bail!("not a Hyprland session"); + } + let out = Command::new("hyprctl").args(["monitors", "-j"]).output()?; + if !out.status.success() || out.stdout.is_empty() { + bail!("hyprctl monitors -j failed"); + } + Ok(serde_json::from_slice(&out.stdout)?) +} + +fn parse_address(address: &str) -> Option { + u64::from_str_radix(address.trim_start_matches("0x"), 16).ok() +} diff --git a/libs/cua-driver/rust/crates/platform-linux/src/lib.rs b/libs/cua-driver/rust/crates/platform-linux/src/lib.rs index 36fbcf15a9..ee10bdfb17 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/lib.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/lib.rs @@ -19,6 +19,18 @@ pub mod pip; #[cfg(target_os = "linux")] pub mod x11; +#[cfg(target_os = "linux")] +pub mod hyprland; + +#[cfg(target_os = "linux")] +pub mod wayland_capture; + +#[cfg(target_os = "linux")] +pub mod video; + +#[cfg(target_os = "linux")] +pub mod recording_hooks; + #[cfg(target_os = "linux")] pub mod input; diff --git a/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs b/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs new file mode 100644 index 0000000000..cd98c13d42 --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs @@ -0,0 +1,143 @@ +//! Recording callbacks for Linux, mirroring +//! `platform_macos::recording_hooks`. +//! +//! - `app_state_json_for` → per-turn `app_state.json`: the AT-SPI tree in +//! the same `{pid, window_id, element_count, tree_markdown}` shape +//! `get_window_state` returns (minus screenshot fields). +//! - `element_window_local_xy` → element center in window-local screenshot +//! pixels, so `click.png` markers and `action.json.click_point` work for +//! element_index-addressed clicks, not just pixel ones. +//! +//! Coordinate spaces: AT-SPI extents and hyprctl geometry are logical +//! compositor coordinates, but toplevel-export screenshots of native +//! Wayland windows are physical pixels at the window's monitor render +//! scale (1.5x fractional scaling is common). X11/XWayland `import` +//! captures are 1:1 with X11 logical coordinates, so they need no scale. +//! +//! Known limitation: element bounds come from a post-action AT-SPI +//! re-walk, so if the action changed the tree the index can drift and the +//! marker lands on the wrong element. macOS avoids this with a process- +//! global element cache; porting that to Linux is the proper fix. + +use std::time::Duration; + +/// Run a hook body on a scratch OS thread with a bounded wait. +/// +/// Recording hooks are invoked synchronously from `write_turn` on a tokio +/// async worker thread (tool impls escape via `spawn_blocking`, the +/// recording path does not). The AT-SPI layer drives its own runtime via +/// `block_on`, which panics when called from inside an async context — +/// so hop to a plain thread first. The join is deadline-bounded so a +/// wedged D-Bus walk cannot stall the tool-response path; on timeout the +/// scratch thread is leaked and finishes (or times out internally) on its +/// own. Recording cadence is human-scale; one short-lived thread per turn +/// is fine. +fn on_scratch_thread(timeout: Duration, f: F) -> Option +where + F: FnOnce() -> Option + Send + 'static, + T: Send + 'static, +{ + let (tx, rx) = std::sync::mpsc::sync_channel(1); + std::thread::Builder::new() + .name("rec-hook".into()) + .spawn(move || { + let _ = tx.send(f()); + }) + .ok()?; + rx.recv_timeout(timeout).ok().flatten() +} + +pub fn app_state_json_for(window_id: Option, pid: Option) -> Option> { + // pid is required, matching the macOS hook: AT-SPI lookup is by pid. + let pid = pid?; + let pid_u32 = u32::try_from(pid).ok()?; + // Bail fast when the action closed the app's last window (OK-button + // clicks, dialog dismissals): an AT-SPI walk against a dying process + // burns multi-second D-Bus timeouts and stalls the tool response. + let windows = crate::x11::list_windows(Some(pid_u32)); + if windows.is_empty() { + return None; + } + // Match macOS/Windows: always emit a numeric window id when one can be + // resolved (first window of the pid when the recorded args had none). + let window_id = window_id.or_else(|| windows.first().map(|w| w.xid)); + on_scratch_thread(Duration::from_secs(12), move || { + // Native AT-SPI only — without the X11-properties fallback the + // shared walk_tree wrapper uses for get_window_state. That + // fallback fabricates a one-node title tree (and truncates + // Hyprland addresses to u32), which is worse than omitting + // app_state.json for the turn. + let (tree_markdown, nodes) = crate::atspi::native::walk_tree(pid_u32).ok().flatten()?; + if tree_markdown.is_empty() { + return None; + } + let payload = serde_json::json!({ + "pid": pid, + "window_id": window_id, + "element_count": nodes.len(), + "tree_markdown": tree_markdown, + }); + serde_json::to_vec_pretty(&payload).ok() + }) +} + +pub fn element_window_local_xy( + window_id: u64, + pid: i64, + element_index: u32, +) -> Option<(f64, f64)> { + let pid_u32 = u32::try_from(pid).ok()?; + on_scratch_thread(Duration::from_secs(8), move || { + element_window_local_xy_blocking(window_id, pid_u32, element_index) + }) +} + +fn element_window_local_xy_blocking( + window_id: u64, + pid_u32: u32, + element_index: u32, +) -> Option<(f64, f64)> { + // Window lookup first: it's cheap, and when the click closed the + // window there's no point burning AT-SPI timeouts on a dead app. + let win = crate::x11::list_windows(Some(pid_u32)) + .into_iter() + .find(|w| w.xid == window_id)?; + + let (x, y, w, h) = crate::atspi::get_element_bounds(pid_u32, element_index as usize).ok()?; + let cx = x as f64 + w as f64 / 2.0; + let cy = y as f64 + h as f64 / 2.0; + + let native_wayland = window_id > u32::MAX as u64; + + // AT-SPI extents are screen coordinates for X11/XWayland apps, but + // toolkits on native Wayland cannot know their global position and + // report window-local coordinates (no Wayland protocol exposes the + // window's place in the layout). Both interpretations are tested for + // in-window containment; when both fit (window near the layout + // origin), native Wayland prefers the window-local reading because + // that is what GTK4/Qt actually emit — the screen-coordinate reading + // only arises from the rare all-zero-extents fallback in + // `component_extents_for_pid`, and mis-picking there costs at most + // the (small) window-origin offset. + let in_window = |lx: f64, ly: f64| { + lx >= 0.0 && ly >= 0.0 && lx <= win.width as f64 && ly <= win.height as f64 + }; + let screen_rel = (cx - win.x as f64, cy - win.y as f64); + let (lx, ly) = if native_wayland && in_window(cx, cy) { + (cx, cy) + } else if in_window(screen_rel.0, screen_rel.1) { + screen_rel + } else { + return None; + }; + + // Native Wayland windows (Hyprland addresses) are captured at physical + // pixel scale; X11/XWayland captures match logical coordinates. + let scale = if native_wayland { + crate::hyprland::monitor_scale_for_window(window_id).unwrap_or(1.0) + } else { + 1.0 + }; + + Some((lx * scale, ly * scale)) +} diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs index 57d31539cf..148f864721 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs @@ -351,6 +351,21 @@ impl Tool for GetWindowStateTool { let capture_mode = args.str_or("capture_mode", &default_mode); let query = args.opt_str("query"); + let requested_pid = pid; + let requested_xid = xid; + let matching_window = tokio::task::spawn_blocking(move || { + crate::x11::list_windows(Some(requested_pid)) + .into_iter() + .any(|w| w.xid == requested_xid) + }) + .await + .unwrap_or(false); + if !matching_window { + return ToolResult::error(format!( + "No window_id {xid} is currently visible for pid {pid}; call list_windows to refresh window state." + )); + } + // "ax" = tree only; "vision" = screenshot only; "som" (default) = both. let do_tree = capture_mode != "vision"; let do_shot = capture_mode != "ax"; @@ -480,6 +495,11 @@ impl Tool for LaunchAppTool { } let result = tokio::task::spawn_blocking(move || -> anyhow::Result { + // Hyprland focuses newly mapped windows; the driver's + // no-foreground contract says the user's active window must not + // change, so watch for a focus steal over the next ~2s and undo + // it (no-op outside Hyprland sessions). + crate::hyprland::spawn_focus_restore_guard(); // Open URLs via xdg-open. if !urls.is_empty() { for url in &urls { @@ -495,7 +515,10 @@ impl Tool for LaunchAppTool { let mut parts = cmd.split_whitespace(); let prog = parts.next().unwrap_or(cmd); let rest: Vec<&str> = parts.collect(); - match std::process::Command::new(prog).args(&rest).spawn() { + let mut command = std::process::Command::new(prog); + command.args(&rest); + prefer_xwayland_for_launched_apps(&mut command); + match command.spawn() { Ok(child) => return Ok(format!("Launched '{}' with pid {}.", cmd, child.id())), Err(_) => { // Fall back to xdg-open for .desktop app names. @@ -517,6 +540,31 @@ impl Tool for LaunchAppTool { // ── shared helpers ──────────────────────────────────────────────────────────── +fn is_native_wayland_window_id(xid: u64) -> bool { + xid > u32::MAX as u64 +} + +fn native_wayland_input_error(action: &str) -> ToolResult { + ToolResult::error(format!( + "{action} is not supported for native Wayland windows yet. Use AT-SPI element_index actions or set_value when the app exposes accessibility actions." + )) +} + +fn prefer_xwayland_for_launched_apps(command: &mut std::process::Command) { + if std::env::var_os("WAYLAND_DISPLAY").is_none() || std::env::var_os("DISPLAY").is_none() { + return; + } + + // The Linux backend currently enumerates and captures X11/XWayland + // windows. When launch_app starts a toolkit app in a Wayland session, GTK + // and Qt default to native Wayland and the very next list_windows call can + // legitimately see nothing. Prefer XWayland for driver-launched apps so the + // launched surface remains automatable. Explicit env in the user's command + // still wins, e.g. env GDK_BACKEND=wayland ... . + command.env("GDK_BACKEND", "x11"); + command.env("QT_QPA_PLATFORM", "xcb"); +} + /// Resolve an AT-SPI element's center in window-local X11 coordinates. /// /// Returns `(xid, window_local_x, window_local_y)`. @@ -537,6 +585,11 @@ fn resolve_element_local_coords(pid: u32, idx: usize, xid_hint: Option) .into_iter().next().map(|w| w.xid) .ok_or_else(|| anyhow::anyhow!("No windows for pid {pid}"))? }; + if is_native_wayland_window_id(xid) { + anyhow::bail!( + "X11 pixel fallback is not supported for native Wayland window_id {xid}; use AT-SPI element_index actions" + ); + } use x11rb::connection::Connection; use x11rb::protocol::xproto::ConnectionExt as _; @@ -786,6 +839,9 @@ impl Tool for ClickTool { Some(v) => v, None => return ToolResult::error("Provide either element_index or window_id + x/y."), }; + if is_native_wayland_window_id(xid) { + return native_wayland_input_error("Pixel click"); + } let from_zoom = args.bool_or("from_zoom", false); let mut x = args.f64_or("x", 0.0); let mut y = args.f64_or("y", 0.0); @@ -899,6 +955,21 @@ impl Tool for TypeTextTool { } } + if is_native_wayland_window_id(xid) { + let text_clone = text.clone(); + let insert_result = tokio::task::spawn_blocking(move || { + crate::atspi::insert_text(pid, &text_clone) + }).await; + return match insert_result { + Ok(Ok(true)) => ToolResult::text(format!( + "Typed {text_len} character(s) (via AT-SPI insert_text)." + )), + Ok(Ok(false)) => native_wayland_input_error("X11 text fallback"), + Ok(Err(e)) => ToolResult::error(e.to_string()), + Err(e) => ToolResult::error(format!("Task error: {e}")), + }; + } + // Qt5 workaround: send synthetic FocusIn to make Qt5's AT-SPI bridge // expose the widget tree, type via AT-SPI, then send FocusOut. // This doesn't change the X11 active window, so the test's focus check passes. @@ -992,6 +1063,9 @@ impl Tool for PressKeyTool { } } }; + if is_native_wayland_window_id(xid) { + return native_wayland_input_error("Key press"); + } let key_for_task = key.clone(); let result = tokio::task::spawn_blocking(move || { if mods.is_empty() && key_for_task.eq_ignore_ascii_case("enter") { @@ -1055,6 +1129,9 @@ impl Tool for HotkeyTool { } } }; + if is_native_wayland_window_id(xid) { + return native_wayland_input_error("Hotkey"); + } // Parse keys array (preferred) or fall back to legacy key+modifiers. let (key, mods) = if let Some(arr) = args.get("keys").and_then(|v| v.as_array()) { @@ -1186,6 +1263,9 @@ impl Tool for ScrollTool { } } }; + if is_native_wayland_window_id(xid) { + return native_wayland_input_error("Scroll"); + } // X11 scroll buttons: 4=up, 5=down, 6=left, 7=right // Note: "page" scroll is still per-click on X11; send more ticks for page. @@ -1264,6 +1344,9 @@ impl Tool for DoubleClickTool { let xid = match args.opt_u64("window_id") { Some(v) => v, None => return ToolResult::error("Provide either element_index or window_id + x/y."), }; + if is_native_wayland_window_id(xid) { + return native_wayland_input_error("Double-click"); + } let from_zoom = args.bool_or("from_zoom", false); let mut x = args.f64_or("x", 0.0); let mut y = args.f64_or("y", 0.0); @@ -1351,6 +1434,9 @@ impl Tool for RightClickTool { let xid = match args.opt_u64("window_id") { Some(v) => v, None => return ToolResult::error("Provide either element_index or window_id + x/y."), }; + if is_native_wayland_window_id(xid) { + return native_wayland_input_error("Right-click"); + } let from_zoom = args.bool_or("from_zoom", false); let mut x = args.f64_or("x", 0.0); let mut y = args.f64_or("y", 0.0); @@ -1419,6 +1505,9 @@ impl Tool for DragTool { let xid = match args.opt_u64("window_id") { Some(v) => v, None => return ToolResult::error("window_id is required on Linux."), }; + if is_native_wayland_window_id(xid) { + return native_wayland_input_error("Drag"); + } let coerce = |key: &str| -> Option { args.opt_f64(key).or_else(|| args.opt_i64(key).map(|i| i as f64)) @@ -2191,6 +2280,9 @@ impl Tool for TypeTextCharsTool { } } }; + if is_native_wayland_window_id(xid) { + return native_wayland_input_error("Character-by-character typing"); + } let text_len = text.chars().count(); let result = tokio::task::spawn_blocking(move || { crate::input::send_type_text_with_delay(xid, &text, delay_ms) diff --git a/libs/cua-driver/rust/crates/platform-linux/src/video.rs b/libs/cua-driver/rust/crates/platform-linux/src/video.rs new file mode 100644 index 0000000000..e77dffa396 --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-linux/src/video.rs @@ -0,0 +1,314 @@ +//! Linux video-recording backend. +//! +//! On Wayland (Hyprland) the stock ffmpeg `x11grab` input only sees the +//! XWayland root — native Wayland surfaces never appear, so screen +//! recordings come out black or XWayland-only. This backend captures real +//! compositor output frames via wlr-screencopy +//! (`crate::wayland_capture::ScreencopyCapturer`) and pipes them to an +//! ffmpeg subprocess as rawvideo on stdin (libx264, same encode settings +//! as the core ffmpeg backend). +//! +//! Frame pacing: the encoder is fed at a fixed nominal FPS against the +//! wall clock. When a capture iteration runs slow, the last good frame is +//! duplicated to catch up, so the mp4's duration tracks wall time and the +//! per-turn `t_ms_from_session_start` timestamps in action.json line up +//! with video time (the render zoom pipeline depends on this). +//! +//! `LinuxVideoBackendFactory` picks per session: Wayland screencopy when +//! `WAYLAND_DISPLAY` is set (falling back to x11grab if the Wayland path +//! fails to start), else the core ffmpeg x11grab backend. + +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result}; +use cua_driver_core::video::{VideoBackend, VideoBackendFactory, VideoMetadata}; + +use crate::wayland_capture::{ffmpeg_pixel_format, FrameInfo, ScreencopyCapturer}; + +const FPS: u32 = 30; +/// Give up on the capture thread after this many consecutive failed frames: +/// the compositor is gone or the output vanished. Each failed attempt can +/// burn up to the 4 s dispatch deadline, so worst case this is ~40 s of +/// retrying, not a frame-count worth of wall time. +const MAX_CONSECUTIVE_FAILURES: u32 = 10; + +pub struct LinuxVideoBackendFactory; + +impl VideoBackendFactory for LinuxVideoBackendFactory { + fn start(&self, output_path: &Path) -> Result> { + if std::env::var_os("WAYLAND_DISPLAY").is_some() { + // No x11grab fallback on Wayland sessions: it records only the + // XWayland root (black / partial), which would silently + // masquerade as a healthy recording. Surfacing the error puts + // it in session.json video.error where the user can see it. + return WaylandVideoBackend::start(output_path) + .map(|b| Box::new(b) as Box) + .map_err(|e| e.context("Wayland screencopy video failed")); + } + cua_driver_core::video_ffmpeg::FfmpegVideoBackendFactory.start(output_path) + } +} + +pub struct WaylandVideoBackend { + child: Child, + output_path: PathBuf, + started_at: Instant, + stop: Arc, + capture_thread: Option>, + stderr_thread: Option>>, +} + +impl WaylandVideoBackend { + fn start(output_path: &Path) -> Result { + let output_path = output_path.to_path_buf(); + + // Open the capture session and grab one frame up front: it proves + // the protocol path works and pins the frame geometry/format for + // the ffmpeg invocation. + let focused = crate::hyprland::focused_monitor_name(); + let mut capturer = ScreencopyCapturer::open(focused.as_deref()) + .context("wlr-screencopy session open failed")?; + let mut first_frame = Vec::new(); + let info = capturer + .capture_frame(true, &mut first_frame) + .context("initial screencopy frame failed")?; + let pix_fmt = ffmpeg_pixel_format(info.format) + .with_context(|| format!("unsupported screencopy pixel format {:?}", info.format))?; + + let ffmpeg = cua_driver_core::video_ffmpeg::find_ffmpeg().context( + "ffmpeg not found on PATH. Install with: apt install ffmpeg (Debian/Ubuntu) \ + or pacman -S ffmpeg (Arch).", + )?; + if let Some(parent) = output_path.parent() { + std::fs::create_dir_all(parent).map_err(|e| { + anyhow::anyhow!( + "failed to create recording output directory {}: {e}", + parent.display() + ) + })?; + } + + let mut cmd = Command::new(&ffmpeg); + cmd.arg("-y") + .arg("-loglevel").arg("error") + .arg("-f").arg("rawvideo") + .arg("-pixel_format").arg(pix_fmt) + .arg("-video_size").arg(format!("{}x{}", info.width, info.height)) + .arg("-framerate").arg(FPS.to_string()) + .arg("-i").arg("pipe:0") + // yuv420p needs even dimensions; pad rather than crop (matches + // the core ffmpeg backend). + .arg("-vf").arg("pad=ceil(iw/2)*2:ceil(ih/2)*2") + .arg("-c:v").arg("libx264") + .arg("-preset").arg("ultrafast") + .arg("-pix_fmt").arg("yuv420p") + .arg("-movflags").arg("+faststart") + .arg("-g").arg("30") + .arg(&output_path); + cmd.stdin(Stdio::piped()).stdout(Stdio::null()).stderr(Stdio::piped()); + + let mut child = cmd + .spawn() + .map_err(|e| anyhow::anyhow!("Failed to spawn ffmpeg ({}): {e}", ffmpeg.display()))?; + + let stderr_thread = child.stderr.take().map(|mut stderr| { + std::thread::spawn(move || -> Vec { + use std::io::Read; + let mut buf = Vec::with_capacity(4096); + let _ = stderr.read_to_end(&mut buf); + let len = buf.len(); + if len > 4096 { + buf.drain(..len - 4096); + } + buf + }) + }); + + let mut stdin = child.stdin.take().context("ffmpeg stdin unavailable")?; + let stop = Arc::new(AtomicBool::new(false)); + let started_at = Instant::now(); + + let capture_thread = { + let stop = Arc::clone(&stop); + let spawned = std::thread::Builder::new() + .name("wl-video-capture".into()) + .spawn(move || { + capture_loop(capturer, info, first_frame, &mut stdin, &stop, started_at); + // Dropping stdin sends EOF — ffmpeg finalizes the mp4. + drop(stdin); + }); + match spawned { + Ok(handle) => handle, + Err(e) => { + // Without the feeder thread ffmpeg would linger as a + // zombie blocked on an empty pipe. + let _ = child.kill(); + let _ = child.wait(); + return Err(e).context("failed to spawn capture thread"); + } + } + }; + + Ok(WaylandVideoBackend { + child, + output_path, + started_at, + stop, + capture_thread: Some(capture_thread), + stderr_thread: Some(stderr_thread).flatten(), + }) + } +} + +/// Feed ffmpeg at a fixed nominal FPS against the wall clock, duplicating +/// the last good frame when capture runs slow and tolerating transient +/// capture failures. +fn capture_loop( + mut capturer: ScreencopyCapturer, + first_info: FrameInfo, + mut latest: Vec, + stdin: &mut std::process::ChildStdin, + stop: &AtomicBool, + started_at: Instant, +) { + let mut scratch: Vec = Vec::with_capacity(latest.len()); + let mut frames_written: u64 = 0; + let mut consecutive_failures: u32 = 0; + let frame_interval = 1.0 / FPS as f64; + + loop { + if stop.load(Ordering::Relaxed) { + return; + } + + // Keep the encoder fed up to the wall clock (duplicates catch up + // after slow captures). Bounded per iteration: when the encoder + // back-pressures (write_all blocks), an unbounded catch-up loop + // would keep accumulating debt and never observe the stop flag. + // Dropping backlog trades a brief slow-motion segment for bounded + // latency; FPS duplicates per pass caps a single iteration's + // writes at one second of video. + let due = (started_at.elapsed().as_secs_f64() / frame_interval) as u64 + 1; + if due - frames_written > FPS as u64 * 2 { + tracing::warn!(target: "recording", + "video encoder back-pressure: dropping {} frames of backlog", + due - frames_written - 1); + frames_written = due - 1; + } + let mut wrote_this_pass = 0u32; + while frames_written < due && wrote_this_pass < FPS { + if stop.load(Ordering::Relaxed) { + return; + } + if stdin.write_all(&latest).is_err() { + // ffmpeg died; nothing more to do. + return; + } + frames_written += 1; + wrote_this_pass += 1; + } + + match capturer.capture_frame(true, &mut scratch) { + Ok(info) if info == first_info => { + std::mem::swap(&mut latest, &mut scratch); + consecutive_failures = 0; + } + Ok(info) => { + tracing::warn!(target: "recording", + "screencopy frame geometry changed ({}x{} -> {}x{}); freezing video frame", + first_info.width, first_info.height, info.width, info.height); + // Output mode/scale changed mid-recording: the rawvideo + // geometry is fixed, so keep duplicating the last good + // frame rather than corrupting the stream. + consecutive_failures = 0; + } + Err(e) => { + consecutive_failures += 1; + if consecutive_failures >= MAX_CONSECUTIVE_FAILURES { + tracing::warn!(target: "recording", + "screencopy capture failing persistently ({e:#}); stopping video frames"); + return; + } + } + } + + // Sleep to the next frame boundary. + let next = started_at + Duration::from_secs_f64(frames_written as f64 * frame_interval); + if let Some(d) = next.checked_duration_since(Instant::now()) { + std::thread::sleep(d); + } + } +} + +impl VideoBackend for WaylandVideoBackend { + fn stop(mut self: Box) -> Result { + let elapsed = self.started_at.elapsed(); + + self.stop.store(true, Ordering::Relaxed); + if let Some(handle) = self.capture_thread.take() { + // Bounded wait: the thread can be stuck in write_all against a + // wedged encoder, and stop() runs under the daemon-wide + // recording mutex — an unbounded join here would hang every + // recorded tool call. Killing ffmpeg breaks the pipe (EPIPE), + // which unblocks write_all and lets the thread exit. + let deadline = Instant::now() + Duration::from_millis(3000); + while !handle.is_finished() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(20)); + } + if !handle.is_finished() { + tracing::warn!(target: "recording", + "video capture thread stuck (encoder back-pressure?); killing ffmpeg"); + let _ = self.child.kill(); + } + let _ = handle.join(); // drops ffmpeg stdin -> EOF + } + + // Encoder flush on EOF; 4K ultrafast flush fits comfortably in 5 s. + let deadline = Instant::now() + Duration::from_millis(5000); + let finalized; + loop { + match self.child.try_wait() { + Ok(Some(status)) => { + finalized = status.success(); + break; + } + Ok(None) => { + if Instant::now() > deadline { + let _ = self.child.kill(); + let _ = self.child.wait(); + finalized = false; + break; + } + std::thread::sleep(Duration::from_millis(80)); + } + Err(_) => { + let _ = self.child.kill(); + let _ = self.child.wait(); + finalized = false; + break; + } + } + } + + if let Some(handle) = self.stderr_thread.take() { + if let Ok(buf) = handle.join() { + if !finalized && !buf.is_empty() { + let tail = String::from_utf8_lossy(&buf); + tracing::warn!(target: "recording", + "ffmpeg did not finalize cleanly. Last stderr tail:\n{tail}"); + } + } + } + + Ok(VideoMetadata { + path: self.output_path, + duration_ms: elapsed.as_millis() as u64, + finalized, + }) + } +} diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland_capture.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland_capture.rs new file mode 100644 index 0000000000..d525dc3318 --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland_capture.rs @@ -0,0 +1,683 @@ +//! Wayland-native capture for Hyprland. +//! +//! Two capture paths, both pure-Rust wayland-client (no grim subprocess): +//! +//! - **Per-window**: `hyprland-toplevel-export-v1` (vendored XML, bindings +//! generated by wayland-scanner). Copies the toplevel's own buffer, so a +//! background window captures *its* content even when occluded or on +//! another workspace — unlike the grim screen-region crop it replaces. +//! The protocol handle is the `hyprctl clients` window address truncated +//! to its low 32 bits (Hyprland's `getWindowFromHandle` compares +//! `ptr & 0xFFFFFFFF`). +//! +//! - **Full output**: `wlr-screencopy-unstable-v1` for repeated frame +//! capture feeding the video recording backend (`crate::video`). +//! +//! Frame dispatch loops are deadline-bounded: `copy` with `ignore_damage=0` +//! waits for window damage and would hang forever on an idle window, and a +//! wedged compositor must not stall the recording hook path. (The initial +//! `connect`/`registry_queue_init`/roundtrip handshakes still use the +//! library's blocking calls — callers on latency-sensitive paths should +//! wrap captures in a bounded scratch thread, as `hyprland.rs` does for +//! the screenshot path.) Captures are one connection per call (cheap, +//! thread-safe); the video capturer keeps one connection plus a reusable +//! shm pool across frames. +//! +//! Caveat (documented in the skill): when Hyprland's +//! `ecosystem:enforce_permissions` is enabled and screencopy is denied, +//! the compositor reports a *successful* capture containing a black +//! "permission denied" frame — there is no failed event to detect. + +use std::fs::File; +use std::os::fd::{AsFd, AsRawFd}; +use std::os::unix::fs::FileExt; +use std::time::{Duration, Instant}; + +use anyhow::{bail, Context, Result}; +use wayland_client::globals::{registry_queue_init, GlobalListContents}; +use wayland_client::protocol::{ + wl_buffer::WlBuffer, + wl_output::{self, WlOutput}, + wl_registry, + wl_shm::{self, WlShm}, + wl_shm_pool::WlShmPool, +}; +use wayland_client::{delegate_noop, Connection, Dispatch, EventQueue, QueueHandle, WEnum}; +use wayland_protocols_wlr::screencopy::v1::client::{ + zwlr_screencopy_frame_v1::{self, ZwlrScreencopyFrameV1}, + zwlr_screencopy_manager_v1::ZwlrScreencopyManagerV1, +}; + +// --------------------------------------------------------------------------- +// Generated bindings for hyprland-toplevel-export-v1 (vendored XML). +// +// The v2 request `capture_toplevel_with_wlr_toplevel_handle` references +// zwlr_foreign_toplevel_handle_v1, so the wlr foreign_toplevel interfaces +// must be in scope for both the __interfaces module and the outer module. +// --------------------------------------------------------------------------- +pub mod hyprland_toplevel_export { + #![allow(non_upper_case_globals, non_camel_case_types, missing_docs)] + use wayland_client; + use wayland_client::protocol::*; + use wayland_protocols_wlr::foreign_toplevel::v1::client::*; + + pub mod __interfaces { + use wayland_client::protocol::__interfaces::*; + use wayland_protocols_wlr::foreign_toplevel::v1::client::__interfaces::*; + wayland_scanner::generate_interfaces!("./protocols/hyprland-toplevel-export-v1.xml"); + } + use self::__interfaces::*; + + wayland_scanner::generate_client_code!("./protocols/hyprland-toplevel-export-v1.xml"); +} + +use hyprland_toplevel_export::{ + hyprland_toplevel_export_frame_v1::{self, HyprlandToplevelExportFrameV1}, + hyprland_toplevel_export_manager_v1::HyprlandToplevelExportManagerV1, +}; + +/// Default deadline for buffer-param negotiation and frame copy. Generous +/// enough for a busy compositor, short enough that a recording hook never +/// stalls a tool call noticeably. +const CAPTURE_DEADLINE: Duration = Duration::from_secs(4); + +// --------------------------------------------------------------------------- +// Capture state machine shared by both frame protocols. +// --------------------------------------------------------------------------- +#[derive(Debug, Default)] +struct CaptureState { + /// (format, width, height, stride) from the frame's `buffer` event. + shm_params: Option<(wl_shm::Format, u32, u32, u32)>, + buffer_done: bool, + y_invert: bool, + ready: bool, + failed: bool, + /// wl_output name events, keyed by the bind-time index user data. + output_names: std::collections::HashMap, +} + +impl CaptureState { + fn reset_frame(&mut self) { + self.shm_params = None; + self.buffer_done = false; + self.y_invert = false; + self.ready = false; + self.failed = false; + } +} + +/// A captured frame with tightly packed rows (`width * 4` bytes per row), +/// y-flip already applied. `format` is the wire format: pixel byte order in +/// memory is B,G,R,[A|X] for `?rgb8888` and R,G,B,[A|X] for `?bgr8888`. +pub struct RawFrame { + pub width: u32, + pub height: u32, + pub format: wl_shm::Format, + pub data: Vec, +} + +impl Dispatch for CaptureState { + fn event( + _state: &mut Self, + _proxy: &wl_registry::WlRegistry, + _event: wl_registry::Event, + _data: &GlobalListContents, + _conn: &Connection, + _qh: &QueueHandle, + ) { + } +} + +impl Dispatch for CaptureState { + fn event( + state: &mut Self, + _proxy: &HyprlandToplevelExportFrameV1, + event: hyprland_toplevel_export_frame_v1::Event, + _data: &(), + _conn: &Connection, + _qh: &QueueHandle, + ) { + use hyprland_toplevel_export_frame_v1::Event; + match event { + Event::Buffer { format: WEnum::Value(format), width, height, stride } => { + state.shm_params = Some((format, width, height, stride)); + } + Event::BufferDone => state.buffer_done = true, + Event::Flags { flags: WEnum::Value(flags) } => { + state.y_invert = + flags.contains(hyprland_toplevel_export_frame_v1::Flags::YInvert); + } + Event::Ready { .. } => state.ready = true, + Event::Failed => state.failed = true, + // damage / linux_dmabuf / unknown-enum values are irrelevant + // for shm capture. + _ => {} + } + } +} + +impl Dispatch for CaptureState { + fn event( + state: &mut Self, + _proxy: &ZwlrScreencopyFrameV1, + event: zwlr_screencopy_frame_v1::Event, + _data: &(), + _conn: &Connection, + _qh: &QueueHandle, + ) { + use zwlr_screencopy_frame_v1::Event; + match event { + Event::Buffer { format: WEnum::Value(format), width, height, stride } => { + state.shm_params = Some((format, width, height, stride)); + } + Event::BufferDone => state.buffer_done = true, + Event::Flags { flags: WEnum::Value(flags) } => { + state.y_invert = flags.contains(zwlr_screencopy_frame_v1::Flags::YInvert); + } + Event::Ready { .. } => state.ready = true, + Event::Failed => state.failed = true, + _ => {} + } + } +} + +/// wl_output bound with a usize index as user data so `name` events can be +/// matched back to the bind order (for focused-monitor selection). +impl Dispatch for CaptureState { + fn event( + state: &mut Self, + _proxy: &WlOutput, + event: wl_output::Event, + data: &usize, + _conn: &Connection, + _qh: &QueueHandle, + ) { + if let wl_output::Event::Name { name } = event { + state.output_names.insert(*data, name); + } + } +} + +delegate_noop!(CaptureState: ignore WlShm); // advertises `format` events +delegate_noop!(CaptureState: WlShmPool); // no events +delegate_noop!(CaptureState: ignore WlBuffer); // `release` event +delegate_noop!(CaptureState: HyprlandToplevelExportManagerV1); // no events +delegate_noop!(CaptureState: ZwlrScreencopyManagerV1); // no events + +// --------------------------------------------------------------------------- +// Deadline-bounded dispatch +// --------------------------------------------------------------------------- + +/// Dispatch events until `done(state)` or the deadline expires. Equivalent to +/// `blocking_dispatch` but with a hard timeout, so a compositor that never +/// answers (or a damage-gated copy) cannot wedge the recording hook path. +fn dispatch_until( + queue: &mut EventQueue, + state: &mut CaptureState, + deadline: Instant, + what: &str, + done: impl Fn(&CaptureState) -> bool, +) -> Result<()> { + loop { + queue.dispatch_pending(state).context("wayland dispatch failed")?; + if done(state) { + return Ok(()); + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { + bail!("wayland capture timed out waiting for {what}"); + } + queue.flush().context("wayland flush failed")?; + let Some(guard) = queue.prepare_read() else { + continue; // events already queued — dispatch them + }; + let fd = guard.connection_fd().as_raw_fd(); + let mut pfd = libc::pollfd { fd, events: libc::POLLIN, revents: 0 }; + let timeout_ms = remaining.as_millis().min(i32::MAX as u128) as i32; + let n = unsafe { libc::poll(&mut pfd, 1, timeout_ms) }; + if n < 0 { + let err = std::io::Error::last_os_error(); + if err.kind() == std::io::ErrorKind::Interrupted { + continue; + } + return Err(err).context("poll on wayland fd failed"); + } + if n == 0 { + bail!("wayland capture timed out waiting for {what}"); + } + match guard.read() { + Ok(_) => {} + // Another reader (none in practice — one connection per capture) + // or a spurious wakeup; loop and retry. + Err(wayland_client::backend::WaylandError::Io(e)) + if e.kind() == std::io::ErrorKind::WouldBlock => {} + Err(e) => return Err(e).context("wayland read failed"), + } + } +} + +// --------------------------------------------------------------------------- +// wl_shm helpers +// --------------------------------------------------------------------------- + +/// Anonymous shared-memory file of `len` bytes (memfd, CLOEXEC). +fn create_shm_file(len: u64) -> Result { + use std::os::fd::FromRawFd; + let name = std::ffi::CString::new("cua-driver-capture").unwrap(); + let fd = unsafe { libc::memfd_create(name.as_ptr(), libc::MFD_CLOEXEC) }; + if fd < 0 { + return Err(std::io::Error::last_os_error()).context("memfd_create failed"); + } + let file = unsafe { File::from_raw_fd(fd) }; + file.set_len(len).context("ftruncate on memfd failed")?; + Ok(file) +} + +struct ShmBuffer { + file: File, + pool: WlShmPool, + buffer: WlBuffer, + params: (wl_shm::Format, u32, u32, u32), +} + +impl ShmBuffer { + fn create( + shm: &WlShm, + qh: &QueueHandle, + params: (wl_shm::Format, u32, u32, u32), + ) -> Result { + let (format, width, height, stride) = params; + let len = stride as u64 * height as u64; + let file = create_shm_file(len)?; + let pool = shm.create_pool(file.as_fd(), len as i32, qh, ()); + let buffer = + pool.create_buffer(0, width as i32, height as i32, stride as i32, format, qh, ()); + Ok(ShmBuffer { file, pool, buffer, params }) + } + + fn destroy(self) { + self.buffer.destroy(); + self.pool.destroy(); + } +} + +/// Copy shm rows into a tightly packed buffer, dropping stride padding and +/// un-flipping when the compositor reported y_invert. +fn pack_rows(raw: &[u8], width: u32, height: u32, stride: u32, y_invert: bool) -> Vec { + let row_bytes = width as usize * 4; + let mut out = Vec::with_capacity(row_bytes * height as usize); + for y in 0..height as usize { + let src_y = if y_invert { height as usize - 1 - y } else { y }; + let start = src_y * stride as usize; + out.extend_from_slice(&raw[start..start + row_bytes]); + } + out +} + +/// Convert a packed frame to RGBA8 in place semantics (returns a new Vec). +/// Returns an error for wl_shm formats outside the four common 32-bit ones. +pub fn frame_to_rgba8(frame: &RawFrame) -> Result> { + let mut rgba = frame.data.clone(); + match frame.format { + // Memory order B,G,R,[A|X] — swap to R,G,B,A. + wl_shm::Format::Argb8888 | wl_shm::Format::Xrgb8888 => { + let opaque = frame.format == wl_shm::Format::Xrgb8888; + for px in rgba.chunks_exact_mut(4) { + px.swap(0, 2); + if opaque { + px[3] = 0xFF; + } + } + } + // Memory order already R,G,B,[A|X]. + wl_shm::Format::Abgr8888 | wl_shm::Format::Xbgr8888 => { + if frame.format == wl_shm::Format::Xbgr8888 { + for px in rgba.chunks_exact_mut(4) { + px[3] = 0xFF; + } + } + } + other => bail!("unsupported wl_shm capture format {other:?}"), + } + Ok(rgba) +} + +/// ffmpeg rawvideo `-pixel_format` name for a wl_shm format, used by the +/// video backend to describe piped frames. +pub fn ffmpeg_pixel_format(format: wl_shm::Format) -> Option<&'static str> { + match format { + wl_shm::Format::Xrgb8888 => Some("bgr0"), + wl_shm::Format::Argb8888 => Some("bgra"), + wl_shm::Format::Xbgr8888 => Some("rgb0"), + wl_shm::Format::Abgr8888 => Some("rgba"), + _ => None, + } +} + +/// The hyprland-toplevel-export protocol handle for a `hyprctl clients` +/// window address: the low 32 bits of the (64-bit) address. +pub fn toplevel_handle(address: u64) -> u32 { + (address & 0xFFFF_FFFF) as u32 +} + +// --------------------------------------------------------------------------- +// One-shot per-window capture (hyprland-toplevel-export-v1) +// --------------------------------------------------------------------------- + +/// Capture one frame of a Hyprland toplevel by its `hyprctl clients` address. +/// Works for occluded windows and windows on other workspaces — the +/// compositor renders the toplevel's own surface tree into the buffer. +/// +/// Fully synchronous; call from a blocking context. Buffer dimensions are +/// physical pixels at the window's render scale (a 1.5x-scaled 800x600 +/// window yields a 1200x900 frame). +pub fn capture_toplevel_frame(address: u64, overlay_cursor: bool) -> Result { + let conn = Connection::connect_to_env().context("WAYLAND_DISPLAY connect failed")?; + let (globals, mut queue) = registry_queue_init::(&conn) + .context("wl_registry global enumeration failed")?; + let qh = queue.handle(); + + let shm: WlShm = globals.bind(&qh, 1..=1, ()).context("compositor lacks wl_shm")?; + let manager: HyprlandToplevelExportManagerV1 = globals + .bind(&qh, 1..=2, ()) + .context("compositor lacks hyprland_toplevel_export_manager_v1 (not Hyprland?)")?; + + let handle = toplevel_handle(address); + let mut state = CaptureState::default(); + let deadline = Instant::now() + CAPTURE_DEADLINE; + let frame = manager.capture_toplevel(overlay_cursor as i32, handle, &qh, ()); + + // Phase 1: buffer params (buffer → buffer_done). + dispatch_until(&mut queue, &mut state, deadline, "toplevel buffer params", |s| { + s.buffer_done || s.failed + })?; + if state.failed { + frame.destroy(); + bail!("toplevel export failed for window 0x{address:x} (handle 0x{handle:08x})"); + } + let Some(params) = state.shm_params else { + frame.destroy(); + bail!("compositor offered no wl_shm buffer params"); + }; + + // Phase 2: copy. ignore_damage=1 — with 0 the copy waits for window + // damage and hangs forever on an idle window. + let shm_buf = match ShmBuffer::create(&shm, &qh, params) { + Ok(b) => b, + Err(e) => { + frame.destroy(); + return Err(e); + } + }; + frame.copy(&shm_buf.buffer, 1); + let copy_result = + dispatch_until(&mut queue, &mut state, deadline, "toplevel frame copy", |s| { + s.ready || s.failed + }); + + frame.destroy(); + let (format, width, height, stride) = params; + let result = (|| { + copy_result?; + if state.failed { + bail!("toplevel export copy failed for window 0x{address:x}"); + } + let mut raw = vec![0u8; stride as usize * height as usize]; + shm_buf + .file + .read_exact_at(&mut raw, 0) + .context("read from shm file failed")?; + Ok(RawFrame { + width, + height, + format, + data: pack_rows(&raw, width, height, stride, state.y_invert), + }) + })(); + shm_buf.destroy(); + result +} + +/// Per-window PNG capture for the screenshot path. Converts to RGBA and +/// encodes with the shared core encoder. +pub fn capture_toplevel_png(address: u64) -> Result> { + let frame = capture_toplevel_frame(address, false)?; + let rgba = frame_to_rgba8(&frame)?; + cua_driver_core::image_utils::encode_rgba_to_png(&rgba, frame.width, frame.height) +} + +// --------------------------------------------------------------------------- +// Repeated full-output capture (wlr-screencopy) for video recording +// --------------------------------------------------------------------------- + +/// Metadata for one captured video frame. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FrameInfo { + pub width: u32, + pub height: u32, + pub format: wl_shm::Format, +} + +/// Persistent screencopy session: one Wayland connection plus a reusable +/// shm pool, capturing the same output repeatedly for video frames. +pub struct ScreencopyCapturer { + _conn: Connection, + queue: EventQueue, + state: CaptureState, + shm: WlShm, + manager: ZwlrScreencopyManagerV1, + output: WlOutput, + shm_buf: Option, + /// Scratch buffer for the raw (strided) shm readback. + raw: Vec, +} + +impl ScreencopyCapturer { + /// Connect and select an output: `prefer_name` (e.g. the focused Hyprland + /// monitor) when it matches a wl_output name, else the first output. + pub fn open(prefer_name: Option<&str>) -> Result { + let conn = Connection::connect_to_env().context("WAYLAND_DISPLAY connect failed")?; + let (globals, mut queue) = registry_queue_init::(&conn) + .context("wl_registry global enumeration failed")?; + let qh = queue.handle(); + + let shm: WlShm = globals.bind(&qh, 1..=1, ()).context("compositor lacks wl_shm")?; + let manager: ZwlrScreencopyManagerV1 = globals + .bind(&qh, 3..=3, ()) + .context("compositor lacks zwlr_screencopy_manager_v1 (v3)")?; + + let registry = globals.registry(); + let outputs: Vec = globals + .contents() + .clone_list() + .into_iter() + .filter(|g| g.interface == "wl_output") + .enumerate() + .map(|(idx, g)| registry.bind(g.name, g.version.min(4), &qh, idx)) + .collect(); + if outputs.is_empty() { + bail!("compositor advertises no wl_output"); + } + + let mut state = CaptureState::default(); + // Collect `name` events so prefer_name can match. + queue.roundtrip(&mut state).context("wl_output roundtrip failed")?; + + let chosen = prefer_name + .and_then(|want| { + state + .output_names + .iter() + .find(|(_, name)| name.as_str() == want) + .map(|(idx, _)| *idx) + }) + .unwrap_or(0); + let output = outputs[chosen].clone(); + + Ok(ScreencopyCapturer { + _conn: conn, + queue, + state, + shm, + manager, + output, + shm_buf: None, + raw: Vec::new(), + }) + } + + /// Capture one frame into `out` (tightly packed rows, y-flip applied). + /// Reuses the shm pool while the advertised params are unchanged; + /// recreates it on mode/scale changes. `overlay_cursor` composites the + /// pointer into the frame (video wants it, like x11grab's draw_mouse). + pub fn capture_frame(&mut self, overlay_cursor: bool, out: &mut Vec) -> Result { + let qh = self.queue.handle(); + self.state.reset_frame(); + let deadline = Instant::now() + CAPTURE_DEADLINE; + let frame = self.manager.capture_output(overlay_cursor as i32, &self.output, &qh, ()); + + let negotiation = dispatch_until( + &mut self.queue, + &mut self.state, + deadline, + "screencopy buffer params", + |s| s.buffer_done || s.failed, + ); + if let Err(e) = negotiation { + frame.destroy(); + return Err(e); + } + if self.state.failed { + frame.destroy(); + bail!("screencopy failed before buffer negotiation"); + } + let Some(params) = self.state.shm_params else { + frame.destroy(); + bail!("compositor offered no wl_shm buffer params"); + }; + + // (Re)create the pool when params change (first frame, mode switch). + let recreate = self.shm_buf.as_ref().map(|b| b.params != params).unwrap_or(true); + if recreate { + if let Some(old) = self.shm_buf.take() { + old.destroy(); + } + match ShmBuffer::create(&self.shm, &qh, params) { + Ok(b) => self.shm_buf = Some(b), + Err(e) => { + frame.destroy(); + return Err(e); + } + } + } + let shm_buf = self.shm_buf.as_ref().unwrap(); + + frame.copy(&shm_buf.buffer); + let copy = dispatch_until( + &mut self.queue, + &mut self.state, + deadline, + "screencopy frame copy", + |s| s.ready || s.failed, + ); + frame.destroy(); + copy?; + if self.state.failed { + // Buffer may have been invalidated (e.g. output resize race); + // drop it so the next call renegotiates. + if let Some(old) = self.shm_buf.take() { + old.destroy(); + } + bail!("screencopy frame copy failed"); + } + + let (format, width, height, stride) = params; + let len = stride as usize * height as usize; + self.raw.resize(len, 0); + shm_buf + .file + .read_exact_at(&mut self.raw, 0) + .context("read from shm file failed")?; + + let row_bytes = width as usize * 4; + out.clear(); + out.reserve(row_bytes * height as usize); + for y in 0..height as usize { + let src_y = if self.state.y_invert { height as usize - 1 - y } else { y }; + let start = src_y * stride as usize; + out.extend_from_slice(&self.raw[start..start + row_bytes]); + } + Ok(FrameInfo { width, height, format }) + } +} + +impl Drop for ScreencopyCapturer { + fn drop(&mut self) { + if let Some(buf) = self.shm_buf.take() { + buf.destroy(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn toplevel_handle_truncates_to_low_32_bits() { + // The protocol doc's own example: address d161e7b0 -> 3512854448. + assert_eq!(toplevel_handle(0xd161e7b0), 3512854448); + // Full 64-bit heap pointer keeps only the low word. + assert_eq!(toplevel_handle(0x56226558b1d0), 0x6558b1d0); + assert_eq!(toplevel_handle(0xFFFF_FFFF_0000_0001), 1); + } + + #[test] + fn pack_rows_drops_stride_padding() { + // 2x2 frame, stride 12 (4 bytes padding per row). + let raw: Vec = vec![ + 1, 1, 1, 1, 2, 2, 2, 2, 99, 99, 99, 99, // row 0 + pad + 3, 3, 3, 3, 4, 4, 4, 4, 99, 99, 99, 99, // row 1 + pad + ]; + let packed = pack_rows(&raw, 2, 2, 12, false); + assert_eq!(packed, vec![1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4]); + } + + #[test] + fn pack_rows_unflips_y_invert() { + let raw: Vec = vec![ + 1, 1, 1, 1, // row 0 + 2, 2, 2, 2, // row 1 + ]; + let packed = pack_rows(&raw, 1, 2, 4, true); + assert_eq!(packed, vec![2, 2, 2, 2, 1, 1, 1, 1]); + } + + #[test] + fn frame_to_rgba8_swaps_bgr_and_forces_alpha() { + let frame = RawFrame { + width: 1, + height: 1, + format: wl_shm::Format::Xrgb8888, + data: vec![0x10, 0x20, 0x30, 0x00], // B,G,R,X + }; + assert_eq!(frame_to_rgba8(&frame).unwrap(), vec![0x30, 0x20, 0x10, 0xFF]); + + let frame = RawFrame { + width: 1, + height: 1, + format: wl_shm::Format::Abgr8888, + data: vec![0x10, 0x20, 0x30, 0x77], // already R,G,B,A + }; + assert_eq!(frame_to_rgba8(&frame).unwrap(), vec![0x10, 0x20, 0x30, 0x77]); + } + + #[test] + fn ffmpeg_pixel_format_covers_common_formats() { + assert_eq!(ffmpeg_pixel_format(wl_shm::Format::Xrgb8888), Some("bgr0")); + assert_eq!(ffmpeg_pixel_format(wl_shm::Format::Argb8888), Some("bgra")); + assert_eq!(ffmpeg_pixel_format(wl_shm::Format::Xbgr8888), Some("rgb0")); + assert_eq!(ffmpeg_pixel_format(wl_shm::Format::Abgr8888), Some("rgba")); + assert_eq!(ffmpeg_pixel_format(wl_shm::Format::Rgb888), None); + } +} diff --git a/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs index 4539f40e09..fbd7cfcaca 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs @@ -22,10 +22,21 @@ pub struct WindowInfo { /// List top-level windows, optionally filtered by pid. pub fn list_windows(filter_pid: Option) -> Vec { - match list_windows_inner(filter_pid) { + let mut windows = match list_windows_inner(filter_pid) { Ok(w) => w, Err(_) => Vec::new(), + }; + for window in crate::hyprland::list_windows(filter_pid) { + if windows.iter().any(|existing| same_window(existing, &window)) { + continue; + } + windows.push(window); } + windows +} + +fn same_window(a: &WindowInfo, b: &WindowInfo) -> bool { + a.xid == b.xid || (a.pid == b.pid && a.title == b.title) } fn list_windows_inner(filter_pid: Option) -> Result> { diff --git a/libs/python/cua-cli/pyproject.toml b/libs/python/cua-cli/pyproject.toml index 081f41b5ba..8784724433 100644 --- a/libs/python/cua-cli/pyproject.toml +++ b/libs/python/cua-cli/pyproject.toml @@ -111,3 +111,8 @@ ignore = ["E501"] [tool.pytest.ini_options] asyncio_mode = "auto" testpaths = ["tests"] + +[tool.uv.sources] +cua-auto = { path = "../cua-auto", editable = true } +cua-computer = { path = "../computer", editable = true } +cua-core = { path = "../core", editable = true } diff --git a/libs/python/cua-cli/tests/commands/test_auth.py b/libs/python/cua-cli/tests/commands/test_auth.py index 457997a6ee..05d58a1ca7 100644 --- a/libs/python/cua-cli/tests/commands/test_auth.py +++ b/libs/python/cua-cli/tests/commands/test_auth.py @@ -1,6 +1,7 @@ """Tests for auth command module.""" import argparse +from types import SimpleNamespace from unittest.mock import patch from cua_cli.commands import auth @@ -95,49 +96,80 @@ def test_login_with_api_key_direct(self, args_namespace): """Test login with --api-key flag and value.""" args = args_namespace(api_key="test-api-key") - with patch.object(auth, "get_api_key", return_value=None): - with patch.object(auth, "save_api_key") as mock_save: - with patch.object(auth, "print_info"): - with patch.object(auth, "print_success"): - result = auth.cmd_login(args) - - mock_save.assert_called_once_with("test-api-key") + with patch.object( + auth, + "_fetch_me", + return_value=( + 200, + { + "workspace": {"slug": "test-workspace", "name": "Test Workspace"}, + "organization": {"name": "Test Org"}, + }, + ), + ): + with patch.object(auth, "save_workspace") as mock_save_workspace: + with patch.object(auth, "set_active_workspace") as mock_set_active: + with patch.object(auth, "print_info"): + with patch.object(auth, "print_success"): + result = auth.cmd_login(args) + + mock_save_workspace.assert_called_once_with( + "test-workspace", + "test-api-key", + "Test Workspace", + "Test Org", + ) + mock_set_active.assert_called_once_with("test-workspace") assert result == 0 - def test_login_already_authenticated(self, args_namespace): - """Test login when already authenticated.""" - args = args_namespace(api_key=None) + def test_login_with_api_key_legacy_response(self, args_namespace): + """Test direct login when /v1/me returns no workspace metadata.""" + args = args_namespace(api_key="test-api-key") - with patch.object(auth, "get_api_key", return_value="existing-key"): - with patch.object(auth, "print_info") as mock_info: - result = auth.cmd_login(args) + with patch.object(auth, "_fetch_me", return_value=(200, {})): + with patch.object(auth, "save_api_key") as mock_save: + with patch.object(auth, "_get_store") as mock_store: + with patch.object(auth, "print_info"): + with patch.object(auth, "print_success"): + result = auth.cmd_login(args) + mock_save.assert_called_once_with("test-api-key") + mock_store.return_value.delete.assert_called_once_with(auth.ACTIVE_WORKSPACE_KEY) assert result == 0 - mock_info.assert_called() def test_login_browser_flow(self, args_namespace): """Test login with browser OAuth flow.""" args = args_namespace(api_key=None) - with patch.object(auth, "get_api_key", return_value=None): - with patch.object(auth, "run_async") as mock_run: - mock_run.return_value = "browser-api-key" - with patch.object(auth, "save_api_key") as mock_save: + with patch.object(auth, "run_async") as mock_run: + mock_run.return_value = SimpleNamespace( + token="browser-api-key", + workspace_slug="test-workspace", + workspace_name="Test Workspace", + org_name="Test Org", + ) + with patch.object(auth, "save_workspace") as mock_save_workspace: + with patch.object(auth, "set_active_workspace") as mock_set_active: with patch.object(auth, "print_success"): result = auth.cmd_login(args) - mock_save.assert_called_once_with("browser-api-key") + mock_save_workspace.assert_called_once_with( + "test-workspace", + "browser-api-key", + "Test Workspace", + "Test Org", + ) + mock_set_active.assert_called_once_with("test-workspace") assert result == 0 def test_login_browser_flow_timeout(self, args_namespace): """Test login when browser flow times out.""" args = args_namespace(api_key=None) - with patch.object(auth, "get_api_key", return_value=None): - with patch.object(auth, "run_async") as mock_run: - mock_run.side_effect = TimeoutError("Authentication timed out") - with patch.object(auth, "print_error"): - result = auth.cmd_login(args) + with patch.object(auth, "run_async") as mock_run: + mock_run.side_effect = TimeoutError("Authentication timed out") + with patch.object(auth, "print_error"): + result = auth.cmd_login(args) assert result == 1 @@ -145,13 +177,14 @@ def test_login_browser_flow_timeout(self, args_namespace): class TestCmdLogout: """Tests for cmd_logout function.""" - def test_logout_clears_credentials(self, args_namespace): - """Test that logout clears credentials.""" + def test_logout_clears_legacy_credentials(self, args_namespace): + """Test that logout clears legacy credentials when no workspace is active.""" args = args_namespace() - with patch.object(auth, "clear_credentials") as mock_clear: - with patch.object(auth, "print_success"): - result = auth.cmd_logout(args) + with patch.object(auth, "get_active_workspace", return_value=None): + with patch.object(auth, "clear_legacy_credentials") as mock_clear: + with patch.object(auth, "print_success"): + result = auth.cmd_logout(args) mock_clear.assert_called_once() assert result == 0 diff --git a/libs/python/cua-cli/uv.lock b/libs/python/cua-cli/uv.lock index 7996b5c95a..3401ebb110 100644 --- a/libs/python/cua-cli/uv.lock +++ b/libs/python/cua-cli/uv.lock @@ -385,17 +385,13 @@ wheels = [ [[package]] name = "cua-auto" version = "0.1.2" -source = { registry = "https://pypi.org/simple" } +source = { editable = "../cua-auto" } dependencies = [ { name = "pillow" }, { name = "pynput" }, { name = "pyperclip" }, { name = "pywinctl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/69/e8/98e6ecc7a3db4d7b4ba60f990423ad3556af018a750beb08900eef47e607/cua_auto-0.1.2.tar.gz", hash = "sha256:2c5ac6002b12d08b03940e1f8ee547ed206f8b20280380dc8a688b0ab4a56d34", size = 12574, upload-time = "2026-02-26T14:12:09.905Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d9/c3/f82cf30b457b9ff6fe3104caa9cd45d35edc1f184ff4b85fbcbbbc1f5262/cua_auto-0.1.2-py3-none-any.whl", hash = "sha256:d4e4bb9d5121791b6daf85779dd223f1aca9801836dd5bf2f322a01983c5328c", size = 13164, upload-time = "2026-02-26T14:12:08.802Z" }, -] [package.optional-dependencies] all = [ @@ -404,9 +400,25 @@ all = [ { name = "pywinpty", marker = "sys_platform == 'win32'" }, ] +[package.metadata] +requires-dist = [ + { name = "cua-auto", extras = ["mss", "windows", "pty"], marker = "extra == 'all'" }, + { name = "mss", marker = "extra == 'mss'", specifier = ">=9.0.0" }, + { name = "pillow", specifier = ">=10.0.0" }, + { name = "pynput", specifier = ">=1.7.0" }, + { name = "pyperclip", specifier = ">=1.9.0" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, + { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, + { name = "pywin32", marker = "sys_platform == 'win32' and extra == 'windows'", specifier = ">=306" }, + { name = "pywinctl", specifier = ">=0.4" }, + { name = "pywinpty", marker = "sys_platform == 'win32' and extra == 'pty'", specifier = ">=2.0.0" }, + { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.1.0" }, +] +provides-extras = ["mss", "windows", "pty", "all", "dev"] + [[package]] name = "cua-cli" -version = "0.1.8" +version = "0.1.11" source = { editable = "." } dependencies = [ { name = "aiohttp" }, @@ -440,12 +452,12 @@ skills = [ [package.metadata] requires-dist = [ { name = "aiohttp", specifier = ">=3.9.0" }, - { name = "cua-auto", extras = ["all"], specifier = ">=0.1.0" }, + { name = "cua-auto", extras = ["all"], editable = "../cua-auto" }, { name = "cua-cli", extras = ["mcp", "skills"], marker = "extra == 'all'" }, - { name = "cua-computer", specifier = ">=0.5.0" }, - { name = "cua-core", specifier = ">=0.1.18" }, + { name = "cua-computer", editable = "../computer" }, + { name = "cua-core", editable = "../core" }, { name = "fastmcp", marker = "extra == 'mcp'", specifier = ">=2.0" }, - { name = "litellm", marker = "extra == 'skills'", specifier = ">=1.74.0" }, + { name = "litellm", marker = "extra == 'skills'", specifier = "==1.80.0" }, { name = "pillow", specifier = ">=10.0.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0.0" }, { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, @@ -459,8 +471,8 @@ provides-extras = ["mcp", "skills", "all", "dev"] [[package]] name = "cua-computer" -version = "0.5.17" -source = { registry = "https://pypi.org/simple" } +version = "0.5.18" +source = { editable = "../computer" } dependencies = [ { name = "aiohttp" }, { name = "cua-core" }, @@ -470,22 +482,47 @@ dependencies = [ { name = "websocket-client" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f9/42/932b8008cd15e40651b3bef4df700feeca7f962f9220fca69c27b800b907/cua_computer-0.5.17.tar.gz", hash = "sha256:8a8881adbbdbf5e65ae7b1cf93e3f1d9c2781fbadd90acd5de757771b5e312f7", size = 101747, upload-time = "2026-03-04T15:47:38.565Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b4/ba/cd691109cba3251302c1b5bdbddfc69a85bb04ccb37984737c4389a12f55/cua_computer-0.5.17-py3-none-any.whl", hash = "sha256:1c4fb866ea8a0ec2c05b853306437fc38bdd7e2acd4772e4811303a80f08697b", size = 117364, upload-time = "2026-03-04T15:47:37.544Z" }, + +[package.metadata] +requires-dist = [ + { name = "aiohttp", specifier = ">=3.9.0" }, + { name = "cua-core", editable = "../core" }, + { name = "datasets", marker = "extra == 'all'", specifier = ">=3.6.0" }, + { name = "datasets", marker = "extra == 'ui'", specifier = ">=3.6.0" }, + { name = "gradio", marker = "extra == 'all'", specifier = ">=5.23.3" }, + { name = "gradio", marker = "extra == 'ui'", specifier = ">=5.23.3" }, + { name = "mslex", specifier = ">=1.3.0" }, + { name = "pillow", specifier = ">=10.0.0" }, + { name = "pydantic", specifier = ">=2.11.1" }, + { name = "python-dotenv", marker = "extra == 'all'", specifier = ">=1.0.1" }, + { name = "python-dotenv", marker = "extra == 'ui'", specifier = ">=1.0.1" }, + { name = "websocket-client", specifier = ">=1.8.0" }, + { name = "websockets", specifier = ">=12.0" }, ] +provides-extras = ["lume", "lumier", "ui", "all"] [[package]] name = "cua-core" -version = "0.1.18" -source = { registry = "https://pypi.org/simple" } +version = "0.3.1" +source = { editable = "../core" } dependencies = [ { name = "posthog" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/62/07/92955e3643fc41f090b41cd7dc699c7dd4a50c873557e060bb2fa19ccc1b/cua_core-0.1.18.tar.gz", hash = "sha256:8c05c96b3942214769d1ef79d09265b416de4a7c7d647d21541878c744221858", size = 10499, upload-time = "2026-02-25T20:43:02.752Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e0/83/719c32bed7741af5759812ea214c3ea5bb7b7878cac86cb390468b2856fc/cua_core-0.1.18-py3-none-any.whl", hash = "sha256:c66ea5b2b8f0a0d1902efe2cf05e068fa34f4fc4acde1e8f9b6be37a314a6237", size = 9825, upload-time = "2026-02-25T20:43:02.013Z" }, + +[package.metadata] +requires-dist = [ + { name = "opentelemetry-api", marker = "extra == 'otel'", specifier = ">=1.20.0" }, + { name = "opentelemetry-api", marker = "extra == 'telemetry'", specifier = ">=1.20.0" }, + { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'otel'", specifier = ">=1.20.0" }, + { name = "opentelemetry-exporter-otlp-proto-http", marker = "extra == 'telemetry'", specifier = ">=1.20.0" }, + { name = "opentelemetry-sdk", marker = "extra == 'otel'", specifier = ">=1.20.0" }, + { name = "opentelemetry-sdk", marker = "extra == 'telemetry'", specifier = ">=1.20.0" }, + { name = "posthog", specifier = ">=3.20.0" }, ] +provides-extras = ["otel", "telemetry"] + +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=8.3.5" }] [[package]] name = "cyclopts" @@ -1004,7 +1041,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.82.1" +version = "1.80.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -1020,9 +1057,9 @@ dependencies = [ { name = "tiktoken" }, { name = "tokenizers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/34/bd/6251e9a965ae2d7bc3342ae6c1a2d25dd265d354c502e63225451b135016/litellm-1.82.1.tar.gz", hash = "sha256:bc8427cdccc99e191e08e36fcd631c93b27328d1af789839eb3ac01a7d281890", size = 17197496, upload-time = "2026-03-10T09:10:04.438Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/8c/48d533affdbc6d485b7ad4221cd3b40b8c12f9f5568edfe0be0b11e7b945/litellm-1.80.0.tar.gz", hash = "sha256:eeac733eb6b226f9e5fb020f72fe13a32b3354b001dc62bcf1bc4d9b526d6231", size = 11591976, upload-time = "2025-11-16T00:03:51.812Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/57/77/0c6eca2cb049793ddf8ce9cdcd5123a35666c4962514788c4fc90edf1d3b/litellm-1.82.1-py3-none-any.whl", hash = "sha256:a9ec3fe42eccb1611883caaf8b1bf33c9f4e12163f94c7d1004095b14c379eb2", size = 15341896, upload-time = "2026-03-10T09:10:00.702Z" }, + { url = "https://files.pythonhosted.org/packages/ea/53/aa31e4d057b3746b3c323ca993003d6cf15ef987e7fe7ceb53681695ae87/litellm-1.80.0-py3-none-any.whl", hash = "sha256:fd0009758f4772257048d74bf79bb64318859adb4ea49a8b66fdbc718cd80b6e", size = 10492975, upload-time = "2025-11-16T00:03:49.182Z" }, ] [[package]]