diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs index ce5e36c50c..77c53d14d7 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs @@ -201,6 +201,7 @@ pub fn default_capabilities_for(tool_name: &str) -> Vec { "screen.capture.region", ], "get_screen_size" => &["screen.dimensions"], + "get_desktop_state" => &["screen.capture", "screen.dimensions"], "get_cursor_position" => &["screen.cursor.position"], // ── accessibility / window state ───────────────────────────── @@ -520,7 +521,8 @@ mod capability_tests { // keyboard "type_text", "type_text_chars", "press_key", "hotkey", "set_value", // screen - "zoom", "get_screen_size", "get_cursor_position", + "zoom", "get_screen_size", "get_desktop_state", + "get_cursor_position", // accessibility "get_accessibility_tree", "get_window_state", // app / window diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs index 66cfb6c0a3..da3e680867 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs @@ -19,11 +19,12 @@ use cursor_overlay::CursorRegistry; #[derive(Clone)] pub struct DriverConfig { pub capture_mode: String, + pub capture_scope: String, pub max_image_dimension: u32, } impl Default for DriverConfig { - fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 1568 } } + fn default() -> Self { Self { capture_mode: "som".into(), capture_scope: "window".into(), max_image_dimension: 1568 } } } /// Load `DriverConfig` from `~/.cua-driver/config.json`, falling back to @@ -36,6 +37,9 @@ pub fn load_driver_config() -> DriverConfig { if let Some(v) = pip_preview::read_config_value("capture_mode").and_then(|v| v.as_str().map(str::to_owned)) { cfg.capture_mode = v; } + if let Some(v) = pip_preview::read_config_value("capture_scope").and_then(|v| v.as_str().map(str::to_owned)) { + cfg.capture_scope = v; + } if let Some(v) = pip_preview::read_config_value("max_image_dimension").and_then(|v| v.as_u64()) { if let Ok(v32) = u32::try_from(v) { cfg.max_image_dimension = v32; } } @@ -2872,33 +2876,11 @@ impl Tool for GetScreenSizeTool { } async fn invoke(&self, _args: Value) -> ToolResult { let result = tokio::task::spawn_blocking(|| { - use x11rb::connection::Connection; - use x11rb::rust_connection::RustConnection; - let (conn, screen_num) = RustConnection::connect(None) - .map_err(|e| anyhow::anyhow!("{e}{}", crate::no_display_hint()))?; - let setup = conn.setup(); - let screen = &setup.roots[screen_num]; - let w = screen.width_in_pixels as u32; - let h = screen.height_in_pixels as u32; - // WSLg / headless XWayland quirk: the X server connects but the - // root screen advertises a 0-px geometry until a real output is - // attached. Returning {width:0,height:0} here would propagate a - // success with zero dimensions to the client, which then either - // divides by zero when scaling or feeds the value into `int(...)` - // after the missing key collapses to None. Fail loudly with an - // actionable, typed error instead (never emit a 0/null where the - // client expects a usable int). See issue #2005. - if w == 0 || h == 0 { - anyhow::bail!( - "X11 connected but reports a 0x0 root screen — no usable \ - display geometry.{}", - crate::no_display_hint() - ); - } // X11 reports pixel dimensions; scale factor on X11 is not // well-defined per-monitor, so report 1.0 (matches DPI-unaware // assumption). Wayland/HiDPI X11 callers should query // `xrandr --query` for true scale. + let (w, h) = x11_screen_size()?; Ok::<(u32, u32, f64), anyhow::Error>((w, h, 1.0)) }).await; match result { @@ -2911,6 +2893,111 @@ impl Tool for GetScreenSizeTool { } } +/// Read the true X11 root-window size in pixels: (width, height). +/// Shared by `get_screen_size` and `get_desktop_state`. +fn x11_screen_size() -> anyhow::Result<(u32, u32)> { + use x11rb::connection::Connection; + use x11rb::rust_connection::RustConnection; + let (conn, screen_num) = RustConnection::connect(None) + .map_err(|e| anyhow::anyhow!("{e}{}", crate::no_display_hint()))?; + let setup = conn.setup(); + let screen = &setup.roots[screen_num]; + let w = screen.width_in_pixels as u32; + let h = screen.height_in_pixels as u32; + // WSLg / headless XWayland quirk: the X server connects but the + // root screen advertises a 0-px geometry until a real output is + // attached. Returning {width:0,height:0} here would propagate a + // success with zero dimensions to the client, which then either + // divides by zero when scaling or feeds the value into `int(...)` + // after the missing key collapses to None. Fail loudly with an + // actionable, typed error instead (never emit a 0/null where the + // client expects a usable int). See issue #2005. + if w == 0 || h == 0 { + anyhow::bail!( + "X11 connected but reports a 0x0 root screen — no usable \ + display geometry.{}", + crate::no_display_hint() + ); + } + Ok((w, h)) +} + +// ── get_desktop_state ───────────────────────────────────────────────────────── + +pub struct GetDesktopStateTool; +static GDS_DEF: std::sync::OnceLock = std::sync::OnceLock::new(); + +#[async_trait] +impl Tool for GetDesktopStateTool { + fn def(&self) -> &ToolDef { + GDS_DEF.get_or_init(|| ToolDef { + name: "get_desktop_state".into(), + description: "Full-display vision screenshot in true screen pixels (no downscale), \ + for capture_scope=\"desktop\" GUI loops. Captures the entire display (root \ + window) as native-size PNG so screen-absolute pixel coordinates land exactly. \ + No AT-SPI walk.".into(), + input_schema: json!({"type":"object","properties":{ + "session":{"type":"string","description":"Optional session id."}, + "screenshot_out_file":{"type":"string","description":"Write PNG here instead of base64."} + },"additionalProperties":false}), + read_only: true, destructive: false, idempotent: false, open_world: false, + }) + } + + async fn invoke(&self, args: Value) -> ToolResult { + let out_file = args.opt_str("screenshot_out_file"); + + let result = tokio::task::spawn_blocking(move || -> anyhow::Result<_> { + // Vision-only: capture the FULL DISPLAY at native size. No downscale + // so screen-absolute pixels land exactly. + let png = crate::capture::screenshot_display_bytes()?; + let (shot_w, shot_h) = crate::capture::png_dimensions_pub(&png)?; + // True screen size from the X11 root window. + let (screen_w, screen_h) = x11_screen_size()?; + // Optional: write PNG to disk instead of returning base64. + let written = if let Some(path) = out_file.as_deref() { + std::fs::write(path, &png)?; + Some(path.to_string()) + } else { + None + }; + use base64::{engine::general_purpose::STANDARD as B64, Engine as _}; + let b64 = if written.is_some() { None } else { Some(B64.encode(&png)) }; + Ok((b64, shot_w, shot_h, screen_w, screen_h, written)) + }).await; + + match result { + Ok(Ok((b64_opt, shot_w, shot_h, screen_w, screen_h, written))) => { + let mut content = Vec::new(); + let mut structured = json!({ + "platform": "linux", + "screenshot_width": shot_w, + "screenshot_height": shot_h, + "screen_width": screen_w, + "screen_height": screen_h, + "screenshot_mime_type": "image/png", + }); + if let Some(b64) = b64_opt { + content.push(cua_driver_core::protocol::Content::image_png(b64)); + } + if let Some(path) = written { + structured["screenshot_file_path"] = json!(path); + content.push(cua_driver_core::protocol::Content::text(format!( + "✅ Desktop screenshot {shot_w}x{shot_h} written to {path} (screen {screen_w}x{screen_h})" + ))); + } else { + content.push(cua_driver_core::protocol::Content::text(format!( + "✅ Desktop screenshot {shot_w}x{shot_h} (screen {screen_w}x{screen_h})" + ))); + } + ToolResult { content, is_error: None, structured_content: Some(structured) } + } + Ok(Err(e)) => ToolResult::error(format!("Capture error: {e}")), + Err(e) => ToolResult::error(format!("Task error: {e}")), + } + } +} + // ── get_cursor_position ─────────────────────────────────────────────────────── pub struct GetCursorPositionTool; @@ -3373,6 +3460,7 @@ impl Tool for GetConfigTool { "version": env!("CARGO_PKG_VERSION"), "platform": "linux", "capture_mode": cfg.capture_mode, + "capture_scope": cfg.capture_scope, "max_image_dimension": cfg.max_image_dimension, "experimental_pip": pip_enabled, "experimental_pip_geometry": pip_geometry @@ -3405,6 +3493,7 @@ impl Tool for SetConfigTool { "key":{"type":"string","description":"Name of a single config field to write ({key, value} shape). Pair with `value`."}, "value":{"description":"New value for `key`. JSON type depends on the key."}, "capture_mode":{"type":"string","enum":["som","vision","ax"],"description":"Legacy per-field shape. Default capture mode for get_window_state."}, + "capture_scope":{"type":"string","enum":["window","desktop"],"description":"Capture scope: single window or whole desktop. Default window."}, "max_image_dimension":{"type":"integer","description":"Legacy per-field shape. Max dimension for screenshot resizing (0 = no limit)."}, "experimental_pip":{"type":"boolean","description":"Enable the experimental PiP preview window (applies next restart; Linux backend stubbed)."}, "experimental_pip_geometry":{"type":"string","description":"PiP window size + optional position in `WxH` or `WxH+X+Y` form."} @@ -3435,6 +3524,17 @@ impl Tool for SetConfigTool { } None => return ToolResult::error(format!("`capture_mode` must be a string, got {val}.")), }, + "capture_scope" => match val.as_str() { + Some(s @ ("window" | "desktop")) => { + cfg.capture_scope = s.to_owned(); + if let Err(e) = pip_preview::write_config_key("capture_scope", Value::String(s.to_owned())) { + tracing::warn!("set_config: failed to persist capture_scope: {e}"); + } + parts.push(format!("capture_scope={s}")); + } + Some(other) => return ToolResult::error(format!("`capture_scope` must be \"window\" or \"desktop\", got \"{other}\".")), + None => return ToolResult::error(format!("`capture_scope` must be a string, got {val}.")), + }, "max_image_dimension" => match val.as_u64() { Some(n) => { cfg.max_image_dimension = n as u32; @@ -3469,7 +3569,7 @@ impl Tool for SetConfigTool { None => return ToolResult::error(format!("`experimental_pip_geometry` must be a string, got {val}.")), }, other => return ToolResult::error(format!( - "Unknown config key `{other}`. Known: capture_mode, max_image_dimension, experimental_pip, experimental_pip_geometry." + "Unknown config key `{other}`. Known: capture_mode, capture_scope, max_image_dimension, experimental_pip, experimental_pip_geometry." )), } } @@ -3481,6 +3581,16 @@ impl Tool for SetConfigTool { parts.push(format!("capture_mode={mode}")); cfg.capture_mode = mode; } + if let Some(scope) = args.opt_str("capture_scope") { + if scope != "window" && scope != "desktop" { + return ToolResult::error(format!("`capture_scope` must be \"window\" or \"desktop\", got \"{scope}\".")); + } + if let Err(e) = pip_preview::write_config_key("capture_scope", Value::String(scope.clone())) { + tracing::warn!("set_config: failed to persist capture_scope: {e}"); + } + parts.push(format!("capture_scope={scope}")); + cfg.capture_scope = scope; + } if let Some(dim) = args.opt_u64("max_image_dimension") { cfg.max_image_dimension = dim as u32; if let Err(e) = pip_preview::write_config_key("max_image_dimension", Value::from(dim)) { @@ -3514,6 +3624,7 @@ impl Tool for SetConfigTool { ToolResult::text(msg) .with_structured(json!({ "capture_mode": cfg.capture_mode, + "capture_scope": cfg.capture_scope, "max_image_dimension": cfg.max_image_dimension, "experimental_pip": pip_enabled, "experimental_pip_geometry": pip_geometry @@ -3867,6 +3978,7 @@ pub fn build_registry(compat: bool) -> ToolRegistry { // screenshot path is `get_window_state` with `capture_mode:"vision"`. let _ = compat; r.register(Box::new(GetScreenSizeTool)); + r.register(Box::new(GetDesktopStateTool)); r.register(Box::new(GetCursorPositionTool)); r.register(Box::new(MoveCursorTool { state: state.clone() })); r.register(Box::new(SetAgentCursorEnabledTool { state: state.clone() })); @@ -3930,3 +4042,13 @@ mod click_button_schema_tests { assert!(lc.contains("wayland"), "description should call out wayland fallback"); } } + +#[cfg(test)] +mod driver_config_tests { + use super::DriverConfig; + + #[test] + fn capture_scope_defaults_to_window() { + assert_eq!(DriverConfig::default().capture_scope, "window"); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_config.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_config.rs index f238a36d9f..0decd59c73 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_config.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_config.rs @@ -37,9 +37,11 @@ impl Tool for GetConfigTool { // sees its own override layered over the global; the anonymous session // (absent `_session_id`) sees the raw global — today's behavior. let session_id = args.opt_str("_session_id"); - let (capture_mode, max_image_dimension) = { + let (capture_mode, max_image_dimension, capture_scope) = { let cfg = self.state.config.read().unwrap(); - self.state.session_config.effective(session_id.as_deref(), &cfg) + let (mode, dim) = self.state.session_config.effective(session_id.as_deref(), &cfg); + let scope = self.state.session_config.effective_scope(session_id.as_deref(), &cfg); + (mode, dim, scope) }; // Report the CALLING session's own cursor enabled-state, not a // nondeterministic HashMap.first(). Resolve the same key the click / @@ -61,6 +63,7 @@ impl Tool for GetConfigTool { "version": env!("CARGO_PKG_VERSION"), "platform": "macos", "capture_mode": capture_mode, + "capture_scope": capture_scope, "max_image_dimension": max_image_dimension, "agent_cursor": { "enabled": cursor_enabled, diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs new file mode 100644 index 0000000000..9413983471 --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_desktop_state.rs @@ -0,0 +1,152 @@ +//! `get_desktop_state` — full-display vision screenshot (macOS). +//! +//! Vision-only desktop capture: grabs the ENTIRE main display at native +//! pixel size (no downscale) so screen-absolute pixel picks land exactly, +//! then reports the true screen size + backing scale. No AX walk, no +//! pid/window_id — this is the capture surface for `capture_scope="desktop"` +//! GUI loops where the agent drives `click(x,y)` / `scroll(x,y)` against +//! screen-absolute coordinates. +//! +//! Mirrors `get_window_state.rs`'s vision ToolResult shape: an `image_png` +//! content part (or a written-out file path), a text summary line, and a +//! `structuredContent` object. + +use async_trait::async_trait; +use cua_driver_core::{protocol::{ToolResult, Content}, tool::{Tool, ToolDef}}; +use serde_json::Value; + +use super::get_screen_size::main_screen_size; + +pub struct GetDesktopStateTool; + +static DEF: std::sync::OnceLock = std::sync::OnceLock::new(); + +fn def() -> &'static ToolDef { + DEF.get_or_init(|| ToolDef { + name: "get_desktop_state".into(), + description: "Capture a full-display vision screenshot in true screen pixels \ + (no downscale), for capture_scope=\"desktop\" GUI loops where the agent then \ + drives click(x,y)/scroll(x,y) with no pid/window_id. Returns the PNG at native \ + display resolution plus the true screen size and backing scale factor so \ + screen-absolute pixel picks land exactly. Vision-only: no AX tree walk." + .into(), + input_schema: serde_json::json!({ + "type": "object", + "properties": { + "session": { "type": "string", "description": "Optional session id." }, + "screenshot_out_file": { "type": "string", "description": "Write PNG here instead of base64." } + }, + "additionalProperties": false + }), + read_only: true, + destructive: false, + idempotent: false, + open_world: false, + }) +} + +#[async_trait] +impl Tool for GetDesktopStateTool { + fn def(&self) -> &ToolDef { def() } + + async fn invoke(&self, args: Value) -> ToolResult { + use cua_driver_core::tool_args::ArgsExt; + + let screenshot_out_file = args.opt_str("screenshot_out_file").map(|s| { + // Expand ~ prefix (mirrors get_window_state). + if s.starts_with("~/") { + let home = std::env::var("HOME").unwrap_or_default(); + format!("{home}/{}", &s[2..]) + } else { + s + } + }); + + // True screen geometry (points + backing scale). Safe off the main thread. + let (screen_width, screen_height, scale_factor) = match main_screen_size() { + Some(t) => t, + None => return ToolResult::error("No main display detected."), + }; + + // Capture the FULL display at native size — no resize. Run the + // blocking screencapture subprocess off the async runtime. + let out_file = screenshot_out_file.clone(); + let res = tokio::task::spawn_blocking( + move || -> anyhow::Result<(Option, Option, u32, u32)> { + use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; + let png = crate::capture::screenshot_display_bytes()?; + let (w, h) = crate::capture::png_dimensions(&png)?; + if let Some(ref path) = out_file { + std::fs::write(path, &png)?; + Ok((None, Some(path.clone()), w, h)) + } else { + Ok((Some(BASE64.encode(&png)), None, w, h)) + } + }, + ) + .await; + + let (b64_opt, file_path, screenshot_width, screenshot_height) = match res { + Ok(Ok(v)) => v, + Ok(Err(e)) => return ToolResult::error(format!("Desktop screenshot failed: {e}")), + Err(e) => return ToolResult::error(format!("Desktop screenshot task error: {e}")), + }; + + let mut content: Vec = Vec::new(); + if let Some(b64) = b64_opt { + content.push(Content::image_png(b64)); + } + let summary = format!( + "desktop screenshot {screenshot_width}x{screenshot_height} px \ + (screen {screen_width}x{screen_height} pts @ {scale_factor}x)" + ); + content.push(Content::text(summary)); + + let mut structured = serde_json::json!({ + "platform": "macos", + "screenshot_width": screenshot_width, + "screenshot_height": screenshot_height, + "screen_width": screen_width, + "screen_height": screen_height, + "scale_factor": scale_factor, + "screenshot_mime_type": "image/png", + }); + if let Some(ref fp) = file_path { + structured["screenshot_file_path"] = serde_json::json!(fp); + } + + ToolResult { content, is_error: None, structured_content: Some(structured) } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn schema_has_no_pid_or_window_id_and_is_read_only() { + let d = def(); + assert!(d.read_only, "get_desktop_state must be read_only"); + assert!(!d.destructive); + assert!(!d.idempotent); + assert!(!d.open_world); + + let props = d.input_schema["properties"].as_object().unwrap(); + assert!(!props.contains_key("pid"), "must not accept pid"); + assert!(!props.contains_key("window_id"), "must not accept window_id"); + assert!(!props.contains_key("capture_mode"), "must not accept capture_mode"); + assert!(props.contains_key("session")); + assert!(props.contains_key("screenshot_out_file")); + assert_eq!(d.input_schema["additionalProperties"], serde_json::json!(false)); + } + + #[test] + fn description_mentions_full_and_screen_or_display() { + let desc = def().description.to_lowercase(); + assert!(desc.contains("full"), "description must mention 'full'"); + assert!( + desc.contains("screen") || desc.contains("display"), + "description must mention 'screen' or 'display'" + ); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_screen_size.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_screen_size.rs index 5fa196c120..dc39dee39a 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_screen_size.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_screen_size.rs @@ -45,7 +45,7 @@ impl Tool for GetScreenSizeTool { /// The previous NSScreen-based implementation required `MainThreadMarker::new()` /// which always returns `None` on async tokio threads, causing the tool to /// return an error even when a display is attached. -fn main_screen_size() -> Option<(i64, i64, f64)> { +pub(crate) fn main_screen_size() -> Option<(i64, i64, f64)> { use core_graphics::display::{CGMainDisplayID, CGDisplayBounds}; // SAFETY: CGMainDisplayID / CGDisplayBounds are thread-safe CG APIs. diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs index 7d4457669e..6abd2db36b 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs @@ -21,6 +21,7 @@ mod scroll; // etc.) live elsewhere under CuaDriverCore::Capture and are reached // through GetWindowStateTool. pub(crate) mod get_screen_size; +mod get_desktop_state; mod get_cursor_position; mod move_cursor; mod cursor_tools; @@ -114,6 +115,9 @@ impl ResizeRegistry { pub struct DriverConfig { /// Default capture_mode for get_window_state when not specified per-call. pub capture_mode: String, + /// Capture scope for get_window_state: "window" (default) crops to the + /// target window; "desktop" captures the full display. + pub capture_scope: String, /// Max screenshot dimension (0 = no limit). Applied during screenshot/zoom. /// Default 1568 matches Swift's `CuaDriverConfig.defaultMaxImageDimension` — /// the long edge is downscaled to this before encoding. @@ -124,6 +128,7 @@ impl Default for DriverConfig { fn default() -> Self { Self { capture_mode: "som".to_owned(), + capture_scope: "window".to_string(), max_image_dimension: 1568, } } @@ -153,6 +158,9 @@ pub fn load_driver_config() -> DriverConfig { if let Some(v) = json.get("capture_mode").and_then(|v| v.as_str()) { cfg.capture_mode = v.to_owned(); } + if let Some(v) = json.get("capture_scope").and_then(|v| v.as_str()) { + cfg.capture_scope = v.to_owned(); + } if let Some(v) = json.get("max_image_dimension").and_then(|v| v.as_u64()) { if let Ok(v32) = u32::try_from(v) { cfg.max_image_dimension = v32; @@ -194,6 +202,7 @@ pub fn write_driver_config_key(key: &str, value: &serde_json::Value) -> Result<( #[derive(Clone, Default)] pub struct ConfigOverrides { pub capture_mode: Option, + pub capture_scope: Option, pub max_image_dimension: Option, } @@ -223,6 +232,9 @@ impl SessionConfigRegistry { if delta.capture_mode.is_some() { entry.capture_mode = delta.capture_mode; } + if delta.capture_scope.is_some() { + entry.capture_scope = delta.capture_scope; + } if delta.max_image_dimension.is_some() { entry.max_image_dimension = delta.max_image_dimension; } @@ -242,6 +254,18 @@ impl SessionConfigRegistry { } } + /// Resolve the effective `capture_scope` for `session`, layering its + /// override over the global `DriverConfig`. Kept separate from `effective()` + /// so existing `(String, u32)` call sites stay unchanged. `session = None` + /// (anonymous) returns the global scope verbatim. + pub fn effective_scope(&self, session_id: Option<&str>, global: &DriverConfig) -> String { + let ov = session_id.and_then(|s| self.inner.lock().unwrap().get(s).cloned()); + match ov { + Some(ov) => ov.capture_scope.unwrap_or_else(|| global.capture_scope.clone()), + None => global.capture_scope.clone(), + } + } + /// Drop `session`'s overrides. No-op for an unknown id (so `session_end` /// for an anonymous / never-set session is harmless). pub fn clear(&self, session: &str) { @@ -330,6 +354,7 @@ pub fn register_all(registry: &mut ToolRegistry, compat: bool) { // screenshot path is `get_window_state` with `capture_mode:"vision"`. let _ = compat; registry.register(Box::new(get_screen_size::GetScreenSizeTool)); + registry.register(Box::new(get_desktop_state::GetDesktopStateTool)); registry.register(Box::new(get_cursor_position::GetCursorPositionTool)); registry.register(Box::new(move_cursor::MoveCursorTool::new(state.clone()))); registry.register(Box::new(cursor_tools::SetAgentCursorEnabledTool::new(state.clone()))); @@ -374,7 +399,35 @@ mod session_config_guard_tests { use cua_driver_core::session::fire_session_end; fn overrides(mode: &str) -> ConfigOverrides { - ConfigOverrides { capture_mode: Some(mode.to_owned()), max_image_dimension: None } + ConfigOverrides { capture_mode: Some(mode.to_owned()), capture_scope: None, max_image_dimension: None } + } + + #[test] + fn default_capture_scope_is_window() { + assert_eq!(DriverConfig::default().capture_scope, "window"); + } + + #[test] + fn effective_scope_uses_session_override_then_global() { + let reg = SessionConfigRegistry::new(); + let global = DriverConfig::default(); + let sid = "wb-scope-live-C1D2E3"; + assert!(!cua_driver_core::session::is_session_ended(sid)); + + // No override yet → falls back to global ("window"). + assert_eq!(reg.effective_scope(Some(sid), &global), "window"); + // Anonymous → global. + assert_eq!(reg.effective_scope(None, &global), "window"); + + // Live session override applies. + reg.set(sid, ConfigOverrides { + capture_mode: None, + capture_scope: Some("desktop".to_owned()), + max_image_dimension: None, + }); + assert_eq!(reg.effective_scope(Some(sid), &global), "desktop"); + // Other sessions / anonymous still see global. + assert_eq!(reg.effective_scope(None, &global), "window"); } #[test] diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/set_config.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/set_config.rs index 345c881dd0..d5c82420c5 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/set_config.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/set_config.rs @@ -31,6 +31,12 @@ fn def() -> &'static ToolDef { "enum": ["som", "vision", "ax"], "description": "Default capture mode for get_window_state." }, + "capture_scope": { + "type": "string", + "enum": ["window", "desktop"], + "description": "Capture scope for get_window_state: 'window' crops to the \ + target window; 'desktop' captures the full display." + }, "max_image_dimension": { "type": "integer", "description": "Max dimension for screenshot resizing (0 = no limit)." @@ -80,10 +86,22 @@ impl Tool for SetConfigTool { }; let capture_mode = args.opt_str("capture_mode"); + // Validate capture_scope up front so both branches share the check and + // we never half-apply an invalid value. + let capture_scope = args.opt_str("capture_scope"); + if let Some(scope) = capture_scope.as_deref() { + if scope != "window" && scope != "desktop" { + return ToolResult::error(format!( + "capture_scope `{scope}` is invalid; expected `window` or `desktop`" + )); + } + } + let (effective_mode, effective_dim) = if let Some(sid) = session_id.as_deref() { // Session-scoped override: in-memory only, no global write, no disk. self.state.session_config.set(sid, ConfigOverrides { capture_mode: capture_mode.clone(), + capture_scope: capture_scope.clone(), max_image_dimension: max_dim, }); self.state.session_config.effective(Some(sid), &self.state.config.read().unwrap()) @@ -97,6 +115,12 @@ impl Tool for SetConfigTool { tracing::warn!("set_config: failed to persist capture_mode: {e}"); } } + if let Some(scope) = capture_scope.clone() { + cfg.capture_scope = scope.clone(); + if let Err(e) = write_driver_config_key("capture_scope", &Value::String(scope)) { + tracing::warn!("set_config: failed to persist capture_scope: {e}"); + } + } if let Some(dim32) = max_dim { cfg.max_image_dimension = dim32; if let Err(e) = write_driver_config_key("max_image_dimension", &Value::Number(u64::from(dim32).into())) { @@ -105,6 +129,13 @@ impl Tool for SetConfigTool { } (cfg.capture_mode.clone(), cfg.max_image_dimension) }; + // Resolve the effective capture_scope for the echo via the same + // session/global precedence (kept separate from `effective()` so its + // `(String, u32)` signature stays unchanged for other call sites). + let effective_scope = self + .state + .session_config + .effective_scope(session_id.as_deref(), &self.state.config.read().unwrap()); // PiP keys persist to the same config.json but take effect only on // next daemon restart — the backend is initialised once at startup. let mut pip_note = String::new(); @@ -134,8 +165,8 @@ impl Tool for SetConfigTool { "" }; ToolResult::text(format!( - "Config updated: capture_mode={}, max_image_dimension={}{}{}", - effective_mode, effective_dim, scope_note, pip_note + "Config updated: capture_mode={}, capture_scope={}, max_image_dimension={}{}{}", + effective_mode, effective_scope, effective_dim, scope_note, pip_note )) } } diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs index d1bb662f88..3209681ca4 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs @@ -15,7 +15,7 @@ pub mod dispatch; pub mod inject; pub use inject::{inject_click_screen, inject_key_cloaked, inject_text_cloaked, NoActivateGuard}; -pub use mouse::{is_chromium_target_window, post_click, post_click_screen, send_click_synthesized}; +pub use mouse::{is_chromium_target_window, post_click, post_click_screen, send_click_synthesized, send_wheel_synthesized}; pub use keyboard::{ is_xaml_host_hwnd, post_char, post_key, post_type_text, post_type_text_with_delay, send_key_synthesized, diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs index 7f8100c7ea..2d1e10c5fb 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs @@ -10,9 +10,10 @@ use std::time::Duration; use windows::Win32::Foundation::{HWND, LPARAM, POINT, WPARAM}; use windows::Win32::Graphics::Gdi::{ClientToScreen, ScreenToClient}; use windows::Win32::UI::Input::KeyboardAndMouse::{ - INPUT, INPUT_0, INPUT_MOUSE, MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP, - MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP, MOUSEEVENTF_MOVE, MOUSEEVENTF_RIGHTDOWN, - MOUSEEVENTF_RIGHTUP, MOUSEEVENTF_VIRTUALDESK, MOUSEINPUT, SendInput, + INPUT, INPUT_0, INPUT_MOUSE, MOUSEEVENTF_ABSOLUTE, MOUSEEVENTF_HWHEEL, MOUSEEVENTF_LEFTDOWN, + MOUSEEVENTF_LEFTUP, MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP, MOUSEEVENTF_MOVE, + MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP, MOUSEEVENTF_VIRTUALDESK, MOUSEEVENTF_WHEEL, + MOUSEINPUT, SendInput, }; use windows::Win32::UI::WindowsAndMessaging::{ ChildWindowFromPointEx, CWP_SKIPDISABLED, CWP_SKIPINVISIBLE, CWP_SKIPTRANSPARENT, @@ -594,3 +595,103 @@ pub fn send_drag_synthesized( Ok(()) } + +/// Standard wheel notch delta. A `mouseData` value of `±WHEEL_DELTA` is one +/// detent of the physical mouse wheel. +const WHEEL_DELTA: i32 = 120; + +/// Compute the `MOUSEINPUT::mouseData` value for a wheel event of `ticks` +/// detents. Positive ticks = wheel forward/up (vertical) or right (horizontal); +/// negative = down / left. `mouseData` is a `u32` field carrying a signed +/// 32-bit delta, so we compute as `i32` then bit-cast to `u32` (this is what +/// the Win32 docs mean by "the value is a multiple of WHEEL_DELTA"). +/// +/// Factored out of [`send_wheel_synthesized`] so the sign/magnitude encoding is +/// unit-testable without a live display / `SendInput`. +fn wheel_mouse_data(ticks: i32) -> u32 { + (WHEEL_DELTA * ticks) as u32 +} + +/// Synthesize a single mouse-wheel event at screen coordinates `(sx, sy)` via +/// `SendInput`. +/// +/// The OS routes wheel input to the window **under the cursor**, not the +/// foreground window, so we `SetCursorPos(sx, sy)` first to place the wheel +/// over the intended target. `ticks` encodes both magnitude and direction: +/// positive scrolls up (vertical) / right (horizontal), negative scrolls down / +/// left — matching the `MOUSEEVENTF_WHEEL` / `MOUSEEVENTF_HWHEEL` convention +/// where `mouseData = WHEEL_DELTA * ticks`. +/// +/// Unlike [`send_click_synthesized`] this does NOT do a foreground swap: wheel +/// delivery follows the cursor, so positioning the cursor is sufficient. The +/// cursor is restored to its previous position afterward. +pub fn send_wheel_synthesized(sx: i32, sy: i32, ticks: i32, horizontal: bool) -> Result<()> { + let flag = if horizontal { MOUSEEVENTF_HWHEEL } else { MOUSEEVENTF_WHEEL }; + let mouse_data = wheel_mouse_data(ticks); + + let wheel_input = INPUT { + r#type: INPUT_MOUSE, + Anonymous: INPUT_0 { + mi: MOUSEINPUT { + dx: 0, + dy: 0, + mouseData: mouse_data, + dwFlags: flag, + time: 0, + dwExtraInfo: 0, + }, + }, + }; + + unsafe { + let mut prev_cursor = POINT::default(); + let _ = GetCursorPos(&mut prev_cursor); + + // Wheel routes to the window under the cursor — place it on the target. + let _ = SetCursorPos(sx, sy); + + let events = [wheel_input]; + let sent = SendInput(&events, std::mem::size_of::() as i32); + if sent as usize != events.len() { + let _ = SetCursorPos(prev_cursor.x, prev_cursor.y); + bail!("SendInput inserted {sent}/{} wheel events", events.len()); + } + + // Brief settle, then restore the cursor. + sleep(Duration::from_millis(20)); + let _ = SetCursorPos(prev_cursor.x, prev_cursor.y); + } + + Ok(()) +} + +#[cfg(test)] +mod wheel_tests { + use super::{wheel_mouse_data, WHEEL_DELTA}; + + #[test] + fn wheel_data_up_is_positive_one_notch() { + // +1 tick (up / right) → +WHEEL_DELTA, bit-cast to u32. + assert_eq!(wheel_mouse_data(1), WHEEL_DELTA as u32); + assert_eq!(wheel_mouse_data(1), 120u32); + } + + #[test] + fn wheel_data_down_is_negative_one_notch() { + // -1 tick (down / left) → -WHEEL_DELTA, bit-cast: 0xFFFFFF88. + assert_eq!(wheel_mouse_data(-1), (-WHEEL_DELTA) as u32); + assert_eq!(wheel_mouse_data(-1), 0xFFFF_FF88); + } + + #[test] + fn wheel_data_scales_with_ticks() { + assert_eq!(wheel_mouse_data(3), (3 * WHEEL_DELTA) as u32); + assert_eq!(wheel_mouse_data(3), 360u32); + assert_eq!(wheel_mouse_data(-3) as i32, -360); + } + + #[test] + fn wheel_data_zero_is_zero() { + assert_eq!(wheel_mouse_data(0), 0); + } +} diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs index 1e4249c6db..c930ad460e 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs @@ -154,16 +154,41 @@ pub(crate) fn resolve_cursor_key(args: &Value) -> String { NO_CURSOR.to_owned() } +/// Returns `true` when a click/scroll invocation should take the **window-less +/// screen-absolute** branch: the caller gave numeric `x` AND `y`, gave NO `pid` +/// and NO `window_id`, and the effective `capture_scope` is `"desktop"`. +/// +/// Pure + arg-shape agnostic (works for both click and scroll args) so it's +/// unit-testable without Win32. When this returns `false` but `x,y` are present +/// with no pid/window_id, the caller returns a `desktop_scope_disabled` +/// structured error telling the agent to `set_config capture_scope=desktop`. +fn is_windowless_desktop_action(args: &serde_json::Value, scope: &str) -> bool { + if scope != "desktop" { + return false; + } + let has_pid = args.get("pid").map(|v| !v.is_null()).unwrap_or(false); + let has_window_id = args.get("window_id").map(|v| !v.is_null()).unwrap_or(false); + if has_pid || has_window_id { + return false; + } + let has_num = |k: &str| args.get(k).map(|v| v.is_number()).unwrap_or(false); + has_num("x") && has_num("y") +} + // ── DriverConfig + ResizeRegistry + ZoomRegistry ───────────────────────────── #[derive(Clone)] pub struct DriverConfig { pub capture_mode: String, + /// Capture scope for vision loops: `"window"` (per-window, the default) or + /// `"desktop"` (full-display). Gates the window-less screen-absolute + /// click/scroll branches — those activate only under `"desktop"`. + pub capture_scope: String, pub max_image_dimension: u32, } impl Default for DriverConfig { - fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 1568 } } + fn default() -> Self { Self { capture_mode: "som".into(), capture_scope: "window".into(), max_image_dimension: 1568 } } } /// Load `DriverConfig` from `~/.cua-driver/config.json`, falling back to @@ -176,6 +201,9 @@ pub fn load_driver_config() -> DriverConfig { if let Some(v) = pip_preview::read_config_value("capture_mode").and_then(|v| v.as_str().map(str::to_owned)) { cfg.capture_mode = v; } + if let Some(v) = pip_preview::read_config_value("capture_scope").and_then(|v| v.as_str().map(str::to_owned)) { + cfg.capture_scope = v; + } if let Some(v) = pip_preview::read_config_value("max_image_dimension").and_then(|v| v.as_u64()) { if let Ok(v32) = u32::try_from(v) { cfg.max_image_dimension = v32; } } @@ -1947,8 +1975,8 @@ impl Tool for ClickTool { tool). The Swift-only `action` / `modifier` / `debug_image_out` schema \ fields aren't supported yet.".into(), input_schema: json!({ - "type":"object","required":["pid"],"properties":{ - "pid":{"type":"integer","description":"Target process ID."}, + "type":"object","properties":{ + "pid":{"type":"integer","description":"Target process ID. Required UNLESS using window-less desktop-scope clicks: with capture_scope=\"desktop\" and no pid/window_id, x/y are treated as TRUE SCREEN pixels (call get_desktop_state first)."}, "window_id":{"type":"integer","description":"HWND for the window whose get_window_state produced the element_index. Required when element_index is used. Optional when element_token is supplied (the token carries it)."}, "element_index":{"type":"integer","description":"Element index from the last get_window_state for the same (pid, window_id)."}, "element_token":{"type":"string","description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."}, @@ -1969,6 +1997,78 @@ impl Tool for ClickTool { use crate::input::dispatch::{DispatchMode, EventKind, background_unavailable_error}; use crate::uia::cache::SnapshotKind; let cursor_key = resolve_cursor_key(&args); + + // ── Window-less screen-absolute branch (capture_scope="desktop") ────── + // When the caller gives x,y with NO pid/window_id, treat x,y as TRUE + // SCREEN pixels. Gate on effective capture_scope: only "desktop" enables + // this; under "window" we return a structured `desktop_scope_disabled` + // error pointing the caller at set_config. + let has_pid = args.get("pid").map(|v| !v.is_null()).unwrap_or(false); + let has_window_id = args.get("window_id").map(|v| !v.is_null()).unwrap_or(false); + let has_xy = args.get("x").map(|v| v.is_number()).unwrap_or(false) + && args.get("y").map(|v| v.is_number()).unwrap_or(false); + if !has_pid && !has_window_id && has_xy { + let scope = self.state.config.read().unwrap().capture_scope.clone(); + if !is_windowless_desktop_action(&args, &scope) { + return ToolResult::error( + "click: x,y given with no pid/window_id, but capture_scope is \ + \"window\". Screen-absolute clicks require desktop scope. Call \ + set_config with capture_scope=desktop (and use get_desktop_state \ + to pick coordinates), or pass a pid/window_id." + ) + .with_structured(json!({ + "code": "desktop_scope_disabled", + "capture_scope": scope, + "suggestion": "set_config capture_scope=desktop", + })); + } + // Resolve button (reuse the same validation as the pid path). + let button_raw = args.str_or("button", "left").to_lowercase(); + if !matches!(button_raw.as_str(), "" | "left" | "right" | "middle") { + return ToolResult::error(format!( + "click: unknown button \"{button_raw}\" — expected one of left, right, middle." + )); + } + let button = if button_raw.is_empty() { "left".to_string() } else { button_raw }; + let count = args.u64_or("count", 1) as usize; + let sx = args.f64_or("x", 0.0) as i32; + let sy = args.f64_or("y", 0.0) as i32; + + // Animate the agent cursor to the screen point, then click. + overlay_glide_to(&cursor_key, sx as f64, sy as f64).await; + crate::overlay::send_command(cursor_key.clone(), cursor_overlay::OverlayCommand::ClickPulse { + x: sx as f64, y: sy as f64, + }); + + // Resolve the HWND that owns this screen pixel and click it via + // send_click_synthesized — it does the foreground-swap + UIPI checks + // on whatever owns the pixel, which is what lands Chromium-content + // clicks. WindowFromPoint walks to the leaf window at the point. + // (send_click_synthesized restores the previous foreground + cursor + // itself ~40ms after the click, so no extra restore guard here.) + let send_result = tokio::task::spawn_blocking(move || -> anyhow::Result { + use windows::Win32::Foundation::POINT; + use windows::Win32::UI::WindowsAndMessaging::WindowFromPoint; + let target = unsafe { WindowFromPoint(POINT { x: sx, y: sy }) }; + if target.0.is_null() { + anyhow::bail!("No window under screen point ({sx},{sy})."); + } + let hwnd_u = target.0 as u64; + crate::input::send_click_synthesized(hwnd_u, sx, sy, count, &button)?; + Ok(hwnd_u) + }).await; + return match send_result { + Ok(Ok(hwnd_u)) => { + let click_word = match count { 2 => "double-click", 3 => "triple-click", _ => "click" }; + ToolResult::text(format!( + "✅ Sent {click_word} via SendInput at screen ({sx},{sy}) on HWND 0x{hwnd_u:x} (desktop scope)." + )) + } + Ok(Err(e)) => ToolResult::error(e.to_string()), + Err(e) => ToolResult::error(format!("Task error: {e}")), + }; + } + let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e }; // Surface 6: element_token / element_index precedence resolution. // Windows uses u64 HWND but the token registry stores u32; truncate @@ -3318,7 +3418,9 @@ impl Tool for SetValueTool { // ── scroll ──────────────────────────────────────────────────────────────────── -pub struct ScrollTool; +pub struct ScrollTool { + state: Arc, +} static SCROLL_DEF: std::sync::OnceLock = std::sync::OnceLock::new(); #[async_trait] @@ -3339,12 +3441,14 @@ impl Tool for ScrollTool { Note: `element_index` is accepted for cross-platform parity but currently \ no-op on Windows (UIA SetFocus not wired up yet — same caveat as `press_key`).".into(), input_schema: json!({ - "type":"object","required":["pid","direction"],"properties":{ - "pid":{"type":"integer","description":"Target process ID."}, + "type":"object","required":["direction"],"properties":{ + "pid":{"type":"integer","description":"Target process ID. Required UNLESS using window-less desktop-scope scroll: with capture_scope=\"desktop\" and no pid/window_id, x/y are TRUE SCREEN pixels and the wheel routes to the window under that point."}, "direction":{"type":"string","enum":["up","down","left","right"]}, "by":{"type":"string","enum":["line","page"],"description":"Scroll granularity. Default: line."}, "amount":{"type":"integer","minimum":1,"maximum":50, "description":"Number of scroll ticks. Default 3."}, + "x":{"type":"number","description":"Screen-absolute X (desktop scope only) — wheel routes to the window under (x,y). Must be paired with y and no pid/window_id."}, + "y":{"type":"number","description":"Screen-absolute Y (desktop scope only). Must be paired with x and no pid/window_id."}, "window_id":{"type":"integer","description":"HWND of the target window. Required when element_index is used; otherwise auto-resolves the pid's first visible window."}, "element_index":{"type":"integer","description":"Optional element_index. Accepted for parity; currently no-op on Windows."}, "element_token":{"type":"string","description":"Opaque per-snapshot element handle from `structuredContent.elements[].element_token`. Takes precedence over element_index when both supplied. Returns an explicit \"stale\" error if the snapshot has been superseded."}, @@ -3361,6 +3465,65 @@ impl Tool for ScrollTool { // scroll-specific structured error (see commit 9e30d2cb). The // dispatch enums are still in use just below. use crate::input::dispatch::{DispatchMode, EventKind}; + use cua_driver_core::tool_args::ArgsExt; + // `direction` is required in both the pid path and the window-less + // desktop path, so resolve it before the pid check. + let direction = match args.get("direction").and_then(|v| v.as_str()) { + Some(d) => d.to_owned(), + None => return ToolResult::error("Missing required string field direction."), + }; + let amount = args.u64_or("amount", 3).clamp(1, 50) as u32; + + // ── Window-less screen-absolute branch (capture_scope="desktop") ────── + // No pid/window_id + numeric x,y + desktop scope → synthesize a wheel + // event at the screen point via SendInput. The wheel routes to whatever + // window is under (x,y). up/down map to a vertical wheel (sign), and + // left/right to a horizontal wheel; `amount` is the tick count. + let has_pid = args.get("pid").map(|v| !v.is_null()).unwrap_or(false); + let has_window_id = args.get("window_id").map(|v| !v.is_null()).unwrap_or(false); + let has_xy = args.get("x").map(|v| v.is_number()).unwrap_or(false) + && args.get("y").map(|v| v.is_number()).unwrap_or(false); + if !has_pid && !has_window_id && has_xy { + let scope = self.state.config.read().unwrap().capture_scope.clone(); + if !is_windowless_desktop_action(&args, &scope) { + return ToolResult::error( + "scroll: x,y given with no pid/window_id, but capture_scope is \ + \"window\". Screen-absolute scroll requires desktop scope. Call \ + set_config with capture_scope=desktop (and use get_desktop_state \ + to pick coordinates), or pass a pid/window_id." + ) + .with_structured(serde_json::json!({ + "code": "desktop_scope_disabled", + "capture_scope": scope, + "suggestion": "set_config capture_scope=desktop", + })); + } + let sx = args.f64_or("x", 0.0) as i32; + let sy = args.f64_or("y", 0.0) as i32; + // Direction → (horizontal?, sign). Positive ticks = up / right. + let (horizontal, sign) = match direction.as_str() { + "up" => (false, 1), + "down" => (false, -1), + "right" => (true, 1), + "left" => (true, -1), + other => return ToolResult::error(format!( + "scroll: unknown direction \"{other}\" — expected up, down, left, right." + )), + }; + let ticks = sign * amount as i32; + let dir_disp = direction.clone(); + let result = tokio::task::spawn_blocking(move || { + crate::input::send_wheel_synthesized(sx, sy, ticks, horizontal) + }).await; + return match result { + Ok(Ok(())) => ToolResult::text(format!( + "✅ Scrolled {dir_disp} via SendInput wheel ({amount} tick(s)) at screen ({sx},{sy}) (desktop scope)." + )), + Ok(Err(e)) => ToolResult::error(e.to_string()), + Err(e) => ToolResult::error(format!("Task error: {e}")), + }; + } + // Swift error wording 1:1. let raw_pid = match args.get("pid").and_then(|v| v.as_i64()) { Some(p) => p, @@ -3368,15 +3531,9 @@ impl Tool for ScrollTool { }; let pid = raw_pid as u32; let dispatch = DispatchMode::from_args(&args); - let direction = match args.get("direction").and_then(|v| v.as_str()) { - Some(d) => d.to_owned(), - None => return ToolResult::error("Missing required string field direction."), - }; - use cua_driver_core::tool_args::ArgsExt; let by = args.str_or("by", "line"); let direction_display = direction.clone(); let by_display = by.clone(); - let amount = args.u64_or("amount", 3).clamp(1, 50) as u32; // Surface 6: element_token / element_index precedence resolution. let resolved = match cua_driver_core::element_token::resolve_element_args( pid as i32, @@ -4286,6 +4443,17 @@ impl Tool for DragTool { // ── get_screen_size ─────────────────────────────────────────────────────────── +/// Read the primary display size in PHYSICAL pixels. +/// +/// With permonitorv2 DPI awareness (set in cua-driver.manifest), +/// `SM_CXSCREEN` / `SM_CYSCREEN` already return physical pixels — the same +/// coordinate space screenshots and pixel clicks use on Windows. Shared by +/// `get_screen_size` and `get_desktop_state`. +fn physical_screen_size() -> (i32, i32) { + use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_CXSCREEN, SM_CYSCREEN}; + unsafe { (GetSystemMetrics(SM_CXSCREEN), GetSystemMetrics(SM_CYSCREEN)) } +} + pub struct GetScreenSizeTool; static GSS_DEF: std::sync::OnceLock = std::sync::OnceLock::new(); @@ -4302,13 +4470,12 @@ impl Tool for GetScreenSizeTool { }) } async fn invoke(&self, _args: Value) -> ToolResult { - use windows::Win32::UI::WindowsAndMessaging::{GetSystemMetrics, SM_CXSCREEN, SM_CYSCREEN}; use windows::Win32::UI::HiDpi::GetDpiForSystem; // With permonitorv2 DPI awareness (set in cua-driver.manifest), // SM_CXSCREEN/SM_CYSCREEN return PHYSICAL pixels — the same // coordinate space screenshots and pixel clicks use on Windows. // Report these as-is, along with the scale factor for reference. - let (w, h) = unsafe { (GetSystemMetrics(SM_CXSCREEN), GetSystemMetrics(SM_CYSCREEN)) }; + let (w, h) = physical_screen_size(); let dpi = unsafe { GetDpiForSystem() }; let scale = if dpi == 0 { 1.0 } else { dpi as f64 / 96.0 }; ToolResult::text(format!("✅ Main display: {w}x{h} pixels @ {scale}x")) @@ -4316,6 +4483,104 @@ impl Tool for GetScreenSizeTool { } } +// ── get_desktop_state ───────────────────────────────────────────────────────── + +/// `get_desktop_state` — full-display vision screenshot (Windows). +/// +/// Vision-only desktop capture: grabs the ENTIRE primary display at native +/// physical-pixel size (no downscale) so screen-absolute pixel picks land +/// exactly, then reports the true screen size. No UIA walk, no pid/window_id — +/// this is the capture surface for `capture_scope="desktop"` GUI loops where +/// the agent drives `click(x,y)` / `scroll(x,y)` against screen-absolute +/// coordinates. +/// +/// Mirrors the `get_window_state` vision branch's ToolResult shape: an +/// `image_png` content part (or a written-out file path), a text summary line, +/// and a `structuredContent` object. +pub struct GetDesktopStateTool; +static GDS_DEF: std::sync::OnceLock = std::sync::OnceLock::new(); + +#[async_trait] +impl Tool for GetDesktopStateTool { + fn def(&self) -> &ToolDef { + GDS_DEF.get_or_init(|| ToolDef { + name: "get_desktop_state".into(), + description: "Capture a full-display vision screenshot in true screen pixels \ + (no downscale), for capture_scope=\"desktop\" GUI loops where the agent then \ + drives click(x,y)/scroll(x,y) with no pid/window_id. Returns the PNG at native \ + display resolution plus the true screen size so screen-absolute pixel picks \ + land exactly. Vision-only: no UIA tree walk.".into(), + input_schema: json!({ + "type": "object", + "properties": { + "session": { "type": "string", "description": "Optional session id." }, + "screenshot_out_file": { "type": "string", "description": "Write PNG here instead of base64." } + }, + "additionalProperties": false + }), + read_only: true, destructive: false, idempotent: false, open_world: false, + }) + } + + async fn invoke(&self, args: Value) -> ToolResult { + use cua_driver_core::tool_args::ArgsExt; + use cua_driver_core::protocol::Content; + + let screenshot_out_file = args.opt_str("screenshot_out_file"); + + // True screen geometry in physical pixels (same space as the capture). + let (screen_width, screen_height) = physical_screen_size(); + + // Capture the FULL display at native size — no resize. Run the + // blocking GDI capture off the async runtime. + let out_file = screenshot_out_file.clone(); + let res = tokio::task::spawn_blocking( + move || -> anyhow::Result<(Option, Option, u32, u32)> { + use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; + let png = crate::capture::screenshot_display_bytes()?; + let (w, h) = crate::capture::png_dimensions_pub(&png)?; + if let Some(ref path) = out_file { + std::fs::write(path, &png)?; + Ok((None, Some(path.clone()), w, h)) + } else { + Ok((Some(BASE64.encode(&png)), None, w, h)) + } + }, + ) + .await; + + let (b64_opt, file_path, screenshot_width, screenshot_height) = match res { + Ok(Ok(v)) => v, + Ok(Err(e)) => return ToolResult::error(format!("Desktop screenshot failed: {e}")), + Err(e) => return ToolResult::error(format!("Desktop screenshot task error: {e}")), + }; + + let mut content: Vec = Vec::new(); + if let Some(b64) = b64_opt { + content.push(Content::image_png(b64)); + } + let summary = format!( + "desktop screenshot {screenshot_width}x{screenshot_height} px \ + (screen {screen_width}x{screen_height} px)" + ); + content.push(Content::text(summary)); + + let mut structured = json!({ + "platform": "windows", + "screenshot_width": screenshot_width, + "screenshot_height": screenshot_height, + "screen_width": screen_width, + "screen_height": screen_height, + "screenshot_mime_type": "image/png", + }); + if let Some(ref fp) = file_path { + structured["screenshot_file_path"] = json!(fp); + } + + ToolResult { content, is_error: None, structured_content: Some(structured) } + } +} + // ── get_cursor_position ─────────────────────────────────────────────────────── pub struct GetCursorPositionTool; @@ -4970,6 +5235,7 @@ impl Tool for GetConfigTool { "version": env!("CARGO_PKG_VERSION"), "platform": "windows", "capture_mode": cfg.capture_mode, + "capture_scope": cfg.capture_scope, "max_image_dimension": cfg.max_image_dimension, "agent_cursor": { "enabled": cursor_enabled }, "experimental_pip": pip_enabled, @@ -5013,6 +5279,7 @@ impl Tool for SetConfigTool { "key":{"type":"string","description":"Dotted snake_case path to a leaf config field (Swift-compatible shape). Pair with `value`."}, "value":{"description":"New value for `key`. JSON type depends on the key."}, "capture_mode":{"type":"string","enum":["som","vision","ax"],"description":"Legacy per-field shape."}, + "capture_scope":{"type":"string","enum":["window","desktop"],"description":"Capture scope: single window (default) or whole desktop. Desktop scope enables window-less screen-absolute click/scroll (no pid/window_id). Accepted in both the {key,value} and legacy per-field shapes."}, "max_image_dimension":{"type":"integer","description":"Legacy per-field shape."}, "experimental_pip":{"type":"boolean","description":"Legacy per-field shape. Enables PiP preview (applies next restart)."}, "experimental_pip_geometry":{"type":"string","description":"Legacy per-field shape. PiP window size + optional position."} @@ -5039,6 +5306,17 @@ impl Tool for SetConfigTool { } None => return ToolResult::error(format!("`capture_mode` must be a string, got {val}.")), }, + "capture_scope" => match val.as_str() { + Some(s @ ("window" | "desktop")) => { + cfg.capture_scope = s.to_owned(); + if let Err(e) = pip_preview::write_config_key("capture_scope", Value::String(s.to_owned())) { + tracing::warn!("set_config: failed to persist capture_scope: {e}"); + } + applied = true; + } + Some(other) => return ToolResult::error(format!("`capture_scope` must be \"window\" or \"desktop\", got \"{other}\".")), + None => return ToolResult::error(format!("`capture_scope` must be a string, got {val}.")), + }, "max_image_dimension" => match val.as_u64() { Some(n) => { cfg.max_image_dimension = n as u32; @@ -5073,7 +5351,7 @@ impl Tool for SetConfigTool { None => return ToolResult::error(format!("`experimental_pip_geometry` must be a string, got {val}.")), }, other => return ToolResult::error(format!( - "Unknown config key `{other}`. Known: capture_mode, max_image_dimension, experimental_pip, experimental_pip_geometry." + "Unknown config key `{other}`. Known: capture_mode, capture_scope, max_image_dimension, experimental_pip, experimental_pip_geometry." )), } } @@ -5085,6 +5363,16 @@ impl Tool for SetConfigTool { } applied = true; } + if let Some(scope) = args.get("capture_scope").and_then(|v| v.as_str()) { + if !matches!(scope, "window" | "desktop") { + return ToolResult::error(format!("`capture_scope` must be \"window\" or \"desktop\", got \"{scope}\".")); + } + cfg.capture_scope = scope.to_owned(); + if let Err(e) = pip_preview::write_config_key("capture_scope", Value::String(scope.to_owned())) { + tracing::warn!("set_config: failed to persist capture_scope: {e}"); + } + applied = true; + } if let Some(dim) = args.get("max_image_dimension").and_then(|v| v.as_u64()) { cfg.max_image_dimension = dim as u32; if let Err(e) = pip_preview::write_config_key("max_image_dimension", Value::from(dim)) { @@ -5124,6 +5412,7 @@ impl Tool for SetConfigTool { "version": env!("CARGO_PKG_VERSION"), "platform": "windows", "capture_mode": cfg.capture_mode, + "capture_scope": cfg.capture_scope, "max_image_dimension": cfg.max_image_dimension, "agent_cursor": { "enabled": cursor_enabled }, "experimental_pip": pip_enabled, @@ -5897,7 +6186,7 @@ pub fn build_registry(compat: bool) -> ToolRegistry { r.register(Box::new(PressKeyTool)); r.register(Box::new(HotkeyTool)); r.register(Box::new(SetValueTool { state: state.clone() })); - r.register(Box::new(ScrollTool)); + r.register(Box::new(ScrollTool { state: state.clone() })); // `screenshot` / `ScreenshotCompatTool` removed from the tool surface // — `get_window_state` with `capture_mode:"vision"` is the single // canonical path for getting a window screenshot. Reasons: @@ -5917,6 +6206,7 @@ pub fn build_registry(compat: bool) -> ToolRegistry { // depend on them via `Tool` trait reflection. let _ = compat; // formerly drove the ScreenshotCompatTool branch r.register(Box::new(GetScreenSizeTool)); + r.register(Box::new(GetDesktopStateTool)); r.register(Box::new(GetCursorPositionTool)); r.register(Box::new(MoveCursorTool { state: state.clone() })); r.register(Box::new(SetAgentCursorEnabledTool { state: state.clone() })); @@ -6210,3 +6500,75 @@ mod click_button_schema_tests { } } } + +#[cfg(test)] +mod desktop_scope_tests { + use super::{is_windowless_desktop_action, DriverConfig, GetDesktopStateTool}; + use cua_driver_core::tool::Tool; + use serde_json::json; + + // ── is_windowless_desktop_action ────────────────────────────────────────── + + #[test] + fn windowless_true_for_xy_under_desktop_scope_click_shape() { + // Click arg shape: {x, y}. + assert!(is_windowless_desktop_action(&json!({"x": 10, "y": 20}), "desktop")); + } + + #[test] + fn windowless_true_for_xy_under_desktop_scope_scroll_shape() { + // Scroll arg shape: {direction, x, y}. + assert!(is_windowless_desktop_action( + &json!({"direction": "down", "x": 10, "y": 20}), + "desktop" + )); + } + + #[test] + fn windowless_false_when_pid_present() { + assert!(!is_windowless_desktop_action(&json!({"x": 10, "y": 20, "pid": 5}), "desktop")); + } + + #[test] + fn windowless_false_when_window_id_present() { + assert!(!is_windowless_desktop_action( + &json!({"x": 10, "y": 20, "window_id": 99}), + "desktop" + )); + } + + #[test] + fn windowless_false_under_window_scope() { + assert!(!is_windowless_desktop_action(&json!({"x": 10, "y": 20}), "window")); + } + + #[test] + fn windowless_false_when_xy_missing() { + assert!(!is_windowless_desktop_action(&json!({"x": 10}), "desktop")); + assert!(!is_windowless_desktop_action(&json!({"y": 20}), "desktop")); + assert!(!is_windowless_desktop_action(&json!({}), "desktop")); + // Non-numeric x/y must not qualify. + assert!(!is_windowless_desktop_action(&json!({"x": "10", "y": "20"}), "desktop")); + } + + // ── DriverConfig default ────────────────────────────────────────────────── + + #[test] + fn default_capture_scope_is_window() { + assert_eq!(DriverConfig::default().capture_scope, "window"); + } + + // ── get_desktop_state schema ────────────────────────────────────────────── + + #[test] + fn get_desktop_state_schema_shape() { + let d = GetDesktopStateTool.def(); + assert!(d.read_only, "get_desktop_state must be read_only"); + let props = d.input_schema["properties"].as_object().unwrap(); + assert!(!props.contains_key("pid"), "must not accept pid"); + assert!(!props.contains_key("window_id"), "must not accept window_id"); + assert!(props.contains_key("session")); + assert!(props.contains_key("screenshot_out_file")); + assert_eq!(d.input_schema["additionalProperties"], json!(false)); + } +}