diff --git a/libs/cua-driver/rust/crates/cua-driver/src/cli.rs b/libs/cua-driver/rust/crates/cua-driver/src/cli.rs index 8598ffdcf5..931f1bf8f5 100644 --- a/libs/cua-driver/rust/crates/cua-driver/src/cli.rs +++ b/libs/cua-driver/rust/crates/cua-driver/src/cli.rs @@ -29,6 +29,12 @@ pub enum Command { /// Override the daemon Unix socket path used by the proxy /// fallback. Defaults to `serve::default_socket_path()`. socket: Option, + /// `--claude-code-computer-use-compat`: register the compat + /// `screenshot` tool (window-scoped, JPEG @ 85%, pid + window_id + /// both required) instead of the full-featured one. Used when + /// the MCP server is wired up as `cua-computer-use` in Claude + /// Code, where this is the documented best-practice install. + claude_code_compat: bool, }, ListTools, Describe(String), @@ -170,6 +176,7 @@ pub fn parse_command() -> Command { } let no_daemon_relaunch = args.iter().any(|a| a == "--no-daemon-relaunch"); + let claude_code_compat = args.iter().any(|a| a == "--claude-code-computer-use-compat"); let mut pos = positionals.into_iter(); match pos.next() { @@ -198,11 +205,13 @@ pub fn parse_command() -> Command { Command::Mcp { no_daemon_relaunch, socket: socket.clone(), + claude_code_compat, } } Some("mcp") => Command::Mcp { no_daemon_relaunch, socket: socket.clone(), + claude_code_compat, }, Some("list-tools") => Command::ListTools, Some("mcp-config") => Command::McpConfig { client: mcp_client }, @@ -669,8 +678,21 @@ pub fn run_mcp_config(client: Option<&str>) { }} }}"#); } - Some("claude") => { - println!("claude mcp add --transport stdio cua-driver -- {binary} mcp"); + Some("claude") | Some("claude-code") => { + // Claude Code wants the MCP server registered as + // `cua-computer-use` and the binary invoked with + // `--claude-code-computer-use-compat` so the regular + // `screenshot` tool is replaced by a window-scoped variant + // (pid + window_id required, JPEG @ 85%, text note pointing + // at pixel tools). See `mcp-server/src/protocol.rs` for the + // server-name reasoning, `updater.rs`/`skills.rs` for the + // matching skill-pack flow, and Skills/cua-driver/SKILL.md + // for the user-facing rationale. + // + // Observed Claude Code behaviour: the exact config key + // "computer-use" is reserved, so external stdio + // registrations use a distinct key — hence `cua-computer-use`. + println!("claude mcp add --transport stdio cua-computer-use -- {binary} mcp --claude-code-computer-use-compat"); } Some("codex") => { println!("codex mcp add cua-driver -- {binary} mcp"); diff --git a/libs/cua-driver/rust/crates/cua-driver/src/main.rs b/libs/cua-driver/rust/crates/cua-driver/src/main.rs index 30ef24d395..0688e4b5c9 100644 --- a/libs/cua-driver/rust/crates/cua-driver/src/main.rs +++ b/libs/cua-driver/rust/crates/cua-driver/src/main.rs @@ -36,6 +36,15 @@ mod updater; mod version_check; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +/// Set by the `Command::Mcp` arm when `--claude-code-computer-use-compat` +/// is on argv. Read by `build_registry` / `build_registry_no_cursor` to +/// pick which `screenshot` tool variant to register. Static keeps the +/// thread of dependency arrows pointed away from the platform crates — +/// they take `compat: bool` directly, but the binary crate decides what +/// to pass without making every Command variant carry the flag. +static CLAUDE_CODE_COMPAT: AtomicBool = AtomicBool::new(false); fn init_logging() { use tracing_subscriber::EnvFilter; @@ -215,7 +224,8 @@ fn main() { cli::run_config_cmd(reg, subcommand.as_deref(), key.as_deref(), value.as_deref(), socket.as_deref()); return; } - cli::Command::Mcp { no_daemon_relaunch, socket } => { + cli::Command::Mcp { no_daemon_relaunch, socket, claude_code_compat } => { + CLAUDE_CODE_COMPAT.store(claude_code_compat, Ordering::SeqCst); // Long-running MCP server — kick off the background update // check before any TCC / daemon-proxy decisions so the // banner can land on stderr in either dispatch path. @@ -285,9 +295,10 @@ fn main() { .enable_all() .build() .expect("tokio runtime"); + let compat = CLAUDE_CODE_COMPAT.load(Ordering::SeqCst); rt.block_on(async move { // Register tools; overlay init has already happened above. - let registry = Arc::new(platform_macos::register_tools()); + let registry = Arc::new(platform_macos::register_tools_with_compat(compat)); // Wire up replay tool's back-reference to the registry. registry.init_self_weak(); if let Err(e) = mcp_server::server::run(registry).await { @@ -430,7 +441,8 @@ fn main() -> anyhow::Result<()> { }).join().ok(); return Ok(()); } - cli::Command::Mcp { no_daemon_relaunch, socket } => { + cli::Command::Mcp { no_daemon_relaunch, socket, claude_code_compat } => { + CLAUDE_CODE_COMPAT.store(claude_code_compat, Ordering::SeqCst); // Long-running MCP server — kick off the background update // check before any daemon-proxy decisions. version_check::maybe_announce_update(); @@ -492,6 +504,7 @@ async fn async_main() -> anyhow::Result<()> { #[cfg(not(target_os = "macos"))] fn build_registry(cursor_cfg: cursor_overlay::CursorConfig) -> mcp_server::tool::ToolRegistry { + let compat = CLAUDE_CODE_COMPAT.load(Ordering::SeqCst); #[cfg(target_os = "windows")] { mcp_server::recording::set_screenshot_fn(|window_id, pid| { @@ -509,7 +522,7 @@ fn build_registry(cursor_cfg: cursor_overlay::CursorConfig) -> mcp_server::tool: mcp_server::recording::set_click_marker_fn(|png_bytes, cx, cy| { platform_windows::capture::crosshair_png_bytes(png_bytes, cx, cy).ok() }); - platform_windows::register_tools_with_cursor(cursor_cfg) + platform_windows::register_tools_with_cursor(cursor_cfg, compat) } #[cfg(target_os = "linux")] { @@ -528,11 +541,12 @@ fn build_registry(cursor_cfg: cursor_overlay::CursorConfig) -> mcp_server::tool: mcp_server::recording::set_click_marker_fn(|png_bytes, cx, cy| { platform_linux::capture::crosshair_png_bytes(png_bytes, cx, cy).ok() }); - platform_linux::register_tools_with_cursor(cursor_cfg) + platform_linux::register_tools_with_cursor(cursor_cfg, compat) } #[cfg(not(any(target_os = "windows", target_os = "linux")))] { let _ = cursor_cfg; + let _ = compat; let mut r = mcp_server::tool::ToolRegistry::new(); r.register(Box::new(crate::stub::UnsupportedPlatformTool)); r @@ -543,6 +557,7 @@ fn build_registry(cursor_cfg: cursor_overlay::CursorConfig) -> mcp_server::tool: /// Used by CLI subcommands (list-tools / describe / call) that don't need the overlay. #[cfg(not(target_os = "macos"))] fn build_registry_no_cursor() -> mcp_server::tool::ToolRegistry { + let compat = CLAUDE_CODE_COMPAT.load(Ordering::SeqCst); #[cfg(target_os = "windows")] { mcp_server::recording::set_screenshot_fn(|window_id, pid| { @@ -560,7 +575,7 @@ fn build_registry_no_cursor() -> mcp_server::tool::ToolRegistry { mcp_server::recording::set_click_marker_fn(|png_bytes, cx, cy| { platform_windows::capture::crosshair_png_bytes(png_bytes, cx, cy).ok() }); - platform_windows::register_tools_with_cursor(cursor_overlay::CursorConfig { enabled: false, ..Default::default() }) + platform_windows::register_tools_with_cursor(cursor_overlay::CursorConfig { enabled: false, ..Default::default() }, compat) } #[cfg(target_os = "linux")] { @@ -579,10 +594,11 @@ fn build_registry_no_cursor() -> mcp_server::tool::ToolRegistry { mcp_server::recording::set_click_marker_fn(|png_bytes, cx, cy| { platform_linux::capture::crosshair_png_bytes(png_bytes, cx, cy).ok() }); - platform_linux::register_tools_with_cursor(cursor_overlay::CursorConfig { enabled: false, ..Default::default() }) + platform_linux::register_tools_with_cursor(cursor_overlay::CursorConfig { enabled: false, ..Default::default() }, compat) } #[cfg(not(any(target_os = "windows", target_os = "linux")))] { + let _ = compat; let mut r = mcp_server::tool::ToolRegistry::new(); r.register(Box::new(crate::stub::UnsupportedPlatformTool)); r diff --git a/libs/cua-driver/rust/crates/platform-linux/src/lib.rs b/libs/cua-driver/rust/crates/platform-linux/src/lib.rs index 77b73de282..3a85d67ea6 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/lib.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/lib.rs @@ -34,13 +34,17 @@ pub mod capture; pub mod atspi; pub fn register_tools() -> ToolRegistry { - tools::build_registry() + tools::build_registry(false) } -pub fn register_tools_with_cursor(cfg: cursor_overlay::CursorConfig) -> ToolRegistry { +/// `compat=true` enables Claude Code computer-use compatibility mode: +/// the regular `screenshot` tool is replaced by a window-scoped variant +/// (pid + window_id required, JPEG @ 85%, text note pointing at pixel +/// tools). See `tools::impl_::ScreenshotCompatTool`. +pub fn register_tools_with_cursor(cfg: cursor_overlay::CursorConfig, compat: bool) -> ToolRegistry { if cfg.enabled { overlay::init(cfg.clone()); overlay::run_on_thread(); } - tools::build_registry() + tools::build_registry(compat) } diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs index 621af43311..718ebf7d55 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs @@ -998,6 +998,98 @@ impl Tool for ScreenshotTool { } } +// ── screenshot (Claude Code computer-use compat) ───────────────────────────── +// +// Drop-in replacement for `ScreenshotTool` selected via the +// `--claude-code-computer-use-compat` flag. Same shape as the Windows / +// macOS compat tools — see those files / SwiftCompatTools.swift for the +// rationale. + +pub struct ScreenshotCompatTool { + state: Arc, +} +static SS_COMPAT_DEF: std::sync::OnceLock = std::sync::OnceLock::new(); + +#[async_trait] +impl Tool for ScreenshotCompatTool { + fn def(&self) -> &ToolDef { + SS_COMPAT_DEF.get_or_init(|| ToolDef { + name: "screenshot".into(), + description: + "Capture a target window and return a JPEG image. Coordinates accepted by \ + CuaDriver's pixel tools are pixels in this window screenshot's coordinate space.\n\n\ + This is the compatibility anchor for Claude Code vision flows: CuaDriver remains \ + window-scoped, and all other tools are the normal CuaDriver tools.".into(), + input_schema: json!({ + "type": "object", + "required": ["pid", "window_id"], + "properties": { + "pid": {"type":"integer","description":"Target process ID from list_windows or launch_app."}, + "window_id": {"type":"integer","description":"Target X11 XID from list_windows or launch_app."} + }, + "additionalProperties": false + }), + read_only: true, destructive: false, idempotent: false, open_world: false, + }) + } + + async fn invoke(&self, args: Value) -> ToolResult { + use mcp_server::tool_args::ArgsExt; + let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e }; + let window_id = match args.require_u64("window_id") { Ok(v) => v, Err(e) => return e }; + + // Validate: window must exist + belong to pid. + let window = tokio::task::spawn_blocking(move || { + crate::x11::list_windows(Some(pid)) + .into_iter() + .find(|w| w.xid == window_id) + }).await.unwrap_or(None); + + let window = match window { + Some(w) => w, + None => return ToolResult::error(format!( + "No visible window {window_id} found for pid {pid}. \ + Use list_windows to choose an on-screen target window." + )), + }; + + let max_dim = self.state.config.read().unwrap().max_image_dimension; + + let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(String, u32, u32)> { + let raw = crate::capture::screenshot_window_bytes(window_id)?; + let png_bytes = crate::capture::resize_png_if_needed(&raw, max_dim)?; + let (w, h) = crate::capture::png_dimensions_pub(&png_bytes)?; + let jpeg = crate::capture::png_bytes_to_jpeg(&png_bytes, 85)?; + use base64::{engine::general_purpose::STANDARD as B64, Engine as _}; + Ok((B64.encode(&jpeg), w, h)) + }).await; + + match result { + Ok(Ok((b64, w, h))) => { + let title = if window.title.is_empty() { "(untitled)".into() } else { window.title }; + let summary = format!( + "Captured window screenshot {w}x{h} for {title} \ + [pid: {pid}, window_id: {window_id}]. \ + Use CuaDriver pixel tools with this window-local coordinate space." + ); + ToolResult { + content: vec![ + mcp_server::protocol::Content::image_jpeg(b64), + mcp_server::protocol::Content::text(summary), + ], + is_error: None, + structured_content: Some(json!({ + "pid": pid, "window_id": window_id, + "width": w, "height": h, "format": "jpeg" + })), + } + } + Ok(Err(e)) => ToolResult::error(format!("Screenshot failed: {e}")), + Err(e) => ToolResult::error(format!("Task error: {e}")), + } + } +} + // ── double_click ────────────────────────────────────────────────────────────── pub struct DoubleClickTool { @@ -1961,7 +2053,7 @@ impl Tool for KillAppTool { // ── registry ───────────────────────────────────────────────────────────────── -pub fn build_registry() -> ToolRegistry { +pub fn build_registry(compat: bool) -> ToolRegistry { let state = ToolState::new(); let mut r = ToolRegistry::new(); r.register(Box::new(ListAppsTool)); @@ -1978,7 +2070,11 @@ pub fn build_registry() -> ToolRegistry { r.register(Box::new(HotkeyTool)); r.register(Box::new(SetValueTool)); r.register(Box::new(ScrollTool)); - r.register(Box::new(ScreenshotTool { state: state.clone() })); + if compat { + r.register(Box::new(ScreenshotCompatTool { state: state.clone() })); + } else { + r.register(Box::new(ScreenshotTool { state: state.clone() })); + } r.register(Box::new(GetScreenSizeTool)); r.register(Box::new(GetCursorPositionTool)); r.register(Box::new(MoveCursorTool { state: state.clone() })); diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/mod.rs index ea79d24754..4ef163cb17 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/tools/mod.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/mod.rs @@ -13,13 +13,16 @@ pub(crate) mod page; #[cfg(not(target_os = "linux"))] mod stubs; -pub fn build_registry() -> ToolRegistry { +pub fn build_registry(compat: bool) -> ToolRegistry { #[cfg(target_os = "linux")] - return impl_::build_registry(); + return impl_::build_registry(compat); #[cfg(not(target_os = "linux"))] - stubs::build_registry() + { + let _ = compat; + stubs::build_registry() + } } // Keep register_all as alias for backwards compat. -pub fn register_all() -> ToolRegistry { build_registry() } +pub fn register_all() -> ToolRegistry { build_registry(false) } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/lib.rs b/libs/cua-driver/rust/crates/platform-macos/src/lib.rs index f2c51cdea0..d0845c856b 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/lib.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/lib.rs @@ -36,14 +36,25 @@ use mcp_server::tool::ToolRegistry; /// Register all macOS tools. For programs that don't restructure `main` /// (e.g. test harnesses), the overlay is skipped. pub fn register_tools() -> ToolRegistry { + register_tools_with_compat(false) +} + +/// Same as `register_tools` but lets the caller pick the Claude Code +/// computer-use compat mode. `compat=true` swaps the regular `screenshot` +/// tool for the window-scoped variant (pid + window_id required, +/// JPEG @ 85%, text note pointing at pixel tools). +pub fn register_tools_with_compat(compat: bool) -> ToolRegistry { #[cfg(target_os = "macos")] { let mut r = ToolRegistry::new(); - tools::register_all(&mut r); + tools::register_all(&mut r, compat); r } #[cfg(not(target_os = "macos"))] - ToolRegistry::new() + { + let _ = compat; + ToolRegistry::new() + } } /// Register all macOS tools and initialise the cursor overlay channel. @@ -51,19 +62,25 @@ pub fn register_tools() -> ToolRegistry { /// After calling this, `main()` must call /// `platform_macos::cursor::overlay::run_on_main_thread()` on the OS /// main thread to actually display the overlay. -pub fn register_tools_with_cursor(cfg: cursor_overlay::CursorConfig) -> ToolRegistry { +/// +/// `compat=true` enables Claude Code computer-use compatibility mode: +/// the regular `screenshot` tool is replaced by a window-scoped variant +/// (pid + window_id required, JPEG @ 85%, text note pointing at pixel +/// tools). See `tools::screenshot_compat`. +pub fn register_tools_with_cursor(cfg: cursor_overlay::CursorConfig, compat: bool) -> ToolRegistry { #[cfg(target_os = "macos")] { if cfg.enabled { cursor::overlay::init(cfg); } let mut r = ToolRegistry::new(); - tools::register_all(&mut r); + tools::register_all(&mut r, compat); r } #[cfg(not(target_os = "macos"))] { let _ = cfg; + let _ = compat; ToolRegistry::new() } } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs index 35bfae9e2e..ea19ad7ddc 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs @@ -15,6 +15,7 @@ mod hotkey; mod set_value; mod scroll; mod screenshot; +mod screenshot_compat; mod get_screen_size; mod get_cursor_position; mod move_cursor; @@ -198,8 +199,11 @@ impl Default for ToolState { } } -/// Register all macOS tools into the registry. -pub fn register_all(registry: &mut ToolRegistry) { +/// Register all macOS tools into the registry. `compat=true` swaps the +/// regular `screenshot` tool for the Claude Code computer-use compat +/// variant — same name, stricter args, window-scoped JPEG @ 85% + a text +/// note telling the caller to use pixel-addressed tools. +pub fn register_all(registry: &mut ToolRegistry, compat: bool) { let state = Arc::new(ToolState::default()); registry.register(Box::new(list_apps::ListAppsTool)); @@ -216,7 +220,11 @@ pub fn register_all(registry: &mut ToolRegistry) { registry.register(Box::new(hotkey::HotkeyTool::new(state.clone()))); registry.register(Box::new(set_value::SetValueTool::new(state.clone()))); registry.register(Box::new(scroll::ScrollTool::new(state.clone()))); - registry.register(Box::new(screenshot::ScreenshotTool { state: state.clone() })); + if compat { + registry.register(Box::new(screenshot_compat::ClaudeCodeCompatScreenshotTool::new(state.clone()))); + } else { + registry.register(Box::new(screenshot::ScreenshotTool { state: state.clone() })); + } registry.register(Box::new(get_screen_size::GetScreenSizeTool)); registry.register(Box::new(get_cursor_position::GetCursorPositionTool)); registry.register(Box::new(move_cursor::MoveCursorTool::new(state.clone()))); diff --git a/libs/cua-driver/rust/crates/platform-windows/src/lib.rs b/libs/cua-driver/rust/crates/platform-windows/src/lib.rs index 3cca789895..663b801b80 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/lib.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/lib.rs @@ -31,13 +31,17 @@ pub mod capture; pub mod launch_uwp; pub fn register_tools() -> ToolRegistry { - tools::build_registry() + tools::build_registry(false) } -pub fn register_tools_with_cursor(cfg: cursor_overlay::CursorConfig) -> ToolRegistry { +/// `compat=true` enables Claude Code computer-use compatibility mode: +/// the regular `screenshot` tool is replaced by a window-scoped variant +/// (pid + window_id required, JPEG @ 85%, text note pointing at pixel +/// tools). See `tools::impl_::ScreenshotCompatTool`. +pub fn register_tools_with_cursor(cfg: cursor_overlay::CursorConfig, compat: bool) -> ToolRegistry { if cfg.enabled { overlay::init(cfg.clone()); overlay::run_on_thread(); } - tools::build_registry() + tools::build_registry(compat) } diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs index 91ea6f7c7b..511158e6ba 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs @@ -2495,6 +2495,107 @@ impl Tool for ScreenshotTool { } } +// ── screenshot (Claude Code computer-use compat) ───────────────────────────── +// +// Drop-in replacement for `ScreenshotTool` selected via the +// `--claude-code-computer-use-compat` flag. Differences: +// - `pid` AND `window_id` BOTH required (the regular tool makes window_id +// optional and falls back to whole-display capture). +// - Validates the target window belongs to that pid and is visible. +// - Always returns JPEG @ 85% (Claude Code vision flows prefer smaller). +// - Includes a follow-up text note pointing the caller at pixel-addressed +// tools so the LLM uses the window's coordinate space. +// +// Mirrors libs/cua-driver/swift/Sources/CuaDriverServer/ +// ClaudeCodeComputerUseCompatTools.swift's `screenshot`. Same name as the +// regular tool — `build_registry(compat)` chooses which to register; both +// are never registered together. + +pub struct ScreenshotCompatTool { + state: Arc, +} +static SCREENSHOT_COMPAT_DEF: std::sync::OnceLock = std::sync::OnceLock::new(); + +#[async_trait] +impl Tool for ScreenshotCompatTool { + fn def(&self) -> &ToolDef { + SCREENSHOT_COMPAT_DEF.get_or_init(|| ToolDef { + name: "screenshot".into(), + description: + "Capture a target window and return a JPEG image. Coordinates accepted by \ + CuaDriver's pixel tools are pixels in this window screenshot's coordinate space.\n\n\ + This is the compatibility anchor for Claude Code vision flows: CuaDriver remains \ + window-scoped, and all other tools are the normal CuaDriver tools.".into(), + input_schema: json!({ + "type": "object", + "required": ["pid", "window_id"], + "properties": { + "pid": {"type":"integer","description":"Target process ID from list_windows or launch_app."}, + "window_id": {"type":"integer","description":"Target HWND from list_windows or launch_app."} + }, + "additionalProperties": false + }), + read_only: true, destructive: false, idempotent: false, open_world: false, + }) + } + + async fn invoke(&self, args: Value) -> ToolResult { + use mcp_server::tool_args::ArgsExt; + let pid = match args.require_u32("pid") { Ok(v) => v, Err(e) => return e }; + let window_id = match args.require_u64("window_id") { Ok(v) => v, Err(e) => return e }; + + // Validate: window must exist, belong to pid, and be visible. + let window = tokio::task::spawn_blocking(move || { + crate::win32::list_windows(Some(pid)) + .into_iter() + .find(|w| w.hwnd == window_id && w.width > 1 && w.height > 1) + }).await.unwrap_or(None); + + let window = match window { + Some(w) => w, + None => return ToolResult::error(format!( + "No visible window {window_id} found for pid {pid}. \ + Use list_windows to choose an on-screen target window." + )), + }; + + let max_dim = self.state.config.read().unwrap().max_image_dimension; + + let result = tokio::task::spawn_blocking(move || -> anyhow::Result<(String, u32, u32)> { + let raw = crate::capture::screenshot_window_bytes(window_id)?; + let png_bytes = crate::capture::resize_png_if_needed(&raw, max_dim)?; + let (w, h) = crate::capture::png_dimensions_pub(&png_bytes)?; + let jpeg = crate::capture::png_bytes_to_jpeg(&png_bytes, 85)?; + use base64::{engine::general_purpose::STANDARD as B64, Engine as _}; + Ok((B64.encode(&jpeg), w, h)) + }).await; + + match result { + Ok(Ok((b64, w, h))) => { + let title = if window.title.is_empty() { "(untitled)".into() } else { window.title }; + let summary = format!( + "Captured window screenshot {w}x{h} for {title} \ + [pid: {pid}, window_id: {window_id}]. \ + Use CuaDriver pixel tools with this window-local coordinate space." + ); + ToolResult { + content: vec![ + mcp_server::protocol::Content::image_jpeg(b64), + mcp_server::protocol::Content::text(summary), + ], + is_error: None, + structured_content: Some(json!({ + "pid": pid, "window_id": window_id, + "width": w, "height": h, "format": "jpeg" + })), + } + } + Ok(Err(e)) => ToolResult::error(format!("Screenshot failed: {e}")), + Err(e) => ToolResult::error(format!("Task error: {e}")), + } + } +} + // ── double_click ────────────────────────────────────────────────────────────── pub struct DoubleClickTool { @@ -4251,7 +4352,7 @@ impl Tool for DebugWindowInfoTool { // ── registry builder ────────────────────────────────────────────────────────── -pub fn build_registry() -> ToolRegistry { +pub fn build_registry(compat: bool) -> ToolRegistry { let state = ToolState::new(); let mut r = ToolRegistry::new(); r.register(Box::new(ListAppsTool)); @@ -4269,7 +4370,11 @@ pub fn build_registry() -> ToolRegistry { r.register(Box::new(HotkeyTool)); r.register(Box::new(SetValueTool { state: state.clone() })); r.register(Box::new(ScrollTool)); - r.register(Box::new(ScreenshotTool { state: state.clone() })); + if compat { + r.register(Box::new(ScreenshotCompatTool { state: state.clone() })); + } else { + r.register(Box::new(ScreenshotTool { state: state.clone() })); + } r.register(Box::new(GetScreenSizeTool)); r.register(Box::new(GetCursorPositionTool)); r.register(Box::new(MoveCursorTool { state: state.clone() })); diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/mod.rs index b8a706a06e..77c15c54cc 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/tools/mod.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/mod.rs @@ -15,10 +15,13 @@ pub(crate) mod page_bookmark; #[cfg(not(target_os = "windows"))] mod stubs; -pub fn build_registry() -> ToolRegistry { +pub fn build_registry(compat: bool) -> ToolRegistry { #[cfg(target_os = "windows")] - return impl_::build_registry(); + return impl_::build_registry(compat); #[cfg(not(target_os = "windows"))] - stubs::build_registry() + { + let _ = compat; + stubs::build_registry() + } }