diff --git a/libs/cua-driver-rs/PARITY.md b/libs/cua-driver-rs/PARITY.md index e1c465765b..faaad766f3 100644 --- a/libs/cua-driver-rs/PARITY.md +++ b/libs/cua-driver-rs/PARITY.md @@ -855,11 +855,15 @@ Windows's `click` takes `{button: enum}` instead. Rationale: Swift verbatim: `"✅ Window screenshot — WxH png [window_id: ID]"` (em-dash, checkmark, window-id suffix). Display fallback uses `"✅ Display screenshot — WxH png"` (Rust-only, see intentional below). -2. **Default JPEG quality** — was 85, Swift defaults 95. Now 95. -3. **Description** — multi-paragraph port from Swift adapted to Windows +2. **Description** — multi-paragraph port from Swift adapted to Windows (BitBlt + PrintWindow transport; no permission gate needed). -4. **`idempotent`** — was `true`; Swift uses `false` (a fresh pixel grab +3. **`idempotent`** — was `true`; Swift uses `false` (a fresh pixel grab every call). Now matches Swift. +4. **`max_image_dimension` default** — was 0 (no cap), Swift uses 1568. + Now 1568 on all 3 Rust platforms (matches + `CuaDriverConfig.defaultMaxImageDimension`). The 0 default was + producing 10MB screenshots on the Windows VM; 1568 caps the long + edge before encoding. ### Intentional Rust-only @@ -868,6 +872,16 @@ Windows's `click` takes `{button: enum}` instead. Rationale: Windows-only convenience that Swift can't easily provide because macOS Screen Recording requires per-window grants. Schema accepts both shapes; description explains. +- **Default `format`** — Swift defaults `png`; all 3 Rust platforms now + default `jpeg`. Rationale: agents typically want compact images for + vision-model context windows; PNG is lossless but multi-MB on screen + content. Schema still accepts both; callers wanting PNG pass + `{"format":"png"}`. Swift may follow; tracked as a follow-up parity + question. +- **Default JPEG `quality`** — Swift defaults 95; Rust defaults 85 + (already Linux's default and the macOS Claude-Code-compat tool's + default). 85 is the typical sweet spot for screen content. Diverges + from Swift only when both sides actually emit JPEG. ### Verified on Windows diff --git a/libs/cua-driver-rs/crates/cua-driver/tests/mcp_protocol_test.rs b/libs/cua-driver-rs/crates/cua-driver/tests/mcp_protocol_test.rs index 76dd577d36..e95ee93ff0 100644 --- a/libs/cua-driver-rs/crates/cua-driver/tests/mcp_protocol_test.rs +++ b/libs/cua-driver-rs/crates/cua-driver/tests/mcp_protocol_test.rs @@ -982,7 +982,7 @@ fn test_browser_eval_no_cdp_error() { #[test] #[cfg(target_os = "macos")] fn test_screenshot_no_window_id() { - //! Call screenshot without window_id — should capture the full display and return a PNG. + //! Call screenshot without window_id — should capture the full display and return image content. let binary = binary_path(); if !binary.exists() { return; } diff --git a/libs/cua-driver-rs/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver-rs/crates/platform-linux/src/tools/impl_.rs index 135e81c019..9a1f7864e6 100644 --- a/libs/cua-driver-rs/crates/platform-linux/src/tools/impl_.rs +++ b/libs/cua-driver-rs/crates/platform-linux/src/tools/impl_.rs @@ -17,7 +17,7 @@ pub struct DriverConfig { } impl Default for DriverConfig { - fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 0 } } + fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 1568 } } } pub struct ResizeRegistry { @@ -953,12 +953,13 @@ impl Tool for ScreenshotTool { SS_DEF.get_or_init(|| ToolDef { name: "screenshot".into(), description: "Capture a screenshot via XGetImage or `import` (ImageMagick). \ - Without window_id captures the full primary display. Supports png and jpeg formats.".into(), + Without window_id captures the full primary display. Supports png and jpeg formats \ + (default jpeg, quality 85).".into(), input_schema: json!({ "type":"object","properties":{ "window_id":{"type":"integer"}, - "format":{"type":"string","enum":["png","jpeg"]}, - "quality":{"type":"integer","minimum":1,"maximum":95} + "format":{"type":"string","enum":["png","jpeg"],"description":"Image format. Default: jpeg."}, + "quality":{"type":"integer","minimum":1,"maximum":95,"description":"JPEG quality 1-95; ignored for png. Default: 85."} },"additionalProperties":false }), read_only: true, destructive: false, idempotent: true, open_world: false, @@ -967,7 +968,7 @@ impl Tool for ScreenshotTool { async fn invoke(&self, args: Value) -> ToolResult { let xid_opt = args.get("window_id").and_then(|v| v.as_u64()); - let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("png").to_owned(); + let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("jpeg").to_owned(); let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(85) as u8; let is_jpeg = format == "jpeg"; let max_dim = self.state.config.read().unwrap().max_image_dimension; diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/mod.rs index 92dd00edea..1a3aa45971 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/tools/mod.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/mod.rs @@ -109,6 +109,8 @@ pub struct DriverConfig { /// Default capture_mode for get_window_state when not specified per-call. pub capture_mode: String, /// Max screenshot dimension (0 = no limit). Applied during screenshot/zoom. + /// Default 1568 matches Swift's `CuaDriverConfig.defaultMaxImageDimension` — + /// the long edge is downscaled to this before encoding. pub max_image_dimension: u32, } @@ -116,7 +118,7 @@ impl Default for DriverConfig { fn default() -> Self { Self { capture_mode: "som".to_owned(), - max_image_dimension: 0, + max_image_dimension: 1568, } } } diff --git a/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot.rs b/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot.rs index eef822ae52..e8d1ba8106 100644 --- a/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot.rs +++ b/libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot.rs @@ -16,7 +16,7 @@ fn def() -> &'static ToolDef { name: "screenshot".into(), description: "Capture a screenshot. Returns base64-encoded image data in the requested format \ - (default png).\n\n\ + (default jpeg, quality 85).\n\n\ Without `window_id`, captures the full main display. With `window_id`, captures \ just that window (pair with `list_windows` or `get_accessibility_tree` which return \ window IDs).\n\n\ @@ -32,13 +32,13 @@ fn def() -> &'static ToolDef { "format": { "type": "string", "enum": ["png", "jpeg"], - "description": "Image format. Default: png." + "description": "Image format. Default: jpeg." }, "quality": { "type": "integer", "minimum": 1, "maximum": 95, - "description": "JPEG quality 1-95; ignored for png. Default: 95." + "description": "JPEG quality 1-95; ignored for png. Default: 85." } }, "additionalProperties": false @@ -56,8 +56,8 @@ impl Tool for ScreenshotTool { async fn invoke(&self, args: Value) -> ToolResult { let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32); - let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("png").to_owned(); - let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(95) as u8; + let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("jpeg").to_owned(); + let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(85) as u8; let use_jpeg = format == "jpeg"; let max_dim = self.state.config.read().unwrap().max_image_dimension; diff --git a/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs index 7d4e1ff0da..dd52507fa0 100644 --- a/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs +++ b/libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs @@ -43,7 +43,7 @@ pub struct DriverConfig { } impl Default for DriverConfig { - fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 0 } } + fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 1568 } } } pub struct ResizeRegistry { @@ -2141,7 +2141,7 @@ impl Tool for ScreenshotTool { // ScreenCaptureKit). description: "Capture a screenshot of a single window via BitBlt + PrintWindow \ (no focus change). Returns base64-encoded image data in the requested format \ - (default png).\n\n\ + (default jpeg, quality 85).\n\n\ `window_id` is recommended (use `list_windows` to find it). When omitted, \ captures the full primary display — a Windows-only convenience for whole-\ screen snapshots without a per-window target.\n\n\ @@ -2149,9 +2149,9 @@ impl Tool for ScreenshotTool { input_schema: json!({ "type":"object","properties":{ "window_id":{"type":"integer","description":"HWND of the window to capture. When omitted, captures the full primary display (Windows-only)."}, - "format":{"type":"string","enum":["png","jpeg"],"description":"Image format. Default: png."}, + "format":{"type":"string","enum":["png","jpeg"],"description":"Image format. Default: jpeg."}, "quality":{"type":"integer","minimum":1,"maximum":95, - "description":"JPEG quality 1-95; ignored for png."} + "description":"JPEG quality 1-95; ignored for png. Default: 85."} },"additionalProperties":false }), read_only: true, destructive: false, idempotent: false, open_world: false, @@ -2160,8 +2160,8 @@ impl Tool for ScreenshotTool { async fn invoke(&self, args: Value) -> ToolResult { let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64()); - let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("png").to_owned(); - let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(95) as u8; + let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("jpeg").to_owned(); + let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(85) as u8; let is_jpeg = format == "jpeg"; let max_dim = self.state.config.read().unwrap().max_image_dimension; diff --git a/libs/cua-driver-rs/tests/integration/test_cursor_visibility.py b/libs/cua-driver-rs/tests/integration/test_cursor_visibility.py index 910b5e0b2c..1bfdccc316 100644 --- a/libs/cua-driver-rs/tests/integration/test_cursor_visibility.py +++ b/libs/cua-driver-rs/tests/integration/test_cursor_visibility.py @@ -255,8 +255,10 @@ def test_custom_cursor_visible_after_click(self) -> None: # 7. Wait for cursor animation + render loop to settle. time.sleep(0.5) - # 8. Take a full-screen screenshot. - scr = client.call_tool("screenshot", {}) + # 8. Take a full-screen screenshot. Force PNG so the per-pixel decode + # below works — the screenshot tool defaults to JPEG, which would be + # lossy and skew the cursor-colour search. + scr = client.call_tool("screenshot", {"format": "png"}) b64 = scr.get("content", [{}])[0].get("data", "") self.assertTrue(b64, "screenshot returned no data") png_bytes = base64.b64decode(b64)