Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 17 additions & 3 deletions libs/cua-driver-rs/PARITY.md
Original file line number Diff line number Diff line change
Expand Up @@ -855,11 +855,15 @@ Windows's `click` takes `{button: enum}` instead. Rationale:
Swift verbatim: `"✅ Window screenshot — WxH png [window_id: ID]"`
(em-dash, checkmark, window-id suffix). Display fallback uses
`"✅ Display screenshot — WxH png"` (Rust-only, see intentional below).
2. **Default JPEG quality** — was 85, Swift defaults 95. Now 95.
3. **Description** — multi-paragraph port from Swift adapted to Windows
2. **Description** — multi-paragraph port from Swift adapted to Windows
(BitBlt + PrintWindow transport; no permission gate needed).
4. **`idempotent`** — was `true`; Swift uses `false` (a fresh pixel grab
3. **`idempotent`** — was `true`; Swift uses `false` (a fresh pixel grab
every call). Now matches Swift.
4. **`max_image_dimension` default** — was 0 (no cap), Swift uses 1568.
Now 1568 on all 3 Rust platforms (matches
`CuaDriverConfig.defaultMaxImageDimension`). The 0 default was
producing 10MB screenshots on the Windows VM; 1568 caps the long
edge before encoding.

### Intentional Rust-only

Expand All @@ -868,6 +872,16 @@ Windows's `click` takes `{button: enum}` instead. Rationale:
Windows-only convenience that Swift can't easily provide because
macOS Screen Recording requires per-window grants. Schema accepts
both shapes; description explains.
- **Default `format`** — Swift defaults `png`; all 3 Rust platforms now
default `jpeg`. Rationale: agents typically want compact images for
vision-model context windows; PNG is lossless but multi-MB on screen
content. Schema still accepts both; callers wanting PNG pass
`{"format":"png"}`. Swift may follow; tracked as a follow-up parity
question.
- **Default JPEG `quality`** — Swift defaults 95; Rust defaults 85
(already Linux's default and the macOS Claude-Code-compat tool's
default). 85 is the typical sweet spot for screen content. Diverges
from Swift only when both sides actually emit JPEG.

### Verified on Windows

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -982,7 +982,7 @@ fn test_browser_eval_no_cdp_error() {
#[test]
#[cfg(target_os = "macos")]
fn test_screenshot_no_window_id() {
//! Call screenshot without window_id — should capture the full display and return a PNG.
//! Call screenshot without window_id — should capture the full display and return image content.
let binary = binary_path();
if !binary.exists() { return; }

Expand Down
11 changes: 6 additions & 5 deletions libs/cua-driver-rs/crates/platform-linux/src/tools/impl_.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ pub struct DriverConfig {
}

impl Default for DriverConfig {
fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 0 } }
fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 1568 } }
}

pub struct ResizeRegistry {
Expand Down Expand Up @@ -953,12 +953,13 @@ impl Tool for ScreenshotTool {
SS_DEF.get_or_init(|| ToolDef {
name: "screenshot".into(),
description: "Capture a screenshot via XGetImage or `import` (ImageMagick). \
Without window_id captures the full primary display. Supports png and jpeg formats.".into(),
Without window_id captures the full primary display. Supports png and jpeg formats \
(default jpeg, quality 85).".into(),
input_schema: json!({
"type":"object","properties":{
"window_id":{"type":"integer"},
"format":{"type":"string","enum":["png","jpeg"]},
"quality":{"type":"integer","minimum":1,"maximum":95}
"format":{"type":"string","enum":["png","jpeg"],"description":"Image format. Default: jpeg."},
"quality":{"type":"integer","minimum":1,"maximum":95,"description":"JPEG quality 1-95; ignored for png. Default: 85."}
},"additionalProperties":false
}),
read_only: true, destructive: false, idempotent: true, open_world: false,
Expand All @@ -967,7 +968,7 @@ impl Tool for ScreenshotTool {

async fn invoke(&self, args: Value) -> ToolResult {
let xid_opt = args.get("window_id").and_then(|v| v.as_u64());
let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("png").to_owned();
let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("jpeg").to_owned();
let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(85) as u8;
let is_jpeg = format == "jpeg";
let max_dim = self.state.config.read().unwrap().max_image_dimension;
Expand Down
4 changes: 3 additions & 1 deletion libs/cua-driver-rs/crates/platform-macos/src/tools/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -109,14 +109,16 @@ pub struct DriverConfig {
/// Default capture_mode for get_window_state when not specified per-call.
pub capture_mode: String,
/// Max screenshot dimension (0 = no limit). Applied during screenshot/zoom.
/// Default 1568 matches Swift's `CuaDriverConfig.defaultMaxImageDimension` —
/// the long edge is downscaled to this before encoding.
pub max_image_dimension: u32,
}

impl Default for DriverConfig {
fn default() -> Self {
Self {
capture_mode: "som".to_owned(),
max_image_dimension: 0,
max_image_dimension: 1568,
}
}
}
Expand Down
10 changes: 5 additions & 5 deletions libs/cua-driver-rs/crates/platform-macos/src/tools/screenshot.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ fn def() -> &'static ToolDef {
name: "screenshot".into(),
description:
"Capture a screenshot. Returns base64-encoded image data in the requested format \
(default png).\n\n\
(default jpeg, quality 85).\n\n\
Without `window_id`, captures the full main display. With `window_id`, captures \
just that window (pair with `list_windows` or `get_accessibility_tree` which return \
window IDs).\n\n\
Expand All @@ -32,13 +32,13 @@ fn def() -> &'static ToolDef {
"format": {
"type": "string",
"enum": ["png", "jpeg"],
"description": "Image format. Default: png."
"description": "Image format. Default: jpeg."
},
"quality": {
"type": "integer",
"minimum": 1,
"maximum": 95,
"description": "JPEG quality 1-95; ignored for png. Default: 95."
"description": "JPEG quality 1-95; ignored for png. Default: 85."
}
},
"additionalProperties": false
Expand All @@ -56,8 +56,8 @@ impl Tool for ScreenshotTool {

async fn invoke(&self, args: Value) -> ToolResult {
let window_id = args.get("window_id").and_then(|v| v.as_u64()).map(|v| v as u32);
let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("png").to_owned();
let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(95) as u8;
let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("jpeg").to_owned();
let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(85) as u8;
let use_jpeg = format == "jpeg";
let max_dim = self.state.config.read().unwrap().max_image_dimension;

Expand Down
12 changes: 6 additions & 6 deletions libs/cua-driver-rs/crates/platform-windows/src/tools/impl_.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ pub struct DriverConfig {
}

impl Default for DriverConfig {
fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 0 } }
fn default() -> Self { Self { capture_mode: "som".into(), max_image_dimension: 1568 } }
}

pub struct ResizeRegistry {
Expand Down Expand Up @@ -2141,17 +2141,17 @@ impl Tool for ScreenshotTool {
// ScreenCaptureKit).
description: "Capture a screenshot of a single window via BitBlt + PrintWindow \
(no focus change). Returns base64-encoded image data in the requested format \
(default png).\n\n\
(default jpeg, quality 85).\n\n\
`window_id` is recommended (use `list_windows` to find it). When omitted, \
captures the full primary display — a Windows-only convenience for whole-\
screen snapshots without a per-window target.\n\n\
Requires no special permissions on Windows.".into(),
input_schema: json!({
"type":"object","properties":{
"window_id":{"type":"integer","description":"HWND of the window to capture. When omitted, captures the full primary display (Windows-only)."},
"format":{"type":"string","enum":["png","jpeg"],"description":"Image format. Default: png."},
"format":{"type":"string","enum":["png","jpeg"],"description":"Image format. Default: jpeg."},
"quality":{"type":"integer","minimum":1,"maximum":95,
"description":"JPEG quality 1-95; ignored for png."}
"description":"JPEG quality 1-95; ignored for png. Default: 85."}
},"additionalProperties":false
}),
read_only: true, destructive: false, idempotent: false, open_world: false,
Expand All @@ -2160,8 +2160,8 @@ impl Tool for ScreenshotTool {

async fn invoke(&self, args: Value) -> ToolResult {
let hwnd_opt = args.get("window_id").and_then(|v| v.as_u64());
let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("png").to_owned();
let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(95) as u8;
let format = args.get("format").and_then(|v| v.as_str()).unwrap_or("jpeg").to_owned();
let quality = args.get("quality").and_then(|v| v.as_u64()).unwrap_or(85) as u8;
let is_jpeg = format == "jpeg";
let max_dim = self.state.config.read().unwrap().max_image_dimension;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -255,8 +255,10 @@ def test_custom_cursor_visible_after_click(self) -> None:
# 7. Wait for cursor animation + render loop to settle.
time.sleep(0.5)

# 8. Take a full-screen screenshot.
scr = client.call_tool("screenshot", {})
# 8. Take a full-screen screenshot. Force PNG so the per-pixel decode
# below works — the screenshot tool defaults to JPEG, which would be
# lossy and skew the cursor-colour search.
scr = client.call_tool("screenshot", {"format": "png"})
b64 = scr.get("content", [{}])[0].get("data", "")
self.assertTrue(b64, "screenshot returned no data")
png_bytes = base64.b64decode(b64)
Expand Down
Loading