From d1e98ccd7484d89c6bf50bc4f402aba1d325d0c5 Mon Sep 17 00:00:00 2001 From: cyq <15000851237@163.com> Date: Wed, 3 Jun 2026 21:00:54 +0800 Subject: [PATCH 1/2] feat(cua-driver-rs): expose macOS element bounds --- .../crates/platform-macos/src/ax/cache.rs | 30 ++- .../rust/crates/platform-macos/src/ax/tree.rs | 151 +++++++++++--- .../src/tools/get_window_state.rs | 187 ++++++++++++++++-- 3 files changed, 319 insertions(+), 49 deletions(-) diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs index 475ef871c0..fc14748ef8 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/cache.rs @@ -83,7 +83,9 @@ pub struct ElementCache { impl ElementCache { pub fn new() -> Self { - Self { core: ElementCacheCore::new() } + Self { + core: ElementCacheCore::new(), + } } /// Replace the snapshot for (pid, window_id) with the nodes from a fresh walk. @@ -93,7 +95,8 @@ impl ElementCache { .filter(|n| n.element_index.is_some()) .map(|n| n.element_ptr) .collect(); - self.core.insert(CacheKey { pid, window_id }, CachedSnapshot { elements }); + self.core + .insert(CacheKey { pid, window_id }, CachedSnapshot { elements }); } /// Look up + `CFRetain` the element for `element_index` in (pid, window_id), @@ -155,6 +158,7 @@ mod tests { identifier: None, help: None, actions: Vec::new(), + screen_rect: None, element_ptr: ptr, } } @@ -176,11 +180,21 @@ mod tests { unsafe { CFRetain(ptr as CFTypeRef) }; let cache = ElementCache::new(); cache.update(1, 2, &[node_with_ptr(ptr)]); - assert_eq!(unsafe { CFGetRetainCount(ptr as CFTypeRef) }, base + 1, "cache owns one retain"); + assert_eq!( + unsafe { CFGetRetainCount(ptr as CFTypeRef) }, + base + 1, + "cache owns one retain" + ); // Borrow the element out for an action. - let guard = cache.get_element_retained(1, 2, 0).expect("element is cached"); - assert_eq!(unsafe { CFGetRetainCount(ptr as CFTypeRef) }, base + 2, "guard adds a retain"); + let guard = cache + .get_element_retained(1, 2, 0) + .expect("element is cached"); + assert_eq!( + unsafe { CFGetRetainCount(ptr as CFTypeRef) }, + base + 2, + "guard adds a retain" + ); // Concurrent get_window_state replaces the snapshot → old one dropped → // CFRelease of the cache's retain. The guard's retain must remain. @@ -193,7 +207,11 @@ mod tests { ); drop(guard); - assert_eq!(unsafe { CFGetRetainCount(ptr as CFTypeRef) }, base, "guard drop releases its retain"); + assert_eq!( + unsafe { CFGetRetainCount(ptr as CFTypeRef) }, + base, + "guard drop releases its retain" + ); } /// A missing index returns None without retaining anything. diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs index 7a2e00012d..d8a64f211e 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/tree.rs @@ -60,6 +60,8 @@ pub struct AXNode { pub identifier: Option, pub help: Option, pub actions: Vec, + /// Element frame in screen points: [x, y, width, height]. + pub screen_rect: Option<[f64; 4]>, /// The raw AXUIElementRef pointer value, for caching. pub element_ptr: usize, } @@ -98,7 +100,11 @@ pub fn walk_tree(pid: i32, window_id: Option, query: Option<&str>) -> TreeW unsafe { let app_elem = AXUIElementCreateApplication(pid); if app_elem.is_null() { - return TreeWalkResult { tree_markdown: String::new(), nodes, truncated: false }; + return TreeWalkResult { + tree_markdown: String::new(), + nodes, + truncated: false, + }; } // Chromium/Electron apps (Arc, VS Code, Electron shells) ship their @@ -110,7 +116,10 @@ pub fn walk_tree(pid: i32, window_id: Option, query: Option<&str>) -> TreeW // read it. Native Cocoa apps reject the attribute, so they pay no // settle cost. This relies on the MAX_ELEMENTS node cap to keep the // now-materialized (potentially large) tree bounded. - let already_enabled = enabled_pids().lock().map(|s| s.contains(&pid)).unwrap_or(false); + let already_enabled = enabled_pids() + .lock() + .map(|s| s.contains(&pid)) + .unwrap_or(false); if !already_enabled && enable_chromium_accessibility(app_elem) { crate::permissions::panel::pump_run_loop_briefly(CHROMIUM_SETTLE_SECONDS); if let Ok(mut set) = enabled_pids().lock() { @@ -137,21 +146,33 @@ pub fn walk_tree(pid: i32, window_id: Option, query: Option<&str>) -> TreeW // Filter: keep non-window children (menu bar) + the target window. let walk_these: Vec = if let Some(wid) = window_id { - top_level.iter().copied().filter(|&child| { - let role = copy_string_attr(child, "AXRole").unwrap_or_default(); - if role != "AXWindow" { - return true; // always keep menu bar and other non-window items - } - // Match AX window element → CGWindowID via private SPI. - ax_get_window_id(child) == Some(wid) - }).collect() + top_level + .iter() + .copied() + .filter(|&child| { + let role = copy_string_attr(child, "AXRole").unwrap_or_default(); + if role != "AXWindow" { + return true; // always keep menu bar and other non-window items + } + // Match AX window element → CGWindowID via private SPI. + ax_get_window_id(child) == Some(wid) + }) + .collect() } else { top_level.iter().copied().collect() }; // Walk each top-level child at depth 0. for child in walk_these { - walk_element(child, 0, &mut nodes, &mut lines, &mut index_counter, &mut visited_count, &mut truncated); + walk_element( + child, + 0, + &mut nodes, + &mut lines, + &mut index_counter, + &mut visited_count, + &mut truncated, + ); } // Release all top-level elements (copy_children / copy_ax_windows both retain). @@ -179,7 +200,11 @@ pub fn walk_tree(pid: i32, window_id: Option, query: Option<&str>) -> TreeW )); } - TreeWalkResult { tree_markdown, nodes, truncated: truncated_flag } + TreeWalkResult { + tree_markdown, + nodes, + truncated: truncated_flag, + } } unsafe fn walk_element( @@ -191,7 +216,9 @@ unsafe fn walk_element( visited_count: &mut usize, truncated: &mut bool, ) { - if depth > MAX_DEPTH { return; } + if depth > MAX_DEPTH { + return; + } // Enforce total-node cap — mirrors Swift's maxElements guard. // Set the truncated flag only when we actually stop early. if *visited_count >= MAX_ELEMENTS { @@ -200,15 +227,22 @@ unsafe fn walk_element( } *visited_count += 1; - let role = copy_string_attr(element, "AXRole") - .unwrap_or_else(|| "AXUnknown".into()); + let role = copy_string_attr(element, "AXRole").unwrap_or_else(|| "AXUnknown".into()); // Skip pure layout containers that have no interesting content. if role == "AXScrollArea" || role == "AXGroup" { // Still recurse — children may be interesting. let children = copy_children(element); for child in children { - walk_element(child, depth, nodes, lines, counter, visited_count, truncated); + walk_element( + child, + depth, + nodes, + lines, + counter, + visited_count, + truncated, + ); CFRelease(child as CFTypeRef); } return; @@ -222,7 +256,8 @@ unsafe fn walk_element( let title = copy_string_attr(element, "AXTitle"); let value = copy_string_attr(element, "AXValue"); // AXPlaceholderValue as fallback for empty text fields. - let value = value.filter(|v| !v.trim().is_empty()) + let value = value + .filter(|v| !v.trim().is_empty()) .or_else(|| copy_string_attr(element, "AXPlaceholderValue")); let description = copy_string_attr(element, "AXDescription"); let identifier = copy_string_attr(element, "AXIdentifier"); @@ -233,15 +268,22 @@ unsafe fn walk_element( let visible_description = description.as_deref().unwrap_or("").trim().to_owned(); let visible_value = value.as_deref().unwrap_or("").trim().to_owned(); - let has_content = !visible_title.is_empty() - || !visible_description.is_empty() - || !visible_value.is_empty(); + let has_content = + !visible_title.is_empty() || !visible_description.is_empty() || !visible_value.is_empty(); let is_actionable = !actions.is_empty(); if !is_actionable && !has_content && role != "AXWindow" && role != "AXSheet" { let children = copy_children(element); for child in children { - walk_element(child, depth + 1, nodes, lines, counter, visited_count, truncated); + walk_element( + child, + depth + 1, + nodes, + lines, + counter, + visited_count, + truncated, + ); CFRelease(child as CFTypeRef); } return; @@ -251,30 +293,57 @@ unsafe fn walk_element( let node = if is_actionable { let idx = *counter; *counter += 1; + let screen_rect = element_screen_rect(element); // Retain so the element stays alive in the cache after `copy_children` // releases the per-child ref at the end of the caller's loop. CFRetain(element as CFTypeRef); AXNode { element_index: Some(idx), role: role.clone(), - title: if visible_title.is_empty() { None } else { Some(visible_title.clone()) }, - value: if visible_value.is_empty() { None } else { Some(visible_value.clone()) }, - description: if visible_description.is_empty() { None } else { Some(visible_description.clone()) }, + title: if visible_title.is_empty() { + None + } else { + Some(visible_title.clone()) + }, + value: if visible_value.is_empty() { + None + } else { + Some(visible_value.clone()) + }, + description: if visible_description.is_empty() { + None + } else { + Some(visible_description.clone()) + }, identifier: identifier.clone(), help: help.clone(), actions: actions.clone(), + screen_rect, element_ptr, } } else { AXNode { element_index: None, role: role.clone(), - title: if visible_title.is_empty() { None } else { Some(visible_title.clone()) }, - value: if visible_value.is_empty() { None } else { Some(visible_value.clone()) }, - description: if visible_description.is_empty() { None } else { Some(visible_description.clone()) }, + title: if visible_title.is_empty() { + None + } else { + Some(visible_title.clone()) + }, + value: if visible_value.is_empty() { + None + } else { + Some(visible_value.clone()) + }, + description: if visible_description.is_empty() { + None + } else { + Some(visible_description.clone()) + }, identifier: identifier.clone(), help: help.clone(), actions: vec![], + screen_rect: None, element_ptr, } }; @@ -285,7 +354,15 @@ unsafe fn walk_element( let children = copy_children(element); for child in children { - walk_element(child, depth + 1, nodes, lines, counter, visited_count, truncated); + walk_element( + child, + depth + 1, + nodes, + lines, + counter, + visited_count, + truncated, + ); CFRelease(child as CFTypeRef); } } @@ -324,7 +401,9 @@ fn format_node_line(node: &AXNode) -> String { attrs.push(format!("help=\"{}\"", h)); } if !node.actions.is_empty() { - let action_str = node.actions.iter() + let action_str = node + .actions + .iter() .map(|a| a.strip_prefix("AX").unwrap_or(a).to_lowercase()) .collect::>() .join(","); @@ -376,8 +455,12 @@ fn filter_tree(markdown: &str, query: &str) -> String { if line.to_lowercase().contains(&needle) { for ancestor_depth in 0..depth { let ancestor = current_ancestor[ancestor_depth]; - if ancestor.is_empty() { continue; } - if last_emitted_at[ancestor_depth] == Some(ancestor) { continue; } + if ancestor.is_empty() { + continue; + } + if last_emitted_at[ancestor_depth] == Some(ancestor) { + continue; + } last_emitted_at[ancestor_depth] = Some(ancestor); output.push(ancestor); } @@ -397,7 +480,11 @@ fn filter_tree(markdown: &str, query: &str) -> String { fn leading_indent_depth(line: &str) -> usize { let mut count = 0; for ch in line.chars() { - if ch == ' ' { count += 1; } else { break; } + if ch == ' ' { + count += 1; + } else { + break; + } } count / 2 } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs index ad1b3e20ad..5a923bf803 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs @@ -1,16 +1,23 @@ use async_trait::async_trait; -use cua_driver_core::{protocol::{ToolResult, Content}, tool::{Tool, ToolDef}}; +use cua_driver_core::{ + protocol::{Content, ToolResult}, + tool::{Tool, ToolDef}, +}; use serde_json::Value; use std::sync::Arc; use super::ToolState; +use crate::ax::AXNode; +use crate::windows::WindowBounds; pub struct GetWindowStateTool { state: Arc, } impl GetWindowStateTool { - pub fn new(state: Arc) -> Self { Self { state } } + pub fn new(state: Arc) -> Self { + Self { state } + } } static DEF: std::sync::OnceLock = std::sync::OnceLock::new(); @@ -54,14 +61,51 @@ fn def() -> &'static ToolDef { }) } +fn build_structured_elements( + nodes: &[AXNode], + bounds: &WindowBounds, + screenshot_width: u32, + screenshot_height: u32, +) -> Vec { + if bounds.width <= 0.0 || bounds.height <= 0.0 { + return Vec::new(); + } + + let scale_x = screenshot_width as f64 / bounds.width; + let scale_y = screenshot_height as f64 / bounds.height; + + nodes + .iter() + .filter_map(|node| { + let idx = node.element_index?; + let [x, y, width, height] = node.screen_rect?; + Some(serde_json::json!({ + "element_index": idx, + "x": ((x - bounds.x) * scale_x).round() as i64, + "y": ((y - bounds.y) * scale_y).round() as i64, + "width": (width * scale_x).round() as i64, + "height": (height * scale_y).round() as i64, + })) + }) + .collect() +} + #[async_trait] impl Tool for GetWindowStateTool { - fn def(&self) -> &ToolDef { def() } + fn def(&self) -> &ToolDef { + def() + } async fn invoke(&self, args: Value) -> ToolResult { use cua_driver_core::tool_args::ArgsExt; - let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e }; - let window_id = match args.require_u32("window_id") { Ok(v) => v, Err(e) => return e }; + let pid = match args.require_i32("pid") { + Ok(v) => v, + Err(e) => return e, + }; + let window_id = match args.require_u32("window_id") { + Ok(v) => v, + Err(e) => return e, + }; let query = args.opt_str("query"); let screenshot_out_file = args.opt_str("screenshot_out_file").map(|s| { // Expand ~ prefix. @@ -77,12 +121,18 @@ impl Tool for GetWindowStateTool { let session_id = args.opt_str("_session_id"); let (default_mode, effective_max_dim) = { let cfg = self.state.config.read().unwrap(); - self.state.session_config.effective(session_id.as_deref(), &cfg) + self.state + .session_config + .effective(session_id.as_deref(), &cfg) }; let capture_mode = args.opt_str("capture_mode").unwrap_or(default_mode); // Walk AX tree (unless vision-only mode). Accept "tree" as deprecated alias for "ax". - let capture_mode = if capture_mode == "tree" { "ax".to_owned() } else { capture_mode }; + let capture_mode = if capture_mode == "tree" { + "ax".to_owned() + } else { + capture_mode + }; let tree_result = if capture_mode != "vision" { let q = query.clone(); // Wrap the blocking AX walk in a 30-second timeout. Heavy webview apps @@ -139,7 +189,9 @@ impl Tool for GetWindowStateTool { // Record resize ratio so ClickTool can scale coordinates back up. if let Some(ow) = orig_w { if w > 0 { - self.state.resize_registry.set_ratio(pid, ow as f64 / w as f64); + self.state + .resize_registry + .set_ratio(pid, ow as f64 / w as f64); } } else { self.state.resize_registry.clear_ratio(pid); @@ -191,11 +243,21 @@ impl Tool for GetWindowStateTool { } if content.is_empty() { - return ToolResult::error("No content produced (neither AX tree nor screenshot succeeded)"); + return ToolResult::error( + "No content produced (neither AX tree nor screenshot succeeded)", + ); } let element_count = self.state.element_cache.element_count(pid, window_id); - let tree_md = tree_result.as_ref().map(|r| r.tree_markdown.clone()).unwrap_or_default(); + let tree_md = tree_result + .as_ref() + .map(|r| r.tree_markdown.clone()) + .unwrap_or_default(); + let structured_elements = match (tree_result.as_ref(), screenshot_dims) { + (Some(result), Some((sw, sh))) => crate::windows::window_bounds_by_id(window_id) + .map(|bounds| build_structured_elements(&result.nodes, &bounds, sw, sh)), + _ => None, + }; let mut structured = serde_json::json!({ "window_id": window_id, "pid": pid, @@ -206,9 +268,112 @@ impl Tool for GetWindowStateTool { structured["screenshot_width"] = serde_json::json!(sw); structured["screenshot_height"] = serde_json::json!(sh); } + if let Some(elements) = structured_elements { + structured["elements"] = serde_json::json!(elements); + } if let Some(ref fp) = screenshot_file_path { structured["screenshot_file_path"] = serde_json::json!(fp); } - ToolResult { content, is_error: None, structured_content: Some(structured) } + ToolResult { + content, + is_error: None, + structured_content: Some(structured), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn node(element_index: Option, screen_rect: Option<[f64; 4]>) -> AXNode { + AXNode { + element_index, + role: "AXButton".to_owned(), + title: None, + value: None, + description: None, + identifier: None, + help: None, + actions: Vec::new(), + screen_rect, + element_ptr: 0, + } + } + + #[test] + fn build_structured_elements_converts_to_window_local_screenshot_pixels() { + let bounds = WindowBounds { + x: 10.0, + y: 20.0, + width: 100.0, + height: 50.0, + }; + let elements = build_structured_elements( + &[node(Some(7), Some([20.0, 24.0, 30.0, 10.0]))], + &bounds, + 150, + 100, + ); + + assert_eq!( + elements, + vec![serde_json::json!({ + "element_index": 7, + "x": 15, + "y": 8, + "width": 45, + "height": 20, + })] + ); + } + + #[test] + fn build_structured_elements_skips_non_actionable_or_missing_geometry_nodes() { + let bounds = WindowBounds { + x: 0.0, + y: 0.0, + width: 100.0, + height: 100.0, + }; + let elements = build_structured_elements( + &[ + node(Some(0), None), + node(None, Some([10.0, 10.0, 5.0, 5.0])), + node(Some(1), Some([10.0, 10.0, 5.0, 5.0])), + ], + &bounds, + 100, + 100, + ); + + assert_eq!( + elements, + vec![serde_json::json!({ + "element_index": 1, + "x": 10, + "y": 10, + "width": 5, + "height": 5, + })] + ); + } + + #[test] + fn build_structured_elements_returns_empty_for_non_positive_window_bounds() { + let bounds = WindowBounds { + x: 0.0, + y: 0.0, + width: 0.0, + height: 120.0, + }; + let elements = build_structured_elements( + &[node(Some(7), Some([10.0, 10.0, 5.0, 5.0]))], + &bounds, + 100, + 100, + ); + + assert!(elements.is_empty()); } } From b6d5e5e6d5cec7097bbb7ff58b74a05b0a9f0ba6 Mon Sep 17 00:00:00 2001 From: cyq <15000851237@163.com> Date: Thu, 4 Jun 2026 01:45:05 +0800 Subject: [PATCH 2/2] fix(cua-driver-rs): clip element bounds to window --- .../src/tools/get_window_state.rs | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs index 5a923bf803..dce3a79a11 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs @@ -78,7 +78,8 @@ fn build_structured_elements( .iter() .filter_map(|node| { let idx = node.element_index?; - let [x, y, width, height] = node.screen_rect?; + let [x, y, width, height] = + intersect_screen_rect_with_bounds(node.screen_rect?, bounds)?; Some(serde_json::json!({ "element_index": idx, "x": ((x - bounds.x) * scale_x).round() as i64, @@ -90,6 +91,24 @@ fn build_structured_elements( .collect() } +fn intersect_screen_rect_with_bounds( + [x, y, width, height]: [f64; 4], + bounds: &WindowBounds, +) -> Option<[f64; 4]> { + let left = x.max(bounds.x); + let top = y.max(bounds.y); + let right = (x + width).min(bounds.x + bounds.width); + let bottom = (y + height).min(bounds.y + bounds.height); + let clipped_width = right - left; + let clipped_height = bottom - top; + + if clipped_width <= 0.0 || clipped_height <= 0.0 { + None + } else { + Some([left, top, clipped_width, clipped_height]) + } +} + #[async_trait] impl Tool for GetWindowStateTool { fn def(&self) -> &ToolDef { @@ -328,6 +347,36 @@ mod tests { ); } + #[test] + fn build_structured_elements_clips_to_window_bounds() { + let bounds = WindowBounds { + x: 10.0, + y: 20.0, + width: 100.0, + height: 50.0, + }; + let elements = build_structured_elements( + &[ + node(Some(1), Some([0.0, 10.0, 30.0, 20.0])), + node(Some(2), Some([200.0, 20.0, 10.0, 10.0])), + ], + &bounds, + 150, + 100, + ); + + assert_eq!( + elements, + vec![serde_json::json!({ + "element_index": 1, + "x": 0, + "y": 0, + "width": 30, + "height": 20, + })] + ); + } + #[test] fn build_structured_elements_skips_non_actionable_or_missing_geometry_nodes() { let bounds = WindowBounds {