diff --git a/docs/content/docs/reference/cua-driver/mcp-tools.mdx b/docs/content/docs/reference/cua-driver/mcp-tools.mdx index ad7e90627d..6df09d65c4 100644 --- a/docs/content/docs/reference/cua-driver/mcp-tools.mdx +++ b/docs/content/docs/reference/cua-driver/mcp-tools.mdx @@ -61,7 +61,7 @@ Walk a running app's AX tree and return BOTH a structured `elements` array (pref INVARIANT: call get_window_state once per turn per (pid, window_id) before any element-indexed action. The index map is replaced by the next snapshot. -PREFERRED CONSUMERS read `structuredContent.elements` (one entry per indexed row with `element_index`, `role`, `label`, `value` (the element's text/AXValue when present — use it to verify what a field holds), `frame: {x,y,w,h}`, `parent_index`, `depth`). The markdown `tree_markdown` stays available and unchanged in shape for existing text-parsing callers — but new fields will only be added to the structured side. +PREFERRED CONSUMERS read `structuredContent.elements` (one entry per indexed row with `element_index`, `role`, `label`, `value` (the element's text/AXValue when present — use it to verify what a field holds), `actions` (names of AX actions exposed by the element, omitted when empty), `frame: {x,y,w,h}`, `parent_index`, `depth`). The markdown `tree_markdown` stays available and unchanged in shape for existing text-parsing callers — but new fields will only be added to the structured side. Always returns BOTH the element tree AND a screenshot — ground on both and cross-check (the tree lies on some surfaces: Electron echo-confirms, Catalyst null values, virtualized off-viewport rows with `h:1` frames). You choose the modality at ACTION time, not here: an element ax action (pass `element_index`/`element_token` → the accessibility rung) or an element px action (pass `x`,`y` → the pixel rung, read straight off this screenshot). `capture_mode` is deprecated and ignored. Pass `include_screenshot:false` to skip the grab and get the tree only — the cheap path when you're just re-indexing before an element ax action. diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs index b9416c1655..b6d079ae63 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs @@ -574,6 +574,67 @@ fn fold_max_dimension(ceiling: u32, per_call: Option) -> u32 { } } +/// Build a single structured element entry for `get_window_state`. +/// Returns `None` when the node has no `element_index` (non-actionable rows). +fn build_element_entry( + n: &crate::atspi::AtspiNode, + snapshot_id: Option, + bounds: Option<(i32, i32, u32, u32)>, +) -> Option { + let idx = n.element_index?; + // `label` mirrors what a human reading the markdown row would call this + // element: name first, then value, then description. + let label = n + .name + .clone() + .or_else(|| n.value.clone()) + .or_else(|| n.description.clone()); + let mut entry = json!({ + "element_index": idx, + "role": n.role, + "depth": n.depth, + }); + if let Some(snapshot_id) = snapshot_id { + entry["element_token"] = json!(cua_driver_core::element_token::token_for(snapshot_id, idx)); + } + if n.in_web_content { + entry["in_web_content"] = json!(true); + } + if let Some(label) = label { + entry["label"] = json!(label); + } + // Surface the element's value separately from `label` (which collapses + // name→value→description): a field with both a name AND typed text would + // otherwise hide the text from a caller reading the structured side, + // leaving it only in tree_markdown. See the macOS get_window_state builder + // for the rationale. + if let Some(value) = n.value.clone().filter(|v| !v.is_empty()) { + entry["value"] = json!(value); + } + if let Some(enabled) = n.enabled { + entry["enabled"] = json!(enabled); + } + if let Some(selected) = n.selected { + entry["selected"] = json!(selected); + } + let actions: Vec = n + .actions + .iter() + .filter(|a| !a.trim().is_empty()) + .cloned() + .collect(); + if !actions.is_empty() { + entry["actions"] = json!(actions); + } + if let Some(parent) = n.parent_element_index { + entry["parent_index"] = json!(parent); + } + if let Some((x, y, w, h)) = bounds { + entry["frame"] = json!({ "x": x, "y": y, "w": w, "h": h }); + } + Some(entry) +} + pub struct GetWindowStateTool { state: Arc, } @@ -592,7 +653,8 @@ impl Tool for GetWindowStateTool { the structured array.\n\n\ PREFERRED CONSUMERS read `structuredContent.elements` (one entry \ per indexed row with `element_index`, `role`, `label`, `value`, \ - `enabled`, `selected`, \ + `enabled`, `selected`, `actions` (names of AT-SPI actions exposed \ + by the element, omitted when empty), \ `frame: {x,y,w,h}` when AT-SPI reports usable bounds, \ `parent_index`, `depth`). The markdown `tree_markdown` stays \ available and unchanged in shape for existing text-parsing \ @@ -857,7 +919,7 @@ impl Tool for GetWindowStateTool { // Structured `elements` array: one entry per actionable node. // Shape: `{element_index, element_token, role, label, - // depth, parent_index?, frame?: {x,y,w,h}}`. Frame is + // depth, actions?, parent_index?, frame?: {x,y,w,h}}`. Frame is // included whenever AT-SPI Component.GetExtents(Screen) // reported usable bounds; omitted otherwise (some // toolkits leave bounds unset on hidden / virtual @@ -871,53 +933,11 @@ impl Tool for GetWindowStateTool { .nodes .iter() .filter_map(|n| { - let idx = n.element_index?; - // `label` mirrors what a human reading the markdown row - // would call this element: name first, then value, - // then description. - let label = n - .name - .clone() - .or_else(|| n.value.clone()) - .or_else(|| n.description.clone()); - let mut entry = json!({ - "element_index": idx, - "role": n.role, - "depth": n.depth, - }); - if let Some(snapshot_id) = snapshot_id { - entry["element_token"] = json!( - cua_driver_core::element_token::token_for(snapshot_id, idx) - ); - } - if n.in_web_content { - entry["in_web_content"] = json!(true); - } - if let Some(label) = label { - entry["label"] = json!(label); - } - // Surface the element's value separately from `label` - // (which collapses name→value→description): a field - // with both a name AND typed text would otherwise hide - // the text from a caller reading the structured side, - // leaving it only in tree_markdown. See the macOS - // get_window_state builder for the rationale. - if let Some(value) = n.value.clone().filter(|v| !v.is_empty()) { - entry["value"] = json!(value); - } - if let Some(enabled) = n.enabled { - entry["enabled"] = json!(enabled); - } - if let Some(selected) = n.selected { - entry["selected"] = json!(selected); - } - if let Some(parent) = n.parent_element_index { - entry["parent_index"] = json!(parent); - } - if let Some((x, y, w, h)) = bounds_by_idx.get(&idx).copied() { - entry["frame"] = json!({ "x": x, "y": y, "w": w, "h": h }); - } - Some(entry) + build_element_entry( + n, + snapshot_id, + bounds_by_idx.get(&n.element_index?).copied(), + ) }) .collect(); let elements = cua_driver_core::element_query::project_elements_for_query( @@ -1093,6 +1113,56 @@ mod get_window_state_capture_tests { } } +#[cfg(test)] +mod get_window_state_actions_tests { + use super::*; + use crate::atspi::AtspiNode; + + fn node(actions: Vec) -> AtspiNode { + AtspiNode { + element_index: Some(1), + role: "button".to_owned(), + name: Some("ok".to_owned()), + value: None, + checked: None, + enabled: Some(true), + selected: None, + description: None, + actions, + element_key: 1, + depth: 0, + parent_element_index: None, + in_web_content: false, + } + } + + #[test] + fn element_entry_includes_actions_when_present() { + let n = node(vec!["Press".to_owned(), "Open".to_owned()]); + let entry = build_element_entry(&n, None, None).unwrap(); + assert_eq!(entry["actions"], json!(["Press", "Open"])); + } + + #[test] + fn element_entry_omits_actions_when_empty() { + let n = node(Vec::new()); + let entry = build_element_entry(&n, None, None).unwrap(); + assert!(entry.get("actions").is_none()); + } + + #[test] + fn element_entry_filters_blank_action_names() { + let n = node(vec![ + "Press".to_owned(), + "".to_owned(), + " ".to_owned(), + "Open".to_owned(), + ]); + let entry = build_element_entry(&n, None, None).unwrap(); + assert_eq!(entry["actions"], json!(["Press", "Open"])); + } +} + pub struct LaunchAppTool; static LAUNCH_DEF: std::sync::OnceLock = std::sync::OnceLock::new(); diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs index f49892b8f1..01e56b5709 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs @@ -33,7 +33,8 @@ fn def() -> &'static ToolDef { PREFERRED CONSUMERS read `structuredContent.elements` (one entry per \ indexed row with `element_index`, `role`, `label`, `value` (the \ element's text/AXValue when present — use it to verify what a field \ - holds), `frame: {x,y,w,h}`, `parent_index`, `depth`). The markdown \ + holds), `actions` (names of AX actions exposed by the element, \ + omitted when empty), `frame: {x,y,w,h}`, `parent_index`, `depth`). The markdown \ `tree_markdown` stays available \ and unchanged in shape for existing text-parsing callers — but new \ fields will only be added to the structured side.\n\n\ @@ -565,8 +566,8 @@ impl Tool for GetWindowStateTool { // (Hermes' regex parser, Codex, Claude Code) and is signalled as // preferred-for-back-compat-only via the `_note` field below. let elements_json: Vec = match (snapshot_id, tree_result.as_ref()) { - (Some(sid), Some(r)) => build_elements_array_with_token(&r.nodes, sid), - (None, Some(r)) if scope_matched => build_elements_array(&r.nodes), + (Some(sid), Some(r)) => build_elements_array_with_token(&r.nodes, Some(sid)), + (None, Some(r)) if scope_matched => build_elements_array_with_token(&r.nodes, None), _ => Vec::new(), }; let elements_json = cua_driver_core::element_query::project_elements_for_query( @@ -841,7 +842,7 @@ fn degradation_for( /// omitted to match the contract on the tool description. pub(crate) fn build_elements_array_with_token( nodes: &[crate::ax::tree::AXNode], - snapshot_id: u32, + snapshot_id: Option, ) -> Vec { nodes .iter() @@ -861,15 +862,18 @@ pub(crate) fn build_elements_array_with_token( .map(|[x, y, w, h]| serde_json::json!({ "x": x, "y": y, "w": w, "h": h })); let mut entry = serde_json::json!({ "element_index": idx, - // Surface 6: opaque token paired to the integer index. - // Tools accept either; the token has explicit validity - // (invalidated when the next snapshot supersedes this - // one in the per-pid LRU). See cua-driver-core's - // `element_token` module. - "element_token": cua_driver_core::element_token::token_for(snapshot_id, idx), "role": node.role, "depth": node.depth, }); + // Surface 6: opaque token paired to the integer index. + // Tools accept either; the token has explicit validity + // (invalidated when the next snapshot supersedes this + // one in the per-pid LRU). See cua-driver-core's + // `element_token` module. + if let Some(sid) = snapshot_id { + entry["element_token"] = + serde_json::json!(cua_driver_core::element_token::token_for(sid, idx)); + } if let Some(label) = label { entry["label"] = serde_json::Value::String(label); } @@ -924,6 +928,9 @@ pub(crate) fn build_elements_array_with_token( if let Some(selected) = selected { entry["selected"] = serde_json::Value::Bool(selected); } + if !node.actions.is_empty() { + entry["actions"] = serde_json::json!(node.actions); + } if node.in_web_content { entry["in_web_content"] = serde_json::Value::Bool(true); } @@ -938,26 +945,6 @@ pub(crate) fn build_elements_array_with_token( .collect() } -/// Back-compat wrapper for callers that don't yet have a snapshot id -/// to pass through. Emits the same fields as the token-aware builder -/// minus `element_token`. New call sites should prefer -/// `build_elements_array_with_token`. -#[allow(dead_code)] -pub(crate) fn build_elements_array(nodes: &[crate::ax::tree::AXNode]) -> Vec { - // Use a snapshot_id of 0 only to satisfy the signature; tokens - // built from id=0 are not registered and would fail the registry's - // stale check — but since this entry point is only kept for - // pre-existing callers (none in production after Surface 6), it - // strips the token field after rendering. - let mut out = build_elements_array_with_token(nodes, 0); - for entry in &mut out { - if let Some(obj) = entry.as_object_mut() { - obj.remove("element_token"); - } - } - out -} - /// Keep the structured response aligned with a query-filtered markdown tree. /// /// The AX walker deliberately keeps the complete node/cache snapshot so the @@ -1136,6 +1123,7 @@ mod tests { use super::*; use crate::ax::tree::AXNode; use cua_driver_core::element_query::project_elements_for_query; + use serde_json::json; fn node( idx: Option, @@ -1144,6 +1132,7 @@ mod tests { depth: usize, parent: Option, frame: Option<[f64; 4]>, + actions: Vec, ) -> AXNode { AXNode { element_index: idx, @@ -1153,7 +1142,7 @@ mod tests { description: None, identifier: None, help: None, - actions: vec![], + actions, element_ptr: 0, depth, parent_element_index: parent, @@ -1179,8 +1168,9 @@ mod tests { 0, None, Some([0.0, 0.0, 800.0, 600.0]), + vec![], ), - node(None, "AXStaticText", Some("hint"), 1, Some(0), None), + node(None, "AXStaticText", Some("hint"), 1, Some(0), None, vec![]), node( Some(1), "AXButton", @@ -1188,6 +1178,7 @@ mod tests { 1, Some(0), Some([10.0, 20.0, 60.0, 24.0]), + vec![], ), node( Some(2), @@ -1196,9 +1187,10 @@ mod tests { 1, Some(0), Some([80.0, 20.0, 60.0, 24.0]), + vec![], ), ]; - let elements = build_elements_array(&nodes); + let elements = build_elements_array_with_token(&nodes, None); assert_eq!( elements.len(), 3, @@ -1218,8 +1210,16 @@ mod tests { #[test] fn query_projection_keeps_only_rendered_actionable_rows() { let nodes = vec![ - node(Some(0), "AXWindow", Some("Document"), 0, None, None), - node(Some(1), "AXMenuItem", Some("Window"), 1, Some(0), None), + node(Some(0), "AXWindow", Some("Document"), 0, None, None, vec![]), + node( + Some(1), + "AXMenuItem", + Some("Window"), + 1, + Some(0), + None, + vec![], + ), node( Some(2), "AXMenuItem", @@ -1227,11 +1227,28 @@ mod tests { 2, Some(1), None, + vec![], + ), + node( + Some(3), + "AXMenuItem", + Some("Left"), + 3, + Some(2), + None, + vec![], + ), + node( + Some(4), + "AXButton", + Some("Unrelated"), + 1, + Some(0), + None, + vec![], ), - node(Some(3), "AXMenuItem", Some("Left"), 3, Some(2), None), - node(Some(4), "AXButton", Some("Unrelated"), 1, Some(0), None), ]; - let elements = build_elements_array(&nodes); + let elements = build_elements_array_with_token(&nodes, None); let filtered_markdown = concat!( "- [0] AXWindow \"Document\"\n", " - [1] AXMenuItem \"Window\"\n", @@ -1250,8 +1267,16 @@ mod tests { #[test] fn query_projection_returns_no_elements_when_markdown_has_no_match() { - let nodes = vec![node(Some(0), "AXButton", Some("Unrelated"), 0, None, None)]; - let elements = build_elements_array(&nodes); + let nodes = vec![node( + Some(0), + "AXButton", + Some("Unrelated"), + 0, + None, + None, + vec![], + )]; + let elements = build_elements_array_with_token(&nodes, None); let projected = project_elements_for_query(elements, Some("zoomLeft"), ""); @@ -1261,10 +1286,10 @@ mod tests { #[test] fn unfiltered_projection_preserves_every_element() { let nodes = vec![ - node(Some(0), "AXButton", Some("One"), 0, None, None), - node(Some(1), "AXButton", Some("Two"), 0, None, None), + node(Some(0), "AXButton", Some("One"), 0, None, None, vec![]), + node(Some(1), "AXButton", Some("Two"), 0, None, None, vec![]), ]; - let elements = build_elements_array(&nodes); + let elements = build_elements_array_with_token(&nodes, None); let projected = project_elements_for_query(elements, None, ""); @@ -1280,8 +1305,9 @@ mod tests { 3, Some(2), Some([1.5, 2.5, 33.0, 44.0]), + vec![], )]; - let entry = &build_elements_array(&nodes)[0]; + let entry = &build_elements_array_with_token(&nodes, None)[0]; assert_eq!(entry["element_index"], 7); assert_eq!(entry["role"], "AXButton"); assert_eq!(entry["label"], "Go"); @@ -1306,9 +1332,10 @@ mod tests { 1, None, None, + vec![], )]; nodes[0].value = Some("i love u".into()); - let entry = &build_elements_array(&nodes)[0]; + let entry = &build_elements_array_with_token(&nodes, None)[0]; assert_eq!(entry["label"], "Compose message", "label stays the title"); assert_eq!( entry["value"], "i love u", @@ -1328,6 +1355,7 @@ mod tests { 1, None, None, + vec![], )]; nodes[0].value_state = Some("8".into()); nodes[0].value_description = Some("8 dB".into()); @@ -1335,7 +1363,7 @@ mod tests { nodes[0].max_value = Some(8.0); nodes[0].enabled = Some(true); nodes[0].selected = Some(false); - let entry = &build_elements_array(&nodes)[0]; + let entry = &build_elements_array_with_token(&nodes, None)[0]; assert_eq!( entry["value"], "8", "numeric AXValue surfaces via value_state" @@ -1356,25 +1384,34 @@ mod tests { 2, None, None, + vec![], )]; nodes[0].in_web_content = true; - let entry = &build_elements_array(&nodes)[0]; + let entry = &build_elements_array_with_token(&nodes, None)[0]; assert_eq!(entry["in_web_content"], true); } #[test] fn checkbox_value_state_normalizes_to_selected() { - let mut nodes = vec![node(Some(0), "AXCheckBox", Some("I agree"), 0, None, None)]; + let mut nodes = vec![node( + Some(0), + "AXCheckBox", + Some("I agree"), + 0, + None, + None, + vec![], + )]; nodes[0].value_state = Some("0".into()); - let entry = &build_elements_array(&nodes)[0]; + let entry = &build_elements_array_with_token(&nodes, None)[0]; assert_eq!(entry["selected"], false); } #[test] fn elements_control_state_fields_omitted_when_absent() { // Stock behaviour is unchanged for elements without control state. - let nodes = vec![node(Some(0), "AXButton", Some("OK"), 0, None, None)]; - let entry = &build_elements_array(&nodes)[0]; + let nodes = vec![node(Some(0), "AXButton", Some("OK"), 0, None, None, vec![])]; + let entry = &build_elements_array_with_token(&nodes, None)[0]; for key in ["value_description", "min", "max", "enabled", "selected"] { assert!(entry.get(key).is_none(), "{key} must be omitted"); } @@ -1384,10 +1421,18 @@ mod tests { fn elements_omit_degenerate_min_max_range() { // WebKit reports AXMinValue/AXMaxValue as 0.0/0.0 on non-range // controls (checkboxes, radios) — a degenerate range is omitted. - let mut nodes = vec![node(Some(0), "AXCheckBox", Some("On"), 0, None, None)]; + let mut nodes = vec![node( + Some(0), + "AXCheckBox", + Some("On"), + 0, + None, + None, + vec![], + )]; nodes[0].min_value = Some(0.0); nodes[0].max_value = Some(0.0); - let entry = &build_elements_array(&nodes)[0]; + let entry = &build_elements_array_with_token(&nodes, None)[0]; assert!(entry.get("min").is_none(), "degenerate min must be omitted"); assert!(entry.get("max").is_none(), "degenerate max must be omitted"); } @@ -1395,9 +1440,9 @@ mod tests { #[test] fn elements_value_state_falls_back_to_string_value() { // String-valued elements keep their `value` even with no value_state. - let mut nodes = vec![node(Some(0), "AXComboBox", None, 0, None, None)]; + let mut nodes = vec![node(Some(0), "AXComboBox", None, 0, None, None, vec![])]; nodes[0].value = Some("Search".into()); - let entry = &build_elements_array(&nodes)[0]; + let entry = &build_elements_array_with_token(&nodes, None)[0]; assert_eq!(entry["value"], "Search"); } @@ -1405,16 +1450,16 @@ mod tests { fn elements_omit_empty_value() { // An empty AXValue must not emit a `value` field (matches the other // optional fields' omit-when-absent contract). - let mut nodes = vec![node(Some(0), "AXButton", Some("OK"), 0, None, None)]; + let mut nodes = vec![node(Some(0), "AXButton", Some("OK"), 0, None, None, vec![])]; nodes[0].value = Some(String::new()); - let entry = &build_elements_array(&nodes)[0]; + let entry = &build_elements_array_with_token(&nodes, None)[0]; assert!(entry.get("value").is_none(), "empty value must be omitted"); } #[test] fn elements_omit_optional_fields_when_missing() { - let nodes = vec![node(Some(0), "AXUnknown", None, 0, None, None)]; - let entry = &build_elements_array(&nodes)[0]; + let nodes = vec![node(Some(0), "AXUnknown", None, 0, None, None, vec![])]; + let entry = &build_elements_array_with_token(&nodes, None)[0]; assert!( entry.get("label").is_none(), "label must be omitted when title/value/desc/id are all empty" @@ -1435,33 +1480,53 @@ mod tests { fn elements_label_fallback_chain() { // title missing → description → value → identifier let nodes = vec![ - node(Some(0), "AXButton", None, 0, None, None), - node(Some(1), "AXButton", None, 0, None, None), - node(Some(2), "AXButton", None, 0, None, None), + node(Some(0), "AXButton", None, 0, None, None, vec![]), + node(Some(1), "AXButton", None, 0, None, None, vec![]), + node(Some(2), "AXButton", None, 0, None, None, vec![]), ]; let mut nodes = nodes; nodes[0].description = Some("from-desc".into()); nodes[1].value = Some("from-val".into()); nodes[2].identifier = Some("from-id".into()); - let elements = build_elements_array(&nodes); + let elements = build_elements_array_with_token(&nodes, None); assert_eq!(elements[0]["label"], "from-desc"); assert_eq!(elements[1]["label"], "from-val"); assert_eq!(elements[2]["label"], "from-id"); } - /// Every element entry carries a non-empty snapshot-bound - /// `element_token` alongside its numeric `element_index`. + #[test] + fn build_elements_array_with_token_emits_actions_when_present() { + let nodes = vec![node( + Some(0), + "AXButton", + Some("OK"), + 1, + None, + None, + vec!["AXPress".to_owned(), "AXShowMenu".to_owned()], + )]; + let entries = build_elements_array_with_token(&nodes, None); + assert_eq!(entries[0]["actions"], json!(["AXPress", "AXShowMenu"])); + } + + #[test] + fn build_elements_array_with_token_omits_actions_when_empty() { + let nodes = vec![node(Some(0), "AXButton", Some("OK"), 1, None, None, vec![])]; + let entries = build_elements_array_with_token(&nodes, None); + assert!(entries[0].get("actions").is_none()); + } + #[test] fn build_elements_array_with_token_emits_element_token_per_row() { let reg = cua_driver_core::element_token::global(); let pid = 0x6abc_0001_i32; let sid = reg.register_snapshot(pid, /* window_id = */ 9, 3); let nodes = vec![ - node(Some(0), "AXButton", Some("A"), 1, None, None), - node(Some(1), "AXButton", Some("B"), 1, None, None), - node(Some(2), "AXButton", Some("C"), 1, None, None), + node(Some(0), "AXButton", Some("A"), 1, None, None, vec![]), + node(Some(1), "AXButton", Some("B"), 1, None, None, vec![]), + node(Some(2), "AXButton", Some("C"), 1, None, None, vec![]), ]; - let entries = build_elements_array_with_token(&nodes, sid); + let entries = build_elements_array_with_token(&nodes, Some(sid)); assert_eq!(entries.len(), 3); // Every entry must have BOTH fields (additive contract). for e in &entries { @@ -1487,17 +1552,23 @@ mod tests { } } - /// Back-compat: `build_elements_array` (the old shim) must NOT emit - /// `element_token` — older callers that never plumb a snapshot id - /// through get a clean shape. #[test] - fn build_elements_array_shim_skips_element_token() { - let nodes = vec![node(Some(0), "AXButton", Some("A"), 1, None, None)]; - let entries = build_elements_array(&nodes); + fn build_elements_array_with_token_observation_only_has_actions_no_token() { + let nodes = vec![node( + Some(0), + "AXButton", + Some("OK"), + 1, + None, + None, + vec!["AXPress".to_owned(), "AXShowMenu".to_owned()], + )]; + let entries = build_elements_array_with_token(&nodes, None); assert_eq!(entries.len(), 1); + assert_eq!(entries[0]["actions"], json!(["AXPress", "AXShowMenu"])); assert!( entries[0].get("element_token").is_none(), - "back-compat shim must NOT emit element_token; got: {}", + "observation-only entries must not emit unregistered element_token: {}", entries[0] ); } diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs index f15bf10ae6..c4aad564c9 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs @@ -1062,6 +1062,48 @@ mod list_windows_z_index_tests { } } +#[cfg(test)] +mod get_window_state_actions_tests { + use super::*; + use crate::uia::UiaNode; + + fn node(actions: Vec) -> UiaNode { + UiaNode { + element_index: Some(1), + control_type: "Button".to_owned(), + name: Some("OK".to_owned()), + value: None, + automation_id: None, + help_text: None, + actions, + enabled: Some(true), + selected: None, + element_ptr: 0, + center_x: 0, + center_y: 0, + rect: None, + msaa_role: None, + depth: 0, + parent_element_index: None, + in_web_content: false, + } + } + + #[test] + fn element_entry_includes_actions_when_present() { + let n = node(vec!["invoke".to_owned(), "toggle".to_owned()]); + let entry = build_element_entry(&n, None).unwrap(); + assert_eq!(entry["actions"], json!(["invoke", "toggle"])); + } + + #[test] + fn element_entry_omits_actions_when_empty() { + let n = node(Vec::new()); + let entry = build_element_entry(&n, None).unwrap(); + assert!(entry.get("actions").is_none()); + } +} + // ── get_window_state ───────────────────────────────────────────────────────── /// Fold a per-call `max_dimension` cap with the configured @@ -1076,6 +1118,65 @@ fn fold_max_dimension(ceiling: u32, per_call: Option) -> u32 { } } +/// Build a single structured element entry for `get_window_state`. +/// Returns `None` when the node has no `element_index` (non-actionable rows). +fn build_element_entry( + n: &crate::uia::UiaNode, + snapshot_id: Option, +) -> Option { + let idx = n.element_index?; + // `label`: name → value → automation_id → help_text. + let label = n + .name + .clone() + .or_else(|| n.value.clone()) + .or_else(|| n.automation_id.clone()) + .or_else(|| n.help_text.clone()); + let mut entry = json!({ + "element_index": idx, + "role": n.control_type, + "depth": n.depth, + }); + if let Some(snapshot_id) = snapshot_id { + entry["element_token"] = json!(cua_driver_core::element_token::token_for(snapshot_id, idx)); + } + if n.in_web_content { + entry["in_web_content"] = json!(true); + } + if let Some(label) = label { + entry["label"] = json!(label); + } + // Surface the element's value separately from `label` (which collapses + // name→value→automation_id→help): a control with both a name AND text + // (a ValuePattern edit holding typed content) would otherwise hide the + // text from a caller reading the structured side. See the macOS + // get_window_state builder for the rationale. + if let Some(value) = n.value.clone().filter(|v| !v.is_empty()) { + entry["value"] = json!(value); + } + if let Some(enabled) = n.enabled { + entry["enabled"] = json!(enabled); + } + if let Some(selected) = n.selected { + entry["selected"] = json!(selected); + } + if !n.actions.is_empty() { + entry["actions"] = json!(n.actions); + } + if let Some(parent) = n.parent_element_index { + entry["parent_index"] = json!(parent); + } + if let Some((l, t, r, b)) = n.rect { + entry["frame"] = json!({ + "x": l, + "y": t, + "w": (r - l).max(0), + "h": (b - t).max(0), + }); + } + Some(entry) +} + pub struct GetWindowStateTool { state: Arc, } @@ -1101,7 +1202,8 @@ impl Tool for GetWindowStateTool { the next snapshot of the same (pid, window_id).\n\n\ PREFERRED CONSUMERS read `structuredContent.elements` (one entry per \ indexed row with `element_index`, `role`, `label`, `value`, `enabled`, \ - `selected`, `frame: {x,y,w,h}`, `parent_index`, `depth`). The markdown \ + `selected`, `actions` (names of UIA patterns exposed as actions, \ + omitted when empty), `frame: {x,y,w,h}`, `parent_index`, `depth`). The markdown \ `tree_markdown` stays available \ and unchanged in shape for existing text-parsing callers — but new \ fields will only be added to the structured side.\n\n\ @@ -1401,64 +1503,12 @@ impl Tool for GetWindowStateTool { // Structured `elements` array — preferred consumption // path. Shape matches the cross-platform spec: // `{element_index, element_token, role, label, depth, - // parent_index?, frame?: {x,y,w,h}}`. Frame is + // actions?, parent_index?, frame?: {x,y,w,h}}`. Frame is // included when UIA reported a usable BoundingRectangle. let elements: Vec = tr .nodes .iter() - .filter_map(|n| { - let idx = n.element_index?; - // `label`: name → value → automation_id → help_text. - let label = n - .name - .clone() - .or_else(|| n.value.clone()) - .or_else(|| n.automation_id.clone()) - .or_else(|| n.help_text.clone()); - let mut entry = json!({ - "element_index": idx, - "role": n.control_type, - "depth": n.depth, - }); - if let Some(snapshot_id) = snapshot_id { - entry["element_token"] = json!( - cua_driver_core::element_token::token_for(snapshot_id, idx) - ); - } - if n.in_web_content { - entry["in_web_content"] = json!(true); - } - if let Some(label) = label { - entry["label"] = json!(label); - } - // Surface the element's value separately from `label` - // (which collapses name→value→automation_id→help): a - // control with both a name AND text (a ValuePattern - // edit holding typed content) would otherwise hide the - // text from a caller reading the structured side. See - // the macOS get_window_state builder for the rationale. - if let Some(value) = n.value.clone().filter(|v| !v.is_empty()) { - entry["value"] = json!(value); - } - if let Some(enabled) = n.enabled { - entry["enabled"] = json!(enabled); - } - if let Some(selected) = n.selected { - entry["selected"] = json!(selected); - } - if let Some(parent) = n.parent_element_index { - entry["parent_index"] = json!(parent); - } - if let Some((l, t, r, b)) = n.rect { - entry["frame"] = json!({ - "x": l, - "y": t, - "w": (r - l).max(0), - "h": (b - t).max(0), - }); - } - Some(entry) - }) + .filter_map(|n| build_element_entry(n, snapshot_id)) .collect(); let elements = cua_driver_core::element_query::project_elements_for_query( elements,