From 4f7ebc86d8d39c0cde89510571de2f018b9d55e0 Mon Sep 17 00:00:00 2001
From: hqhq1025 <1506751656@qq.com>
Date: Tue, 28 Jul 2026 15:22:59 +0800
Subject: [PATCH] feat(cua-driver): expose element AX actions in structured
 elements
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`get_window_state` already collects each element's exposed AX actions —
`AXNode.actions` is populated by the walk and rendered into
`tree_markdown` as `actions=[...]`. The structured `elements` array never
carried them, so a caller reading `structuredContent` has no way to know
which action a given element supports and must guess a name for
`perform_action`. A wrong guess is indistinguishable from an element that
legitimately refused.

This is the same gap that was already closed for `value`. The comment
above that field says it plainly:

    the value is shadowed and invisible to a caller reading the structured
    side — it only showed up in `tree_markdown`, forcing a markdown grep to
    verify what landed. Emit it explicitly so the verify-then-escalate loop
    can read the typed text structurally.

`actions` has exactly that shape: present in the markdown, absent from the
structured side, forcing a markdown grep for something the walk already
knows.

The key is omitted when the element exposes no actions, so rows for inert
nodes (AXStaticText and friends) do not grow. Additive only — no existing
field changes shape.

macOS only for now: the Linux (AT-SPI) and Windows (UIA) element builders
do not collect an equivalent action list today, so there is nothing to
emit there yet. The field is therefore optional per platform rather than a
new cross-platform contract.

Verified:
- `cargo test -p platform-macos`: 184 passed, 0 failed
- `cargo fmt -p platform-macos -- --check`: clean
- `cua-contract-gen all --check`: generated manifest already up to date
---
 .../src/tools/get_window_state.rs             | 41 ++++++++++++++++++-
 1 file changed, 40 insertions(+), 1 deletion(-)

diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs
index 025f66eb8b..d16fedab44 100644
--- a/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs
+++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/get_window_state.rs
@@ -33,7 +33,10 @@ fn def() -> &'static ToolDef {
             PREFERRED CONSUMERS read `structuredContent.elements` (one entry per \
             indexed row with `element_index`, `role`, `label`, `value` (the \
             element's text/AXValue when present — use it to verify what a field \
-            holds), `frame: {x,y,w,h}`, `parent_index`, `depth`). The markdown \
+            holds), `actions` (the AX actions this element exposes, e.g. \
+            AXPress/AXShowMenu — pass one of these to `perform_action` rather \
+            than guessing a name; omitted for non-actionable rows), \
+            `frame: {x,y,w,h}`, `parent_index`, `depth`). The markdown \
             `tree_markdown` stays available \
             and unchanged in shape for existing text-parsing callers — but new \
             fields will only be added to the structured side.\n\n\
@@ -471,6 +474,15 @@ pub(crate) fn build_elements_array_with_token(
             if let Some(parent) = node.parent_element_index {
                 entry["parent_index"] = serde_json::json!(parent);
             }
+            // The AX actions this element actually exposes (AXPress, AXShowMenu,
+            // AXIncrement, …). The walk already collects them; without emitting
+            // them a caller has to guess an action name for `perform_action`,
+            // and a wrong guess is indistinguishable from an element that
+            // simply refused. Omitted when empty so rows for non-actionable
+            // nodes stay small.
+            if !node.actions.is_empty() {
+                entry["actions"] = serde_json::json!(node.actions);
+            }
             Some(entry)
         })
         .collect()
@@ -736,6 +748,33 @@ mod tests {
     /// Surface 6: every element entry must carry a non-empty
     /// `element_token` alongside its integer `element_index`. The
     /// integer field stays unchanged — the token is purely additive.
+    #[test]
+    fn build_elements_array_emits_exposed_actions() {
+        let reg = cua_driver_core::element_token::global();
+        let pid = 0x6abc_0002_i32;
+        let sid = reg.register_snapshot(pid, /* window_id = */ 11, 2);
+        let mut pressable = node(Some(0), "AXButton", Some("Go"), 1, None, None);
+        pressable.actions = vec!["AXPress".to_string(), "AXShowMenu".to_string()];
+        let inert = node(Some(1), "AXStaticText", Some("Label"), 1, None, None);
+
+        let entries = build_elements_array_with_token(&[pressable, inert], sid);
+        assert_eq!(entries.len(), 2);
+
+        let actions = entries[0]
+            .get("actions")
+            .and_then(|v| v.as_array())
+            .expect("an actionable element must expose its AX actions");
+        let names: Vec<&str> = actions.iter().filter_map(|v| v.as_str()).collect();
+        assert_eq!(names, vec!["AXPress", "AXShowMenu"]);
+
+        // Rows for non-actionable nodes stay small: the key is omitted rather
+        // than emitted as an empty array.
+        assert!(
+            entries[1].get("actions").is_none(),
+            "element with no AX actions must not carry an empty actions key"
+        );
+    }
+
     #[test]
     fn build_elements_array_with_token_emits_element_token_per_row() {
         let reg = cua_driver_core::element_token::global();