Skip to content

feat(cua-driver): expose action names in get_window_state elements #3617

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
injaneity merged 2 commits into from
Sep 7, 2026
Merged

feat(cua-driver): expose action names in get_window_state elements #3617

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9d2fd50
feat(cua-driver): expose action names in get_window_state elements
Wangxiaoxiaoa Sep 3, 2026
bd99fba
docs(cua-driver): regenerate action reference on macOS
injaneity Sep 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/content/docs/reference/cua-driver/mcp-tools.mdx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ Walk a running app's AX tree and return BOTH a structured `elements` array (pref

INVARIANT: call get_window_state once per turn per (pid, window_id) before any element-indexed action. The index map is replaced by the next snapshot.

PREFERRED CONSUMERS read `structuredContent.elements` (one entry per indexed row with `element_index`, `role`, `label`, `value` (the element's text/AXValue when present — use it to verify what a field holds), `frame: {x,y,w,h}`, `parent_index`, `depth`). The markdown `tree_markdown` stays available and unchanged in shape for existing text-parsing callers — but new fields will only be added to the structured side.
PREFERRED CONSUMERS read `structuredContent.elements` (one entry per indexed row with `element_index`, `role`, `label`, `value` (the element's text/AXValue when present — use it to verify what a field holds), `actions` (names of AX actions exposed by the element, omitted when empty), `frame: {x,y,w,h}`, `parent_index`, `depth`). The markdown `tree_markdown` stays available and unchanged in shape for existing text-parsing callers — but new fields will only be added to the structured side.

Always returns BOTH the element tree AND a screenshot — ground on both and cross-check (the tree lies on some surfaces: Electron echo-confirms, Catalyst null values, virtualized off-viewport rows with `h:1` frames). You choose the modality at ACTION time, not here: an element ax action (pass `element_index`/`element_token` → the accessibility rung) or an element px action (pass `x`,`y` → the pixel rung, read straight off this screenshot). `capture_mode` is deprecated and ignored. Pass `include_screenshot:false` to skip the grab and get the tree only — the cheap path when you're just re-indexing before an element ax action.

Expand Down
168 changes: 119 additions & 49 deletions libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -574,6 +574,67 @@ fn fold_max_dimension(ceiling: u32, per_call: Option<u32>) -> u32 {
}
}

/// Build a single structured element entry for `get_window_state`.
/// Returns `None` when the node has no `element_index` (non-actionable rows).
fn build_element_entry(
n: &crate::atspi::AtspiNode,
snapshot_id: Option<u32>,
bounds: Option<(i32, i32, u32, u32)>,
) -> Option<serde_json::Value> {
let idx = n.element_index?;
// `label` mirrors what a human reading the markdown row would call this
// element: name first, then value, then description.
let label = n
.name
.clone()
.or_else(|| n.value.clone())
.or_else(|| n.description.clone());
let mut entry = json!({
"element_index": idx,
"role": n.role,
"depth": n.depth,
});
if let Some(snapshot_id) = snapshot_id {
entry["element_token"] = json!(cua_driver_core::element_token::token_for(snapshot_id, idx));
}
if n.in_web_content {
entry["in_web_content"] = json!(true);
}
if let Some(label) = label {
entry["label"] = json!(label);
}
// Surface the element's value separately from `label` (which collapses
// name→value→description): a field with both a name AND typed text would
// otherwise hide the text from a caller reading the structured side,
// leaving it only in tree_markdown. See the macOS get_window_state builder
// for the rationale.
if let Some(value) = n.value.clone().filter(|v| !v.is_empty()) {
entry["value"] = json!(value);
}
if let Some(enabled) = n.enabled {
entry["enabled"] = json!(enabled);
}
if let Some(selected) = n.selected {
entry["selected"] = json!(selected);
}
let actions: Vec<String> = n
.actions
.iter()
.filter(|a| !a.trim().is_empty())
.cloned()
.collect();
if !actions.is_empty() {
entry["actions"] = json!(actions);
}
if let Some(parent) = n.parent_element_index {
entry["parent_index"] = json!(parent);
}
if let Some((x, y, w, h)) = bounds {
entry["frame"] = json!({ "x": x, "y": y, "w": w, "h": h });
}
Some(entry)
}

pub struct GetWindowStateTool {
state: Arc<ToolState>,
}
Expand All @@ -592,7 +653,8 @@ impl Tool for GetWindowStateTool {
the structured array.\n\n\
PREFERRED CONSUMERS read `structuredContent.elements` (one entry \
per indexed row with `element_index`, `role`, `label`, `value`, \
`enabled`, `selected`, \
`enabled`, `selected`, `actions` (names of AT-SPI actions exposed \
by the element, omitted when empty), \
`frame: {x,y,w,h}` when AT-SPI reports usable bounds, \
`parent_index`, `depth`). The markdown `tree_markdown` stays \
available and unchanged in shape for existing text-parsing \
Expand Down Expand Up @@ -857,7 +919,7 @@ impl Tool for GetWindowStateTool {

// Structured `elements` array: one entry per actionable node.
// Shape: `{element_index, element_token, role, label,
// depth, parent_index?, frame?: {x,y,w,h}}`. Frame is
// depth, actions?, parent_index?, frame?: {x,y,w,h}}`. Frame is
// included whenever AT-SPI Component.GetExtents(Screen)
// reported usable bounds; omitted otherwise (some
// toolkits leave bounds unset on hidden / virtual
Expand All @@ -871,53 +933,11 @@ impl Tool for GetWindowStateTool {
.nodes
.iter()
.filter_map(|n| {
let idx = n.element_index?;
// `label` mirrors what a human reading the markdown row
// would call this element: name first, then value,
// then description.
let label = n
.name
.clone()
.or_else(|| n.value.clone())
.or_else(|| n.description.clone());
let mut entry = json!({
"element_index": idx,
"role": n.role,
"depth": n.depth,
});
if let Some(snapshot_id) = snapshot_id {
entry["element_token"] = json!(
cua_driver_core::element_token::token_for(snapshot_id, idx)
);
}
if n.in_web_content {
entry["in_web_content"] = json!(true);
}
if let Some(label) = label {
entry["label"] = json!(label);
}
// Surface the element's value separately from `label`
// (which collapses name→value→description): a field
// with both a name AND typed text would otherwise hide
// the text from a caller reading the structured side,
// leaving it only in tree_markdown. See the macOS
// get_window_state builder for the rationale.
if let Some(value) = n.value.clone().filter(|v| !v.is_empty()) {
entry["value"] = json!(value);
}
if let Some(enabled) = n.enabled {
entry["enabled"] = json!(enabled);
}
if let Some(selected) = n.selected {
entry["selected"] = json!(selected);
}
if let Some(parent) = n.parent_element_index {
entry["parent_index"] = json!(parent);
}
if let Some((x, y, w, h)) = bounds_by_idx.get(&idx).copied() {
entry["frame"] = json!({ "x": x, "y": y, "w": w, "h": h });
}
Some(entry)
build_element_entry(
n,
snapshot_id,
bounds_by_idx.get(&n.element_index?).copied(),
)
})
.collect();
let elements = cua_driver_core::element_query::project_elements_for_query(
Expand Down Expand Up @@ -1093,6 +1113,56 @@ mod get_window_state_capture_tests {
}
}

#[cfg(test)]
mod get_window_state_actions_tests {
use super::*;
use crate::atspi::AtspiNode;

fn node(actions: Vec<String>) -> AtspiNode {
AtspiNode {
element_index: Some(1),
role: "button".to_owned(),
name: Some("ok".to_owned()),
value: None,
checked: None,
enabled: Some(true),
selected: None,
description: None,
actions,
element_key: 1,
depth: 0,
parent_element_index: None,
in_web_content: false,
}
}

#[test]
fn element_entry_includes_actions_when_present() {
let n = node(vec!["Press".to_owned(), "Open".to_owned()]);
let entry = build_element_entry(&n, None, None).unwrap();
assert_eq!(entry["actions"], json!(["Press", "Open"]));
}

#[test]
fn element_entry_omits_actions_when_empty() {
let n = node(Vec::new());
let entry = build_element_entry(&n, None, None).unwrap();
assert!(entry.get("actions").is_none());
}

#[test]
fn element_entry_filters_blank_action_names() {
let n = node(vec![
"Press".to_owned(),
"".to_owned(),
" ".to_owned(),
"Open".to_owned(),
]);
let entry = build_element_entry(&n, None, None).unwrap();
assert_eq!(entry["actions"], json!(["Press", "Open"]));
}
}

pub struct LaunchAppTool;
static LAUNCH_DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new();

Expand Down
Loading
Loading