diff --git a/docs/content/docs/reference/cua-driver/mcp-tools.mdx b/docs/content/docs/reference/cua-driver/mcp-tools.mdx index e12a95f47a..bbbac596c2 100644 --- a/docs/content/docs/reference/cua-driver/mcp-tools.mdx +++ b/docs/content/docs/reference/cua-driver/mcp-tools.mdx @@ -211,7 +211,7 @@ from_zoom: set true after a zoom call to auto-translate zoom-image pixel coordin - `button` (string, optional): Mouse button. Default: "left" — omit for legacy left-click behaviour. Pixel path uses the matching CGEvent primitive; AX path maps "right" to AXShowMenu and falls back to a pixel middle-click at the element's center for "middle". - `count` (integer, optional): Click count (pixel path only). Default 1. - `debug_image_out` (string, optional): Optional file path. When set on a pixel-addressed click, captures a fresh screenshot, draws a red crosshair at (x, y), and writes the PNG. Use to verify coordinate spaces. Requires window_id; incompatible with from_zoom. -- `delivery_mode` (string, optional): Best-effort-background ladder rung for a PIXEL click (default "background"). "background": post the CGEvent to the pid without fronting. "foreground": briefly front the window, click, restore the prior frontmost — the explicit last resort for surfaces that drop background synthetic clicks. Requires window_id. A click is never driver-verifiable (no read-back), so both report verified:false — confirm the effect via screenshot. Use the agent loop: background AX (element_index) → screenshot → background pixel (x/y) → screenshot → delivery_mode:"foreground". +- `delivery_mode` (string, optional): Best-effort-background ladder rung (default "background"). "background": perform the AX action or post the CGEvent without fronting. "foreground": briefly front the window, act, let transient UI settle, then restore the prior frontmost app. Requires window_id. A click is never driver-verifiable (no read-back), so both report verified:false — confirm the effect via screenshot. Use the agent loop: background AX (element_index) → screenshot → background pixel (x/y) → screenshot → delivery_mode:"foreground". - `element_index` (integer, optional): Element index from last get_window_state. REQUIRES `pid` and `window_id` to be passed alongside it — element_index alone (no pid) fails fast with "Missing required integer field: pid"; it is not a silent no-op. - `element_token` (string, optional): Opaque per-snapshot element handle from `structuredContent.elements[].element_token` of the last get_window_state. Takes precedence over element_index when both supplied. Returns an explicit "stale" error if the snapshot has been superseded — re-snapshot in that case. - `from_zoom` (boolean, optional): When true, x and y are in the last zoom image for this pid; driver translates back to full-window coordinates. diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rs index ad6b9171d3..3dd32c7288 100644 --- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rs +++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rs @@ -167,6 +167,22 @@ impl ForegroundSentinel { } } + /// Re-establish the observation boundary after video capture starts. + /// Capture backends may briefly perturb foreground state while they attach; + /// those setup events are not part of the action under test. + pub fn prepare_background_observation( + &self, + driver: &mut impl Driver, + target: TargetWindow, + ) -> Result<(), String> { + activate_native_foreground(driver, self.target); + wait_for_native_focus_stable(self.target); + std::thread::sleep(Duration::from_millis(100)); + fs::write(&self.journal_path, "") + .map_err(|error| format!("reset foreground sentinel journal: {error}"))?; + self.assert_background_posture(target) + } + /// Run one background action while checking the native desktop and the /// sentinel journal. The returned oracle list is suitable for a typed E2E /// result; any unsupported observation or side effect is an error. @@ -261,7 +277,6 @@ fn is_wayland_session() -> bool { .is_ok_and(|session| session.eq_ignore_ascii_case("wayland")) } -#[cfg(any(target_os = "windows", target_os = "linux"))] fn activate_native_foreground(driver: &mut impl Driver, target: TargetWindow) { let response = driver.call( "bring_to_front", @@ -331,9 +346,6 @@ fn physically_focus_windows_sentinel(target: TargetWindow) { } } -#[cfg(not(any(target_os = "windows", target_os = "linux")))] -fn activate_native_foreground(_driver: &mut impl Driver, _target: TargetWindow) {} - #[cfg(any(target_os = "windows", target_os = "linux"))] fn wait_for_native_focus_stable(target: TargetWindow) { use crate::observer::{ObserverBackend, TargetZ}; @@ -373,6 +385,7 @@ pub fn run_with_background_oracles( let sentinel = ForegroundSentinel::launch(driver); sentinel.assert_background_posture(target)?; driver.start_behavior_recording(); + sentinel.prepare_background_observation(driver, target)?; sentinel.observe_background(target, || action(driver)) } diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/capture_contract_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/capture_contract_test.rs index d5d8b64a20..01b0c936c6 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/capture_contract_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/capture_contract_test.rs @@ -417,6 +417,15 @@ fn background_screenshot_preserves_desktop() { }) .expect("establish capture background posture before recording"); driver.start_behavior_recording(); + sentinel + .prepare_background_observation( + &mut driver, + TargetWindow { + pid, + native_id: wid, + }, + ) + .expect("re-establish capture background posture after recording setup"); let (response, mut passed) = sentinel .observe_background( TargetWindow { diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs index 49c617b949..f4f1236ac9 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs @@ -170,6 +170,17 @@ where .expect("establish background posture before recording"); } fixture.driver.start_behavior_recording(); + if let Some(sentinel) = &sentinel { + sentinel + .prepare_background_observation( + &mut fixture.driver, + TargetWindow { + pid: fixture.pid, + native_id: fixture.wid, + }, + ) + .expect("re-establish background posture after recording setup"); + } let mut observation = if delivery == Delivery::Background { let sentinel = sentinel.as_ref().expect("background sentinel"); let (mut observation, passed) = sentinel diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_macos_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_macos_test.rs similarity index 54% rename from libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_macos_test.rs rename to libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_macos_test.rs index d1cef67d7c..9cf4a123fd 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_macos_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_macos_test.rs @@ -1,6 +1,6 @@ //! macOS **desktop-scope** (vision/foreground) modality, exercised through the -//! SAME cua-driver interface as the Windows `modality_desktop_scope_test`: -//! `set_config capture_scope=desktop` + a window-less screen-absolute `click` +//! SAME cua-driver interface as the Windows `desktop_scope_windows_test`: +//! a window-less screen-absolute `click` with `scope=desktop` //! (no `pid`, no `window_id`, no `list_windows`). The macOS actuator resolves //! the frontmost on-screen window under the point (the `WindowFromPoint` peer, //! via `CGWindowList`) and clicks it through the proven window-local pixel path, @@ -14,24 +14,21 @@ //! asserts the `window`-scope gate rejects a window-less click //! (`desktop_scope_disabled`), matching the Windows contract. //! -//! `set_config` is made session-scoped (a `session` arg → `_session_id`), so it -//! is in-memory only and never writes the developer's `~/.cua-driver/config.json`. -//! //! #[ignore] (needs a real desktop session + TCC Accessibility + the AppKit //! harness). Run: -//! cargo test -p cua-driver --test modality_desktop_scope_macos_test -- --ignored --nocapture --test-threads=1 +//! cargo test -p cua-driver --test desktop_scope_macos_test -- --ignored --nocapture --test-threads=1 #![cfg(target_os = "macos")] use std::process::{Command, Stdio}; use std::time::{Duration, Instant}; +use cua_driver_testkit::e2e::{ + execute_case, recording_evidence, CaseSpec, Delivery, DriverRoute, Evidence, Observation, + OracleKind, Scope, Targeting, +}; use cua_driver_testkit::{harness_app, Driver, McpDriver}; -/// Session id so `set_config capture_scope=desktop` is session-scoped (no disk -/// write) and the `click` resolves the same scope override. -const SESSION: &str = "vf-desktop"; - fn harness_exe() -> std::path::PathBuf { std::env::var("HARNESS_APPKIT_APP") .map(std::path::PathBuf::from) @@ -46,22 +43,36 @@ fn harness_exe() -> std::path::PathBuf { fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> { let exe = harness_exe(); if !exe.exists() { + if std::env::var_os("CUA_TEST_REQUIRE_FIXTURES").is_some() { + panic!("required AppKit harness is missing at {exe:?}"); + } eprintln!("[desktop-mac] AppKit harness not built ({exe:?}) — run tests/fixtures/build/macos.sh; skipping"); return None; } - driver - .reaper() - .spawn( - Command::new(&exe) - .stdout(Stdio::null()) - .stderr(Stdio::null()), - ) - .ok()?; + let child = match cua_driver_testkit::spawn_in_job( + Command::new(&exe) + .stdout(Stdio::null()) + .stderr(Stdio::null()), + ) { + Ok(child) => child, + Err(error) => { + if std::env::var_os("CUA_TEST_REQUIRE_FIXTURES").is_some() { + panic!("failed to launch required AppKit harness {exe:?}: {error}"); + } + eprintln!("[desktop-mac] AppKit harness launch failed: {error}; skipping"); + return None; + } + }; + let launched_pid = child.id(); + driver.reaper().push(child); let deadline = Instant::now() + Duration::from_secs(14); while Instant::now() < deadline { let r = driver.call("list_windows", serde_json::json!({})); if let Some(wins) = r.structured()["windows"].as_array() { for w in wins { + if w["pid"].as_u64() != Some(launched_pid as u64) { + continue; + } if w["title"] .as_str() .unwrap_or("") @@ -70,7 +81,6 @@ fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> { let pid = w["pid"].as_u64().unwrap_or(0) as u32; let wid = w["window_id"].as_u64().unwrap_or(0); if pid != 0 && wid != 0 { - driver.reaper().track_pid(pid); return Some((pid, wid)); } } @@ -78,6 +88,9 @@ fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> { } std::thread::sleep(Duration::from_millis(400)); } + if std::env::var_os("CUA_TEST_REQUIRE_FIXTURES").is_some() { + panic!("required AppKit harness window never appeared"); + } eprintln!( "[desktop-mac] harness window never appeared — graphical session available? skipping" ); @@ -156,67 +169,68 @@ fn activate_pid(pid: u32) { #[test] #[ignore] fn desktop_scope_windowless_click_lands_on_control() { - let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else { - return; - }; - let Some((pid, wid)) = launch(&mut driver) else { - return; - }; + let cell_id = "macos-appkit-desktop-left-click-px-foreground"; + let case = CaseSpec::delivered( + cell_id, + "appkit", + "appkit", + "left_click", + Targeting::Px, + Delivery::Foreground, + Scope::Desktop, + DriverRoute::MacosCgEventHid, + vec![OracleKind::FixtureState], + ); + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_macos_daemon_proxy_named(cell_id) + .expect("start installed macOS daemon proxy"); + *evidence = recording_evidence(driver.recording_dir()); + let (pid, wid) = launch(&mut driver).expect("required AppKit harness did not launch"); - // Settle for the AppKit AX tree to register the button + its frame. - let mut snap = ax_snapshot(&mut driver, pid, wid); - let mut center = increment_center(&snap); - let deadline = Instant::now() + Duration::from_secs(8); - while center.is_none() && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(400)); - snap = ax_snapshot(&mut driver, pid, wid); - center = increment_center(&snap); - } - let Some((cx, cy)) = center else { - eprintln!("[desktop-mac] increment button frame not found (TCC Accessibility missing?) — skipping"); - return; - }; - let pre = counter(&snap).unwrap_or(0); - println!("[desktop-mac] increment button screen-center=({cx},{cy}) pre-counter={pre}"); + // Settle for the AppKit AX tree to register the button + its frame. + let mut snap = ax_snapshot(&mut driver, pid, wid); + let mut center = increment_center(&snap); + let deadline = Instant::now() + Duration::from_secs(8); + while center.is_none() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(400)); + snap = ax_snapshot(&mut driver, pid, wid); + center = increment_center(&snap); + } + let Some((cx, cy)) = center else { + panic!("increment button frame not found in required AppKit AX tree"); + }; + let pre = counter(&snap).unwrap_or(0); + println!("[desktop-mac] increment button screen-center=({cx},{cy}) pre-counter={pre}"); - // Desktop scope clicks the frontmost window at the point — put the harness there. - activate_pid(pid); + // Desktop scope clicks the frontmost window at the point — put the harness there. + activate_pid(pid); + driver.start_behavior_recording(); - // Window-less screen-absolute click — no pid, no window_id; scope per-call. - let clicked = driver.call( - "click", - serde_json::json!({ "x": cx, "y": cy, "scope": "desktop", "session": SESSION }), - ); - assert!( - !clicked.is_error(), - "desktop-scope click errored: {}", - clicked.text() - ); - assert!( - clicked.text().to_lowercase().contains("desktop scope"), - "click not reported as desktop-scope: {}", - clicked.text() - ); - println!("[desktop-mac] {}", clicked.text()); + // Window-less screen-absolute click — no pid, no window_id; scope per-call. + let clicked = driver.call( + "click", + serde_json::json!({ "x": cx, "y": cy, "scope": "desktop" }), + ); + assert!( + !clicked.is_error(), + "desktop-scope click errored: {}", + clicked.text() + ); + assert!( + clicked.text().to_lowercase().contains("desktop scope"), + "click not reported as desktop-scope: {}", + clicked.text() + ); + println!("[desktop-mac] {}", clicked.text()); - std::thread::sleep(Duration::from_millis(600)); - let post = counter(&ax_snapshot(&mut driver, pid, wid)).unwrap_or(pre); - if post > pre { - println!("✅ desktop_scope_windowless_click_lands_on_control: counter {pre} → {post}"); - return; - } - // The desktop click lands on whatever window is *visually frontmost* at the - // point — that is the contract. On a busy desktop another window can cover - // the harness (and `activate` may not beat a floating panel), so a - // non-advance here means the harness was not frontmost at the point, NOT a - // driver fault. The click is confirmed to have resolved a real window (see - // its result text above). Skip rather than false-fail; a clean session - // asserts the landing, as the Linux peer does end-to-end. - eprintln!( - "[desktop-mac] counter did not advance ({pre}→{post}) — the harness was not the \ - frontmost window at ({cx},{cy}) on this desktop (another window covering it). \ - Skipping the landing assertion; run on a clean GUI session to assert it." - ); + std::thread::sleep(Duration::from_millis(600)); + let post = counter(&ax_snapshot(&mut driver, pid, wid)).unwrap_or(pre); + assert!( + post > pre, + "desktop click did not advance AppKit counter at ({cx},{cy}): {pre} -> {post}" + ); + Observation::delivered_with_fixture_state(Vec::new()) + }); } /// Negative gate: a window-less screen-absolute click while `capture_scope=window` @@ -225,19 +239,33 @@ fn desktop_scope_windowless_click_lands_on_control() { #[test] #[ignore] fn window_scope_rejects_windowless_click() { - let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else { - return; - }; - // Default scope is "window" — a window-less click must be rejected. - let r = driver.call( - "click", - serde_json::json!({ "x": 100, "y": 100, "scope": "window", "session": SESSION }), - ); - let txt = r.text().to_lowercase(); - assert!( - r.is_error() || txt.contains("desktop scope") || txt.contains("desktop_scope_disabled"), - "window-scope window-less click was NOT rejected: {}", - r.text() + let cell_id = "macos-window-scope-gate-px-not-applicable"; + let case = CaseSpec::delivered( + cell_id, + "desktop", + "quartz", + "window_scope_gate", + Targeting::Px, + Delivery::NotApplicable, + Scope::Window, + DriverRoute::Composite, + vec![OracleKind::Protocol], ); - println!("✅ window_scope_rejects_windowless_click: window-less click correctly gated"); + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_macos_daemon_proxy_named(cell_id) + .expect("start installed macOS daemon proxy"); + *evidence = recording_evidence(driver.recording_dir()); + // Default scope is "window" — a window-less click must be rejected. + driver.start_behavior_recording(); + let r = driver.call( + "click", + serde_json::json!({ "x": 100, "y": 100, "scope": "window" }), + ); + assert!( + r.is_error() && r.structured()["code"].as_str() == Some("desktop_scope_disabled"), + "window-scope window-less click was NOT rejected: {}", + r.text() + ); + Observation::delivered(vec![OracleKind::Protocol], Evidence::default()) + }); } diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs index 65711e784f..5fe217f093 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs @@ -19,10 +19,8 @@ //! //! Tests are `#[ignore]` so they don't run in plain `cargo test`. //! -//! **TCC caveat:** on a fresh Mac, the cua-driver process needs -//! Accessibility permission for AX queries to return non-empty trees. -//! These tests print a TCC hint and exit cleanly (PASS-by-skip) when the -//! AX tree is empty rather than misreporting as a test failure. +//! The macOS lane preflight verifies the installed daemon identity and TCC +//! grants before these tests run. Missing fixtures or AX trees fail here too. #![cfg(target_os = "macos")] @@ -30,7 +28,13 @@ use std::path::PathBuf; use std::process::{Child, Command, Stdio}; use std::time::Duration; -use cua_driver_testkit::ax::{element_index_by_id, has_id, looks_empty}; +use cua_driver_testkit::ax::{element_index_by_id, element_index_containing, has_id, looks_empty}; +use cua_driver_testkit::e2e::{ + execute_case, native_background_case, native_foreground_case, native_readonly_case, + recording_evidence, DriverRoute, Evidence, Observation, OracleKind, RefusalCode, Targeting, +}; +use cua_driver_testkit::observer::TargetWindow; +use cua_driver_testkit::sentinel::run_with_background_oracles; use cua_driver_testkit::{Driver, McpDriver, ToolResponse}; // ── paths ──────────────────────────────────────────────────────────────────── @@ -57,15 +61,12 @@ struct Harness { } impl Harness { - fn launch() -> Option { + fn launch() -> Self { let exe = harness_exe(); - if !exe.exists() { - eprintln!( - "harness exe not found at {exe:?} \ - — run libs/cua-driver/tests/fixtures/build/macos.sh first" - ); - return None; - } + assert!( + exe.exists(), + "required AppKit harness is missing at {exe:?}; run the fixture build" + ); // Launch the binary directly (not via `open`) so we control the pid // and can kill it cleanly on Drop. The app still installs an AppKit // window via NSApp.run(). @@ -73,11 +74,11 @@ impl Harness { .stdout(Stdio::null()) .stderr(Stdio::null()) .spawn() - .ok()?; + .unwrap_or_else(|error| panic!("launch AppKit harness {exe:?}: {error}")); let pid = app.id(); // Settle for window creation + activation. std::thread::sleep(Duration::from_millis(800)); - Some(Self { _app: app, pid }) + Self { _app: app, pid } } } @@ -102,74 +103,148 @@ fn snapshot_elements(driver: &mut McpDriver, pid: u32, window_id: u64) -> ToolRe ) } -// ── tests ──────────────────────────────────────────────────────────────────── +fn element_pixel_frame(snapshot: &ToolResponse, identifier: &str) -> (f64, f64, f64, f64) { + let index = element_index_by_id(snapshot.tree_text(), identifier) + .unwrap_or_else(|| panic!("{identifier} element_index not found")); + let elements = snapshot.structured()["elements"] + .as_array() + .expect("AppKit structured elements"); + let element = elements + .iter() + .find(|element| element["element_index"].as_u64() == Some(index)) + .unwrap_or_else(|| panic!("{identifier} element frame not found")); + let window = elements + .iter() + .find(|element| element["role"].as_str() == Some("AXWindow")) + .expect("AppKit window frame"); + let scale = snapshot.structured()["screenshot_width"] + .as_f64() + .unwrap_or(1.0) + / window["frame"]["w"].as_f64().unwrap_or(1.0).max(1.0); + ( + (element["frame"]["x"].as_f64().unwrap_or(0.0) + - window["frame"]["x"].as_f64().unwrap_or(0.0)) + * scale, + (element["frame"]["y"].as_f64().unwrap_or(0.0) + - window["frame"]["y"].as_f64().unwrap_or(0.0)) + * scale, + element["frame"]["w"].as_f64().unwrap_or(0.0) * scale, + element["frame"]["h"].as_f64().unwrap_or(0.0) * scale, + ) +} -#[test] -#[ignore] -fn harness_appkit_smoke() { - let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else { - return; - }; - let harness = match Harness::launch() { - Some(h) => h, - None => { - eprintln!("harness not built — skipping"); - return; +fn run_case( + case: cua_driver_testkit::e2e::CaseSpec, + test: impl FnOnce(u32, u64, &mut McpDriver) -> Observation, +) { + let cell_id = case.cell_id.clone(); + let delivery = case.delivery; + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_macos_daemon_proxy_named(&cell_id) + .expect("start installed macOS daemon proxy"); + *evidence = recording_evidence(driver.recording_dir()); + let harness = Harness::launch(); + let (wid, _) = driver + .find_window(harness.pid as i64, "CuaTestHarness AppKit") + .expect("AppKit main window not found"); + if delivery != cua_driver_testkit::e2e::Delivery::Background { + driver.start_behavior_recording(); } - }; - println!("harness pid={}", harness.pid); + test(harness.pid, wid, &mut driver) + }); +} - let (wid, title) = driver - .find_window(harness.pid as i64, "CuaTestHarness AppKit") - .expect("main window not found via list_windows"); - println!("main window: id={wid} title={title:?}"); +fn run_background_case( + action: &str, + route: DriverRoute, + test: impl FnOnce(u32, u64, &mut McpDriver), +) { + run_background_case_targeting(action, Targeting::Ax, route, test); +} - let snap = snapshot_elements(&mut driver, harness.pid, wid); +fn run_background_case_targeting( + action: &str, + targeting: Targeting, + route: DriverRoute, + test: impl FnOnce(u32, u64, &mut McpDriver), +) { + run_case( + native_background_case("appkit", action, targeting, route), + |pid, wid, driver| { + let (_, passed) = run_with_background_oracles( + driver, + TargetWindow { + pid, + native_id: wid, + }, + |driver| test(pid, wid, driver), + ) + .unwrap_or_else(|error| panic!("background desktop contract failed: {error}")); + Observation::delivered_with_fixture_state(passed) + }, + ); +} - if looks_empty(snap.tree_text()) { - eprintln!( - "AX tree empty — likely TCC Accessibility not granted to the test runner. \ - Skipping element-assertion phase. To enable: System Settings → Privacy & \ - Security → Accessibility → add the binary running `cargo test`." - ); - return; - } +// ── tests ──────────────────────────────────────────────────────────────────── - let text = snap.tree_text(); - println!("snapshot:\n{text}"); - - // AppKit AX quirk (mirrors the WPF behavior documented in - // harness_wpf_test.rs::harness_wpf_smoke): NSTextField in label mode - // and other AXStaticText leaves do NOT propagate - // setAccessibilityIdentifier into the AX tree's identifier slot, so - // we don't assert on ids for labels. We assert on text-presence for - // those, and on AX ids only for actionable controls (Buttons, - // TextFields). - for aid in [ - "wnd-main", // NSWindow - "btn-increment", - "btn-reset", // NSButton - "txt-input", // editable NSTextField - "menu-test-item", // NSMenuItem (Mac-specific) - "btn-exit", - ] { - assert!( - has_id(snap.tree_text(), aid), - "missing AX identifier {aid} in AppKit snapshot" - ); - } +#[test] +#[ignore] +fn harness_appkit_smoke() { + run_case( + native_readonly_case( + "appkit", + "ax_tree", + Targeting::Ax, + DriverRoute::AxRead, + vec![OracleKind::AxState], + ), + |pid, wid, driver| { + let snap = snapshot_elements(driver, pid, wid); + + assert!( + !looks_empty(snap.tree_text()), + "required AppKit AX tree is empty" + ); - // text_body marker carried by the visible string of the NSTextField - assert!( - text.contains("HARNESS_TEXT_MARKER_v1"), - "text_body marker not in AppKit snapshot" - ); - // The two label-mode NSTextFields under click_target render as - // AXStaticText nodes — assert on their starting text instead of ids. - assert!(text.contains("clicks=0"), "click_count label missing"); - assert!( - text.contains("last_action=none"), - "last_action label missing" + let text = snap.tree_text(); + println!("snapshot:\n{text}"); + + // AppKit AX quirk (mirrors the WPF behavior documented in + // harness_wpf_test.rs::harness_wpf_smoke): NSTextField in label mode + // and other AXStaticText leaves do NOT propagate + // setAccessibilityIdentifier into the AX tree's identifier slot, so + // we don't assert on ids for labels. We assert on text-presence for + // those, and on AX ids only for actionable controls (Buttons, + // TextFields). + for aid in [ + "wnd-main", // NSWindow + "btn-increment", + "btn-reset", // NSButton + "txt-input", // editable NSTextField + "menu-test-item", // NSMenuItem (Mac-specific) + "btn-exit", + ] { + assert!( + has_id(snap.tree_text(), aid), + "missing AX identifier {aid} in AppKit snapshot" + ); + } + + // text_body marker carried by the visible string of the NSTextField + assert!( + text.contains("HARNESS_TEXT_MARKER_v1"), + "text_body marker not in AppKit snapshot" + ); + // The two label-mode NSTextFields under click_target render as + // AXStaticText nodes — assert on their starting text instead of ids. + assert!(text.contains("counter=0"), "counter label missing"); + assert!(text.contains("clicks=0"), "click_count label missing"); + assert!( + text.contains("last_action=none"), + "last_action label missing" + ); + Observation::delivered(vec![OracleKind::AxState], Evidence::default()) + }, ); } @@ -179,47 +254,40 @@ fn harness_appkit_smoke() { #[test] #[ignore] fn harness_appkit_text_input() { - let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else { - return; - }; - let harness = match Harness::launch() { - Some(h) => h, - None => { - eprintln!("harness not built — skipping"); - return; - } - }; - - let (wid, _) = driver - .find_window(harness.pid as i64, "CuaTestHarness AppKit") - .expect("main window not found"); - let snap_pre = snapshot_elements(&mut driver, harness.pid, wid); - if looks_empty(snap_pre.tree_text()) { - eprintln!("AX empty — TCC not granted; skipping"); - return; - } - let idx = element_index_by_id(snap_pre.tree_text(), "txt-input") - .expect("txt-input element_index not found"); - - // set_value via AX is the deterministic background path; type_text would - // also work but races with cursor focus on cold-launched windows. - let resp = driver.call( + run_background_case( "set_value", - serde_json::json!({ - "pid": harness.pid as i64, - "window_id": wid, - "element_index": idx, - "value": "hello-cua" - }), - ); - println!("set_value resp: {}", resp.text()); - - std::thread::sleep(Duration::from_millis(250)); - let snap_post = snapshot_elements(&mut driver, harness.pid, wid); - let post_text = snap_post.tree_text().to_owned(); - assert!( - post_text.contains("hello-cua"), - "text_input value did not propagate to mirror; snapshot:\n{post_text}" + DriverRoute::MacosAxValue, + |pid, wid, driver| { + let snap_pre = snapshot_elements(driver, pid, wid); + assert!( + !looks_empty(snap_pre.tree_text()), + "required AppKit AX tree is empty" + ); + let idx = element_index_by_id(snap_pre.tree_text(), "txt-input") + .expect("txt-input element_index not found"); + + // set_value via AX is the deterministic background path; type_text would + // also work but races with cursor focus on cold-launched windows. + let resp = driver.call( + "set_value", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "element_index": idx, + "value": "hello-cua" + }), + ); + assert!(!resp.is_error(), "AppKit set_value failed: {}", resp.text()); + println!("set_value resp: {}", resp.text()); + + std::thread::sleep(Duration::from_millis(250)); + let snap_post = snapshot_elements(driver, pid, wid); + let post_text = snap_post.tree_text().to_owned(); + assert!( + post_text.contains("hello-cua"), + "text_input value did not propagate to mirror; snapshot:\n{post_text}" + ); + }, ); } @@ -228,150 +296,125 @@ fn harness_appkit_text_input() { /// dispatch chain reaches a backgrounded Cocoa text input. #[test] #[ignore] -fn harness_appkit_type_text_keystroke() { - let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else { - return; - }; - let harness = match Harness::launch() { - Some(h) => h, - None => { - eprintln!("harness not built — skipping"); - return; - } - }; - - let (wid, _) = driver - .find_window(harness.pid as i64, "CuaTestHarness AppKit") - .expect("main window not found"); - let snap_pre = snapshot_elements(&mut driver, harness.pid, wid); - if looks_empty(snap_pre.tree_text()) { - eprintln!("AX empty — TCC not granted; skipping"); - return; - } - let idx: u64 = if let Some(i) = element_index_by_id(snap_pre.tree_text(), "txt-input") { - i - } else { - eprintln!("txt-input not found; skipping"); - return; - }; - - // Focus the field first so the keystrokes land in it. AX press on - // a text field has the side effect of giving it keyboard focus. - let _ = driver.call( - "click", - serde_json::json!({ - "pid": harness.pid as i64, "window_id": wid, - "element_index": idx, "action": "press" - }), - ); - std::thread::sleep(Duration::from_millis(150)); - - // CGEvent-based type_text against the focused field (does NOT use - // set_value — exercises the keystroke synthesis chain). - let resp = driver.call( +fn harness_appkit_type_text_background() { + run_background_case( "type_text", - serde_json::json!({ - "pid": harness.pid as i64, "window_id": wid, - "text": "kbd-cua" - }), - ); - println!("type_text resp: {}", resp.text()); - std::thread::sleep(Duration::from_millis(250)); - - let snap_post = snapshot_elements(&mut driver, harness.pid, wid); - let post = snap_post.tree_text().to_owned(); - assert!( - post.contains("kbd-cua"), - "type_text keystroke did not land in the text field; snapshot:\n{post}" + DriverRoute::MacosAxValue, + |pid, wid, driver| { + let snap_pre = snapshot_elements(driver, pid, wid); + assert!( + !looks_empty(snap_pre.tree_text()), + "required AppKit AX tree is empty" + ); + let idx = element_index_by_id(snap_pre.tree_text(), "txt-input") + .expect("txt-input element_index not found"); + + // Address the field through type_text itself. AXTextField does not + // advertise AXPress, so a preparatory click would test an invalid + // action and fail before the keyboard/value delivery path runs. + let resp = driver.call( + "type_text", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "text": "kbd-cua", "delivery_mode": "background" + }), + ); + assert!(!resp.is_error(), "AppKit type_text failed: {}", resp.text()); + println!("type_text resp: {}", resp.text()); + std::thread::sleep(Duration::from_millis(250)); + + let snap_post = snapshot_elements(driver, pid, wid); + let post = snap_post.tree_text().to_owned(); + assert!( + post.contains("kbd-cua"), + "type_text keystroke did not land in the text field; snapshot:\n{post}" + ); + }, ); } -/// scroll: scroll the NSScrollView downward, verify the offset label -/// changes (it mirrors the clip view's documentVisibleRect.origin.y). -/// -/// **Status:** EXPECTED-FAIL today (see notes below). The `scroll` tool -/// itself works at the API level — `libs/cua-driver/tests/fixtures/smoke/macos.sh` confirms it -/// PASSes against the same harness window. What this test would -/// verify additionally is that the scroll event actually moved the -/// scroll view's bounds (state-change observation, not just API -/// success). -/// -/// Why it's expected-fail: on macOS, `CGEvent.scroll` requires the -/// cursor position to lie inside the target NSScrollView for the event -/// to be routed to it (Cocoa scroll-routing is cursor-anchored). The -/// `move_cursor` tool we expose is overlay-only — it doesn't move the -/// OS hardware cursor on macOS. So state-change tests for scroll need -/// either (a) an OS-cursor warp (intentionally not exposed) or (b) a -/// different scroll dispatch primitive (NSEvent.otherEvent -/// keyDown(.swipeUp) on focused window, or an AXScrollAreaScrollTo -/// action). Both are open implementation work; tracking in the journal's -/// "Open items" section. #[test] #[ignore] -#[should_panic(expected = "scroll offset label did not advance from 0")] -fn harness_appkit_scroll_expected_fail() { - let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else { - return; - }; - let harness = match Harness::launch() { - Some(h) => h, - None => { - eprintln!("harness not built — skipping"); - return; - } - }; - - let (wid, _) = driver - .find_window(harness.pid as i64, "CuaTestHarness AppKit") - .expect("main window not found"); - let snap_pre = snapshot_elements(&mut driver, harness.pid, wid); - if looks_empty(snap_pre.tree_text()) { - eprintln!("AX empty — TCC not granted; skipping"); - return; - } - // Pre-condition: offset label should be at "0" (the controller - // initial state). The label text appears as an AXStaticText leaf - // immediately after the AXTextArea body in the rendered tree. - let pre = snap_pre.tree_text().to_owned(); - let pre_has_zero_offset = pre.lines().any(|l| l.trim() == "- AXStaticText = \"0\""); - assert!( - pre_has_zero_offset, - "scroll offset label not at 0 pre-scroll" +fn harness_appkit_scroll_foreground() { + run_case( + native_foreground_case( + "appkit", + "scroll", + Targeting::Ax, + DriverRoute::MacosAxAction, + ), + |pid, wid, driver| { + let pre = snapshot_elements(driver, pid, wid); + assert!(pre.tree_text().contains("scroll_offset=0")); + let index = element_index_by_id(pre.tree_text(), "scroll-tall") + .or_else(|| element_index_containing(pre.tree_text(), "SCROLL_TOP_MARKER_v1")) + .unwrap_or_else(|| { + panic!("scroll-tall element_index not found:\n{}", pre.tree_text()) + }); + let response = driver.call( + "scroll", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "element_index": index, + "direction": "down", + "amount": 5, + "delivery_mode": "foreground" + }), + ); + assert!( + !response.is_error(), + "AppKit foreground scroll failed: {}; raw={}", + response.text(), + response.raw + ); + std::thread::sleep(Duration::from_millis(300)); + let post = snapshot_elements(driver, pid, wid); + assert!( + !post.tree_text().contains("scroll_offset=0"), + "AppKit foreground scroll did not move the NSScrollView; response={}; raw={}", + response.text(), + response.raw + ); + Observation::delivered_with_fixture_state(Vec::new()) + }, ); +} - // Scroll the scroll view down a few ticks. The scroll tool takes - // window-local pixel coords; pick a point inside the scroller - // (the scroll target sits roughly mid-window). - let resp = driver.call( - "scroll", - serde_json::json!({ - "pid": harness.pid as i64, "window_id": wid, - "x": 180, "y": 450, // inside the scroll view - "direction": "down", - "amount": 5 - }), - ); - println!("scroll resp: {}", resp.text()); - std::thread::sleep(Duration::from_millis(250)); - - let snap_post = snapshot_elements(&mut driver, harness.pid, wid); - let post = snap_post.tree_text().to_owned(); - // After scroll, the offset label should no longer be "0" — any - // positive integer indicates the bounds-change notification fired - // and the label updated. We don't pin a specific value (scroll - // wheel pixel delta varies by macOS version + accessibility setting). - let still_zero = post.lines().any(|l| l.trim() == "- AXStaticText = \"0\""); - let unchanged_count = post.matches("- AXStaticText = \"0\"").count(); - let pre_count = pre.matches("- AXStaticText = \"0\"").count(); - // Counter label is also "0" so the bare presence isn't a signal — - // instead check the COUNT decreased by 1 (only the offset label - // moved off zero, not the counter). - assert!( - unchanged_count < pre_count || !still_zero, - "scroll offset label did not advance from 0; pre: {} \"0\" leaves; post: {} \"0\" leaves", - pre_count, - unchanged_count - ); +#[test] +#[ignore] +fn harness_appkit_scroll_background() { + run_background_case("scroll", DriverRoute::MacosAxAction, |pid, wid, driver| { + let pre = snapshot_elements(driver, pid, wid); + assert!(pre.tree_text().contains("scroll_offset=0")); + let index = element_index_by_id(pre.tree_text(), "scroll-tall") + .or_else(|| element_index_containing(pre.tree_text(), "SCROLL_TOP_MARKER_v1")) + .unwrap_or_else(|| panic!("scroll-tall element_index not found:\n{}", pre.tree_text())); + let response = driver.call( + "scroll", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "element_index": index, + "direction": "down", + "amount": 5, + "delivery_mode": "background" + }), + ); + assert!( + !response.is_error(), + "AppKit background scroll failed: {}; raw={}", + response.text(), + response.raw + ); + std::thread::sleep(Duration::from_millis(200)); + assert!( + !snapshot_elements(driver, pid, wid) + .tree_text() + .contains("scroll_offset=0"), + "AppKit background AX scroll did not move the NSScrollView" + ); + }); } /// counter: click the increment button via element_index, verify the @@ -379,52 +422,352 @@ fn harness_appkit_scroll_expected_fail() { #[test] #[ignore] fn harness_appkit_counter() { - let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else { - return; - }; - let harness = match Harness::launch() { - Some(h) => h, - None => { - eprintln!("harness not built — skipping"); - return; - } - }; - - let (wid, _) = driver - .find_window(harness.pid as i64, "CuaTestHarness AppKit") - .expect("main window not found"); - let snap_pre = snapshot_elements(&mut driver, harness.pid, wid); - if looks_empty(snap_pre.tree_text()) { - eprintln!("AX empty — TCC not granted; skipping"); - return; - } - let pre_text = snap_pre.tree_text().to_owned(); - assert!( - pre_text.contains("\"0\""), - "counter not 0 pre-click; snapshot:\n{pre_text}" + run_background_case( + "left_click", + DriverRoute::MacosAxAction, + |pid, wid, driver| { + let snap_pre = snapshot_elements(driver, pid, wid); + assert!( + !looks_empty(snap_pre.tree_text()), + "required AppKit AX tree is empty" + ); + let pre_text = snap_pre.tree_text().to_owned(); + assert!( + pre_text.contains("counter=0"), + "counter not 0 pre-click; snapshot:\n{pre_text}" + ); + + let idx = element_index_by_id(snap_pre.tree_text(), "btn-increment") + .expect("btn-increment element_index not found"); + + let click_resp = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "element_index": idx, + "action": "press", + "delivery_mode": "background" + }), + ); + assert!( + !click_resp.is_error(), + "AppKit counter click failed: {}", + click_resp.text() + ); + println!("click resp: {}", click_resp.text()); + + // Let the AppKit run-loop process the press and refresh the label. + std::thread::sleep(Duration::from_millis(200)); + + let snap_post = snapshot_elements(driver, pid, wid); + let post_text = snap_post.tree_text().to_owned(); + assert!( + post_text.contains("counter=1"), + "counter did not advance to 1 after press; post snapshot:\n{post_text}" + ); + }, ); +} - let idx = element_index_by_id(snap_pre.tree_text(), "btn-increment") - .expect("btn-increment element_index not found"); +/// Resolve the native AppKit button from a screenshot-space PX target, then +/// deliver through the background-safe AX hit-test bridge while another app +/// remains fully foreground. +#[test] +#[ignore] +fn harness_appkit_counter_px_background() { + run_background_case_targeting( + "left_click", + Targeting::Px, + DriverRoute::MacosAxAction, + |pid, wid, driver| { + let pre = snapshot_elements(driver, pid, wid); + let (x, y, width, height) = element_pixel_frame(&pre, "btn-increment"); + let response = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "x": x + width / 2.0, + "y": y + height / 2.0, + "delivery_mode": "background" + }), + ); + assert!( + !response.is_error(), + "AppKit PX background click failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(200)); + assert!( + snapshot_elements(driver, pid, wid) + .tree_text() + .contains("counter=1"), + "AppKit PX background click did not advance counter" + ); + }, + ); +} - let click_resp = driver.call( - "click", - serde_json::json!({ - "pid": harness.pid as i64, - "window_id": wid, - "element_index": idx, - "action": "press" - }), +#[test] +#[ignore] +fn harness_appkit_right_click_px_foreground() { + run_case( + native_foreground_case( + "appkit", + "right_click", + Targeting::Px, + DriverRoute::MacosCgEventHid, + ), + |pid, wid, driver| { + let pre = snapshot_elements(driver, pid, wid); + let (x, y, width, height) = element_pixel_frame(&pre, "btn-clicktarget"); + let response = driver.call( + "right_click", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "x": x + width / 2.0, + "y": y + height / 2.0, + "delivery_mode": "foreground" + }), + ); + assert!( + !response.is_error(), + "AppKit right click failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(250)); + assert!( + snapshot_elements(driver, pid, wid) + .tree_text() + .contains("last_action=right_click"), + "AppKit right-click handler did not fire" + ); + Observation::delivered_with_fixture_state(Vec::new()) + }, ); - println!("click resp: {}", click_resp.text()); +} - // Let the AppKit run-loop process the press and refresh the label. - std::thread::sleep(Duration::from_millis(200)); +#[test] +#[ignore] +fn harness_appkit_right_click_px_background() { + run_background_case_targeting( + "right_click", + Targeting::Px, + DriverRoute::MacosCgEventPid, + |pid, wid, driver| { + let pre = snapshot_elements(driver, pid, wid); + let (x, y, width, height) = element_pixel_frame(&pre, "btn-clicktarget"); + let response = driver.call( + "right_click", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "x": x + width / 2.0, + "y": y + height / 2.0, + "delivery_mode": "background" + }), + ); + assert!( + !response.is_error(), + "AppKit right click failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(250)); + assert!( + snapshot_elements(driver, pid, wid) + .tree_text() + .contains("last_action=right_click"), + "AppKit background right-click handler did not fire" + ); + }, + ); +} - let snap_post = snapshot_elements(&mut driver, harness.pid, wid); - let post_text = snap_post.tree_text().to_owned(); - assert!( - post_text.contains("\"1\""), - "counter did not advance to 1 after press; post snapshot:\n{post_text}" +#[test] +#[ignore] +fn harness_appkit_double_click_px_foreground() { + run_case( + native_foreground_case( + "appkit", + "double_click", + Targeting::Px, + DriverRoute::MacosCgEventHid, + ), + |pid, wid, driver| { + let pre = snapshot_elements(driver, pid, wid); + let (x, y, width, height) = element_pixel_frame(&pre, "btn-clicktarget"); + let response = driver.call( + "double_click", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "x": x + width / 2.0, + "y": y + height / 2.0, + "delivery_mode": "foreground" + }), + ); + assert!( + !response.is_error(), + "AppKit double click failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(250)); + assert!( + snapshot_elements(driver, pid, wid) + .tree_text() + .contains("last_action=double_click"), + "AppKit double-click handler did not fire" + ); + Observation::delivered_with_fixture_state(Vec::new()) + }, + ); +} + +#[test] +#[ignore] +fn harness_appkit_double_click_px_background() { + run_background_case_targeting( + "double_click", + Targeting::Px, + DriverRoute::MacosCgEventPid, + |pid, wid, driver| { + let pre = snapshot_elements(driver, pid, wid); + let (x, y, width, height) = element_pixel_frame(&pre, "btn-clicktarget"); + let response = driver.call( + "double_click", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "x": x + width / 2.0, + "y": y + height / 2.0, + "delivery_mode": "background" + }), + ); + assert!( + !response.is_error(), + "AppKit double click failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(250)); + assert!( + snapshot_elements(driver, pid, wid) + .tree_text() + .contains("last_action=double_click"), + "AppKit background double-click handler did not fire" + ); + }, + ); +} + +#[test] +#[ignore] +fn harness_appkit_slider_drag_px_foreground() { + run_case( + native_foreground_case( + "appkit", + "slider_drag", + Targeting::Px, + DriverRoute::MacosCgEventHid, + ), + |pid, wid, driver| { + let pre = snapshot_elements(driver, pid, wid); + assert!(pre.tree_text().contains("slider_value=0")); + let (x, y, width, height) = element_pixel_frame(&pre, "sld-value"); + let response = driver.call( + "drag", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "from_x": x + width * 0.05, + "from_y": y + height / 2.0, + "to_x": x + width * 0.90, + "to_y": y + height / 2.0, + "duration_ms": 500, + "steps": 30, + "delivery_mode": "foreground" + }), + ); + assert!( + !response.is_error(), + "AppKit slider drag failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(300)); + assert!( + !snapshot_elements(driver, pid, wid) + .tree_text() + .contains("slider_value=0"), + "AppKit foreground drag did not move the slider" + ); + Observation::delivered_with_fixture_state(Vec::new()) + }, ); } + +#[test] +#[ignore] +fn harness_appkit_slider_drag_px_background() { + let case = native_background_case( + "appkit", + "slider_drag", + Targeting::Px, + DriverRoute::MacosCgEventPid, + ) + .expecting_refusal(vec![RefusalCode::BackgroundUnavailable]); + run_case(case, |pid, wid, driver| { + let pre = snapshot_elements(driver, pid, wid); + assert!(pre.tree_text().contains("slider_value=0")); + let (x, y, width, height) = element_pixel_frame(&pre, "sld-value"); + let (response, mut passed) = run_with_background_oracles( + driver, + TargetWindow { + pid, + native_id: wid, + }, + |driver| { + driver.call( + "drag", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "from_x": x + width * 0.05, + "from_y": y + height / 2.0, + "to_x": x + width * 0.90, + "to_y": y + height / 2.0, + "duration_ms": 500, + "steps": 30, + "delivery_mode": "background" + }), + ) + }, + ) + .unwrap_or_else(|error| panic!("background desktop contract failed: {error}")); + assert!( + response.is_error(), + "AppKit background drag unexpectedly reported delivery: {}", + response.text() + ); + assert_eq!( + response.structured()["code"].as_str(), + Some("background_unavailable"), + "AppKit background drag returned the wrong refusal: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(200)); + assert!( + snapshot_elements(driver, pid, wid) + .tree_text() + .contains("slider_value=0"), + "refused AppKit background drag changed the slider" + ); + passed.push(OracleKind::FixtureState); + Observation::refused( + RefusalCode::BackgroundUnavailable, + passed, + response.text(), + Evidence::default(), + ) + }); +} diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_swiftui_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_swiftui_test.rs index 27435a4f47..5df102bdcd 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_swiftui_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_swiftui_test.rs @@ -21,6 +21,12 @@ use std::process::{Child, Command, Stdio}; use std::time::Duration; use cua_driver_testkit::ax::{element_index_by_id, has_id, looks_empty}; +use cua_driver_testkit::e2e::{ + execute_case, native_background_case, native_foreground_case, native_readonly_case, + recording_evidence, DriverRoute, Evidence, Observation, OracleKind, Targeting, +}; +use cua_driver_testkit::observer::TargetWindow; +use cua_driver_testkit::sentinel::run_with_background_oracles; use cua_driver_testkit::{harness_app, Driver, McpDriver, ToolResponse}; fn harness_exe() -> PathBuf { @@ -42,20 +48,17 @@ struct Harness { } impl Harness { - fn launch() -> Option { + fn launch() -> Self { let exe = harness_exe(); - if !exe.exists() { - eprintln!("harness exe not found at {exe:?}"); - return None; - } + assert!(exe.exists(), "required SwiftUI harness is missing: {exe:?}"); let app = Command::new(&exe) .stdout(Stdio::null()) .stderr(Stdio::null()) .spawn() - .ok()?; + .unwrap_or_else(|error| panic!("launch SwiftUI harness {exe:?}: {error}")); let pid = app.id(); std::thread::sleep(Duration::from_millis(900)); - Some(Self { _app: app, pid }) + Self { _app: app, pid } } } @@ -78,134 +81,252 @@ fn snapshot_elements(driver: &mut McpDriver, pid: u32, window_id: u64) -> ToolRe ) } +fn run_case( + case: cua_driver_testkit::e2e::CaseSpec, + test: impl FnOnce(u32, u64, &mut McpDriver) -> Observation, +) { + let cell_id = case.cell_id.clone(); + let delivery = case.delivery; + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_macos_daemon_proxy_named(&cell_id) + .expect("start installed macOS daemon proxy"); + *evidence = recording_evidence(driver.recording_dir()); + let harness = Harness::launch(); + let (wid, _) = driver + .find_window(harness.pid as i64, "CuaTestHarness SwiftUI") + .expect("SwiftUI main window not found"); + if delivery != cua_driver_testkit::e2e::Delivery::Background { + driver.start_behavior_recording(); + } + test(harness.pid, wid, &mut driver) + }); +} + +fn run_foreground_case(action: &str, test: impl FnOnce(u32, u64, &mut McpDriver)) { + run_case( + native_foreground_case("swiftui", action, Targeting::Ax, DriverRoute::MacosAxAction), + |pid, wid, driver| { + test(pid, wid, driver); + Observation::delivered_with_fixture_state(Vec::new()) + }, + ); +} + +fn run_background_case( + action: &str, + route: DriverRoute, + test: impl FnOnce(u32, u64, &mut McpDriver), +) { + run_case( + native_background_case("swiftui", action, Targeting::Ax, route), + |pid, wid, driver| { + let (_, passed) = run_with_background_oracles( + driver, + TargetWindow { + pid, + native_id: wid, + }, + |driver| test(pid, wid, driver), + ) + .unwrap_or_else(|error| panic!("background desktop contract failed: {error}")); + Observation::delivered_with_fixture_state(passed) + }, + ); +} + #[test] #[ignore] fn harness_swiftui_smoke() { - let harness = match Harness::launch() { - Some(h) => h, - None => { - eprintln!("harness not built — skipping"); - return; - } - }; - println!("harness pid={}", harness.pid); - - let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else { - return; - }; - - let (wid, title) = driver - .find_window(harness.pid as i64, "CuaTestHarness SwiftUI") - .expect("main window not found"); - println!("main window: id={wid} title={title:?}"); - - let snap = snapshot_elements(&mut driver, harness.pid, wid); - if looks_empty(snap.tree_text()) { - eprintln!("AX empty — TCC not granted; skipping element-assertions"); - return; - } - let text = snap.tree_text(); - println!("snapshot:\n{text}"); - - // SwiftUI Text views render as AXStaticText leaves and don't propagate - // accessibilityIdentifier into the AX tree's identifier slot (same - // quirk as AppKit's NSTextField label mode + WPF's TextBlock). Assert - // on text content for labels, AX-id only for actionable controls. - for aid in [ - "btn-increment", - "btn-reset", - "txt-input", - "btn-open-popover", - "btn-exit", - ] { - assert!( - has_id(snap.tree_text(), aid), - "missing AX identifier {aid} in SwiftUI snapshot" - ); - } + run_case( + native_readonly_case( + "swiftui", + "ax_tree", + Targeting::Ax, + DriverRoute::AxRead, + vec![OracleKind::AxState], + ), + |pid, wid, driver| { + let snap = snapshot_elements(driver, pid, wid); + assert!( + !looks_empty(snap.tree_text()), + "required SwiftUI AX tree is empty" + ); + let text = snap.tree_text(); + println!("snapshot:\n{text}"); + + // SwiftUI Text views render as AXStaticText leaves and don't propagate + // accessibilityIdentifier into the AX tree's identifier slot (same + // quirk as AppKit's NSTextField label mode + WPF's TextBlock). Assert + // on text content for labels, AX-id only for actionable controls. + for aid in [ + "btn-increment", + "btn-reset", + "txt-input", + "btn-open-popover", + "btn-exit", + ] { + assert!( + has_id(snap.tree_text(), aid), + "missing AX identifier {aid} in SwiftUI snapshot" + ); + } - assert!( - text.contains("HARNESS_TEXT_MARKER_v1"), - "text_body marker not in SwiftUI snapshot" + assert!( + text.contains("HARNESS_TEXT_MARKER_v1"), + "text_body marker not in SwiftUI snapshot" + ); + Observation::delivered(vec![OracleKind::AxState], Evidence::default()) + }, ); } -/// popover: click the popover trigger, verify the popover body text appears -/// in the AX tree after the open. SwiftUI's analogue of WinUI3 CommandBarFlyout. #[test] #[ignore] -fn harness_swiftui_popover() { - let harness = match Harness::launch() { - Some(h) => h, - None => { - eprintln!("harness not built — skipping"); - return; - } - }; - - let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else { - return; - }; - - let (wid, _) = driver - .find_window(harness.pid as i64, "CuaTestHarness SwiftUI") - .expect("main window not found"); - let snap_pre = snapshot_elements(&mut driver, harness.pid, wid); - if looks_empty(snap_pre.tree_text()) { - eprintln!("AX empty — TCC not granted; skipping"); - return; - } - // Verify popover body is NOT yet in the tree. - let pre_text = snap_pre.tree_text().to_owned(); - assert!( - !pre_text.contains("POPOVER_MARKER_v1"), - "popover body unexpectedly present BEFORE open" +fn harness_swiftui_counter_background() { + run_background_case( + "left_click", + DriverRoute::MacosAxAction, + |pid, wid, driver| { + let pre = snapshot_elements(driver, pid, wid); + assert!(pre.tree_text().contains("counter=0")); + let index = element_index_by_id(pre.tree_text(), "btn-increment") + .expect("btn-increment element_index not found"); + let response = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "element_index": index, + "action": "press", + "delivery_mode": "background" + }), + ); + assert!( + !response.is_error(), + "SwiftUI counter click failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(200)); + assert!( + snapshot_elements(driver, pid, wid) + .tree_text() + .contains("counter=1"), + "SwiftUI background AX click did not advance counter" + ); + }, ); +} - let trigger_idx: u64 = - if let Some(i) = element_index_by_id(snap_pre.tree_text(), "btn-open-popover") { - i - } else { - eprintln!("popover trigger not found, skipping"); - return; - }; - let click = driver.call( - "click", - serde_json::json!({ - "pid": harness.pid as i64, - "window_id": wid, - "element_index": trigger_idx, - "action": "press" - }), +#[test] +#[ignore] +fn harness_swiftui_set_value_background() { + run_background_case( + "set_value", + DriverRoute::MacosAxValue, + |pid, wid, driver| { + let pre = snapshot_elements(driver, pid, wid); + let index = element_index_by_id(pre.tree_text(), "txt-input") + .expect("txt-input element_index not found"); + let response = driver.call( + "set_value", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "element_index": index, + "value": "swiftui-cua" + }), + ); + assert!( + !response.is_error(), + "SwiftUI set_value failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(200)); + assert!( + snapshot_elements(driver, pid, wid) + .tree_text() + .contains("swiftui-cua"), + "SwiftUI background AX value did not reach the field" + ); + }, ); - println!("popover trigger click: {}", click.text()); - - std::thread::sleep(Duration::from_millis(300)); - - // Popovers may live in a separate AXWindow on macOS — try the main - // window first, then list_windows for additional candidates. - let snap_post = snapshot_elements(&mut driver, harness.pid, wid); - let mut found_marker = snap_post.tree_text().contains("POPOVER_MARKER_v1"); - if !found_marker { - // Walk any new windows for the same pid. - let resp = driver.call( - "list_windows", - serde_json::json!({ "pid": harness.pid as i64 }), +} + +/// Popover activation: click the trigger and verify fixture-owned state changes. +/// Transient-window AX discovery is observed separately so it cannot hide a +/// correctly delivered action. +#[test] +#[ignore] +fn harness_swiftui_popover_foreground() { + run_foreground_case("popover_open", |pid, wid, driver| { + let snap_pre = snapshot_elements(driver, pid, wid); + assert!( + !looks_empty(snap_pre.tree_text()), + "required SwiftUI AX tree is empty" ); - if let Some(wins) = resp.structured()["windows"].as_array() { - for w in wins { - if let Some(other_wid) = w["window_id"].as_u64() { - if other_wid == wid { - continue; - } - let s = snapshot_elements(&mut driver, harness.pid, other_wid); - if s.tree_text().contains("POPOVER_MARKER_v1") { - found_marker = true; - break; + // Verify popover body is NOT yet in the tree. + let pre_text = snap_pre.tree_text().to_owned(); + assert!( + !pre_text.contains("POPOVER_MARKER_v1"), + "popover body unexpectedly present BEFORE open" + ); + assert!( + pre_text.contains("popover_open=false"), + "popover state was not false before open" + ); + + let trigger_idx = element_index_by_id(snap_pre.tree_text(), "btn-open-popover") + .expect("popover trigger not found"); + let click = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "element_index": trigger_idx, + "action": "press", + "delivery_mode": "foreground" + }), + ); + assert!( + !click.is_error(), + "SwiftUI popover click failed: {}", + click.text() + ); + println!("popover trigger click: {}", click.text()); + + // First prove the button action reached SwiftUI's state independently + // of whether AX can enumerate the transient panel. + let deadline = std::time::Instant::now() + Duration::from_secs(3); + let mut state_open = false; + let mut found_marker = false; + while !state_open && std::time::Instant::now() < deadline { + let owner = snapshot_elements(driver, pid, wid); + state_open = owner.tree_text().contains("popover_open=true"); + found_marker = owner.tree_text().contains("POPOVER_MARKER_v1"); + let resp = driver.call("list_windows", serde_json::json!({ "pid": pid as i64 })); + if let Some(wins) = resp.structured()["windows"].as_array() { + for w in wins { + if let Some(other_wid) = w["window_id"].as_u64() { + if other_wid == wid { + continue; + } + let s = snapshot_elements(driver, pid, other_wid); + if s.tree_text().contains("POPOVER_MARKER_v1") { + found_marker = true; + break; + } } } } + if !state_open { + std::thread::sleep(Duration::from_millis(100)); + } } - } - assert!(found_marker, "popover body marker not found after open"); + assert!(state_open, "popover trigger did not change fixture state"); + if !found_marker { + eprintln!( + "SwiftUI popover opened, but its transient panel remains absent from targeted AX enumeration" + ); + } + }); } diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_launch_focus_macos_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/installed_app_launch_macos_test.rs similarity index 61% rename from libs/cua-driver/rust/crates/cua-driver/tests/modality_launch_focus_macos_test.rs rename to libs/cua-driver/rust/crates/cua-driver/tests/installed_app_launch_macos_test.rs index 7fee4f4deb..3058a21c45 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/modality_launch_focus_macos_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/installed_app_launch_macos_test.rs @@ -1,11 +1,11 @@ -//! Optional macOS real-app launch/focus checks. +//! macOS real-app launch/focus checks. //! //! These are Rust ports of the old Python focus-steal parity coverage for -//! built-in macOS apps. They are not part of the canonical harness run-all path: -//! they exercise external apps and a live user desktop, so run them explicitly. +//! built-in macOS apps. The canonical macOS lane runs them in a logged-in, +//! TCC-authorized desktop session. //! //! Run: -//! cargo test -p cua-driver --test modality_launch_focus_macos_test -- --ignored --nocapture --test-threads=1 +//! cargo test -p cua-driver --test installed_app_launch_macos_test -- --ignored --nocapture --test-threads=1 #![cfg(target_os = "macos")] @@ -13,6 +13,10 @@ use std::process::Command; use std::thread::sleep; use std::time::{Duration, Instant}; +use cua_driver_testkit::e2e::{ + execute_case, recording_evidence, CaseSpec, Delivery, DriverRoute, Observation, OracleKind, + Scope, Targeting, +}; use cua_driver_testkit::{Driver, McpDriver}; const FINDER_BUNDLE: &str = "com.apple.finder"; @@ -58,18 +62,15 @@ fn wait_for_frontmost(bundle_id: &str, timeout: Duration) -> bool { false } -fn ensure_finder_frontmost() -> bool { +fn ensure_finder_frontmost() { for _ in 0..3 { activate_bundle(FINDER_BUNDLE); if wait_for_frontmost(FINDER_BUNDLE, Duration::from_secs(2)) { - return true; + return; } sleep(Duration::from_millis(500)); } - eprintln!( - "[launch-focus] could not make Finder frontmost; skipping optional real-app test" - ); - false + panic!("could not make Finder frontmost for installed-app launch validation"); } fn kill_app_process(process_name: &str) { @@ -106,35 +107,52 @@ fn launch_and_assert_frontmost_unchanged(driver: &mut McpDriver, bundle_id: &str ); } -fn driver() -> Option { - McpDriver::spawn_macos_daemon_proxy().or_else(McpDriver::spawn) +fn run_launch_case(cell_id: &str, bundle_id: &str, process_name: &str, label: &str) { + let case = CaseSpec::delivered( + cell_id, + label, + "appkit", + "launch_app", + Targeting::NotApplicable, + Delivery::Background, + Scope::Desktop, + DriverRoute::Composite, + vec![OracleKind::Focus, OracleKind::Protocol], + ); + execute_case(case, |evidence| { + kill_app_process(process_name); + ensure_finder_frontmost(); + let mut driver = McpDriver::spawn_macos_daemon_proxy_named(cell_id) + .expect("start installed macOS daemon proxy"); + *evidence = recording_evidence(driver.recording_dir()); + driver.start_behavior_recording(); + ensure_finder_frontmost(); + launch_and_assert_frontmost_unchanged(&mut driver, bundle_id, label); + Observation::delivered( + vec![OracleKind::Focus, OracleKind::Protocol], + Default::default(), + ) + }); } #[test] #[ignore] fn textedit_launch_preserves_finder_frontmost() { - if !ensure_finder_frontmost() { - return; - } - kill_app_process("TextEdit"); - if !ensure_finder_frontmost() { - return; - } - - let Some(mut driver) = driver() else { return }; - launch_and_assert_frontmost_unchanged(&mut driver, TEXTEDIT_BUNDLE, "TextEdit launch"); + run_launch_case( + "macos-textedit-launch-background", + TEXTEDIT_BUNDLE, + "TextEdit", + "textedit", + ); } #[test] #[ignore] -fn calculator_then_textedit_launches_preserve_finder_frontmost() { - kill_app_process("Calculator"); - kill_app_process("TextEdit"); - if !ensure_finder_frontmost() { - return; - } - - let Some(mut driver) = driver() else { return }; - launch_and_assert_frontmost_unchanged(&mut driver, CALCULATOR_BUNDLE, "Calculator launch"); - launch_and_assert_frontmost_unchanged(&mut driver, TEXTEDIT_BUNDLE, "TextEdit launch"); +fn calculator_launch_preserves_finder_frontmost() { + run_launch_case( + "macos-calculator-launch-background", + CALCULATOR_BUNDLE, + "Calculator", + "calculator", + ); } diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/installed_app_textedit_macos_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/installed_app_textedit_macos_test.rs new file mode 100644 index 0000000000..a09abbe93e --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver/tests/installed_app_textedit_macos_test.rs @@ -0,0 +1,117 @@ +//! TextEdit background-delivery integration check for macOS. +//! +//! Covers the `{path, verified}` structured outcome on a real Cocoa app. +//! +//! The schema contract lives in `protocol_schema_test.rs`. This installed-app +//! check runs in the canonical logged-in macOS desktop lane. + +#![cfg(target_os = "macos")] + +// ── End-to-end ladder behavior (interactive; needs a GUI session) ──────────── + +/// On a NATIVE Cocoa field (TextEdit), `delivery_mode:"background"` lands via the +/// AX value-write and the driver confirms it: `path:"ax", verified:true`. This is +/// the driver-verifiable happy path — no foreground needed, no screenshot needed. +#[test] +#[ignore] +fn background_type_on_native_cocoa_is_ax_verified() { + use cua_driver_testkit::e2e::{ + execute_case, recording_evidence, CaseSpec, Delivery, DriverRoute, Observation, OracleKind, + Scope, Targeting, + }; + use cua_driver_testkit::observer::TargetWindow; + use cua_driver_testkit::sentinel::run_with_background_oracles; + use cua_driver_testkit::{Driver, McpDriver}; + + let cell_id = "macos-textedit-type-text-ax-background"; + let case = CaseSpec::delivered( + cell_id, + "textedit", + "appkit", + "type_text", + Targeting::Ax, + Delivery::Background, + Scope::Window, + DriverRoute::MacosAxValue, + vec![ + OracleKind::AxState, + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::Cursor, + OracleKind::NoLeakedInput, + ], + ); + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_macos_daemon_proxy_named(cell_id) + .expect("start installed macOS daemon proxy"); + *evidence = recording_evidence(driver.recording_dir()); + + // Launch TextEdit and open a blank document. + let launch = driver.call( + "launch_app", + serde_json::json!({ "bundle_id": "com.apple.TextEdit" }), + ); + assert!( + !launch.is_error(), + "could not launch TextEdit: {}", + launch.text() + ); + let pid = launch.structured()["pid"].as_i64().expect("TextEdit pid"); + let windows = launch.structured()["windows"] + .as_array() + .cloned() + .unwrap_or_default(); + let wid = windows + .first() + .and_then(|window| window["window_id"].as_u64()) + .expect("TextEdit opened no window"); + + // Find the AXTextArea. + let state = driver.call( + "get_window_state", + serde_json::json!({ "pid": pid, "window_id": wid, "capture_mode": "ax" }), + ); + let el = state.structured()["elements"] + .as_array() + .and_then(|elements| { + elements + .iter() + .find(|element| element["role"] == "AXTextArea") + .and_then(|element| element["element_index"].as_u64()) + }) + .expect("TextEdit AXTextArea"); + + let (typed, mut passed) = run_with_background_oracles( + &mut driver, + TargetWindow { + pid: pid as u32, + native_id: wid, + }, + |driver| { + driver.call( + "type_text", + serde_json::json!({ + "pid": pid, "window_id": wid, "element_index": el, + "text": "ladder", "delivery_mode": "background" + }), + ) + }, + ) + .unwrap_or_else(|error| panic!("background TextEdit contract failed: {error}")); + assert!(!typed.is_error(), "type_text errored: {}", typed.text()); + assert_eq!( + typed.path(), + Some("ax"), + "native Cocoa field should land via AX: {}", + typed.text() + ); + assert_eq!( + typed.verified(), + Some(true), + "AX write should read back as verified: {}", + typed.text() + ); + passed.push(OracleKind::AxState); + Observation::delivered(passed, Default::default()) + }); +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs index b4e4c8902d..5391d26228 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs @@ -3,7 +3,12 @@ //! We call the C-level AX API directly rather than using a crate wrapper, //! because most available crates are incomplete or unmaintained. -#![allow(non_upper_case_globals, non_camel_case_types, non_snake_case, dead_code)] +#![allow( + non_upper_case_globals, + non_camel_case_types, + non_snake_case, + dead_code +)] use core_foundation::{ array::CFArrayRef, @@ -54,14 +59,14 @@ extern "C" { element: AXUIElementRef, names: *mut CFArrayRef, ) -> AXError; - pub fn AXUIElementCopyActionNames( - element: AXUIElementRef, - names: *mut CFArrayRef, - ) -> AXError; - pub fn AXUIElementPerformAction( - element: AXUIElementRef, - action: CFStringRef, + pub fn AXUIElementCopyActionNames(element: AXUIElementRef, names: *mut CFArrayRef) -> AXError; + pub fn AXUIElementCopyElementAtPosition( + application: AXUIElementRef, + x: f32, + y: f32, + element: *mut AXUIElementRef, ) -> AXError; + pub fn AXUIElementPerformAction(element: AXUIElementRef, action: CFStringRef) -> AXError; pub fn AXUIElementSetAttributeValue( element: AXUIElementRef, attribute: CFStringRef, @@ -73,27 +78,42 @@ extern "C" { /// `{kAXTrustedCheckOptionPrompt: true}` raises the system Accessibility /// prompt if the process isn't already trusted. Returns the post-prompt /// trust state (may still be false if the user dismissed the prompt). - pub fn AXIsProcessTrustedWithOptions(options: core_foundation::dictionary::CFDictionaryRef) -> bool; + pub fn AXIsProcessTrustedWithOptions( + options: core_foundation::dictionary::CFDictionaryRef, + ) -> bool; /// Private SPI: maps an AX window element to its CGWindowID. /// Stable since macOS 10.9; used by yabai, Hammerspoon, Accessibility Inspector. pub fn _AXUIElementGetWindow(element: AXUIElementRef, window_id: *mut u32) -> AXError; } +/// Hit-test one process's accessibility tree at a screen point. The returned +/// element is retained and must be released by the caller. +pub unsafe fn element_at_screen_position(pid: i32, x: f64, y: f64) -> Option { + let application = AXUIElementCreateApplication(pid); + if application.is_null() { + return None; + } + let mut element = std::ptr::null_mut(); + let error = AXUIElementCopyElementAtPosition(application, x as f32, y as f32, &mut element); + CFRelease(application as CFTypeRef); + (error == kAXErrorSuccess && !element.is_null()).then_some(element) +} + // ── AXValue functions ──────────────────────────────────────────────────────── #[link(name = "ApplicationServices", kind = "framework")] extern "C" { pub fn AXValueGetType(value: AXValueRef) -> AXValueType; - pub fn AXValueGetValue(value: AXValueRef, the_type: AXValueType, value_ptr: *mut c_void) -> bool; + pub fn AXValueGetValue( + value: AXValueRef, + the_type: AXValueType, + value_ptr: *mut c_void, + ) -> bool; } // ── Helper functions ────────────────────────────────────────────────────────── -use core_foundation::{ - array::CFArray, - base::TCFType, - string::CFString as CFStr, -}; +use core_foundation::{array::CFArray, base::TCFType, string::CFString as CFStr}; /// Copy a string attribute from an AX element. Returns `None` on any error. pub unsafe fn copy_string_attr(element: AXUIElementRef, attr_name: &str) -> Option { @@ -162,7 +182,10 @@ pub unsafe fn element_screen_center(element: AXUIElementRef) -> Option<(f64, f64 return None; } #[repr(C)] - struct CGPoint { x: f64, y: f64 } + struct CGPoint { + x: f64, + y: f64, + } let mut pos = CGPoint { x: 0.0, y: 0.0 }; let ok = AXValueGetValue( pos_ref as AXValueRef, @@ -170,7 +193,9 @@ pub unsafe fn element_screen_center(element: AXUIElementRef) -> Option<(f64, f64 &mut pos as *mut _ as *mut std::ffi::c_void, ); CFRelease(pos_ref); - if !ok { return None; } + if !ok { + return None; + } // AXSize → CGSize let sz_attr = CFStr::new("AXSize"); @@ -180,7 +205,10 @@ pub unsafe fn element_screen_center(element: AXUIElementRef) -> Option<(f64, f64 return None; } #[repr(C)] - struct CGSize { w: f64, h: f64 } + struct CGSize { + w: f64, + h: f64, + } let mut sz = CGSize { w: 0.0, h: 0.0 }; let ok2 = AXValueGetValue( sz_ref as AXValueRef, @@ -188,7 +216,9 @@ pub unsafe fn element_screen_center(element: AXUIElementRef) -> Option<(f64, f64 &mut sz as *mut _ as *mut std::ffi::c_void, ); CFRelease(sz_ref); - if !ok2 || sz.w < 1.0 || sz.h < 1.0 { return None; } + if !ok2 || sz.w < 1.0 || sz.h < 1.0 { + return None; + } Some((pos.x + sz.w / 2.0, pos.y + sz.h / 2.0)) } @@ -204,7 +234,10 @@ pub unsafe fn element_screen_rect(element: AXUIElementRef) -> Option<[f64; 4]> { return None; } #[repr(C)] - struct CGPoint { x: f64, y: f64 } + struct CGPoint { + x: f64, + y: f64, + } let mut pos = CGPoint { x: 0.0, y: 0.0 }; let ok = AXValueGetValue( pos_ref as AXValueRef, @@ -212,7 +245,9 @@ pub unsafe fn element_screen_rect(element: AXUIElementRef) -> Option<[f64; 4]> { &mut pos as *mut _ as *mut std::ffi::c_void, ); CFRelease(pos_ref); - if !ok { return None; } + if !ok { + return None; + } // AXSize → CGSize let sz_attr = CFStr::new("AXSize"); @@ -222,7 +257,10 @@ pub unsafe fn element_screen_rect(element: AXUIElementRef) -> Option<[f64; 4]> { return None; } #[repr(C)] - struct CGSize { w: f64, h: f64 } + struct CGSize { + w: f64, + h: f64, + } let mut sz = CGSize { w: 0.0, h: 0.0 }; let ok2 = AXValueGetValue( sz_ref as AXValueRef, @@ -230,7 +268,9 @@ pub unsafe fn element_screen_rect(element: AXUIElementRef) -> Option<[f64; 4]> { &mut sz as *mut _ as *mut std::ffi::c_void, ); CFRelease(sz_ref); - if !ok2 || sz.w < 1.0 || sz.h < 1.0 { return None; } + if !ok2 || sz.w < 1.0 || sz.h < 1.0 { + return None; + } Some([pos.x, pos.y, sz.w, sz.h]) } @@ -287,6 +327,25 @@ pub unsafe fn copy_children(element: AXUIElementRef) -> Vec { .collect() } +/// Copy an AX element-valued attribute. The returned element is retained and +/// must be released by the caller. +pub unsafe fn copy_element_attr( + element: AXUIElementRef, + attr_name: &str, +) -> Option { + let attr = CFStr::new(attr_name); + let mut value: CFTypeRef = std::ptr::null(); + let err = AXUIElementCopyAttributeValue(element, attr.as_concrete_TypeRef(), &mut value); + if err != kAXErrorSuccess || value.is_null() { + return None; + } + if core_foundation::base::CFGetTypeID(value) != AXUIElementGetTypeID() { + CFRelease(value); + return None; + } + Some(value as AXUIElementRef) +} + /// Perform an AX action using a string attribute name. pub unsafe fn perform_action(element: AXUIElementRef, action_name: &str) -> AXError { let action = CFStr::new(action_name); @@ -352,7 +411,11 @@ pub unsafe fn enable_chromium_accessibility(app_element: AXUIElementRef) -> bool pub unsafe fn ax_get_window_id(element: AXUIElementRef) -> Option { let mut wid: u32 = 0; let err = _AXUIElementGetWindow(element, &mut wid); - if err == kAXErrorSuccess && wid != 0 { Some(wid) } else { None } + if err == kAXErrorSuccess && wid != 0 { + Some(wid) + } else { + None + } } /// Read the `AXWindows` attribute of an application element. diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs index 47c30f9775..83a01a78a1 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs @@ -202,9 +202,8 @@ fn click_at_xy_inner( /// Full Chromium-compatible left-click recipe matching Swift's `clickViaAuthSignedPost`. /// -/// The focus-without-raise prologue makes the target window key without changing -/// its z-order, which Chromium requires before it accepts a background pixel -/// mouseDown. The cursor overlay is re-pinned by the click tool after dispatch. +/// The sequence stays PID/window-routed throughout. It must not make the target +/// key: changing key-window ownership violates background delivery. /// 1. Stamped `mouseMoved` at target coords (f0=2, cursor-state primer). /// 2. Off-screen primer down/up at (-1, -1) (f0=1/2) — satisfies Chromium's /// user-activation gate without hitting any DOM element. @@ -234,13 +233,6 @@ pub fn click_at_xy_chromium( ) -> anyhow::Result<()> { use std::time::{SystemTime, UNIX_EPOCH}; - // Chromium's first-mouse handling rejects a background click delivered to - // a non-key window. This SkyLight focus record keys the requested window - // without raising it or moving the user's cursor. - if crate::input::skylight::activate_without_raise(pid as libc::pid_t, wid) { - std::thread::sleep(std::time::Duration::from_millis(50)); - } - let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState) .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?; let target = CGPoint::new(screen_x, screen_y); diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs index f18b8a36c8..7cef8a554f 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs @@ -23,11 +23,12 @@ use std::sync::Arc; use crate::apps; use crate::ax::bindings::{ - copy_action_names, copy_children, copy_string_attr, element_screen_rect, AXUIElementRef, + copy_action_names, copy_children, copy_string_attr, element_at_screen_position, + element_screen_rect, kAXErrorSuccess, AXUIElementPerformAction, AXUIElementRef, }; use crate::focus_guard; use crate::window_change_detector::WindowChangeDetector; -use core_foundation::base::CFRelease; +use core_foundation::base::{CFRelease, TCFType}; use super::ToolState; @@ -111,7 +112,7 @@ fn def() -> &'static ToolDef { "delivery_mode": { "type": "string", "enum": ["background", "foreground"], - "description": "Best-effort-background ladder rung for a PIXEL click (default \"background\"). \"background\": post the CGEvent to the pid without fronting. \"foreground\": briefly front the window, click, restore the prior frontmost — the explicit last resort for surfaces that drop background synthetic clicks. Requires window_id. A click is never driver-verifiable (no read-back), so both report verified:false — confirm the effect via screenshot. Use the agent loop: background AX (element_index) → screenshot → background pixel (x/y) → screenshot → delivery_mode:\"foreground\"." + "description": "Best-effort-background ladder rung (default \"background\"). \"background\": perform the AX action or post the CGEvent without fronting. \"foreground\": briefly front the window, act, let transient UI settle, then restore the prior frontmost app. Requires window_id. A click is never driver-verifiable (no read-back), so both report verified:false — confirm the effect via screenshot. Use the agent loop: background AX (element_index) → screenshot → background pixel (x/y) → screenshot → delivery_mode:\"foreground\"." }, "scope": { "type": "string", @@ -224,85 +225,13 @@ impl Tool for ClickTool { .cursor_registry .update_position(&cursor_key, sx, sy); - // Resolve the frontmost on-screen window under the point (the macOS - // peer of Windows' WindowFromPoint). When found, click THAT pid via - // the proven SkyLight path (`click_at_xy`, screen coords) — reliable - // on AppKit/Chromium where a bare HID post can miss. Only when no - // app window owns the pixel (desktop background, etc.) fall back to - // the cursor-warp + HID post. - // Resolve as (pid, window_id, win_origin_x, win_origin_y) so the - // click can stamp the window-LOCAL point — AppKit hit-tests the - // stamped window-local coordinate, not the bare screen point, so a - // plain screen-coord post misses. - // Exclude our OWN windows (the agent-cursor overlay we just glided to - // the point sits on top of the target — never resolve the click to it). - let own_pid = std::process::id() as i32; - let target = { - let mut wins = crate::windows::visible_windows(); - // visible_windows() assigns HIGHER z_index = MORE FRONT - // (z_index = total - idx over CGWindowList's front-to-back order). - // Sort DESCENDING so the first match is the FRONTMOST window under - // the point — the one the agent actually sees in the screenshot. - // (Ascending picked the BACKMOST occluded window — a real miss when - // windows overlap, e.g. resolving a click to a buried app.) - wins.sort_by(|a, b| b.z_index.cmp(&a.z_index)); // front-to-back - wins.into_iter() - .find(|w| { - w.layer == 0 - && w.pid != own_pid - && sx >= w.bounds.x - && sx < w.bounds.x + w.bounds.width - && sy >= w.bounds.y - && sy < w.bounds.y + w.bounds.height - }) - .map(|w| (w.pid, w.window_id, w.bounds.x, w.bounds.y)) - }; let btn = button.clone(); - let result = tokio::task::spawn_blocking(move || -> anyhow::Result> { - match target { - Some((pid, wid, ox, oy)) => { - let (wx, wy) = (sx - ox, sy - oy); - // Honor `btn` on the window-resolved path too: a windowless - // right/middle click over an app window must stay a - // right/middle click, not silently degrade to left. Route to - // the window-local right/middle primitives (single-pair, same - // as the pixel path); `count` only repeats on the left path. - match btn.as_str() { - "right" => crate::input::mouse::right_click_at_xy_with_window_local( - pid, - sx, - sy, - wx, - wy, - wid, - &[], - )?, - "middle" => crate::input::mouse::middle_click_at_xy_with_window_local( - pid, - sx, - sy, - wx, - wy, - &[], - )?, - _ => crate::input::mouse::click_at_xy_with_window_local( - pid, - sx, - sy, - wx, - wy, - wid, - count, - &[], - )?, - } - Ok(Some(pid)) - } - None => { - crate::input::mouse::click_at_xy_desktop(sx, sy, count, &btn)?; - Ok(None) - } - } + let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { + // Desktop scope is explicitly foreground and vision-driven: post + // at the global HID tap so WindowServer delivers to the window + // actually visible at this point. PID-posting here would silently + // turn the foreground contract back into background delivery. + crate::input::mouse::click_at_xy_desktop(sx, sy, count, &btn) }) .await; let button_label = match button.as_str() { @@ -311,18 +240,12 @@ impl Tool for ClickTool { _ => "click", }; return match result { - Ok(Ok(Some(pid))) => ToolResult::text(format!( - "✅ Sent {button_label} at desktop-pixel ({sx_shot:.0},{sy_shot:.0}) \ - → screen-point ({sx:.0},{sy:.0}) on pid {pid} (desktop scope; \ - not driver-verified — confirm via screenshot)." - )) - .with_structured(serde_json::json!({ "path": "cgevent", "verified": false, "effect": "unverifiable" })), - Ok(Ok(None)) => ToolResult::text(format!( + Ok(Ok(())) => ToolResult::text(format!( "✅ Sent screen-absolute {button_label} at desktop-pixel \ ({sx_shot:.0},{sy_shot:.0}) → screen-point ({sx:.0},{sy:.0}) \ - (desktop scope, no window under point; not driver-verified)." + (desktop scope; not driver-verified)." )) - .with_structured(serde_json::json!({ "path": "cgevent", "verified": false, "effect": "unverifiable" })), + .with_structured(serde_json::json!({ "path": "cgevent_hid", "verified": false, "effect": "unverifiable" })), Ok(Err(e)) => ToolResult::error(format!("desktop-scope click failed: {e}")), Err(e) => ToolResult::error(format!("task error: {e}")), }; @@ -374,9 +297,8 @@ impl Tool for ClickTool { // "middle" has no AX equivalent and falls back to a pixel middle-click // at the element's screen-space center. let button_str = args.str_or("button", "left").to_lowercase(); - // delivery_mode: per-call ladder rung. foreground only applies to the - // pixel path and needs a window_id to front (else it degrades to - // background). A click is never driver-verifiable either way. + // delivery_mode: per-call ladder rung. Foreground briefly activates the + // target for both AX and pixel paths, then restores the prior app. let delivery_mode = super::DeliveryMode::parse(args.opt_str("delivery_mode").as_deref()); // Reject unknown buttons explicitly so silent left-click fall-through can't // mask a typo. Keep "" → default left for old clients that never sent the field. @@ -494,7 +416,12 @@ impl Tool for ClickTool { // new-window / foreground side-effects and append a one-liner // suffix matching Swift's wording. let prior_front = apps::frontmost_pid(); - let snapshot = WindowChangeDetector::snapshot(prior_front); + let foreground = delivery_mode.is_foreground(); + let snapshot = if foreground { + WindowChangeDetector::snapshot_without_suppression(prior_front) + } else { + WindowChangeDetector::snapshot(prior_front) + }; // Run AX work on a blocking thread (can't block async executor). // Use `effective_action` so button=right rewrites press → show_menu. @@ -505,12 +432,37 @@ impl Tool for ClickTool { // and stomp default for a non-default session). let ck = cursor_key.clone(); let result = focus_guard::with_focus_suppressed( - Some(pid), + if foreground { None } else { Some(pid) }, prior_front, "click.AXPress", || async move { tokio::task::spawn_blocking(move || { - perform_ax_click(element_ptr, idx, pid, wid, &action_clone, &ck) + if foreground { + let mut outcome = None; + let fronted = crate::input::skylight::with_foreground_assist( + pid as libc::pid_t, + wid, + || { + outcome = Some(perform_ax_click( + element_ptr, + idx, + pid, + wid, + &action_clone, + &ck, + )?); + std::thread::sleep(std::time::Duration::from_millis(150)); + Ok(()) + }, + )?; + let outcome = outcome.ok_or_else(|| { + anyhow::anyhow!("foreground AX click did not execute") + })?; + Ok((outcome, fronted)) + } else { + perform_ax_click(element_ptr, idx, pid, wid, &action_clone, &ck) + .map(|outcome| (outcome, false)) + } }) .await }, @@ -521,7 +473,7 @@ impl Tool for ClickTool { let changes = snapshot.detect_async().await; match result { - Ok(Ok((mut msg, needs_webkit_delay, suspected_noop))) => { + Ok(Ok(((mut msg, needs_webkit_delay, suspected_noop), fronted))) => { // For text inputs, wait 800ms for WebKit DOM focus to settle // before returning — matches the Swift reference behaviour. if needs_webkit_delay { @@ -538,7 +490,7 @@ impl Tool for ClickTool { // * unverifiable — dispatched fine, driver just can't confirm; // the caller verifies via screenshot. let mut structured = serde_json::json!({ - "path": "ax", + "path": if fronted { "ax_fg" } else { "ax" }, "verified": false, "effect": if suspected_noop { "suspected_noop" } else { "unverifiable" }, }); @@ -677,6 +629,62 @@ impl Tool for ClickTool { (cx, cy, cx, cy) }; + // A background PX action can still use an accessibility delivery + // backend after resolving the requested screen point. This keeps + // targeting (PX) orthogonal to delivery (AX) and avoids making a + // Chromium/AppKit window key merely to satisfy first-mouse rules. + if !delivery_mode.is_foreground() + && window_id.is_some() + && button_str == "left" + && count == 1 + && modifiers.is_empty() + { + let focus_only = action == "focus"; + let ax_result = tokio::task::spawn_blocking(move || unsafe { + let Some(element) = element_at_screen_position(pid, screen_x, screen_y) else { + return Ok::(false); + }; + let delivered = if focus_only { + crate::input::ax_actions::focus_element(element as usize).is_ok() + } else { + let press = core_foundation::string::CFString::new("AXPress"); + AXUIElementPerformAction(element, press.as_concrete_TypeRef()) + == kAXErrorSuccess + }; + CFRelease(element as _); + Ok(delivered) + }) + .await; + match ax_result { + Ok(Ok(true)) => { + let label = if focus_only { "focused" } else { "pressed" }; + return ToolResult::text(format!( + "✅ PX hit-test {label} the background element via AX." + )) + .with_structured(serde_json::json!({ + "path": "ax", + "verified": false, + "effect": "unverifiable" + })); + } + Ok(Ok(false)) if focus_only => { + return ToolResult::error( + "Background PX focus is unavailable at the requested point.".to_owned(), + ) + .with_structured(serde_json::json!({ + "code": "background_unavailable" + })); + } + Ok(Err(error)) if focus_only => { + return ToolResult::error(format!("Background PX focus failed: {error}")) + .with_structured(serde_json::json!({ + "code": "background_unavailable" + })); + } + _ => {} + } + } + // Pin the overlay above the target window BEFORE animating so // the cursor is already sandwiched correctly while it glides in. if let Some(wid) = window_id { diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs index 8deff2249d..c07221adaf 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs @@ -118,6 +118,13 @@ impl Tool for DragTool { // that drop background CGEvents), via the same skylight assist click // uses. Requires a window_id to have a window to front. let delivery_mode = super::DeliveryMode::parse(args.opt_str("delivery_mode").as_deref()); + if !delivery_mode.is_foreground() { + return ToolResult::error( + "Background drag is unavailable on macOS; use delivery_mode:\"foreground\"." + .to_owned(), + ) + .with_structured(serde_json::json!({ "code": "background_unavailable" })); + } let cursor_key = super::cursor_tools::resolve_cursor_key(&args); // Coerce integer or float from JSON for coordinate fields. diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs index ba90c36015..d8db786e73 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs @@ -117,6 +117,7 @@ pub(crate) async fn focus_by_pixel( let mut click_args = serde_json::json!({ "pid": pid, "x": x, "y": y, "delivery_mode": if foreground { "foreground" } else { "background" }, + "action": if foreground { "press" } else { "focus" }, }); if let Some(wid) = window_id { click_args["window_id"] = serde_json::json!(wid); } if let Some(s) = session { click_args["session"] = serde_json::json!(s); } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs index 38d8ae59f7..ad69f68a41 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs @@ -1,10 +1,17 @@ use async_trait::async_trait; -use cua_driver_core::{protocol::ToolResult, tool::{Tool, ToolDef}}; +use core_foundation::base::{CFRelease, CFTypeRef}; +use cua_driver_core::{ + protocol::ToolResult, + tool::{Tool, ToolDef}, +}; use serde_json::Value; use std::sync::Arc; use crate::apps; -use crate::ax::bindings::{element_screen_center, AXUIElementRef}; +use crate::ax::bindings::{ + copy_children, copy_element_attr, copy_string_attr, element_screen_center, kAXErrorSuccess, + perform_action, AXUIElementRef, +}; use crate::focus_guard; use crate::window_change_detector::WindowChangeDetector; @@ -30,7 +37,9 @@ pub struct ScrollTool { } impl ScrollTool { - pub fn new(state: Arc) -> Self { Self { state } } + pub fn new(state: Arc) -> Self { + Self { state } + } } static DEF: std::sync::OnceLock = std::sync::OnceLock::new(); @@ -96,22 +105,37 @@ fn def() -> &'static ToolDef { #[async_trait] impl Tool for ScrollTool { - fn def(&self) -> &ToolDef { def() } + fn def(&self) -> &ToolDef { + def() + } async fn invoke(&self, args: Value) -> ToolResult { use cua_driver_core::tool_args::ArgsExt; - let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e }; + let pid = match args.require_i32("pid") { + Ok(v) => v, + Err(e) => return e, + }; // delivery_mode: foreground briefly fronts the window before the // pixel-wheel dispatch (the explicit last resort for surfaces that drop // background CGEvents). Only the pixel-wheel path honors it; the // keystroke path is background-by-design and untouched. let delivery_mode = super::DeliveryMode::parse(args.opt_str("delivery_mode").as_deref()); - let direction = match args.require_str("direction") { Ok(v) => v, Err(e) => return e }; + if !delivery_mode.is_foreground() && crate::browser::ElectronJs::is_electron(pid) { + return ToolResult::error( + "Background scroll is unavailable for Electron/Chromium windows on macOS." + .to_owned(), + ) + .with_structured(serde_json::json!({ "code": "background_unavailable" })); + } + let direction = match args.require_str("direction") { + Ok(v) => v, + Err(e) => return e, + }; let by = args.str_or("by", "line"); let amount = args.u64_or("amount", 3) as usize; // Surface 6: element_token / element_index precedence. let element_token_arg = args.opt_str("element_token"); - let window_id_arg = args.opt_u64("window_id").map(|v| v as u32); + let window_id_arg = args.opt_u64("window_id").map(|v| v as u32); let element_index_arg = args.opt_u64("element_index").map(|v| v as usize); let resolved = match cua_driver_core::element_token::resolve_element_args( pid, @@ -126,7 +150,9 @@ impl Tool for ScrollTool { let (element_index, window_id) = match resolved { cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg), cua_driver_core::element_token::ResolvedElement::Element { - window_id: wid, element_index: idx, via_token: _, + window_id: wid, + element_index: idx, + via_token: _, } => (Some(idx), wid), }; @@ -155,6 +181,79 @@ impl Tool for ScrollTool { } } + // AppKit exposes vertical scroll-bar buttons beneath the text area's + // AXScrollArea parent. Pressing those controls is a true + // background-safe scroll: no activation, z-order change, or cursor move. + if matches!(direction.as_str(), "up" | "down") { + if let (Some(index), Some(wid)) = (element_index, window_id) { + let native_element_guard = self + .state + .element_cache + .get_element_retained(pid, wid, index); + let direction_for_ax = direction.clone(); + let by_for_ax = by.clone(); + let foreground = delivery_mode.is_foreground(); + let ax_result = + tokio::task::spawn_blocking(move || -> anyhow::Result<(bool, bool)> { + let Some(element_guard) = native_element_guard else { + return Ok((false, false)); + }; + if foreground { + let mut delivered = false; + let fronted = crate::input::skylight::with_foreground_assist( + pid as libc::pid_t, + wid, + || { + delivered = unsafe { + scroll_native_text_area( + element_guard.as_ptr() as AXUIElementRef, + &direction_for_ax, + &by_for_ax, + amount, + ) + }; + std::thread::sleep(std::time::Duration::from_millis(100)); + Ok(()) + }, + )?; + Ok((delivered, fronted)) + } else { + Ok(( + unsafe { + scroll_native_text_area( + element_guard.as_ptr() as AXUIElementRef, + &direction_for_ax, + &by_for_ax, + amount, + ) + }, + false, + )) + } + }) + .await; + match ax_result { + Ok(Ok((true, fronted))) => { + return ToolResult::text(format!( + "✅ Scrolled native macOS control {direction} by {by} × {amount} through AX." + )) + .with_structured(serde_json::json!({ + "path": if fronted { "ax_fg" } else { "ax" }, + "verified": false, + "effect": "unverifiable" + })); + } + Ok(Ok((false, _))) => {} + Ok(Err(error)) => { + return ToolResult::error(format!("Native AX scroll failed: {error}")); + } + Err(error) => { + return ToolResult::error(format!("Native AX scroll task failed: {error}")); + } + } + } + } + // ── Targeted wheel path ───────────────────────────────────────────── // A target — element (preferred) OR window-local x,y — routes the scroll // through a synthesized mouse-wheel event at that screen point, so the @@ -162,19 +261,27 @@ impl Tool for ScrollTool { // cursor. This is the ONLY way to scroll a nested overflow:auto region // that never takes keyboard focus (the keystroke path below no-ops on // it). No user-facing flag: presence of a target IS the switch. - let x_arg = args.opt_f64("x").or_else(|| args.opt_i64("x").map(|v| v as f64)); - let y_arg = args.opt_f64("y").or_else(|| args.opt_i64("y").map(|v| v as f64)); + let x_arg = args + .opt_f64("x") + .or_else(|| args.opt_i64("x").map(|v| v as f64)); + let y_arg = args + .opt_f64("y") + .or_else(|| args.opt_i64("y").map(|v| v as f64)); // Per-notch step + direction→delta mapping (sign convention lives // here; the mouse primitive stays sign-agnostic). macOS: +y reveals // content ABOVE, -y reveals BELOW; +x reveals LEFT, -x reveals RIGHT. - let step = if by == "page" { WHEEL_STEP_PAGE_PX } else { WHEEL_STEP_LINE_PX }; + let step = if by == "page" { + WHEEL_STEP_PAGE_PX + } else { + WHEEL_STEP_LINE_PX + }; let (delta_y, delta_x): (i32, i32) = match direction.as_str() { - "down" => (-step, 0), - "up" => ( step, 0), + "down" => (-step, 0), + "up" => (step, 0), "right" => (0, -step), - "left" => (0, step), - _ => (-step, 0), + "left" => (0, step), + _ => (-step, 0), }; // Resolve a screen-space wheel target, if a target was supplied. @@ -201,7 +308,12 @@ impl Tool for ScrollTool { let win_local = wid .and_then(crate::windows::window_bounds_by_id) .map(|b| (cx - b.x, cy - b.y)); - WheelTarget { screen_x: cx, screen_y: cy, win_local, wid } + WheelTarget { + screen_x: cx, + screen_y: cy, + win_local, + wid, + } }) }) .await @@ -213,8 +325,7 @@ impl Tool for ScrollTool { // Without one, refuse rather than scrolling at screen-absolute coords. if window_id.is_none() { return ToolResult::error( - "window_id is required when scrolling by window-local x,y pixels." - .to_string(), + "window_id is required when scrolling by window-local x,y pixels.".to_string(), ); } // Pixel path: x,y are window-local screenshot pixels. Mirror the @@ -231,21 +342,39 @@ impl Tool for ScrollTool { let scale: f64 = if let Some(ref b) = bounds { if let Ok(png) = crate::capture::screenshot_window_bytes(wid) { if png.len() >= 24 { - let pw = u32::from_be_bytes([png[16], png[17], png[18], png[19]]) as f64; - if b.width > 0.0 && pw > b.width { pw / b.width } else { 1.0 } - } else { 1.0 } - } else { 1.0 } - } else { 1.0 }; + let pw = + u32::from_be_bytes([png[16], png[17], png[18], png[19]]) as f64; + if b.width > 0.0 && pw > b.width { + pw / b.width + } else { + 1.0 + } + } else { + 1.0 + } + } else { + 1.0 + } + } else { + 1.0 + }; if let Some(b) = bounds { let (wx, wy) = (cx / scale, cy / scale); return WheelTarget { - screen_x: b.x + wx, screen_y: b.y + wy, - win_local: Some((wx, wy)), wid: Some(wid), + screen_x: b.x + wx, + screen_y: b.y + wy, + win_local: Some((wx, wy)), + wid: Some(wid), }; } } // No window_id → treat x,y as screen coordinates. - WheelTarget { screen_x: cx, screen_y: cy, win_local: None, wid: None } + WheelTarget { + screen_x: cx, + screen_y: cy, + win_local: None, + wid: None, + } }) .await .ok() @@ -264,15 +393,26 @@ impl Tool for ScrollTool { ); } crate::cursor::overlay::animate_cursor_to( - cursor_key.clone(), target.screen_x, target.screen_y, - ).await; - self.state.cursor_registry - .update_position(&cursor_key, target.screen_x, target.screen_y); + cursor_key.clone(), + target.screen_x, + target.screen_y, + ) + .await; + self.state.cursor_registry.update_position( + &cursor_key, + target.screen_x, + target.screen_y, + ); let prior_front = apps::frontmost_pid(); let snapshot = WindowChangeDetector::snapshot(prior_front); - let WheelTarget { screen_x, screen_y, win_local, wid } = target; + let WheelTarget { + screen_x, + screen_y, + win_local, + wid, + } = target; let amount_ticks = amount; let fg = delivery_mode.is_foreground() && wid.is_some(); let result = focus_guard::with_focus_suppressed( @@ -283,14 +423,24 @@ impl Tool for ScrollTool { tokio::task::spawn_blocking(move || -> anyhow::Result<()> { let do_it = move || -> anyhow::Result<()> { crate::input::mouse::scroll_wheel_at_xy( - pid, screen_x, screen_y, win_local, wid, - delta_y, delta_x, amount_ticks, + pid, + screen_x, + screen_y, + win_local, + wid, + delta_y, + delta_x, + amount_ticks, ) }; // Foreground rung: brief front → wheel → restore prior frontmost. match (fg, wid) { (true, Some(w)) => { - crate::input::skylight::with_foreground_assist(pid as libc::pid_t, w, do_it)?; + crate::input::skylight::with_foreground_assist( + pid as libc::pid_t, + w, + do_it, + )?; Ok(()) } _ => do_it(), @@ -302,7 +452,11 @@ impl Tool for ScrollTool { .await; let changes = snapshot.detect_async().await; - let mode_label = if fg { " (delivery_mode:foreground)" } else { "" }; + let mode_label = if fg { + " (delivery_mode:foreground)" + } else { + "" + }; return match result { Ok(Ok(())) => ToolResult::text(format!( "✅ Sent {direction} scroll by {by} × {amount} via pixel wheel at \ @@ -319,13 +473,13 @@ impl Tool for ScrollTool { } let key = match (by.as_str(), direction.as_str()) { - ("page", "down") | (_, "down") if by == "page" => "pagedown", - ("page", "up") | (_, "up") if by == "page" => "pageup", - ("line", "down") | (_, "down") => "down", - ("line", "up") | (_, "up") => "up", - (_, "left") => "left", - (_, "right") => "right", - _ => "down", + ("page", "down") | (_, "down") if by == "page" => "pagedown", + ("page", "up") | (_, "up") if by == "page" => "pageup", + ("line", "down") | (_, "down") => "down", + ("line", "up") | (_, "up") => "up", + (_, "left") => "left", + (_, "right") => "right", + _ => "down", }; let key = key.to_owned(); @@ -350,7 +504,8 @@ impl Tool for ScrollTool { if let Some(element_ptr) = pre_focus_ptr { let _ = tokio::task::spawn_blocking(move || { crate::input::ax_actions::focus_element(element_ptr) - }).await; + }) + .await; tokio::time::sleep(std::time::Duration::from_millis(30)).await; } @@ -382,3 +537,69 @@ impl Tool for ScrollTool { } } } + +unsafe fn scroll_native_text_area( + element: AXUIElementRef, + direction: &str, + by: &str, + amount: usize, +) -> bool { + if copy_string_attr(element, "AXRole").as_deref() != Some("AXTextArea") { + return false; + } + let Some(scroll_area) = copy_element_attr(element, "AXParent") else { + return false; + }; + if copy_string_attr(scroll_area, "AXRole").as_deref() != Some("AXScrollArea") { + CFRelease(scroll_area as CFTypeRef); + return false; + } + let mut buttons = Vec::new(); + collect_ax_buttons(scroll_area, 0, &mut buttons); + CFRelease(scroll_area as CFTypeRef); + if buttons.is_empty() { + return false; + } + + let reverse = direction == "up"; + let base = if by == "page" && buttons.len() >= 4 { + 2 + } else { + 0 + }; + let index = base + usize::from(reverse); + let mut delivered = false; + if let Some(target) = buttons.get(index).copied() { + for _ in 0..amount.max(1) { + if perform_action(target, "AXPress") != kAXErrorSuccess { + break; + } + delivered = true; + std::thread::sleep(std::time::Duration::from_millis(30)); + } + } + for button in buttons { + CFRelease(button as CFTypeRef); + } + delivered +} + +unsafe fn collect_ax_buttons( + element: AXUIElementRef, + depth: usize, + buttons: &mut Vec, +) { + if depth >= 4 || buttons.len() >= 4 { + return; + } + for child in copy_children(element) { + if buttons.len() >= 4 { + CFRelease(child as CFTypeRef); + } else if copy_string_attr(child, "AXRole").as_deref() == Some("AXButton") { + buttons.push(child); + } else { + collect_ax_buttons(child, depth + 1, buttons); + CFRelease(child as CFTypeRef); + } + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs index 2d9ef6d610..49def31da7 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs @@ -476,6 +476,13 @@ fn cgevent_type_verified( std::thread::sleep(std::time::Duration::from_millis(settle_ms)); } if clear_first { + if settle_ms > 0 { + // Some renderer focus proxies discard the first printable event + // after activation even after their AX focus is visible. Prime + // that channel with disposable text, then clear it before the + // requested payload. Never do this for a nonempty field. + let _ = crate::input::keyboard::type_text_with_delay(pid, " ", delay_ms); + } let _ = crate::input::keyboard::press_key(pid, "a", &["cmd"]); let _ = crate::input::keyboard::press_key(pid, "delete", &[]); } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs b/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs index 974d21045d..8836344897 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs @@ -179,6 +179,17 @@ impl WindowChangeDetector { /// Safe to call from any thread — `CGWindowListCopyWindowInfo` is /// documented as thread-safe. pub fn snapshot(prior_front: Option) -> Snapshot { + Self::capture(prior_front, true) + } + + /// Capture the same before-state without arming reactive focus suppression. + /// Foreground delivery owns its temporary activation and restoration, so a + /// wildcard lease would race the target while the action is settling. + pub fn snapshot_without_suppression(prior_front: Option) -> Snapshot { + Self::capture(prior_front, false) + } + + fn capture(prior_front: Option, suppress_focus: bool) -> Snapshot { let window_ids: HashSet = windows::visible_windows() .into_iter() .filter(|w| w.layer == 0) @@ -190,7 +201,7 @@ impl WindowChangeDetector { // (any other pid). If there's no frontmost (rare — screensaver, // login window), we skip the lease; foreground-change tracking // still runs. - let lease = prior_front.map(|restore_to| { + let lease = prior_front.filter(|_| suppress_focus).map(|restore_to| { focus_steal::begin_suppression( None, // wildcard restore_to, @@ -471,7 +482,10 @@ mod tests { foreground_changed: false, }; // No title → just the app name, no parentheses. - assert_eq!(c.result_suffix(), "\n\n🪟 Action opened new window(s): Finder."); + assert_eq!( + c.result_suffix(), + "\n\n🪟 Action opened new window(s): Finder." + ); } /// Regression: `snapshot(prior_front)` must store the caller's diff --git a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift index eb5bec9d1d..b97f4b3ee9 100644 --- a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift +++ b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift @@ -52,7 +52,7 @@ let kMenuItemTitle = "Harness Test Item" final class HarnessWindowController: NSObject, NSTextFieldDelegate { let window: NSWindow - let counterLabel = NSTextField(labelWithString: "0") + let counterLabel = NSTextField(labelWithString: "counter=0") var counterValue = 0 let textInput = NSTextField(string: "") let textInputMirror = NSTextField(labelWithString: "") @@ -67,7 +67,7 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate { // Pinned content size — every launch MUST produce a byte-identical window // so screenshot dimensions (and the hardcoded pixel coords the harness tests // rely on) never drift. - static let kContentSize = NSSize(width: 720, height: 1080) + static let kContentSize = NSSize(width: 720, height: 860) override init() { let rect = NSRect(origin: NSPoint(x: 100, y: 100), size: HarnessWindowController.kContentSize) @@ -103,8 +103,8 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate { let content = NSStackView() content.orientation = .vertical content.alignment = .leading - content.spacing = 16 - content.edgeInsets = NSEdgeInsets(top: 20, left: 20, bottom: 20, right: 20) + content.spacing = 8 + content.edgeInsets = NSEdgeInsets(top: 12, left: 20, bottom: 12, right: 20) content.translatesAutoresizingMaskIntoConstraints = false // counter @@ -232,7 +232,7 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate { let scrollWrap = NSStackView() scrollWrap.orientation = .horizontal scrollWrap.spacing = 12 - let scroller = NSScrollView(frame: NSRect(x: 0, y: 0, width: 360, height: 200)) + let scroller = NSScrollView(frame: NSRect(x: 0, y: 0, width: 360, height: 120)) scroller.hasVerticalScroller = true scroller.borderType = .lineBorder let bodyText = NSTextView(frame: NSRect(x: 0, y: 0, width: 340, height: 600)) @@ -296,12 +296,12 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate { @objc private func onIncrement() { counterValue += 1 - counterLabel.stringValue = String(counterValue) + counterLabel.stringValue = "counter=\(counterValue)" } @objc private func onReset() { counterValue = 0 - counterLabel.stringValue = "0" + counterLabel.stringValue = "counter=0" } @objc private func onExit() { diff --git a/libs/cua-driver/tests/fixtures/apps/macos/swiftui/main.swift b/libs/cua-driver/tests/fixtures/apps/macos/swiftui/main.swift index 7ee28048da..6aa61d0ad1 100644 --- a/libs/cua-driver/tests/fixtures/apps/macos/swiftui/main.swift +++ b/libs/cua-driver/tests/fixtures/apps/macos/swiftui/main.swift @@ -46,6 +46,7 @@ let kScrollOffsetAID = "lbl-scroll-offset" let kScrollTopMarker = "SCROLL_TOP_MARKER_v1" let kScrollBottomMarker = "SCROLL_BOTTOM_MARKER_v1" let kPopupTriggerAID = "btn-open-popover" +let kPopupStateAID = "lbl-popover-state" let kPopupTextAID = "txt-popover-body" let kPopupMarker = "POPOVER_MARKER_v1" let kExitButtonAID = "btn-exit" @@ -107,7 +108,7 @@ struct HarnessRootView: View { .accessibilityIdentifier(kIncrementButtonAID) Button("Reset") { counter = 0 } .accessibilityIdentifier(kResetButtonAID) - Text("\(counter)") + Text("counter=\(counter)") .font(.system(size: 18, weight: .semibold, design: .monospaced)) .accessibilityIdentifier(kCounterLabelAID) } @@ -216,6 +217,9 @@ struct HarnessRootView: View { .padding() .accessibilityIdentifier(kPopupTextAID) } + Text("popover_open=" + String(showPopover)) + .font(.system(.body, design: .monospaced)) + .accessibilityIdentifier(kPopupStateAID) } Spacer(minLength: 24) diff --git a/libs/cua-driver/tests/fixtures/apps/macos/wkwebview/main.swift b/libs/cua-driver/tests/fixtures/apps/macos/wkwebview/main.swift index f070f20eb4..e7283ec984 100644 --- a/libs/cua-driver/tests/fixtures/apps/macos/wkwebview/main.swift +++ b/libs/cua-driver/tests/fixtures/apps/macos/wkwebview/main.swift @@ -35,6 +35,15 @@ final class HarnessAppDelegate: NSObject, NSApplicationDelegate, WKNavigationDel window.setContentSize(kContentSize) let config = WKWebViewConfiguration() + let journalURL = ProcessInfo.processInfo.environment["CUA_E2E_FIXTURE_JOURNAL_URL"] ?? "" + if let encoded = try? JSONEncoder().encode(journalURL), + let literal = String(data: encoded, encoding: .utf8) { + let source = "window.__CUA_E2E_FIXTURE_JOURNAL_URL = \(literal);" + config.userContentController.addUserScript(WKUserScript( + source: source, + injectionTime: .atDocumentStart, + forMainFrameOnly: false)) + } webView = WKWebView(frame: NSRect(origin: .zero, size: kContentSize), configuration: config) webView.autoresizingMask = [.width, .height] webView.navigationDelegate = self diff --git a/libs/cua-driver/tests/fixtures/build/macos.sh b/libs/cua-driver/tests/fixtures/build/macos.sh index f2747c7f54..8c50f2d375 100755 --- a/libs/cua-driver/tests/fixtures/build/macos.sh +++ b/libs/cua-driver/tests/fixtures/build/macos.sh @@ -9,7 +9,8 @@ # Usage: # ./macos.sh # build all macOS-runnable harnesses # ./macos.sh --skip swiftui # skip one target (appkit|swiftui|wkwebview|electron|tauri) -# ./macos.sh --clean # remove staged outputs first +# ./macos.sh --only wkwebview # build just one target +# ./macos.sh --clean # archive staged outputs first set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -17,10 +18,12 @@ HARNESS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" STAGE_DIR="$(cd "$HARNESS_DIR/../../rust/test-apps" && pwd)" SKIP="" +ONLY="" CLEAN=0 while [[ $# -gt 0 ]]; do case "$1" in --skip) SKIP="$2"; shift 2;; + --only) ONLY="$2"; shift 2;; --clean) CLEAN=1; shift;; -h|--help) sed -n '2,11p' "$0"; exit 0;; @@ -28,8 +31,22 @@ while [[ $# -gt 0 ]]; do esac done +archive_existing() { + local target="$1" + [[ -e "$target" ]] || return 0 + local archive_root="${TMPDIR:-/tmp}/cua-driver-fixture-build-archive" + local stamp + stamp="$(date +%Y%m%d-%H%M%S)-$$" + mkdir -p "$archive_root" + mv "$target" "$archive_root/$(basename "$target").$stamp" +} + if [[ "$CLEAN" == "1" ]]; then - rm -rf "$STAGE_DIR/harness-appkit" "$STAGE_DIR/harness-swiftui" "$STAGE_DIR/harness-wkwebview" "$STAGE_DIR/harness-electron" "$STAGE_DIR/harness-tauri" + archive_existing "$STAGE_DIR/harness-appkit" + archive_existing "$STAGE_DIR/harness-swiftui" + archive_existing "$STAGE_DIR/harness-wkwebview" + archive_existing "$STAGE_DIR/harness-electron" + archive_existing "$STAGE_DIR/harness-tauri" mkdir -p "$STAGE_DIR/harness-appkit" "$STAGE_DIR/harness-swiftui" "$STAGE_DIR/harness-wkwebview" "$STAGE_DIR/harness-electron" "$STAGE_DIR/harness-tauri" echo "==> Cleaned stage dirs" fi @@ -41,7 +58,7 @@ build_app() { local plist="$bundle/Contents/Info.plist" echo "==> Building $name" - rm -rf "$bundle" + archive_existing "$bundle" mkdir -p "$bundle/Contents/MacOS" # shellcheck disable=SC2086 # word-splitting on $frameworks is intentional @@ -77,21 +94,21 @@ EOF echo " → $bundle" } -if [[ "$SKIP" != "appkit" ]]; then +if [[ -z "$ONLY" || "$ONLY" == "appkit" ]] && [[ "$SKIP" != "appkit" ]]; then build_app "CuaTestHarness.AppKit" \ "$HARNESS_DIR/apps/macos/appkit" \ "" \ "harness-appkit" fi -if [[ "$SKIP" != "swiftui" ]]; then +if [[ -z "$ONLY" || "$ONLY" == "swiftui" ]] && [[ "$SKIP" != "swiftui" ]]; then build_app "CuaTestHarness.SwiftUI" \ "$HARNESS_DIR/apps/macos/swiftui" \ "" \ "harness-swiftui" fi -if [[ "$SKIP" != "wkwebview" ]]; then +if [[ -z "$ONLY" || "$ONLY" == "wkwebview" ]] && [[ "$SKIP" != "wkwebview" ]]; then build_app "CuaTestHarness.WKWebView" \ "$HARNESS_DIR/apps/macos/wkwebview" \ "-framework WebKit" \ @@ -105,11 +122,11 @@ if [[ "$SKIP" != "wkwebview" ]]; then echo " → bundled shared/web/index.html into Resources/web/" fi -if [[ "$SKIP" != "electron" ]]; then +if [[ -z "$ONLY" || "$ONLY" == "electron" ]] && [[ "$SKIP" != "electron" ]]; then "$HARNESS_DIR/apps/cross-platform/electron/build.sh" fi -if [[ "$SKIP" != "tauri" ]]; then +if [[ -z "$ONLY" || "$ONLY" == "tauri" ]] && [[ "$SKIP" != "tauri" ]]; then "$HARNESS_DIR/apps/cross-platform/tauri/build.sh" fi diff --git a/libs/cua-driver/tests/fixtures/smoke/macos.sh b/libs/cua-driver/tests/fixtures/smoke/macos.sh index 5537558b32..17bebb7986 100755 --- a/libs/cua-driver/tests/fixtures/smoke/macos.sh +++ b/libs/cua-driver/tests/fixtures/smoke/macos.sh @@ -21,7 +21,7 @@ # because the call exits cleanly — flagged in output) set -u -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd)" +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../../.." && pwd)" DRIVER="$ROOT/libs/cua-driver/rust/target/release/cua-driver" [[ -x "$DRIVER" ]] || DRIVER="$ROOT/libs/cua-driver/rust/target/debug/cua-driver" HARNESS_APP="$ROOT/libs/cua-driver/rust/test-apps/harness-appkit/CuaTestHarness.AppKit.app" @@ -43,8 +43,9 @@ run_tool() { # Same trap I hit in scripts/linux-smoke.sh. local args if [[ -z "${2-}" ]]; then args='{}'; else args="$2"; fi - local out code + local out code first_line out=$("$DRIVER" call "$tool" "$args" 2>&1) ; code=$? + first_line="${out%%$'\n'*}" # Treat documented "this tool is intentionally a per-platform stub" # responses as SKIP rather than FAIL — they indicate the tool was # called correctly but isn't meaningful on this OS by design. @@ -54,12 +55,12 @@ run_tool() { fi if [[ $code -eq 0 ]]; then if [[ "$out" == "❌"* || "$out" == "Error:"* ]]; then - record "$tool" "FAIL" "exit0+❌: $(echo "$out" | head -1 | cut -c1-100)" + record "$tool" "FAIL" "exit0+❌: $(printf '%.100s' "$first_line")" else - record "$tool" "PASS" "$(echo "$out" | head -1 | cut -c1-100)" + record "$tool" "PASS" "$(printf '%.100s' "$first_line")" fi else - record "$tool" "FAIL" "exit=$code: $(echo "$out" | head -1 | cut -c1-100)" + record "$tool" "FAIL" "exit=$code: $(printf '%.100s' "$first_line")" fi } @@ -116,7 +117,7 @@ run_tool set_config '{"max_image_dimension":1024}' run_tool set_agent_cursor_enabled '{"enabled":true}' run_tool set_agent_cursor_style '{"style":"default"}' run_tool set_agent_cursor_motion '{}' -run_tool set_recording '{"enabled":false}' +run_tool stop_recording '{}' # ── group 3: app lifecycle ────────────────────────────────────────────────── echo "" @@ -158,7 +159,7 @@ else run_tool click "{\"pid\":$HARNESS_PID,\"window_id\":$WIN_ID,\"x\":120,\"y\":80}" run_tool double_click "{\"pid\":$HARNESS_PID,\"window_id\":$WIN_ID,\"x\":120,\"y\":80}" run_tool right_click "{\"pid\":$HARNESS_PID,\"window_id\":$WIN_ID,\"x\":120,\"y\":80}" - run_tool drag "{\"pid\":$HARNESS_PID,\"window_id\":$WIN_ID,\"from_x\":120,\"from_y\":80,\"to_x\":180,\"to_y\":120}" + run_tool drag "{\"pid\":$HARNESS_PID,\"window_id\":$WIN_ID,\"from_x\":120,\"from_y\":80,\"to_x\":180,\"to_y\":120,\"delivery_mode\":\"foreground\"}" run_tool scroll "{\"pid\":$HARNESS_PID,\"window_id\":$WIN_ID,\"x\":200,\"y\":400,\"direction\":\"down\"}" run_tool type_text "{\"pid\":$HARNESS_PID,\"window_id\":$WIN_ID,\"text\":\"hi\"}" run_tool press_key "{\"pid\":$HARNESS_PID,\"window_id\":$WIN_ID,\"key\":\"a\"}" @@ -210,3 +211,5 @@ echo "Interpretation:" echo " PASS = tool ran cleanly (exit 0, no ❌ in output)" echo " FAIL = error or non-zero exit" echo " SKIP = intentionally not probed (covered by integration tests, missing fixture, etc.)" + +[[ "$fail" -eq 0 ]] diff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.sh new file mode 100755 index 0000000000..ddc647fbcc --- /dev/null +++ b/scripts/ci/macos/run-rust-e2e.sh @@ -0,0 +1,241 @@ +#!/usr/bin/env bash +# Run the canonical Rust desktop matrix in a logged-in macOS user session. +# macOS harness tests use the installed, TCC-authorized cua-driver daemon path. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +DRIVER_ROOT="${REPO_ROOT}/libs/cua-driver" +RUST_ROOT="${DRIVER_ROOT}/rust" +SUITE="${CUA_E2E_INTERNAL_LANE:-all}" +BUILD_FIXTURES=1 + +usage() { + cat <<'EOF' +Usage: run-rust-e2e.sh [--no-build] + +Run from a logged-in macOS desktop after install-local and TCC authorization. +The testkit proxies MCP calls through the installed CuaDriver daemon. +The contributor-facing command always runs the complete matrix. +EOF +} + +while (($#)); do + case "$1" in + --no-build) BUILD_FIXTURES=0 ;; + -h|--help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;; + esac + shift +done + +case "$SUITE" in + shared|native|capture|all) ;; + *) echo "unsupported internal lane: $SUITE" >&2; exit 2 ;; +esac + +if ! git -C "${REPO_ROOT}" diff --quiet || ! git -C "${REPO_ROOT}" diff --cached --quiet; then + echo "macOS canonical E2E requires a clean tracked working tree" >&2 + exit 2 +fi +if [[ -z "${CUA_E2E_SOURCE_SHA:-}" ]]; then + CUA_E2E_SOURCE_SHA="$(git -C "${REPO_ROOT}" rev-parse HEAD)" +fi +if [[ ! "${CUA_E2E_SOURCE_SHA}" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "CUA_E2E_SOURCE_SHA must be a full 40-character commit SHA" >&2 + exit 2 +fi +export CUA_E2E_SOURCE_SHA + +ARTIFACT_DIR="${REPO_ROOT}/artifacts/cua-driver/macos" +RECORDING_ROOT="${ARTIFACT_DIR}/recordings" +if [[ -e "${RECORDING_ROOT}" ]]; then + RECORDING_ARCHIVE="$(mktemp -d "${TMPDIR:-/tmp}/cua-macos-e2e-recordings.XXXXXX")" + mv "${RECORDING_ROOT}" "${RECORDING_ARCHIVE}/recordings" + echo "Previous recordings preserved at ${RECORDING_ARCHIVE}/recordings" +fi +mkdir -p "${RECORDING_ROOT}" +RESULTS_FILE="${ARTIFACT_DIR}/results.jsonl" +DECLARATIONS_FILE="${ARTIFACT_DIR}/cases.jsonl" +ENVIRONMENT_FILE="${ARTIFACT_DIR}/environment.jsonl" +SUMMARY_FILE="${ARTIFACT_DIR}/summary.md" +mkdir -p "${ARTIFACT_DIR}" +: > "${DECLARATIONS_FILE}" +: > "${ENVIRONMENT_FILE}" +: > "${RESULTS_FILE}" +rm -f "${SUMMARY_FILE}" + +export CUA_E2E_DECLARATIONS_FILE="${DECLARATIONS_FILE}" +export CUA_E2E_ENVIRONMENT_FILE="${ENVIRONMENT_FILE}" +export CUA_E2E_RESULTS_FILE="${RESULTS_FILE}" +export CUA_E2E_RECORDINGS_ROOT="${RECORDING_ROOT}" +export CUA_TEST_WORKSPACE_ROOT="${RUST_ROOT}" +export CUA_TEST_DRIVER_BIN="${RUST_ROOT}/target/release/cua-driver" +export CUA_TEST_APPS_ROOT="${RUST_ROOT}/test-apps" +export CUA_TEST_REQUIRE_FIXTURES=1 +export CUA_TEST_DRIVER_STDERR=1 + +command -v ffmpeg >/dev/null || { echo "ffmpeg is required for E2E trajectory videos" >&2; exit 1; } +command -v ffprobe >/dev/null || { echo "ffprobe is required for E2E trajectory validation" >&2; exit 1; } +command -v jq >/dev/null || { echo "jq is required for E2E ownership validation" >&2; exit 1; } + +if [[ "${BUILD_FIXTURES}" == 1 ]]; then + cargo build --release -p cua-driver --manifest-path "${RUST_ROOT}/Cargo.toml" + bash "${DRIVER_ROOT}/tests/fixtures/build/macos.sh" +fi + +if [[ ! -x "${CUA_TEST_DRIVER_BIN}" ]]; then + echo "Required driver binary was not built: ${CUA_TEST_DRIVER_BIN}" >&2 + exit 1 +fi + +required_fixtures=() +required_fixtures+=("${CUA_TEST_APPS_ROOT}/harness-electron/CuaTestHarness.Electron.app") +if [[ "${SUITE}" == shared || "${SUITE}" == all ]]; then + required_fixtures+=( + "${CUA_TEST_APPS_ROOT}/harness-tauri/CuaTestHarness.Tauri.app" + "${CUA_TEST_APPS_ROOT}/harness-wkwebview/CuaTestHarness.WKWebView.app" + ) +fi +if [[ "${SUITE}" == native || "${SUITE}" == all ]]; then + required_fixtures+=( + "${CUA_TEST_APPS_ROOT}/harness-appkit/CuaTestHarness.AppKit.app" + "${CUA_TEST_APPS_ROOT}/harness-swiftui/CuaTestHarness.SwiftUI.app" + ) +fi +for fixture in "${required_fixtures[@]}"; do + [[ -d "${fixture}" ]] || { echo "Required fixture missing: ${fixture}" >&2; exit 1; } +done + +FAILURE_COUNT=0 + +run_report() { + (cd "${RUST_ROOT}" && cargo run -p cua-driver-testkit --bin cua-e2e-report -- \ + --declarations "${DECLARATIONS_FILE}" \ + --environment "${ENVIRONMENT_FILE}" \ + --results "${RESULTS_FILE}" \ + --artifact-root "${ARTIFACT_DIR}" \ + --require-video \ + --output "${SUMMARY_FILE}") +} + +echo "[PREFLIGHT] macOS daemon identity, fixture, AX, capture, and video" +set +e +(cd "${RUST_ROOT}" && cargo test -p cua-driver --test e2e_environment_preflight_test -- \ + --ignored --exact canonical_e2e_environment_is_ready --nocapture --test-threads=1) \ + 2>&1 | tee "${ARTIFACT_DIR}/environment-preflight.log" +PREFLIGHT_EXIT=${PIPESTATUS[0]} +set -e +if [[ "${PREFLIGHT_EXIT}" != 0 ]]; then + set +e + run_report + set -e + echo "macOS E2E environment preflight failed" >&2 + exit 1 +fi + +run_test() { + local name="$1"; shift + echo "[RUN] ${name}" + set +e + (cd "${RUST_ROOT}" && "$@") 2>&1 | tee "${ARTIFACT_DIR}/${name}.log" + local exit_code=${PIPESTATUS[0]} + set -e + if [[ "${exit_code}" != 0 ]]; then + FAILURE_COUNT=$((FAILURE_COUNT + 1)) + fi +} + +if [[ "${SUITE}" == shared || "${SUITE}" == all ]]; then + run_test shared-app-matrix cargo test -p cua-driver --test cross_platform_behavior_test -- \ + --ignored --exact shared_web_action_matrix_is_state_verified \ + --nocapture --test-threads=1 +fi +if [[ "${SUITE}" == native || "${SUITE}" == all ]]; then + for appkit_test in \ + harness_appkit_smoke \ + harness_appkit_text_input \ + harness_appkit_type_text_background \ + harness_appkit_scroll_foreground \ + harness_appkit_scroll_background \ + harness_appkit_counter \ + harness_appkit_counter_px_background \ + harness_appkit_right_click_px_foreground \ + harness_appkit_right_click_px_background \ + harness_appkit_double_click_px_foreground \ + harness_appkit_double_click_px_background \ + harness_appkit_slider_drag_px_foreground \ + harness_appkit_slider_drag_px_background; do + run_test "appkit-${appkit_test}" cargo test -p cua-driver --test harness_appkit_test -- \ + --ignored --exact "${appkit_test}" --nocapture --test-threads=1 + done + for swiftui_test in \ + harness_swiftui_smoke \ + harness_swiftui_counter_background \ + harness_swiftui_set_value_background \ + harness_swiftui_popover_foreground; do + run_test "swiftui-${swiftui_test}" cargo test -p cua-driver --test harness_swiftui_test -- \ + --ignored --exact "${swiftui_test}" --nocapture --test-threads=1 + done + run_test installed-app-launch cargo test -p cua-driver --test installed_app_launch_macos_test -- \ + --ignored --nocapture --test-threads=1 + run_test installed-app-textedit cargo test -p cua-driver --test installed_app_textedit_macos_test -- \ + --ignored --exact background_type_on_native_cocoa_is_ax_verified \ + --nocapture --test-threads=1 +fi +if [[ "${SUITE}" == capture || "${SUITE}" == all ]]; then + run_test capture-contract cargo test -p cua-driver --test capture_contract_test -- \ + --ignored --nocapture --test-threads=1 + run_test desktop-scope cargo test -p cua-driver --test desktop_scope_macos_test -- \ + --ignored --nocapture --test-threads=1 +fi + +video_count=0 +while IFS= read -r -d '' video; do + video_count=$((video_count + 1)) + if ! ffprobe -v error -show_entries format=duration \ + -of default=noprint_wrappers=1:nokey=1 "${video}" >/dev/null; then + echo "[VIDEO FAIL] Unplayable trajectory: ${video}" >&2 + FAILURE_COUNT=$((FAILURE_COUNT + 1)) + fi +done < <(find "${RECORDING_ROOT}" -type f -name recording.mp4 -print0) + +OWNED_VIDEOS="$(mktemp)" +jq -r 'select(.evidence.video != null) | .evidence.video' "${RESULTS_FILE}" > "${OWNED_VIDEOS}" +while IFS= read -r -d '' video; do + relative="${video#${ARTIFACT_DIR}/}" + if [[ "${relative}" == recordings/environment-preflight-*/recording.mp4 ]]; then + continue + fi + if ! grep -Fxq -- "${relative}" "${OWNED_VIDEOS}"; then + echo "[VIDEO FAIL] Orphan trajectory has no typed result row: ${relative}" >&2 + FAILURE_COUNT=$((FAILURE_COUNT + 1)) + fi +done < <(find "${RECORDING_ROOT}" -type f -name recording.mp4 -print0) +rm -f "${OWNED_VIDEOS}" + +while IFS= read -r -d '' error_file; do + echo "[VIDEO FAIL] ${error_file}" >&2 + cat "${error_file}" >&2 + FAILURE_COUNT=$((FAILURE_COUNT + 1)) +done < <(find "${RECORDING_ROOT}" -type f -name recording-error.txt -print0) + +if [[ "${video_count}" == 0 ]]; then + echo "[VIDEO FAIL] No E2E trajectory videos were produced" >&2 + FAILURE_COUNT=$((FAILURE_COUNT + 1)) +fi + +set +e +run_report +REPORT_EXIT=$? +set -e +if [[ "${REPORT_EXIT}" != 0 ]]; then + echo "macOS E2E result validation failed" >&2 + FAILURE_COUNT=$((FAILURE_COUNT + 1)) +fi + +if [[ "${FAILURE_COUNT}" != 0 ]]; then + echo "macOS Rust E2E suite had ${FAILURE_COUNT} failing lane(s)" >&2 + exit 1 +fi +echo "macOS Rust E2E suite completed: ${SUITE}"