From 52c3c2127c12da8b463d8589db16232f5cb21dc0 Mon Sep 17 00:00:00 2001 From: Francesco Bonacci Date: Mon, 25 May 2026 23:01:36 +0000 Subject: [PATCH 1/9] feat(cua-driver-rs)(test-harness): WebView2 + Electron hosts, shared HTML, slider/combo/check/menu coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2 of the Windows test harness adds: **WebView2 + Electron hosts** loading a single shared HTML page (`shared-web/index.html`), so cua-driver's `page` tool can be exercised against two Chromium-based hosts with identical expected behaviour: CuaTestHarness.WebView/ .NET 8 + Microsoft.Web.WebView2.Wpf Exposes --remote-debugging-port via AdditionalBrowserArguments (CDP listener needs WebView2-config follow-up — TODO). CuaTestHarness.Electron/ Electron 31 unpacked-runtime stage (electron-builder portable mode needs admin for winCodeSign symlinks; flat stage of node_modules/electron/dist renamed to CuaTestHarness.Electron.exe). CDP listener confirmed via --remote-debugging-port flag. shared-web/index.html counter, text_input, slider, click_target, checkable_controls, combo_box, navigation anchor. data-cua-id attrs match the AutomationIds used by the WPF host. **Slider scenario + drag tool coverage:** - WPF Slider doesn't surface its parent AutomationId in the flat UIA index list; the SliderAutomationPeer reports its DecreaseLarge / IncreaseLarge / Thumb child parts. - `harness_wpf_slider_drag_tool_returns` exercises the drag tool but doesn't assert value-change — a documented gap caused by PostMessage WM_LBUTTONDOWN not updating OS mouse-button state visible to GetKeyState (WPF Slider Thumb polls this). - `harness_wpf_slider_increase_large` covers the slider via UIA Invoke on the IncreaseLarge sub-button — works on a backgrounded window. **Additional WPF control coverage:** - checkable_controls: CheckBox toggle + RadioButton group select - combo_box: expand + click-item (WPF ComboBox doesn't expose ValuePattern at the parent) - list_box: UIA SelectionItem.Select on a ListBoxItem - menus: top-level Menu expand + invoke; ContextMenu via right-click **Documented cua-driver gaps** (tests assert error shape so the gaps are tracked, not silently regressed): - CDP `/json` HTTP read uses `stream.read_to_end()` but Chromium keeps the socket alive (ignoring `Connection: close`); discovery hangs to the 10 s timeout. Confirmed against Electron 31 on port 9223 via `harness_electron_page_tool_documented_gap`. - WebView2 `--remote-debugging-port` in AdditionalBrowserArguments appears to be filtered — no CDP listener on the WebView helper processes. Tracked as a harness-config TODO. Local Win11 verification: cargo test --test harness_wpf_test -- --ignored --test-threads=1 -> 18 passed; 0 failed (was 11, added 7) cargo test --test harness_web_test -- --ignored --test-threads=1 -> 3 passed; 0 failed (smoke + Electron-gap regression guard) Co-Authored-By: Claude Opus 4.7 --- .../cua-driver/tests/harness_web_test.rs | 208 ++++++++++++++++++ .../cua-driver/tests/harness_wpf_test.rs | 206 +++++++++++++++++ libs/cua-driver/rust/test-apps/.gitignore | 2 + .../CuaTestHarness.Electron/.gitignore | 4 + .../CuaTestHarness.Electron/build.ps1 | 55 +++++ .../CuaTestHarness.Electron/main.js | 48 ++++ .../CuaTestHarness.Electron/package.json | 37 ++++ .../CuaTestHarness.WebView/.gitignore | 4 + .../CuaTestHarness.WebView/App.xaml | 6 + .../CuaTestHarness.WebView/App.xaml.cs | 7 + .../CuaTestHarness.WebView.csproj | 28 +++ .../CuaTestHarness.WebView/MainWindow.xaml | 27 +++ .../CuaTestHarness.WebView/MainWindow.xaml.cs | 51 +++++ .../CuaTestHarness.WebView/app.manifest | 23 ++ .../CuaTestHarness.Wpf/MainWindow.xaml | 131 ++++++++++- .../CuaTestHarness.Wpf/MainWindow.xaml.cs | 51 +++++ .../test-harness/CuaTestHarness.sln | 6 + libs/cua-driver/test-harness/build.ps1 | 23 +- .../test-harness/scenarios/scenarios.json | 80 +++++++ .../test-harness/shared-web/index.html | 175 +++++++++++++++ 20 files changed, 1170 insertions(+), 2 deletions(-) create mode 100644 libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs create mode 100644 libs/cua-driver/test-harness/CuaTestHarness.Electron/.gitignore create mode 100644 libs/cua-driver/test-harness/CuaTestHarness.Electron/build.ps1 create mode 100644 libs/cua-driver/test-harness/CuaTestHarness.Electron/main.js create mode 100644 libs/cua-driver/test-harness/CuaTestHarness.Electron/package.json create mode 100644 libs/cua-driver/test-harness/CuaTestHarness.WebView/.gitignore create mode 100644 libs/cua-driver/test-harness/CuaTestHarness.WebView/App.xaml create mode 100644 libs/cua-driver/test-harness/CuaTestHarness.WebView/App.xaml.cs create mode 100644 libs/cua-driver/test-harness/CuaTestHarness.WebView/CuaTestHarness.WebView.csproj create mode 100644 libs/cua-driver/test-harness/CuaTestHarness.WebView/MainWindow.xaml create mode 100644 libs/cua-driver/test-harness/CuaTestHarness.WebView/MainWindow.xaml.cs create mode 100644 libs/cua-driver/test-harness/CuaTestHarness.WebView/app.manifest create mode 100644 libs/cua-driver/test-harness/shared-web/index.html diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs new file mode 100644 index 0000000000..44e6359970 --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs @@ -0,0 +1,208 @@ +//! Integration tests against the CuaTestHarness.WebView (WPF + WebView2) +//! and CuaTestHarness.Electron hosts. Both load the same +//! `test-harness/shared-web/index.html`, so the same `page` tool flows +//! are exercised against two Chromium-based hosts. +//! +//! Run via: +//! cargo test --test harness_web_test -- --ignored --nocapture +//! +//! ## Known cua-driver gaps these tests document +//! +//! - **CDP `/json` HTTP read uses `read_to_end`** — `mcp-server/src/cdp.rs` +//! sends `Connection: close` and then calls `stream.read_to_end()`, but +//! Chromium's CDP HTTP server ignores `Connection: close` and keeps the +//! socket alive, so `read_to_end` hangs until the 10 s discovery timeout. +//! Confirmed against Electron 31 on port 9223 (verified manually via +//! curl: instant 200, JSON body present). Fix: parse `Content-Length` +//! and `read_exact` that many bytes, or honour `Transfer-Encoding: +//! chunked`. Tracked in this test as a structural assertion (window +//! discoverable) rather than a behavioural one (page tool round-trip). +//! +//! - **WebView2 `--remote-debugging-port` ignored** — passing +//! `AdditionalBrowserArguments = "--remote-debugging-port=9222"` via +//! `CoreWebView2EnvironmentOptions` does not open a CDP listener on the +//! WebView2 helper processes. WebView2 may be filtering the flag. +//! Tracked here as a TODO for the harness rather than a cua-driver +//! issue (since this is a WebView2 configuration concern). + +#![cfg(target_os = "windows")] + +use std::io::{BufRead, BufReader, Write}; +use std::path::PathBuf; +use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio}; +use std::time::Duration; + +// ── workspace paths ────────────────────────────────────────────────────────── + +fn workspace_root() -> PathBuf { + let manifest = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR"); + PathBuf::from(manifest).parent().unwrap().parent().unwrap().to_owned() +} +fn driver_binary() -> PathBuf { workspace_root().join("target/debug/cua-driver.exe") } + +fn webview_exe() -> PathBuf { + if let Ok(p) = std::env::var("HARNESS_WEBVIEW_EXE") { + let pb = PathBuf::from(p); + if pb.exists() { return pb; } + } + workspace_root().join("test-apps/harness-webview/CuaTestHarness.WebView.exe") +} +fn electron_exe() -> PathBuf { + if let Ok(p) = std::env::var("HARNESS_ELECTRON_EXE") { + let pb = PathBuf::from(p); + if pb.exists() { return pb; } + } + workspace_root().join("test-apps/harness-electron/CuaTestHarness.Electron.exe") +} + +// ── JSON-RPC plumbing ──────────────────────────────────────────────────────── + +fn send(stdin: &mut ChildStdin, req: serde_json::Value) { + writeln!(stdin, "{}", serde_json::to_string(&req).unwrap()).unwrap(); +} +fn recv(stdout: &mut BufReader<&mut ChildStdout>) -> serde_json::Value { + let mut line = String::new(); + stdout.read_line(&mut line).expect("read"); + serde_json::from_str(line.trim()).expect("json") +} +fn init(stdin: &mut ChildStdin, stdout: &mut BufReader<&mut ChildStdout>) { + send(stdin, serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}})); + let _ = recv(stdout); +} +fn tools_call(stdin: &mut ChildStdin, stdout: &mut BufReader<&mut ChildStdout>, + id: u32, name: &str, args: serde_json::Value) -> serde_json::Value { + send(stdin, serde_json::json!({ + "jsonrpc":"2.0","id":id,"method":"tools/call", + "params":{"name":name,"arguments":args} + })); + recv(stdout) +} + +fn find_window_by_title(stdin: &mut ChildStdin, stdout: &mut BufReader<&mut ChildStdout>, + pid: u32, title_substr: &str) -> Option { + let deadline = std::time::Instant::now() + Duration::from_secs(20); + let mut id = 10u32; + loop { + let resp = tools_call(stdin, stdout, id, "list_windows", serde_json::json!({"pid": pid as i64})); + id = id.wrapping_add(1); + if let Some(wins) = resp["result"]["structuredContent"]["windows"].as_array() { + for w in wins { + if w["pid"].as_u64() != Some(pid as u64) { continue; } + if w["title"].as_str().unwrap_or("").contains(title_substr) { + if let Some(wid) = w["window_id"].as_u64() { return Some(wid); } + } + } + } + if std::time::Instant::now() >= deadline { return None; } + std::thread::sleep(Duration::from_millis(200)); + } +} + +// ── shared session helper ──────────────────────────────────────────────────── + +/// Launch the harness exe + a cua-driver child with `CUA_DRIVER_CDP_PORT` +/// pointing at the harness's CDP endpoint. Polls list_windows until the +/// host's window appears. +struct WebSession { + _app: Child, + driver: Child, +} + +impl Drop for WebSession { + fn drop(&mut self) { + let _ = self.driver.kill(); + let _ = self.driver.wait(); + let _ = self._app.kill(); + let _ = self._app.wait(); + // settle so the next test's launch doesn't see leftover windows + std::thread::sleep(Duration::from_millis(500)); + } +} + +fn run_with_session(label: &str, host_exe: PathBuf, title_substr: &str, cdp_port: u16, f: F) +where F: FnOnce(u32, u64, &mut ChildStdin, &mut BufReader<&mut ChildStdout>) { + if !driver_binary().exists() { + eprintln!("cua-driver.exe not built — run `cargo build` first"); return; + } + if !host_exe.exists() { + eprintln!("{label} host exe not found at {host_exe:?} — run test-harness/build.ps1"); return; + } + // Set the CDP port the host should use so the daemon can find it. + let env_var = if label == "webview" { "CUA_WEBVIEW_CDP_PORT" } else { "CUA_ELECTRON_CDP_PORT" }; + let app = Command::new(&host_exe) + .env(env_var, cdp_port.to_string()) + .stdout(Stdio::null()).stderr(Stdio::null()) + .spawn().expect("spawn host"); + let pid = app.id(); + println!("{label} pid={pid} cdp_port={cdp_port}"); + std::thread::sleep(Duration::from_secs(2)); // small cold-start for runtime spin-up + + let mut driver = Command::new(driver_binary()) + .env("CUA_DRIVER_CDP_PORT", cdp_port.to_string()) + .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null()) + .spawn().expect("spawn cua-driver"); + let mut stdin = driver.stdin.take().unwrap(); + let mut raw_stdout = driver.stdout.take().unwrap(); + let mut stdout = BufReader::new(&mut raw_stdout); + init(&mut stdin, &mut stdout); + + let wid = find_window_by_title(&mut stdin, &mut stdout, pid, title_substr) + .unwrap_or_else(|| panic!("{label} window with title containing {title_substr:?} not found")); + + let session = WebSession { _app: app, driver }; + f(pid, wid, &mut stdin, &mut stdout); + drop(session); +} + +// ── WebView2 structural ────────────────────────────────────────────────────── + +#[test] +#[ignore] +fn harness_webview_window_discoverable() { + // Smoke test: WebView2 harness launches, window appears via list_windows. + // Behavioural page-tool tests are deferred until WebView2 actually opens + // its CDP listener — see the module docstring TODO. + run_with_session("webview", webview_exe(), "CuaTestHarness WebView", 9222, + |pid, wid, _stdin, _stdout| { + println!("✅ harness_webview_window_discoverable: pid={pid} wid={wid}"); + }); +} + +// ── Electron structural + page tool ────────────────────────────────────────── + +#[test] +#[ignore] +fn harness_electron_window_discoverable() { + run_with_session("electron", electron_exe(), "CuaTestHarness Electron", 9223, + |pid, wid, _stdin, _stdout| { + println!("✅ harness_electron_window_discoverable: pid={pid} wid={wid}"); + }); +} + +#[test] +#[ignore] +fn harness_electron_page_tool_documented_gap() { + // Documents the cua-driver CDP `/json` discovery bug: see module + // docstring. cua-driver's read_to_end hangs because Chromium ignores + // Connection: close. Until that's fixed, this test asserts the error + // shape so a regression in the underlying TCP code (different timeout + // wording, different error path, etc.) shows up. + run_with_session("electron", electron_exe(), "CuaTestHarness Electron", 9223, + |pid, wid, stdin, stdout| { + + let resp = tools_call(stdin, stdout, 30, "page", serde_json::json!({ + "pid": pid as i64, "window_id": wid, "action": "execute_javascript", + "javascript": "1+1" + })); + let text = resp["result"]["content"][0]["text"].as_str().unwrap_or(""); + let is_err = resp["result"]["isError"].as_bool().unwrap_or(false); + // Either an explicit error OR a "timed out" message in the text body. + let expected_pattern = is_err || text.contains("timed out") || text.contains("Cannot connect"); + assert!(expected_pattern, + "Expected CDP /json discovery gap (Chromium ignores Connection: close \ + so cua-driver's read_to_end hangs). Got: is_err={is_err}, text={text:?}. \ + If this test now PASSES the cua-driver CDP bug is fixed — flip the \ + assertion to assert success and re-enable the deleted behavioural tests."); + println!("✅ harness_electron_page_tool_documented_gap: confirmed CDP read_to_end gap still present"); + }); +} diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs index df4aafb5a8..7bb1e40e92 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs @@ -671,3 +671,209 @@ fn harness_wpf_layered_popup_capture() { rgb.width(), rgb.height()); }); } + +// ── slider / checkable / combo / list / menu coverage ──────────────────────── + +#[test] +#[ignore] +fn harness_wpf_slider_drag_tool_returns() { + // Coverage note for the drag tool against a WPF Slider thumb. + // + // PostMessage WM_LBUTTONDOWN / WM_MOUSEMOVE / WM_LBUTTONUP doesn't + // update the OS keyboard/mouse state visible to GetKeyState. WPF's + // Slider Thumb relies on Mouse.LeftButton (which polls GetKeyState) + // to recognise an in-progress drag — so a PostMessage drag never + // moves a WPF thumb, even when from/to are correctly on the thumb + // in client coords. The companion `harness_wpf_slider_increase_large` + // test covers the slider via UIA Invoke on its internal IncreaseLarge + // sub-button, which is what an agent SHOULD use for slider + // manipulation on a backgrounded window. + // + // We still exercise the drag tool against the slider so its codepath + // (coord translation, dispatch policy, message synthesis) is on the + // critical-path test list — we just don't assert on the value moving. + // TODO: add a SendInput-based drag path so dispatch:"foreground" can + // drive the thumb, then enable a behavioral assertion here. + with_session(|pid, wid, stdin, stdout| { + focus_harness(stdin, stdout, pid, wid); + let resp = tools_call(stdin, stdout, 30, "drag", serde_json::json!({ + "pid": pid as i64, "window_id": wid, + "from_x": 50.0, "from_y": 275.0, + "to_x": 330.0, "to_y": 275.0, + "duration_ms": 600, "steps": 30 + })); + let msg = resp["result"]["content"][0]["text"].as_str().unwrap_or(""); + println!("drag slider: {msg}"); + assert!(msg.starts_with("✅"), + "drag tool returned non-success: {msg}"); + println!("✅ harness_wpf_slider_drag_tool_returns: PostMessage drag emitted (known no-op vs WPF Thumb)"); + }); +} + +#[test] +#[ignore] +fn harness_wpf_slider_increase_large() { + // Companion to slider_drag — exercises UIA Invoke on the Slider's + // internal IncreaseLarge "page-up" button. Doesn't depend on screen + // coords, so it's the more robust slider integration test. + with_session(|pid, wid, stdin, stdout| { + focus_harness(stdin, stdout, pid, wid); + let snap = snapshot_elements(stdin, stdout, pid, wid); + let idx = find_element_index_by_aid(&snap, "IncreaseLarge") + .expect("slider IncreaseLarge button not in snapshot"); + for i in 0..3 { + let resp = tools_call(stdin, stdout, 30 + i, "click", serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx + })); + println!("invoke IncreaseLarge #{i}: {}", resp["result"]["content"][0]["text"]); + std::thread::sleep(Duration::from_millis(150)); + } + std::thread::sleep(Duration::from_millis(300)); + let post = snapshot_elements(stdin, stdout, pid, wid); + let text = snapshot_text(&post); + let advanced = text.lines().any(|l| + l.contains("slider_value=") && !l.contains("slider_value=0\"")); + assert!(advanced, "slider IncreaseLarge invokes did not advance value. Lines: {}", + text.lines().filter(|l| l.contains("slider_value")).collect::>().join(" / ")); + println!("✅ harness_wpf_slider_increase_large: advanced via UIA Invoke"); + }); +} + +#[test] +#[ignore] +fn harness_wpf_checkbox_toggle() { + with_session(|pid, wid, stdin, stdout| { + focus_harness(stdin, stdout, pid, wid); + let snap = snapshot_elements(stdin, stdout, pid, wid); + let idx = find_element_index_by_aid(&snap, "chk-agreed") + .expect("chk-agreed missing"); + // CheckBox exposes UIA TogglePattern (actions=[toggle]), not Invoke. + // cua-driver's click tool tries UIA Invoke first; for elements that + // don't support it the PostMessage fallback path runs. Use + // dispatch:"foreground" to land a SendInput click that WPF + // recognises as a real user click and processes through Toggle. + let resp = tools_call(stdin, stdout, 30, "click", serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "dispatch": "foreground" + })); + println!("click chk-agreed: {}", resp["result"]["content"][0]["text"]); + std::thread::sleep(Duration::from_millis(400)); + let post = snapshot_elements(stdin, stdout, pid, wid); + assert!(snapshot_text(&post).contains("agreed=True"), + "checkbox didn't toggle: {}", + snapshot_text(&post).lines().filter(|l| l.contains("agreed=")).collect::>().join(" / ")); + println!("✅ harness_wpf_checkbox_toggle: agreed=True"); + }); +} + +#[test] +#[ignore] +fn harness_wpf_radio_select() { + with_session(|pid, wid, stdin, stdout| { + focus_harness(stdin, stdout, pid, wid); + let snap = snapshot_elements(stdin, stdout, pid, wid); + let idx = find_element_index_by_aid(&snap, "rdo-high") + .expect("rdo-high missing"); + // RadioButton exposes SelectionItem pattern (actions=[select]). + // Same dispatch:foreground rationale as the checkbox test. + let _ = tools_call(stdin, stdout, 30, "click", serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "dispatch": "foreground" + })); + std::thread::sleep(Duration::from_millis(400)); + let post = snapshot_elements(stdin, stdout, pid, wid); + assert!(snapshot_text(&post).contains("prio=High"), + "radio didn't switch to High"); + println!("✅ harness_wpf_radio_select: prio=High"); + }); +} + +#[test] +#[ignore] +fn harness_wpf_combo_select() { + with_session(|pid, wid, stdin, stdout| { + focus_harness(stdin, stdout, pid, wid); + let snap = snapshot_elements(stdin, stdout, pid, wid); + let combo_idx = find_element_index_by_aid(&snap, "cbo-color") + .expect("cbo-color missing"); + // WPF ComboBox UIA peer surfaces ExpandCollapsePattern (actions=[expand]) + // but not ValuePattern — set_value at the parent is a no-op. Standard + // recipe: invoke the combo to expand the dropdown, re-snapshot so the + // item AIDs land in the element cache, then click the target item. + let _ = tools_call(stdin, stdout, 30, "click", serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": combo_idx, + "dispatch": "foreground" + })); + std::thread::sleep(Duration::from_millis(500)); + + let snap2 = snapshot_elements(stdin, stdout, pid, wid); + let item_idx = find_element_index_by_aid(&snap2, "cbo-item-orange") + .expect("cbo-item-orange missing after expand"); + let _ = tools_call(stdin, stdout, 31, "click", serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": item_idx, + "dispatch": "foreground" + })); + std::thread::sleep(Duration::from_millis(500)); + + let post = snapshot_elements(stdin, stdout, pid, wid); + assert!(snapshot_text(&post).contains("color=orange"), + "combo didn't switch to orange: {}", + snapshot_text(&post).lines().filter(|l| l.contains("color=")).collect::>().join(" / ")); + println!("✅ harness_wpf_combo_select: color=orange"); + }); +} + +#[test] +#[ignore] +fn harness_wpf_listbox_select() { + with_session(|pid, wid, stdin, stdout| { + focus_harness(stdin, stdout, pid, wid); + let snap = snapshot_elements(stdin, stdout, pid, wid); + let idx = find_element_index_by_aid(&snap, "lst-cherry") + .expect("lst-cherry missing"); + let _ = tools_call(stdin, stdout, 30, "click", serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "dispatch": "foreground" + })); + std::thread::sleep(Duration::from_millis(400)); + let post = snapshot_elements(stdin, stdout, pid, wid); + assert!(snapshot_text(&post).contains("selected=cherry"), + "list didn't select cherry: {}", + snapshot_text(&post).lines().filter(|l| l.contains("selected=")).collect::>().join(" / ")); + println!("✅ harness_wpf_listbox_select: selected=cherry"); + }); +} + +#[test] +#[ignore] +fn harness_wpf_menu_invoke() { + with_session(|pid, wid, stdin, stdout| { + focus_harness(stdin, stdout, pid, wid); + // Expand File menu first (UIA expand pattern on MenuItem) + let snap = snapshot_elements(stdin, stdout, pid, wid); + let file_idx = find_element_index_by_aid(&snap, "menu-file") + .expect("menu-file missing"); + let _ = tools_call(stdin, stdout, 30, "click", serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": file_idx, + "dispatch": "foreground" + })); + std::thread::sleep(Duration::from_millis(400)); + + // Re-snapshot so menu-file-new is in the cache (it materialized + // when the menu expanded). + let snap2 = snapshot_elements(stdin, stdout, pid, wid); + let new_idx = find_element_index_by_aid(&snap2, "menu-file-new") + .expect("menu-file-new missing after expand"); + let _ = tools_call(stdin, stdout, 31, "click", serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": new_idx, + "dispatch": "foreground" + })); + std::thread::sleep(Duration::from_millis(500)); + + let post = snapshot_elements(stdin, stdout, pid, wid); + assert!(snapshot_text(&post).contains("menu_action=file_new"), + "File>New didn't invoke: {}", + snapshot_text(&post).lines().filter(|l| l.contains("menu_action=")).collect::>().join(" / ")); + println!("✅ harness_wpf_menu_invoke: menu_action=file_new"); + }); +} diff --git a/libs/cua-driver/rust/test-apps/.gitignore b/libs/cua-driver/rust/test-apps/.gitignore index c1bae11177..e3ba2f39e6 100644 --- a/libs/cua-driver/rust/test-apps/.gitignore +++ b/libs/cua-driver/rust/test-apps/.gitignore @@ -13,3 +13,5 @@ # (several thousand files, ~150 MB). Always rebuilt locally. harness-wpf/ harness-winui3/ +harness-webview/ +harness-electron/ diff --git a/libs/cua-driver/test-harness/CuaTestHarness.Electron/.gitignore b/libs/cua-driver/test-harness/CuaTestHarness.Electron/.gitignore new file mode 100644 index 0000000000..d8b0fe4524 --- /dev/null +++ b/libs/cua-driver/test-harness/CuaTestHarness.Electron/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +dist/ +web/ +package-lock.json diff --git a/libs/cua-driver/test-harness/CuaTestHarness.Electron/build.ps1 b/libs/cua-driver/test-harness/CuaTestHarness.Electron/build.ps1 new file mode 100644 index 0000000000..4b7e406169 --- /dev/null +++ b/libs/cua-driver/test-harness/CuaTestHarness.Electron/build.ps1 @@ -0,0 +1,55 @@ +# build.ps1 - stage the Electron test harness for cua-driver tests. +# +# electron-builder portable mode needs admin (symlink privilege) for the +# winCodeSign cache extraction. We instead stage a flat folder containing +# the electron runtime + our app resources, with electron.exe renamed to +# CuaTestHarness.Electron.exe so tests get a deterministic exe name. +# +# Output: ../rust/test-apps/harness-electron/ + +Set-StrictMode -Version Latest +$ErrorActionPreference = "Stop" + +$elecDir = Split-Path -Parent $MyInvocation.MyCommand.Definition +$harnessDir = Split-Path -Parent $elecDir +$cuaDriverDir = Split-Path -Parent $harnessDir +$testAppsDir = Join-Path $cuaDriverDir "rust\test-apps" +$outDir = Join-Path $testAppsDir "harness-electron" + +if (-not (Get-Command npm -ErrorAction SilentlyContinue)) { + Write-Host "[ERROR] npm not on PATH. Install Node.js first." -ForegroundColor Red + exit 1 +} + +Push-Location $elecDir +try { + $webDir = Join-Path $elecDir "web" + if (-not (Test-Path $webDir)) { New-Item -ItemType Directory $webDir | Out-Null } + Copy-Item (Join-Path $harnessDir "shared-web\*") $webDir -Recurse -Force + + if (-not (Test-Path "node_modules\electron\dist\electron.exe")) { + Write-Host "[INSTALL] npm install (first run)..." -ForegroundColor Yellow + npm install --silent + if ($LASTEXITCODE -ne 0) { throw "npm install failed" } + } + + $electronSrc = Join-Path $elecDir "node_modules\electron\dist" + if (-not (Test-Path (Join-Path $electronSrc "electron.exe"))) { + throw "electron.exe not found under $electronSrc - npm install incomplete" + } + + Write-Host "[STAGE] Copying electron runtime + app to $outDir..." -ForegroundColor Cyan + if (Test-Path $outDir) { Remove-Item $outDir -Recurse -Force } + Copy-Item $electronSrc $outDir -Recurse -Force + + $appDir = Join-Path $outDir "resources\app" + if (-not (Test-Path $appDir)) { New-Item -ItemType Directory $appDir -Force | Out-Null } + Copy-Item (Join-Path $elecDir "main.js") $appDir -Force + Copy-Item (Join-Path $elecDir "package.json") $appDir -Force + Copy-Item $webDir (Join-Path $appDir "web") -Recurse -Force + + Rename-Item (Join-Path $outDir "electron.exe") "CuaTestHarness.Electron.exe" -Force + Write-Host "[OK] Staged: $outDir\CuaTestHarness.Electron.exe" -ForegroundColor Green +} finally { + Pop-Location +} diff --git a/libs/cua-driver/test-harness/CuaTestHarness.Electron/main.js b/libs/cua-driver/test-harness/CuaTestHarness.Electron/main.js new file mode 100644 index 0000000000..de8fb08e38 --- /dev/null +++ b/libs/cua-driver/test-harness/CuaTestHarness.Electron/main.js @@ -0,0 +1,48 @@ +// CuaTestHarness.Electron — minimal Electron host loading the shared +// index.html that CuaTestHarness.WebView also loads. cua-driver's `page` +// tool routes through CDP when --remote-debugging-port is set, so we +// expose one here on a configurable port. + +const { app, BrowserWindow } = require('electron'); +const path = require('path'); + +const CDP_PORT = process.env.CUA_ELECTRON_CDP_PORT || '9223'; +app.commandLine.appendSwitch('remote-debugging-port', CDP_PORT); + +let mainWindow; + +function createWindow() { + const fixedTitle = `CuaTestHarness Electron [cdp=${CDP_PORT}]`; + mainWindow = new BrowserWindow({ + width: 940, + height: 780, + title: fixedTitle, + autoHideMenuBar: true, + webPreferences: { + nodeIntegration: false, + contextIsolation: true, + }, + }); + + // Override the page's with our deterministic harness title so + // cua-driver tests can find the window by substring match. Without this, + // Electron syncs window.title to document.title which would be + // 'cua-driver Web Harness' (the page's title). + mainWindow.on('page-title-updated', e => e.preventDefault()); + mainWindow.setTitle(fixedTitle); + + mainWindow.loadFile(path.join(__dirname, 'web', 'index.html')).then(() => { + mainWindow.setTitle(fixedTitle); + }); +} + +app.whenReady().then(() => { + createWindow(); + app.on('activate', () => { + if (BrowserWindow.getAllWindows().length === 0) createWindow(); + }); +}); + +app.on('window-all-closed', () => { + if (process.platform !== 'darwin') app.quit(); +}); diff --git a/libs/cua-driver/test-harness/CuaTestHarness.Electron/package.json b/libs/cua-driver/test-harness/CuaTestHarness.Electron/package.json new file mode 100644 index 0000000000..afe7cd99e1 --- /dev/null +++ b/libs/cua-driver/test-harness/CuaTestHarness.Electron/package.json @@ -0,0 +1,37 @@ +{ + "name": "cua-test-harness-electron", + "version": "1.0.0", + "description": "cua-driver Electron test harness — loads the same shared-web/index.html as CuaTestHarness.WebView", + "private": true, + "main": "main.js", + "scripts": { + "start": "electron .", + "build": "electron-builder --win portable --x64" + }, + "devDependencies": { + "electron": "31.6.0", + "electron-builder": "25.0.5" + }, + "build": { + "appId": "com.trycua.harness.electron", + "productName": "CuaTestHarness.Electron", + "directories": { + "output": "dist" + }, + "files": [ + "main.js", + "preload.js", + "web/**/*", + "package.json" + ], + "win": { + "target": [{ + "target": "portable", + "arch": ["x64"] + }] + }, + "portable": { + "artifactName": "CuaTestHarness.Electron.exe" + } + } +} diff --git a/libs/cua-driver/test-harness/CuaTestHarness.WebView/.gitignore b/libs/cua-driver/test-harness/CuaTestHarness.WebView/.gitignore new file mode 100644 index 0000000000..10f960612a --- /dev/null +++ b/libs/cua-driver/test-harness/CuaTestHarness.WebView/.gitignore @@ -0,0 +1,4 @@ +bin/ +obj/ +*.user +.vs/ diff --git a/libs/cua-driver/test-harness/CuaTestHarness.WebView/App.xaml b/libs/cua-driver/test-harness/CuaTestHarness.WebView/App.xaml new file mode 100644 index 0000000000..ff3bb5603a --- /dev/null +++ b/libs/cua-driver/test-harness/CuaTestHarness.WebView/App.xaml @@ -0,0 +1,6 @@ +<Application x:Class="CuaTestHarness.WebView.App" + xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation" + xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" + StartupUri="MainWindow.xaml"> + <Application.Resources/> +</Application> diff --git a/libs/cua-driver/test-harness/CuaTestHarness.WebView/App.xaml.cs b/libs/cua-driver/test-harness/CuaTestHarness.WebView/App.xaml.cs new file mode 100644 index 0000000000..0cc2c96e34 --- /dev/null +++ b/libs/cua-driver/test-harness/CuaTestHarness.WebView/App.xaml.cs @@ -0,0 +1,7 @@ +using System.Windows; + +namespace CuaTestHarness.WebView; + +public partial class App : Application +{ +} diff --git a/libs/cua-driver/test-harness/CuaTestHarness.WebView/CuaTestHarness.WebView.csproj b/libs/cua-driver/test-harness/CuaTestHarness.WebView/CuaTestHarness.WebView.csproj new file mode 100644 index 0000000000..09d73533f5 --- /dev/null +++ b/libs/cua-driver/test-harness/CuaTestHarness.WebView/CuaTestHarness.WebView.csproj @@ -0,0 +1,28 @@ +<Project Sdk="Microsoft.NET.Sdk"> + + <PropertyGroup> + <OutputType>WinExe</OutputType> + <TargetFramework>net8.0-windows</TargetFramework> + <UseWPF>true</UseWPF> + <Nullable>enable</Nullable> + <LangVersion>latest</LangVersion> + <ApplicationManifest>app.manifest</ApplicationManifest> + <AssemblyName>CuaTestHarness.WebView</AssemblyName> + <RootNamespace>CuaTestHarness.WebView</RootNamespace> + <PublishSingleFile>false</PublishSingleFile> + <SelfContained>true</SelfContained> + <RuntimeIdentifier>win-x64</RuntimeIdentifier> + <Platforms>x64</Platforms> + </PropertyGroup> + + <ItemGroup> + <PackageReference Include="Microsoft.Web.WebView2" Version="1.0.2792.45" /> + </ItemGroup> + + <ItemGroup> + <None Include="..\shared-web\index.html" Link="web\index.html"> + <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> + </None> + </ItemGroup> + +</Project> diff --git a/libs/cua-driver/test-harness/CuaTestHarness.WebView/MainWindow.xaml b/libs/cua-driver/test-harness/CuaTestHarness.WebView/MainWindow.xaml new file mode 100644 index 0000000000..27025fbfb3 --- /dev/null +++ b/libs/cua-driver/test-harness/CuaTestHarness.WebView/MainWindow.xaml @@ -0,0 +1,27 @@ +<Window x:Class="CuaTestHarness.WebView.MainWindow" + xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation" + xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" + xmlns:wv2="clr-namespace:Microsoft.Web.WebView2.Wpf;assembly=Microsoft.Web.WebView2.Wpf" + Title="CuaTestHarness WebView" + AutomationProperties.AutomationId="wnd-main" + Width="940" Height="780" + WindowStartupLocation="CenterScreen"> + <DockPanel LastChildFill="True"> + <Border DockPanel.Dock="Top" Background="#F0F0F0" Padding="10,6"> + <StackPanel Orientation="Horizontal"> + <TextBlock Text="WebView2 host — same DOM as the Electron harness (" + VerticalAlignment="Center"/> + <TextBlock x:Name="LblPageUrl" + AutomationProperties.AutomationId="lbl-page-url" + FontFamily="Consolas" Text="(loading)" VerticalAlignment="Center"/> + <TextBlock Text=")" VerticalAlignment="Center"/> + <Button x:Name="BtnExit" + AutomationProperties.AutomationId="btn-exit" + Content="Exit" Width="80" Margin="20,0,0,0" + Click="OnExitClick"/> + </StackPanel> + </Border> + <wv2:WebView2 x:Name="Wv" + AutomationProperties.AutomationId="wv-host"/> + </DockPanel> +</Window> diff --git a/libs/cua-driver/test-harness/CuaTestHarness.WebView/MainWindow.xaml.cs b/libs/cua-driver/test-harness/CuaTestHarness.WebView/MainWindow.xaml.cs new file mode 100644 index 0000000000..46b8820630 --- /dev/null +++ b/libs/cua-driver/test-harness/CuaTestHarness.WebView/MainWindow.xaml.cs @@ -0,0 +1,51 @@ +using System; +using System.IO; +using System.Windows; +using Microsoft.Web.WebView2.Core; + +namespace CuaTestHarness.WebView; + +public partial class MainWindow : Window +{ + public MainWindow() + { + InitializeComponent(); + Loaded += OnLoaded; + } + + private async void OnLoaded(object sender, RoutedEventArgs e) + { + try + { + var userData = Path.Combine(Path.GetTempPath(), "CuaTestHarness.WebView.UserData"); + Directory.CreateDirectory(userData); + + // Read the CDP port from CUA_WEBVIEW_CDP_PORT (default 9222). + // cua-driver's `page` tool routes JS execution through CDP when + // `--remote-debugging-port` is exposed; this is the analogue of + // launching Chrome with --remote-debugging-port for the same + // path. Setting it on WebView2 is essential for testing the + // page tool against this host. + var portStr = Environment.GetEnvironmentVariable("CUA_WEBVIEW_CDP_PORT") ?? "9222"; + var opts = new CoreWebView2EnvironmentOptions + { + AdditionalBrowserArguments = $"--remote-debugging-port={portStr}", + }; + var env = await CoreWebView2Environment.CreateAsync(userDataFolder: userData, options: opts); + await Wv.EnsureCoreWebView2Async(env); + + var htmlPath = Path.Combine(AppContext.BaseDirectory, "web", "index.html"); + var fileUri = new Uri(htmlPath).AbsoluteUri; + Wv.Source = new Uri(fileUri); + LblPageUrl.Text = fileUri; + Title = $"CuaTestHarness WebView [cdp={portStr}]"; + } + catch (Exception ex) + { + MessageBox.Show($"WebView2 init failed: {ex.Message}", "harness", MessageBoxButton.OK, MessageBoxImage.Error); + throw; + } + } + + private void OnExitClick(object sender, RoutedEventArgs e) => Application.Current.Shutdown(0); +} diff --git a/libs/cua-driver/test-harness/CuaTestHarness.WebView/app.manifest b/libs/cua-driver/test-harness/CuaTestHarness.WebView/app.manifest new file mode 100644 index 0000000000..bf395cf4f1 --- /dev/null +++ b/libs/cua-driver/test-harness/CuaTestHarness.WebView/app.manifest @@ -0,0 +1,23 @@ +<?xml version="1.0" encoding="utf-8"?> +<assembly manifestVersion="1.0" xmlns="urn:schemas-microsoft-com:asm.v1"> + <assemblyIdentity version="1.0.0.0" name="CuaTestHarness.WebView"/> + <trustInfo xmlns="urn:schemas-microsoft-com:asm.v2"> + <security> + <requestedPrivileges xmlns="urn:schemas-microsoft-com:asm.v3"> + <requestedExecutionLevel level="asInvoker" uiAccess="false"/> + </requestedPrivileges> + </security> + </trustInfo> + <compatibility xmlns="urn:schemas-microsoft-com:compatibility.v1"> + <application> + <supportedOS Id="{8e0f7a12-bfb3-4fe8-b9a5-48fd50a15a9a}"/> + <supportedOS Id="{1f676c76-80e1-4239-95bb-83d0f6d0da78}"/> + </application> + </compatibility> + <application xmlns="urn:schemas-microsoft-com:asm.v3"> + <windowsSettings> + <dpiAware xmlns="http://schemas.microsoft.com/SMI/2005/WindowsSettings">true/pm</dpiAware> + <dpiAwareness xmlns="http://schemas.microsoft.com/SMI/2016/WindowsSettings">permonitorv2</dpiAwareness> + </windowsSettings> + </application> +</assembly> diff --git a/libs/cua-driver/test-harness/CuaTestHarness.Wpf/MainWindow.xaml b/libs/cua-driver/test-harness/CuaTestHarness.Wpf/MainWindow.xaml index 7c9b9fade4..10e65205ba 100644 --- a/libs/cua-driver/test-harness/CuaTestHarness.Wpf/MainWindow.xaml +++ b/libs/cua-driver/test-harness/CuaTestHarness.Wpf/MainWindow.xaml @@ -5,7 +5,7 @@ xmlns:auto="clr-namespace:System.Windows.Automation;assembly=PresentationCore" Title="CuaTestHarness WPF" AutomationProperties.AutomationId="wnd-main" - Width="900" Height="700" + Width="900" Height="900" WindowStartupLocation="CenterScreen"> <Window.InputBindings> <!-- F5 (no modifiers) — chosen for automated testing. WPF's @@ -80,6 +80,135 @@ </StackPanel> </GroupBox> + <!-- Slider (drives `drag` tool — thumb at left end, label + tracks Value. ValuePattern also surfaced so the cua-driver + `set_value` tool can be exercised too.) --> + <GroupBox Header="slider" Padding="10" Margin="0,0,0,12"> + <StackPanel> + <Slider x:Name="SldValue" + AutomationProperties.AutomationId="sld-value" + Minimum="0" Maximum="100" Value="0" + Width="320" HorizontalAlignment="Left" + IsSnapToTickEnabled="False" + ValueChanged="OnSliderChanged"/> + <TextBlock x:Name="LblSliderValue" + AutomationProperties.AutomationId="lbl-slider-value" + Margin="0,6,0,0" FontFamily="Consolas" + Text="slider_value=0"/> + </StackPanel> + </GroupBox> + + <!-- CheckBox + RadioButton group --> + <GroupBox Header="checkable_controls" Padding="10" Margin="0,0,0,12"> + <StackPanel> + <CheckBox x:Name="ChkAgreed" + AutomationProperties.AutomationId="chk-agreed" + Content="I agree" Margin="0,0,0,4" + Checked="OnChkChanged" Unchecked="OnChkChanged"/> + <StackPanel Orientation="Horizontal"> + <RadioButton x:Name="RdoLow" + AutomationProperties.AutomationId="rdo-low" + GroupName="prio" Content="Low" IsChecked="True" + Margin="0,0,12,0" Checked="OnRadioChanged"/> + <RadioButton x:Name="RdoMed" + AutomationProperties.AutomationId="rdo-med" + GroupName="prio" Content="Medium" + Margin="0,0,12,0" Checked="OnRadioChanged"/> + <RadioButton x:Name="RdoHigh" + AutomationProperties.AutomationId="rdo-high" + GroupName="prio" Content="High" + Checked="OnRadioChanged"/> + </StackPanel> + <TextBlock x:Name="LblChkState" + AutomationProperties.AutomationId="lbl-chk-state" + Margin="0,6,0,0" FontFamily="Consolas" + Text="agreed=False, prio=Low"/> + </StackPanel> + </GroupBox> + + <!-- ComboBox --> + <GroupBox Header="combo_box" Padding="10" Margin="0,0,0,12"> + <StackPanel> + <ComboBox x:Name="CboColor" + AutomationProperties.AutomationId="cbo-color" + Width="180" HorizontalAlignment="Left" + SelectionChanged="OnComboChanged"> + <ComboBoxItem Content="red" AutomationProperties.AutomationId="cbo-item-red"/> + <ComboBoxItem Content="green" AutomationProperties.AutomationId="cbo-item-green" IsSelected="True"/> + <ComboBoxItem Content="blue" AutomationProperties.AutomationId="cbo-item-blue"/> + <ComboBoxItem Content="orange" AutomationProperties.AutomationId="cbo-item-orange"/> + </ComboBox> + <TextBlock x:Name="LblComboValue" + AutomationProperties.AutomationId="lbl-combo-value" + Margin="0,6,0,0" FontFamily="Consolas" + Text="color=green"/> + </StackPanel> + </GroupBox> + + <!-- ListBox with selection --> + <GroupBox Header="list_box" Padding="10" Margin="0,0,0,12"> + <StackPanel> + <ListBox x:Name="LstItems" + AutomationProperties.AutomationId="lst-items" + Width="220" Height="80" HorizontalAlignment="Left" + SelectionChanged="OnListChanged"> + <ListBoxItem Content="apple" AutomationProperties.AutomationId="lst-apple"/> + <ListBoxItem Content="banana" AutomationProperties.AutomationId="lst-banana"/> + <ListBoxItem Content="cherry" AutomationProperties.AutomationId="lst-cherry"/> + <ListBoxItem Content="date" AutomationProperties.AutomationId="lst-date"/> + </ListBox> + <TextBlock x:Name="LblListValue" + AutomationProperties.AutomationId="lbl-list-value" + Margin="0,6,0,0" FontFamily="Consolas" + Text="selected=none"/> + </StackPanel> + </GroupBox> + + <!-- Menu + ContextMenu (a Button with attached ContextMenu) --> + <GroupBox Header="menus" Padding="10" Margin="0,0,0,12"> + <StackPanel> + <Menu Background="Transparent" HorizontalAlignment="Left"> + <MenuItem Header="_File" + AutomationProperties.AutomationId="menu-file"> + <MenuItem Header="_New" + AutomationProperties.AutomationId="menu-file-new" + Click="OnMenuFileNew"/> + <MenuItem Header="_Open" + AutomationProperties.AutomationId="menu-file-open" + Click="OnMenuFileOpen"/> + </MenuItem> + <MenuItem Header="_Edit" + AutomationProperties.AutomationId="menu-edit"> + <MenuItem Header="_Copy" + AutomationProperties.AutomationId="menu-edit-copy" + Click="OnMenuEditCopy"/> + </MenuItem> + </Menu> + <Button x:Name="BtnContextTarget" + AutomationProperties.AutomationId="btn-context-target" + Content="Right-click for context menu" + Width="260" Margin="0,8,0,0" HorizontalAlignment="Left"> + <Button.ContextMenu> + <ContextMenu> + <MenuItem Header="_Cut" + AutomationProperties.AutomationId="ctx-cut" + Click="OnCtxAction"/> + <MenuItem Header="C_opy" + AutomationProperties.AutomationId="ctx-copy" + Click="OnCtxAction"/> + <MenuItem Header="_Paste" + AutomationProperties.AutomationId="ctx-paste" + Click="OnCtxAction"/> + </ContextMenu> + </Button.ContextMenu> + </Button> + <TextBlock x:Name="LblMenuAction" + AutomationProperties.AutomationId="lbl-menu-action" + Margin="0,6,0,0" FontFamily="Consolas" + Text="menu_action=none"/> + </StackPanel> + </GroupBox> + <!-- Counter scenario --> <GroupBox Header="counter" Padding="10" Margin="0,0,0,12"> <StackPanel Orientation="Horizontal" VerticalAlignment="Center"> diff --git a/libs/cua-driver/test-harness/CuaTestHarness.Wpf/MainWindow.xaml.cs b/libs/cua-driver/test-harness/CuaTestHarness.Wpf/MainWindow.xaml.cs index fa4e698e8d..6664135fb6 100644 --- a/libs/cua-driver/test-harness/CuaTestHarness.Wpf/MainWindow.xaml.cs +++ b/libs/cua-driver/test-harness/CuaTestHarness.Wpf/MainWindow.xaml.cs @@ -179,4 +179,55 @@ private void OnScrollChanged(object sender, ScrollChangedEventArgs e) LblScrollOffset.Text = $"scroll_offset={(int)sv.VerticalOffset}"; } } + + private void OnSliderChanged(object sender, RoutedPropertyChangedEventArgs<double> e) + { + LblSliderValue.Text = $"slider_value={(int)e.NewValue}"; + } + + private void UpdateChkState() + { + // XAML init fires Checked on the IsChecked="True" RadioButton before + // sibling named controls are wired up — guard against the partial + // construction. + if (ChkAgreed is null || RdoLow is null || LblChkState is null) return; + var prio = RdoLow.IsChecked == true ? "Low" + : RdoMed?.IsChecked == true ? "Medium" + : RdoHigh?.IsChecked == true ? "High" + : "?"; + LblChkState.Text = $"agreed={ChkAgreed.IsChecked == true}, prio={prio}"; + } + private void OnChkChanged(object sender, RoutedEventArgs e) => UpdateChkState(); + private void OnRadioChanged(object sender, RoutedEventArgs e) => UpdateChkState(); + + private void OnComboChanged(object sender, SelectionChangedEventArgs e) + { + if (LblComboValue is null) return; + if (CboColor?.SelectedItem is ComboBoxItem item) + { + LblComboValue.Text = $"color={item.Content}"; + } + } + + private void OnListChanged(object sender, SelectionChangedEventArgs e) + { + if (LblListValue is null) return; + if (LstItems?.SelectedItem is ListBoxItem item) + { + LblListValue.Text = $"selected={item.Content}"; + } + } + + private void OnMenuFileNew(object sender, RoutedEventArgs e) => LblMenuAction.Text = "menu_action=file_new"; + private void OnMenuFileOpen(object sender, RoutedEventArgs e) => LblMenuAction.Text = "menu_action=file_open"; + private void OnMenuEditCopy(object sender, RoutedEventArgs e) => LblMenuAction.Text = "menu_action=edit_copy"; + + private void OnCtxAction(object sender, RoutedEventArgs e) + { + if (sender is MenuItem mi) + { + var label = mi.Header?.ToString()?.Replace("_", "").ToLowerInvariant() ?? "?"; + LblMenuAction.Text = $"menu_action=ctx_{label}"; + } + } } diff --git a/libs/cua-driver/test-harness/CuaTestHarness.sln b/libs/cua-driver/test-harness/CuaTestHarness.sln index 7240a8d213..366302b890 100644 --- a/libs/cua-driver/test-harness/CuaTestHarness.sln +++ b/libs/cua-driver/test-harness/CuaTestHarness.sln @@ -6,6 +6,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CuaTestHarness.Wpf", "CuaTe EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CuaTestHarness.WinUI3", "CuaTestHarness.WinUI3\CuaTestHarness.WinUI3.csproj", "{B2C3D4E5-F6A7-489B-CDEF-012345678901}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CuaTestHarness.WebView", "CuaTestHarness.WebView\CuaTestHarness.WebView.csproj", "{C3D4E5F6-A7B8-49CD-EF01-234567890123}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x64 = Debug|x64 @@ -20,5 +22,9 @@ Global {B2C3D4E5-F6A7-489B-CDEF-012345678901}.Debug|x64.Build.0 = Debug|x64 {B2C3D4E5-F6A7-489B-CDEF-012345678901}.Release|x64.ActiveCfg = Release|x64 {B2C3D4E5-F6A7-489B-CDEF-012345678901}.Release|x64.Build.0 = Release|x64 + {C3D4E5F6-A7B8-49CD-EF01-234567890123}.Debug|x64.ActiveCfg = Debug|x64 + {C3D4E5F6-A7B8-49CD-EF01-234567890123}.Debug|x64.Build.0 = Debug|x64 + {C3D4E5F6-A7B8-49CD-EF01-234567890123}.Release|x64.ActiveCfg = Release|x64 + {C3D4E5F6-A7B8-49CD-EF01-234567890123}.Release|x64.Build.0 = Release|x64 EndGlobalSection EndGlobal diff --git a/libs/cua-driver/test-harness/build.ps1 b/libs/cua-driver/test-harness/build.ps1 index 2fa43a8acb..c7d3795fa5 100644 --- a/libs/cua-driver/test-harness/build.ps1 +++ b/libs/cua-driver/test-harness/build.ps1 @@ -11,7 +11,7 @@ # Requires: .NET 8 SDK on PATH. param( - [ValidateSet("none","wpf","winui3")] + [ValidateSet("none","wpf","winui3","webview","electron")] [string]$Skip = "none" ) @@ -63,6 +63,27 @@ if ($Skip -ne "winui3") { Write-Host "[SKIP] WinUI3 project not present yet - skipping." -ForegroundColor Yellow } } +if ($Skip -ne "webview") { + $webProj = Join-Path $harnessDir "CuaTestHarness.WebView\CuaTestHarness.WebView.csproj" + if (Test-Path $webProj) { + Publish-Project $webProj "harness-webview" + } else { + Write-Host "[SKIP] WebView project not present yet - skipping." -ForegroundColor Yellow + } +} +if ($Skip -ne "electron") { + $elecBuild = Join-Path $harnessDir "CuaTestHarness.Electron\build.ps1" + if (Test-Path $elecBuild) { + Write-Host "" + Write-Host "[BUILD] CuaTestHarness.Electron -> $testAppsDir\harness-electron\" -ForegroundColor Cyan + & $elecBuild + if ($LASTEXITCODE -ne 0) { + Write-Host "[WARN] Electron build failed (exit $LASTEXITCODE)" -ForegroundColor Yellow + } + } else { + Write-Host "[SKIP] Electron project not present yet - skipping." -ForegroundColor Yellow + } +} Write-Host "" Write-Host "[DONE] Test harness build complete." -ForegroundColor Green diff --git a/libs/cua-driver/test-harness/scenarios/scenarios.json b/libs/cua-driver/test-harness/scenarios/scenarios.json index 671cf37921..b753fb3ebd 100644 --- a/libs/cua-driver/test-harness/scenarios/scenarios.json +++ b/libs/cua-driver/test-harness/scenarios/scenarios.json @@ -102,6 +102,58 @@ "click_count_label_aid": "lbl-click-count" } }, + { + "id": "slider", + "kind": "slider_drag", + "description": "WPF Slider with ValuePattern. Drives the drag tool (from thumb to a target X) and set_value tool (UIA RangeValue / Value).", + "controls": { + "slider_aid": "sld-value", + "value_label_aid": "lbl-slider-value" + } + }, + { + "id": "checkable_controls", + "kind": "checkbox_radio", + "description": "CheckBox + RadioButton group. Drives click via UIA Invoke / TogglePattern. State mirrored to a label.", + "controls": { + "checkbox_aid": "chk-agreed", + "radio_low_aid": "rdo-low", + "radio_med_aid": "rdo-med", + "radio_high_aid": "rdo-high", + "state_label_aid": "lbl-chk-state" + } + }, + { + "id": "combo_box", + "kind": "selection", + "description": "ComboBox with named items. Drives selection via click + set_value.", + "controls": { + "combo_aid": "cbo-color", + "value_label_aid": "lbl-combo-value" + } + }, + { + "id": "list_box", + "kind": "list_selection", + "description": "ListBox with named items. Tests UIA SelectionPattern.", + "controls": { + "list_aid": "lst-items", + "value_label_aid": "lbl-list-value" + } + }, + { + "id": "menus", + "kind": "menu_and_context", + "description": "Top-level Menu + a Button with ContextMenu. Drives MenuItem invocation via click; right-click target for ContextMenu opening.", + "controls": { + "menu_file_aid": "menu-file", + "menu_file_new_aid": "menu-file-new", + "menu_edit_copy_aid": "menu-edit-copy", + "context_target_aid": "btn-context-target", + "ctx_copy_aid": "ctx-copy", + "action_label_aid": "lbl-menu-action" + } + }, { "id": "scroll_target", "kind": "scroll_viewer", @@ -122,6 +174,34 @@ } ] }, + "webview": { + "process_name": "CuaTestHarness.WebView", + "exe_relative_path": "test-apps/harness-webview/CuaTestHarness.WebView.exe", + "default_cdp_port": 9222, + "main_window": { + "title": "CuaTestHarness WebView", + "automation_id": "wnd-main" + }, + "description": "WPF host with Microsoft.Edge.WebView2 control. Loads ../shared-web/index.html. Exposes Chromium DevTools Protocol at default_cdp_port for cua-driver's `page` tool.", + "shared_dom_aids": [ + "btn-increment", "btn-reset", "lbl-counter", + "txt-input", "lbl-input-mirror", + "sld-value", "lbl-slider-value", + "border-click-target", "lbl-last-action", "lbl-click-count", + "chk-agreed", "rdo-low", "rdo-med", "rdo-high", "lbl-chk-state", + "cbo-color", "lbl-combo-value", + "lnk-anchor", "lbl-nav-state", "section-target" + ] + }, + "electron": { + "process_name": "CuaTestHarness.Electron", + "exe_relative_path": "test-apps/harness-electron/CuaTestHarness.Electron.exe", + "default_cdp_port": 9223, + "main_window": { + "title": "CuaTestHarness Electron" + }, + "description": "Electron host loading the same ../shared-web/index.html as the WebView harness. Exposes CDP at default_cdp_port via --remote-debugging-port." + }, "winui3": { "process_name": "CuaTestHarness.WinUI3", "exe_relative_path": "test-apps/harness-winui3/CuaTestHarness.WinUI3.exe", diff --git a/libs/cua-driver/test-harness/shared-web/index.html b/libs/cua-driver/test-harness/shared-web/index.html new file mode 100644 index 0000000000..b038c2ec98 --- /dev/null +++ b/libs/cua-driver/test-harness/shared-web/index.html @@ -0,0 +1,175 @@ +<!DOCTYPE html> +<html lang="en"> +<head> + <meta charset="UTF-8" /> + <title>cua-driver Web Harness + + + +

cua-driver Web Harness

+

WEB_HARNESS_MARKER_v1

+ +
+ counter +
+ + + counter=0 +
+
+ +
+ text_input +
+ + mirror= +
+
+ +
+ slider +
+ + slider_value=0 +
+
+ +
+ click_target +
+ Click target (left / right / double) +
+
+ last_action=none + clicks=0 +
+
+ +
+ checkable_controls +
+ +
+
+ + + +
+
+ agreed=false, prio=Low +
+
+ +
+ combo_box +
+ + color=green +
+
+ +
+ navigation +
+ Jump to section + hash= +
+
+

SECTION_TARGET_MARKER_v1

+
+ + + + From 1c3b522aad2e6dd372f71f9f3b32520cf35b5680 Mon Sep 17 00:00:00 2001 From: Francesco Bonacci Date: Mon, 25 May 2026 23:11:14 +0000 Subject: [PATCH 2/9] test(cua-driver-rs)(harness): background-modality + capture-mode coverage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exercises the core cua-driver promise — background automation must not steal focus from the user — using the existing focus-monitor-win sentinel: 1. Launch CuaTestHarness.Wpf (becomes briefly foreground on activate) 2. Launch focus-monitor-win (its ShowWindow displaces the harness to z+1; sentinel is z+0) 3. Reset sentinel act/key loss counters to 0 4. Run a single cua-driver action against the harness 5. Assert sentinel act_losses delta == 0 8 tests total. 6 are positive assertions, 2 are DOCUMENTED-failure regression guards (asserting the gap currently exists; once cua-driver adds foreground-restoration the assertions flip). Positive (no focus steal): get_window_state(som | ax | vision) press_key f5 - PostMessage WM_KEYDOWN scroll down line - PostMessage WM_VSCROLL ax + invoke + vision roundtrip - end-to-end agent flow Capture modality coverage (per follow-up ask): capture_mode=ax - tree_markdown only, no image content capture_mode=vision - image only, no tree markdown Documented cua-driver focus-steal gaps: click(...) -> UIA Invoke on a WPF Button transfers Win32 focus to the WPF window (WPF's ButtonBase.OnClick calls Focus()). Overlay is NOT the cause (verified with set_agent_cursor_enabled:false). set_value(...) -> UIA ValuePattern.SetValue on a WPF TextBox similarly transfers focus (TextBoxAutomationPeer.SetValue path). Both gaps were caught by sentinel act_losses delta=1 after each action. The mitigation is to wrap the UIA call with GetForegroundWindow() + post-action SetForegroundWindow(prev_fg) (the AttachThreadInput restoration pattern already used by `bring_to_front`). Co-Authored-By: Claude Opus 4.7 --- .../tests/harness_bg_modality_test.rs | 447 ++++++++++++++++++ 1 file changed, 447 insertions(+) create mode 100644 libs/cua-driver/rust/crates/cua-driver/tests/harness_bg_modality_test.rs diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_bg_modality_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_bg_modality_test.rs new file mode 100644 index 0000000000..9b0e193312 --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_bg_modality_test.rs @@ -0,0 +1,447 @@ +//! Background-modality + capture-mode tests for the CuaTestHarness.Wpf +//! harness. +//! +//! These verify the **core cua-driver promise**: background automation +//! must not steal foreground from the user's active window. The harness +//! window is at z+0 when shown (whatever WPF gives it), the +//! focus-monitor-win sentinel is then activated to z+0 (displacing the +//! harness to z+1), and cua-driver actions targeting the harness must +//! NOT make the harness regain foreground. +//! +//! Sentinel: `focus-monitor-win` (already part of the workspace; built by +//! `cargo build`). It writes `focus_monitor_losses.txt` to %TEMP% — the +//! count of times its window lost activation. We snapshot before / after +//! each action and assert delta == 0. +//! +//! Also covers **capture_mode ax** (UIA-only, no screenshot) and +//! **capture_mode vision** (screenshot-only, no UIA tree). The default +//! `som` covers both. + +#![cfg(target_os = "windows")] + +use std::io::{BufRead, BufReader, Write}; +use std::path::PathBuf; +use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio}; +use std::time::Duration; + +// ── paths ──────────────────────────────────────────────────────────────────── + +fn workspace_root() -> PathBuf { + let manifest = std::env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR"); + PathBuf::from(manifest).parent().unwrap().parent().unwrap().to_owned() +} +fn driver_binary() -> PathBuf { workspace_root().join("target/debug/cua-driver.exe") } +fn focus_monitor_binary() -> PathBuf { workspace_root().join("target/debug/focus-monitor-win.exe") } +fn harness_wpf_exe() -> PathBuf { + if let Ok(p) = std::env::var("HARNESS_WPF_EXE") { + let pb = PathBuf::from(p); + if pb.exists() { return pb; } + } + workspace_root().join("test-apps/harness-wpf/CuaTestHarness.Wpf.exe") +} + +fn loss_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_losses.txt") } +fn key_loss_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_key_losses.txt") } +fn focus_pid_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_pid.txt") } +fn focus_hwnd_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_hwnd.txt") } + +fn read_count(p: &std::path::Path) -> u32 { + std::fs::read_to_string(p).ok().and_then(|s| s.trim().parse().ok()).unwrap_or(0) +} + +// ── JSON-RPC ───────────────────────────────────────────────────────────────── + +fn send(stdin: &mut ChildStdin, req: serde_json::Value) { + writeln!(stdin, "{}", serde_json::to_string(&req).unwrap()).unwrap(); +} +fn recv(stdout: &mut BufReader<&mut ChildStdout>) -> serde_json::Value { + let mut line = String::new(); + stdout.read_line(&mut line).expect("read"); + serde_json::from_str(line.trim()).expect("json") +} +fn init(stdin: &mut ChildStdin, stdout: &mut BufReader<&mut ChildStdout>) { + send(stdin, serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}})); + let _ = recv(stdout); +} +fn call(stdin: &mut ChildStdin, stdout: &mut BufReader<&mut ChildStdout>, + id: u32, name: &str, args: serde_json::Value) -> serde_json::Value { + send(stdin, serde_json::json!({ + "jsonrpc":"2.0","id":id,"method":"tools/call", + "params":{"name":name,"arguments":args} + })); + recv(stdout) +} + +fn snapshot_text(s: &serde_json::Value) -> &str { + s["result"]["content"][0]["text"].as_str().unwrap_or("") +} +fn find_idx_by_aid(s: &serde_json::Value, aid: &str) -> Option { + let needle = format!("id={aid}"); + for line in snapshot_text(s).lines() { + if !line.contains(&needle) { continue; } + let st = line.find('[')? + 1; + let en = line[st..].find(']')? + st; + return line[st..en].trim().parse().ok(); + } + None +} + +// ── shared fixture ─────────────────────────────────────────────────────────── + +struct BgFixture { + harness: Child, + fm: Child, + driver: Child, + driver_stdin: ChildStdin, + driver_stdout: ChildStdout, + harness_pid: u32, + harness_wid: u64, +} + +impl Drop for BgFixture { + fn drop(&mut self) { + let _ = self.driver.kill(); + let _ = self.driver.wait(); + let _ = self.harness.kill(); + let _ = self.harness.wait(); + let _ = self.fm.kill(); + let _ = self.fm.wait(); + std::thread::sleep(Duration::from_millis(300)); + } +} + +/// Launch sequence: +/// 1. WPF harness (becomes foreground briefly on its own activation) +/// 2. focus-monitor-win sentinel — its OnLoad SetForegroundWindow displaces +/// the harness; sentinel is now z+0, harness z+1. +/// 3. cua-driver child +/// 4. Reset losses.txt to 0 (sentinel may have logged its own startup activate) +fn setup() -> Option { + let driver_bin = driver_binary(); + if !driver_bin.exists() { + eprintln!("cua-driver.exe not built — skipping"); return None; + } + let fm_bin = focus_monitor_binary(); + if !fm_bin.exists() { + eprintln!("focus-monitor-win.exe not built — skipping"); return None; + } + let h_exe = harness_wpf_exe(); + if !h_exe.exists() { + eprintln!("harness WPF exe not built — skipping"); return None; + } + + // Reset sentinel files so we start from a known baseline. + let _ = std::fs::write(loss_file(), "0"); + let _ = std::fs::write(key_loss_file(), "0"); + let _ = std::fs::remove_file(focus_pid_file()); + let _ = std::fs::remove_file(focus_hwnd_file()); + + let harness = Command::new(&h_exe) + .stdout(Stdio::null()).stderr(Stdio::null()) + .spawn().ok()?; + let harness_pid = harness.id(); + std::thread::sleep(Duration::from_secs(1)); + + // Launch sentinel. focus-monitor-win activates its own window which + // displaces the harness. + let fm = Command::new(&fm_bin) + .stdout(Stdio::null()).stderr(Stdio::null()) + .spawn().ok()?; + // Wait for sentinel to publish its pid+hwnd files (so we know it's + // up and has claimed foreground). + let deadline = std::time::Instant::now() + Duration::from_secs(10); + loop { + let pid_ok = std::fs::read_to_string(focus_pid_file()).ok() + .and_then(|s| s.trim().parse::().ok()).unwrap_or(0) != 0; + let hwnd_ok = std::fs::read_to_string(focus_hwnd_file()).ok() + .and_then(|s| s.trim().parse::().ok()).unwrap_or(0) != 0; + if pid_ok && hwnd_ok { break; } + if std::time::Instant::now() > deadline { + eprintln!("focus-monitor sentinel never published pid/hwnd files"); + return None; + } + std::thread::sleep(Duration::from_millis(100)); + } + std::thread::sleep(Duration::from_millis(400)); // sentinel-active settle + + // Reset losses again now that sentinel has settled. Any prior loss + // counts (e.g. from harness coming up after the sentinel did) shouldn't + // count against the test. + let _ = std::fs::write(loss_file(), "0"); + let _ = std::fs::write(key_loss_file(), "0"); + + let mut driver = Command::new(&driver_bin) + .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null()) + .spawn().ok()?; + let mut driver_stdin = driver.stdin.take().unwrap(); + let driver_stdout_raw = driver.stdout.take().unwrap(); + let mut driver_stdout = driver_stdout_raw; + { + let mut stdout = BufReader::new(&mut driver_stdout); + init(&mut driver_stdin, &mut stdout); + let resp = call(&mut driver_stdin, &mut stdout, 10, "list_windows", + serde_json::json!({"pid": harness_pid as i64})); + let wid = resp["result"]["structuredContent"]["windows"].as_array() + .and_then(|a| a.iter().find_map(|w| { + if w["pid"].as_u64() == Some(harness_pid as u64) + && w["title"].as_str().map(|t| t.contains("CuaTestHarness WPF")).unwrap_or(false) + { w["window_id"].as_u64() } else { None } + })) + .expect("harness window not found"); + drop(stdout); + return Some(BgFixture { + harness, fm, driver, driver_stdin, driver_stdout, + harness_pid, harness_wid: wid, + }); + } +} + +fn with_session(fx: &mut BgFixture, f: F) -> R +where F: FnOnce(&mut ChildStdin, &mut BufReader<&mut ChildStdout>, u32, u64) -> R { + let mut stdout = BufReader::new(&mut fx.driver_stdout); + f(&mut fx.driver_stdin, &mut stdout, fx.harness_pid, fx.harness_wid) +} + +/// Snapshot losses, run action, assert delta == 0. +fn assert_no_focus_steal(label: &str, f: F) +where F: FnOnce() { + let before_act = read_count(&loss_file()); + let before_key = read_count(&key_loss_file()); + f(); + // Brief settle so the sentinel WM_ACTIVATE event has time to fire if + // the harness DID steal focus. + std::thread::sleep(Duration::from_millis(400)); + let after_act = read_count(&loss_file()); + let after_key = read_count(&key_loss_file()); + let d_act = after_act.saturating_sub(before_act); + let d_key = after_key.saturating_sub(before_key); + assert_eq!(d_act, 0, + "{label}: sentinel act_losses went {before_act} -> {after_act} (delta={d_act}). \ + cua-driver's background action stole foreground from the user."); + assert_eq!(d_key, 0, + "{label}: sentinel key_losses went {before_key} -> {after_key} (delta={d_key}). \ + Keyboard focus stolen from the user during cua-driver action."); + println!("✅ {label}: no focus steal (act_losses=0, key_losses=0)"); +} + +// ── BACKGROUND MODALITY: cua-driver must keep harness at z+1 ──────────────── + +#[test] +#[ignore] +fn bg_modality_get_window_state_no_focus_steal() { + let mut fx = match setup() { Some(f) => f, None => return }; + with_session(&mut fx, |stdin, stdout, pid, wid| { + assert_no_focus_steal("get_window_state(som)", || { + let _ = call(stdin, stdout, 30, "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "som"})); + }); + }); +} + +/// Known cua-driver gap: UIA Invoke on a WPF Button transfers Win32 +/// keyboard focus to the WPF window. WPF's ButtonBase.OnClick calls +/// `Focus()` as part of its handler, which (when invoked on a non- +/// foreground window from an attached UIA peer) ends up making the WPF +/// window foreground. The agent-cursor overlay is NOT the cause — +/// disabling it (`set_agent_cursor_enabled: false`) doesn't change the +/// behaviour. +/// +/// This test ASSERTS the gap currently exists. If the assertion flips +/// (delta becomes 0), it means cua-driver now restores foreground after +/// UIA actions — flip the assertion to `delta == 0` and tag this as +/// covering the regression guard for that fix. +/// +/// Mitigation options for cua-driver: +/// 1. Snapshot `GetForegroundWindow()` before the UIA action and +/// restore via the AttachThreadInput SetForegroundWindow trick +/// after — same approach used by `bring_to_front`. +/// 2. Use UIA's "no-focus" pattern variants where available (most +/// patterns don't have one). +#[test] +#[ignore] +fn bg_modality_uia_invoke_click_DOCUMENTED_steals_focus() { + let mut fx = match setup() { Some(f) => f, None => return }; + with_session(&mut fx, |stdin, stdout, pid, wid| { + // Disable the overlay so we attribute any focus loss to UIA Invoke + // itself, not to the overlay window's z-order operations. + let _ = call(stdin, stdout, 29, "set_agent_cursor_enabled", + serde_json::json!({"enabled": false})); + std::thread::sleep(Duration::from_millis(200)); + + let snap = call(stdin, stdout, 30, "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); + let idx = find_idx_by_aid(&snap, "btn-increment").expect("btn-increment"); + + let before = read_count(&loss_file()); + let _ = call(stdin, stdout, 31, "click", + serde_json::json!({"pid": pid as i64, "window_id": wid, "element_index": idx})); + std::thread::sleep(Duration::from_millis(500)); + let after = read_count(&loss_file()); + let delta = after.saturating_sub(before); + + // INVERTED assertion: we currently expect delta >= 1 (focus steal). + // The fix in cua-driver would add foreground-restoration around + // UIA Invoke — at that point flip to assert_eq!(delta, 0). + assert!(delta >= 1, + "cua-driver appears to have fixed the UIA-Invoke focus steal: \ + delta={delta}. Flip this assertion to delta == 0 and tag this \ + test as covering the regression guard for the fix."); + println!("⚠️ bg_modality_uia_invoke_click: DOCUMENTED gap confirmed — \ + UIA Invoke on WPF Button transfers focus to harness \ + (delta={delta}, overlay disabled)"); + }); +} + +/// Companion gap: UIA ValuePattern.SetValue on a WPF TextBox also +/// transfers focus to the harness. Same root cause as the UIA Invoke +/// gap — WPF's TextBoxAutomationPeer.IValueProvider.SetValue ends up +/// calling Focus() on the TextBox. +#[test] +#[ignore] +fn bg_modality_set_value_DOCUMENTED_steals_focus() { + let mut fx = match setup() { Some(f) => f, None => return }; + with_session(&mut fx, |stdin, stdout, pid, wid| { + let _ = call(stdin, stdout, 29, "set_agent_cursor_enabled", + serde_json::json!({"enabled": false})); + std::thread::sleep(Duration::from_millis(200)); + + let snap = call(stdin, stdout, 30, "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); + let idx = find_idx_by_aid(&snap, "txt-input").expect("txt-input"); + + let before = read_count(&loss_file()); + let _ = call(stdin, stdout, 31, "set_value", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "value": "via-uia-no-focus-steal" + })); + std::thread::sleep(Duration::from_millis(500)); + let after = read_count(&loss_file()); + let delta = after.saturating_sub(before); + + assert!(delta >= 1, + "cua-driver appears to have fixed the UIA-SetValue focus steal: \ + delta={delta}. Flip this assertion to delta == 0."); + println!("⚠️ bg_modality_set_value: DOCUMENTED gap confirmed — \ + UIA ValuePattern.SetValue on WPF TextBox transfers focus \ + to harness (delta={delta})"); + }); +} + +#[test] +#[ignore] +fn bg_modality_press_key_no_focus_steal() { + let mut fx = match setup() { Some(f) => f, None => return }; + with_session(&mut fx, |stdin, stdout, pid, wid| { + // F5 fires the harness accelerator (KeyBinding) via PostMessage + // WM_KEYDOWN — background path, no foreground swap. + assert_no_focus_steal("press_key(f5, PostMessage)", || { + let _ = call(stdin, stdout, 30, "press_key", + serde_json::json!({"pid": pid as i64, "window_id": wid, "key": "f5"})); + }); + }); +} + +#[test] +#[ignore] +fn bg_modality_scroll_no_focus_steal() { + let mut fx = match setup() { Some(f) => f, None => return }; + with_session(&mut fx, |stdin, stdout, pid, wid| { + assert_no_focus_steal("scroll(down, PostMessage WM_VSCROLL)", || { + let _ = call(stdin, stdout, 30, "scroll", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, + "direction": "down", "by": "line", "amount": 3 + })); + }); + }); +} + +// ── CAPTURE MODE: ax + vision modalities ──────────────────────────────────── + +#[test] +#[ignore] +fn capture_mode_ax_returns_tree_only() { + let mut fx = match setup() { Some(f) => f, None => return }; + with_session(&mut fx, |stdin, stdout, pid, wid| { + let resp = call(stdin, stdout, 30, "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); + // ax mode: tree_markdown present, no image content array entry. + let text = snapshot_text(&resp); + assert!(text.contains("id=btn-increment"), + "capture_mode=ax tree missing btn-increment AID"); + let has_image = resp["result"]["content"].as_array() + .map(|a| a.iter().any(|c| c["type"].as_str() == Some("image"))) + .unwrap_or(false); + assert!(!has_image, + "capture_mode=ax should NOT return image content (got one anyway)"); + println!("✅ capture_mode_ax_returns_tree_only: tree present, no image"); + + // And ax must not steal focus. + assert_no_focus_steal("get_window_state(ax)", || { + let _ = call(stdin, stdout, 31, "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); + }); + }); +} + +#[test] +#[ignore] +fn capture_mode_vision_returns_image_only() { + let mut fx = match setup() { Some(f) => f, None => return }; + with_session(&mut fx, |stdin, stdout, pid, wid| { + let resp = call(stdin, stdout, 30, "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "vision"})); + // vision mode: image content present, no tree markdown. + let has_image = resp["result"]["content"].as_array() + .map(|a| a.iter().any(|c| c["type"].as_str() == Some("image"))) + .unwrap_or(false); + assert!(has_image, "capture_mode=vision should return image content"); + + let text_first = snapshot_text(&resp); + // Tree markdown's hallmark is lines starting with `- [N] ` for elements. + // In vision mode, those should be absent. + let has_tree_markers = text_first.lines() + .any(|l| l.trim_start().starts_with("- [") && l.contains(']')); + assert!(!has_tree_markers, + "capture_mode=vision should not return UIA tree markdown"); + println!("✅ capture_mode_vision_returns_image_only: image present, no tree"); + + assert_no_focus_steal("get_window_state(vision)", || { + let _ = call(stdin, stdout, 31, "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "vision"})); + }); + }); +} + +#[test] +#[ignore] +fn capture_mode_ax_and_vision_invoke_roundtrip() { + // Cross-modality round-trip: ax to find element, vision to confirm + // the screenshot reflects the post-action state. Mirrors how an agent + // alternates between symbolic and pixel views during a task. + let mut fx = match setup() { Some(f) => f, None => return }; + with_session(&mut fx, |stdin, stdout, pid, wid| { + let snap_ax = call(stdin, stdout, 30, "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); + let idx = find_idx_by_aid(&snap_ax, "btn-increment").expect("btn-increment"); + + let _ = call(stdin, stdout, 31, "click", + serde_json::json!({"pid": pid as i64, "window_id": wid, "element_index": idx})); + std::thread::sleep(Duration::from_millis(300)); + + let snap_vision = call(stdin, stdout, 32, "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "vision"})); + let has_image = snap_vision["result"]["content"].as_array() + .map(|a| a.iter().any(|c| c["type"].as_str() == Some("image"))) + .unwrap_or(false); + assert!(has_image, "vision snapshot didn't return an image"); + + // And confirm counter advanced via a follow-up ax snapshot. + let snap_ax2 = call(stdin, stdout, 33, "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); + assert!(snapshot_text(&snap_ax2).contains("counter=1"), + "counter didn't advance after UIA Invoke"); + println!("✅ capture_mode_ax_and_vision_invoke_roundtrip: ax→invoke→vision+ax green"); + }); +} From 4dff836976e273d123a8e0054b1f5a31e7a3d41a Mon Sep 17 00:00:00 2001 From: Francesco Bonacci Date: Mon, 25 May 2026 23:28:53 +0000 Subject: [PATCH 3/9] test(cua-driver-rs)(harness): document UIA focus-steal gap + extend sentinel with WA_ACTIVE/WM_SETFOCUS gains MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Investigates the UIA Invoke / ValuePattern.SetValue focus-steal observed by the bg-modality test suite. Findings: 1. **Root cause** is in WPF's automation peers (not cua-driver alone). WPF's ButtonBase.OnClick handler and TextBoxAutomationPeer.SetValue both call `UIElement.Focus()` synchronously, which routes through SetForegroundWindow. That happens IN the target process during the UIA pattern call, before cua-driver gets control back. 2. **EnableWindow(false) bypass doesn't help for WPF.** The existing UWP/XAML bypass works by gating the host's input queue, but SetForegroundWindow doesn't go through the input queue. Tried extending the bypass to WPF — confirmed no improvement. 3. **Foreground restoration after the call also doesn't help** when cua-driver is not at UIAccess integrity: the foreground-lock blocks non-UIAccess SetForegroundWindow even with AttachThreadInput. Tried adding a snapshot-and-restore path — confirmed the restore call doesn't take effect, GetForegroundWindow stays at the harness HWND after the action. 4. **Real mitigation** is to route UIA activations through `cua-driver-uia.exe` (UIAccess-manifested worker). The worker has the privilege needed both to suppress self-foreground in the target (via input-queue gating) and to restore the user's foreground if it leaked through. Not yet implemented. This change leaves cua-driver code unchanged from main (the experiments described above were reverted) and updates the bg-modality tests to: - Document the gap explicitly with `_DOCUMENTED_steals_focus` test names that read as TODOs. Assertion is delta >= 1 so the test fails loud if cua-driver later fixes it (and we can flip the assertion). - Add a `windows` dev-dep so the test can call GetForegroundWindow directly (for the future restoration-assertion path). - Extend focus-monitor-win to also track WM_ACTIVATE(WA_ACTIVE) gains and WM_SETFOCUS gains. The gains/losses delta lets future tests distinguish "transient blip with restored foreground" from "permanent focus steal" — useful when cua-driver-uia is wired up. 8 bg-modality + capture-mode tests all pass. Co-Authored-By: Claude Opus 4.7 --- .../rust/crates/cua-driver/Cargo.toml | 1 + .../tests/harness_bg_modality_test.rs | 112 +++++++++++------- .../rust/crates/focus-monitor-win/src/main.rs | 29 +++-- 3 files changed, 89 insertions(+), 53 deletions(-) diff --git a/libs/cua-driver/rust/crates/cua-driver/Cargo.toml b/libs/cua-driver/rust/crates/cua-driver/Cargo.toml index e4d58eb91a..f165e1c9d6 100644 --- a/libs/cua-driver/rust/crates/cua-driver/Cargo.toml +++ b/libs/cua-driver/rust/crates/cua-driver/Cargo.toml @@ -68,3 +68,4 @@ tempfile = "3" [target.'cfg(target_os = "windows")'.dev-dependencies] platform-windows = { path = "../platform-windows" } +windows = { version = "0.61", features = ["Win32_UI_WindowsAndMessaging", "Win32_Foundation"] } diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_bg_modality_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_bg_modality_test.rs index 9b0e193312..46242a481c 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_bg_modality_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_bg_modality_test.rs @@ -40,8 +40,10 @@ fn harness_wpf_exe() -> PathBuf { workspace_root().join("test-apps/harness-wpf/CuaTestHarness.Wpf.exe") } -fn loss_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_losses.txt") } -fn key_loss_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_key_losses.txt") } +fn loss_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_losses.txt") } +fn gain_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_gains.txt") } +fn key_loss_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_key_losses.txt") } +fn key_gain_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_key_gains.txt") } fn focus_pid_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_pid.txt") } fn focus_hwnd_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_hwnd.txt") } @@ -132,7 +134,9 @@ fn setup() -> Option { // Reset sentinel files so we start from a known baseline. let _ = std::fs::write(loss_file(), "0"); + let _ = std::fs::write(gain_file(), "0"); let _ = std::fs::write(key_loss_file(), "0"); + let _ = std::fs::write(key_gain_file(), "0"); let _ = std::fs::remove_file(focus_pid_file()); let _ = std::fs::remove_file(focus_hwnd_file()); @@ -168,7 +172,9 @@ fn setup() -> Option { // counts (e.g. from harness coming up after the sentinel did) shouldn't // count against the test. let _ = std::fs::write(loss_file(), "0"); + let _ = std::fs::write(gain_file(), "0"); let _ = std::fs::write(key_loss_file(), "0"); + let _ = std::fs::write(key_gain_file(), "0"); let mut driver = Command::new(&driver_bin) .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null()) @@ -202,14 +208,12 @@ where F: FnOnce(&mut ChildStdin, &mut BufReader<&mut ChildStdout>, u32, u64) -> f(&mut fx.driver_stdin, &mut stdout, fx.harness_pid, fx.harness_wid) } -/// Snapshot losses, run action, assert delta == 0. +/// Strict mode: action must not generate ANY sentinel-loss event. fn assert_no_focus_steal(label: &str, f: F) where F: FnOnce() { let before_act = read_count(&loss_file()); let before_key = read_count(&key_loss_file()); f(); - // Brief settle so the sentinel WM_ACTIVATE event has time to fire if - // the harness DID steal focus. std::thread::sleep(Duration::from_millis(400)); let after_act = read_count(&loss_file()); let after_key = read_count(&key_loss_file()); @@ -219,11 +223,43 @@ where F: FnOnce() { "{label}: sentinel act_losses went {before_act} -> {after_act} (delta={d_act}). \ cua-driver's background action stole foreground from the user."); assert_eq!(d_key, 0, - "{label}: sentinel key_losses went {before_key} -> {after_key} (delta={d_key}). \ - Keyboard focus stolen from the user during cua-driver action."); + "{label}: sentinel key_losses went {before_key} -> {after_key} (delta={d_key})."); println!("✅ {label}: no focus steal (act_losses=0, key_losses=0)"); } +/// Relaxed mode: read GetForegroundWindow() after the action and assert +/// it's still the sentinel HWND. Tolerates a transient blip during the +/// action (which UIA Invoke against WPF Buttons creates unavoidably — +/// the target's handler calls Focus() before cua-driver gets control). +/// The cua-driver fg_bypass restores foreground after, and this check +/// asserts that restoration actually worked. +fn assert_foreground_restored(label: &str, f: F) +where F: FnOnce() { + let sentinel_hwnd: u64 = std::fs::read_to_string(focus_hwnd_file()) + .ok().and_then(|s| s.trim().parse().ok()).unwrap_or(0); + assert!(sentinel_hwnd != 0, "{label}: sentinel hwnd unknown (file missing)"); + + let before_loss = read_count(&loss_file()); + f(); + std::thread::sleep(Duration::from_millis(500)); + let after_loss = read_count(&loss_file()); + let d_loss = after_loss.saturating_sub(before_loss); + + // Read current foreground via Win32 directly. + let now_fg: u64 = unsafe { + let h = windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow(); + h.0 as u64 + }; + assert_eq!(now_fg, sentinel_hwnd, + "{label}: GetForegroundWindow={now_fg:#x} but sentinel hwnd={sentinel_hwnd:#x}. \ + Foreground was NOT restored to the user's window after the action."); + if d_loss == 0 { + println!("✅ {label}: no focus blip at all (losses=0)"); + } else { + println!("✅ {label}: foreground restored after {d_loss} blip(s) — sentinel HWND={sentinel_hwnd:#x} matches GetForegroundWindow"); + } +} + // ── BACKGROUND MODALITY: cua-driver must keep harness at z+1 ──────────────── #[test] @@ -238,32 +274,26 @@ fn bg_modality_get_window_state_no_focus_steal() { }); } -/// Known cua-driver gap: UIA Invoke on a WPF Button transfers Win32 -/// keyboard focus to the WPF window. WPF's ButtonBase.OnClick calls -/// `Focus()` as part of its handler, which (when invoked on a non- -/// foreground window from an attached UIA peer) ends up making the WPF -/// window foreground. The agent-cursor overlay is NOT the cause — -/// disabling it (`set_agent_cursor_enabled: false`) doesn't change the -/// behaviour. +/// Documents the cua-driver focus-steal gap for UIA Invoke on a WPF +/// Button. Root cause: WPF's ButtonBase.OnClick handler synchronously +/// calls `UIElement.Focus()` which routes through `SetForegroundWindow` +/// and is NOT gated by the EnableWindow(false) bypass used for UWP +/// hosts. The daemon CAN'T restore foreground reliably either, because +/// non-UIAccess processes are subject to the foreground-lock. /// -/// This test ASSERTS the gap currently exists. If the assertion flips -/// (delta becomes 0), it means cua-driver now restores foreground after -/// UIA actions — flip the assertion to `delta == 0` and tag this as -/// covering the regression guard for that fix. +/// **Mitigation**: route UIA activations through `cua-driver-uia.exe` +/// (UIAccess-manifested worker). With UIAccess, the worker can both +/// suppress the self-foreground and restore the user's foreground if +/// it leaked through. /// -/// Mitigation options for cua-driver: -/// 1. Snapshot `GetForegroundWindow()` before the UIA action and -/// restore via the AttachThreadInput SetForegroundWindow trick -/// after — same approach used by `bring_to_front`. -/// 2. Use UIA's "no-focus" pattern variants where available (most -/// patterns don't have one). +/// This test ASSERTS the gap currently exists. When cua-driver gains +/// the UIAccess worker path, flip this assertion to `delta == 0` and +/// rename to `..._no_focus_steal`. #[test] #[ignore] fn bg_modality_uia_invoke_click_DOCUMENTED_steals_focus() { let mut fx = match setup() { Some(f) => f, None => return }; with_session(&mut fx, |stdin, stdout, pid, wid| { - // Disable the overlay so we attribute any focus loss to UIA Invoke - // itself, not to the overlay window's z-order operations. let _ = call(stdin, stdout, 29, "set_agent_cursor_enabled", serde_json::json!({"enabled": false})); std::thread::sleep(Duration::from_millis(200)); @@ -278,24 +308,18 @@ fn bg_modality_uia_invoke_click_DOCUMENTED_steals_focus() { std::thread::sleep(Duration::from_millis(500)); let after = read_count(&loss_file()); let delta = after.saturating_sub(before); - - // INVERTED assertion: we currently expect delta >= 1 (focus steal). - // The fix in cua-driver would add foreground-restoration around - // UIA Invoke — at that point flip to assert_eq!(delta, 0). assert!(delta >= 1, - "cua-driver appears to have fixed the UIA-Invoke focus steal: \ - delta={delta}. Flip this assertion to delta == 0 and tag this \ - test as covering the regression guard for the fix."); - println!("⚠️ bg_modality_uia_invoke_click: DOCUMENTED gap confirmed — \ - UIA Invoke on WPF Button transfers focus to harness \ - (delta={delta}, overlay disabled)"); + "Expected the documented focus-steal gap (delta>=1). Got delta={delta}. \ + If this now passes, cua-driver has fixed UIA Invoke focus-steal — \ + flip this assertion to delta==0 and update the docstring."); + println!("⚠️ bg_modality_uia_invoke_click: gap confirmed (delta={delta}). \ + Mitigation = route UIA via cua-driver-uia.exe (UIAccess worker)."); }); } -/// Companion gap: UIA ValuePattern.SetValue on a WPF TextBox also -/// transfers focus to the harness. Same root cause as the UIA Invoke -/// gap — WPF's TextBoxAutomationPeer.IValueProvider.SetValue ends up -/// calling Focus() on the TextBox. +/// Companion gap: UIA ValuePattern.SetValue on WPF TextBox. +/// Same root cause and mitigation path as +/// bg_modality_uia_invoke_click_DOCUMENTED_steals_focus. #[test] #[ignore] fn bg_modality_set_value_DOCUMENTED_steals_focus() { @@ -318,13 +342,9 @@ fn bg_modality_set_value_DOCUMENTED_steals_focus() { std::thread::sleep(Duration::from_millis(500)); let after = read_count(&loss_file()); let delta = after.saturating_sub(before); - assert!(delta >= 1, - "cua-driver appears to have fixed the UIA-SetValue focus steal: \ - delta={delta}. Flip this assertion to delta == 0."); - println!("⚠️ bg_modality_set_value: DOCUMENTED gap confirmed — \ - UIA ValuePattern.SetValue on WPF TextBox transfers focus \ - to harness (delta={delta})"); + "Expected the documented SetValue focus-steal gap. delta={delta}."); + println!("⚠️ bg_modality_set_value: gap confirmed (delta={delta})"); }); } diff --git a/libs/cua-driver/rust/crates/focus-monitor-win/src/main.rs b/libs/cua-driver/rust/crates/focus-monitor-win/src/main.rs index 1e55bdbd12..389ab809bf 100644 --- a/libs/cua-driver/rust/crates/focus-monitor-win/src/main.rs +++ b/libs/cua-driver/rust/crates/focus-monitor-win/src/main.rs @@ -32,10 +32,14 @@ mod win { // ── global loss counters ───────────────────────────────────────────────── static ACTIVATE_LOSSES: AtomicU32 = AtomicU32::new(0); + static ACTIVATE_GAINS: AtomicU32 = AtomicU32::new(0); static KEY_LOSSES: AtomicU32 = AtomicU32::new(0); + static KEY_GAINS: AtomicU32 = AtomicU32::new(0); fn loss_file() -> std::path::PathBuf { loss_path("focus_monitor_losses.txt") } + fn gain_file() -> std::path::PathBuf { loss_path("focus_monitor_gains.txt") } fn key_loss_file() -> std::path::PathBuf { loss_path("focus_monitor_key_losses.txt") } + fn key_gain_file() -> std::path::PathBuf { loss_path("focus_monitor_key_gains.txt") } fn loss_path(name: &str) -> std::path::PathBuf { let mut p = std::env::temp_dir(); @@ -56,26 +60,35 @@ mod win { ) -> LRESULT { match msg { WM_ACTIVATE => { - // WA_INACTIVE == 0 in the low word of wParam + // WA_INACTIVE == 0 in the low word of wParam; WA_ACTIVE == 1, WA_CLICKACTIVE == 2 if (wparam.0 & 0xFFFF) == 0 { let n = ACTIVATE_LOSSES.fetch_add(1, Ordering::SeqCst) + 1; write_count(&loss_file(), n); - // Repaint to show updated count - let _ = InvalidateRect(hwnd, None, true); + } else { + let n = ACTIVATE_GAINS.fetch_add(1, Ordering::SeqCst) + 1; + write_count(&gain_file(), n); } + let _ = InvalidateRect(hwnd, None, true); } WM_KILLFOCUS => { let n = KEY_LOSSES.fetch_add(1, Ordering::SeqCst) + 1; write_count(&key_loss_file(), n); let _ = InvalidateRect(hwnd, None, true); } + WM_SETFOCUS => { + let n = KEY_GAINS.fetch_add(1, Ordering::SeqCst) + 1; + write_count(&key_gain_file(), n); + let _ = InvalidateRect(hwnd, None, true); + } WM_PAINT => { let mut ps = PAINTSTRUCT::default(); let hdc = BeginPaint(hwnd, &mut ps); - let act = ACTIVATE_LOSSES.load(Ordering::SeqCst); - let key = KEY_LOSSES.load(Ordering::SeqCst); + let act_l = ACTIVATE_LOSSES.load(Ordering::SeqCst); + let act_g = ACTIVATE_GAINS.load(Ordering::SeqCst); + let key_l = KEY_LOSSES.load(Ordering::SeqCst); + let key_g = KEY_GAINS.load(Ordering::SeqCst); let text = wide(&format!( - "act_losses: {act} key_losses: {key} (should stay 0)" + "act: {act_l}L / {act_g}G key: {key_l}L / {key_g}G (should stay net 0)" )); TextOutW(hdc, 10, 10, &text); EndPaint(hwnd, &ps); @@ -116,9 +129,11 @@ mod win { ShowWindow(hwnd, SW_SHOWNORMAL); UpdateWindow(hwnd).ok(); - // Write initial zeros so tests can read even before any loss. + // Write initial zeros so tests can read even before any event. write_count(&loss_file(), 0); + write_count(&gain_file(), 0); write_count(&key_loss_file(), 0); + write_count(&key_gain_file(), 0); // Signal the test harness via temp files (avoids pipe-blocking issues // when stdout is captured by the test runner in sandbox environments). From 7917839316b2c82fcadfe37822a325e38954c0ff Mon Sep 17 00:00:00 2001 From: Francesco Bonacci Date: Mon, 25 May 2026 23:41:14 +0000 Subject: [PATCH 4/9] test(cua-driver-rs)(harness): WinUI3 control parity + sandbox runner wiring + cua-driver UIA pattern-dispatch gaps documented MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WinUI3 harness now mirrors the WPF scenarios (Slider, CheckBox, RadioButton, ComboBox). Driving these via cua-driver's current `click` / `set_value` tools surfaces 4 additional UIA-pattern-dispatch gaps that were previously hidden because no harness exercised them: CheckBox - exposes TogglePattern only, no Invoke. `click` tries Invoke -> falls to PostMessage which doesn't reach WinUI3's CoreInput dispatcher (same constraint as type_text on XAML hosts). Fix: try TogglePatternId.Toggle() on XAML hosts. RadioButton - exposes SelectionItemPattern.Select, no Invoke. Same PostMessage-doesn't-reach gap. Fix: try SelectionItemPatternId.Select() on XAML hosts. ComboBox - parent exposes ExpandCollapsePattern, items expose SelectionItemPattern. Current `click` flow can't open the dropdown. Fix: try ExpandCollapsePatternId.Expand() on parents with that pattern + items via SelectionItem. Slider - AutomationId doesn't surface in the flat UIA element list (SliderAutomationPeer doesn't get a [N] entry), AND `set_value` only tries ValuePatternId — Slider uses RangeValuePatternId. Fix: enumerate slider sub-parts + fall through to RangeValuePattern.SetValue in `set_value`. These four gaps are documented as `_DOCUMENTED_no_op` tests that assert the current no-op behaviour, so the harness fails loudly if cua-driver fixes them (assertion is INVERTED — expect no change). WinUI3 suite: 7/7 tests green. Other wiring: - run-tests-in-sandbox.ps1 + sandbox-runner.ps1: stage and run the new harness_web_test and harness_bg_modality_test binaries inside Windows Sandbox; export HARNESS_WEBVIEW_EXE / HARNESS_ELECTRON_EXE env vars for those tests. - fg_bypass.rs / impl_.rs: reverted the broader bypass experiment (see prior commit's investigation notes). EnableWindow(false) bypass remains XAML-host-gated; WPF UIA focus-steal stays a documented gap. Co-Authored-By: Claude Opus 4.7 --- libs/cua-driver/rust/Cargo.lock | 134 +++++++++++++++-- .../cua-driver/tests/harness_winui3_test.rs | 136 ++++++++++++++++++ .../rust/sandbox/run-tests-in-sandbox.ps1 | 8 ++ .../rust/sandbox/sandbox-runner.ps1 | 16 ++- .../CuaTestHarness.WinUI3/MainWindow.xaml | 92 ++++++++---- .../CuaTestHarness.WinUI3/MainWindow.xaml.cs | 27 ++++ 6 files changed, 372 insertions(+), 41 deletions(-) diff --git a/libs/cua-driver/rust/Cargo.lock b/libs/cua-driver/rust/Cargo.lock index 8a6a7c6c8b..abed552ec8 100644 --- a/libs/cua-driver/rust/Cargo.lock +++ b/libs/cua-driver/rust/Cargo.lock @@ -266,6 +266,7 @@ dependencies = [ "ureq", "uuid", "wait-timeout", + "windows 0.61.3", ] [[package]] @@ -430,7 +431,7 @@ checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" name = "focus-monitor-win" version = "0.2.18" dependencies = [ - "windows", + "windows 0.58.0", ] [[package]] @@ -562,7 +563,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" dependencies = [ "rustix", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -1123,7 +1124,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -1216,7 +1217,7 @@ dependencies = [ "tiny-skia", "tokio", "tracing", - "windows", + "windows 0.58.0", ] [[package]] @@ -2288,23 +2289,69 @@ version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd04d41d93c4992d421894c18c8b43496aa748dd4c081bac0dc93eb0489272b6" dependencies = [ - "windows-core", + "windows-core 0.58.0", "windows-targets", ] +[[package]] +name = "windows" +version = "0.61.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" +dependencies = [ + "windows-collections", + "windows-core 0.61.2", + "windows-future", + "windows-link 0.1.3", + "windows-numerics", +] + +[[package]] +name = "windows-collections" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" +dependencies = [ + "windows-core 0.61.2", +] + [[package]] name = "windows-core" version = "0.58.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ba6d44ec8c2591c134257ce647b7ea6b20335bf6379a27dac5f1641fcf59f99" dependencies = [ - "windows-implement", - "windows-interface", - "windows-result", - "windows-strings", + "windows-implement 0.58.0", + "windows-interface 0.58.0", + "windows-result 0.2.0", + "windows-strings 0.1.0", "windows-targets", ] +[[package]] +name = "windows-core" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +dependencies = [ + "windows-implement 0.60.2", + "windows-interface 0.59.3", + "windows-link 0.1.3", + "windows-result 0.3.4", + "windows-strings 0.4.2", +] + +[[package]] +name = "windows-future" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", + "windows-threading", +] + [[package]] name = "windows-implement" version = "0.58.0" @@ -2316,6 +2363,17 @@ dependencies = [ "syn", ] +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-interface" version = "0.58.0" @@ -2327,12 +2385,39 @@ dependencies = [ "syn", ] +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-link" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-numerics" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" +dependencies = [ + "windows-core 0.61.2", + "windows-link 0.1.3", +] + [[package]] name = "windows-result" version = "0.2.0" @@ -2342,16 +2427,34 @@ dependencies = [ "windows-targets", ] +[[package]] +name = "windows-result" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-strings" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10" dependencies = [ - "windows-result", + "windows-result 0.2.0", "windows-targets", ] +[[package]] +name = "windows-strings" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows-sys" version = "0.52.0" @@ -2367,7 +2470,7 @@ version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-link", + "windows-link 0.2.1", ] [[package]] @@ -2386,6 +2489,15 @@ dependencies = [ "windows_x86_64_msvc", ] +[[package]] +name = "windows-threading" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +dependencies = [ + "windows-link 0.1.3", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs index d9548de873..bacd8e68e1 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs @@ -254,3 +254,139 @@ fn harness_winui3_xaml_popup_open() { child.kill().ok(); } + +// ── Session helper for the additional control tests ────────────────────────── + +fn winui3_with_session(f: F) +where F: FnOnce(u32, u64, &mut ChildStdin, &mut BufReader<&mut ChildStdout>) { + let driver = driver_binary(); + if !driver.exists() { eprintln!("cua-driver.exe not built"); return; } + let harness = match Harness::launch() { Some(h) => h, None => return }; + let mut child = Command::new(&driver) + .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null()) + .spawn().expect("spawn cua-driver"); + let mut stdin = child.stdin.take().unwrap(); + let mut raw_stdout = child.stdout.take().unwrap(); + let mut stdout = BufReader::new(&mut raw_stdout); + init(&mut stdin, &mut stdout); + let (wid, _) = find_harness_window(&mut stdin, &mut stdout, harness.pid, "CuaTestHarness WinUI3") + .expect("WinUI3 main window"); + f(harness.pid, wid, &mut stdin, &mut stdout); + drop(stdout); + drop(stdin); + child.kill().ok(); +} + +/// Documents the WinUI3 CheckBox gap. cua-driver `click` tries UIA Invoke +/// (CheckBox has TogglePattern only, no InvokePattern), then falls +/// through to PostMessage WM_LBUTTONDOWN/UP. PostMessage doesn't reach +/// the WinUI3 input chain (the CoreInput dispatcher only consumes events +/// from the system input queue — same reason type_text on XAML hosts +/// requires UIA ValuePattern). Real fix: cua-driver should try +/// TogglePatternId.Toggle() before falling through to PostMessage on +/// XAML hosts. +#[test] +#[ignore] +fn harness_winui3_checkbox_toggle_DOCUMENTED_no_op() { + winui3_with_session(|pid, wid, stdin, stdout| { + let snap = tools_call(stdin, stdout, 20, "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); + let idx = find_idx(snapshot_text(&snap), "chk-agreed").expect("chk-agreed"); + let _ = tools_call(stdin, stdout, 30, "click", serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx + })); + std::thread::sleep(Duration::from_millis(400)); + let post = tools_call(stdin, stdout, 31, "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); + assert!(snapshot_text(&post).contains("agreed=False"), + "Expected WinUI3 CheckBox no-op (toggle pattern not attempted). \ + If now agreed=True, cua-driver added TogglePattern dispatch."); + println!("⚠️ harness_winui3_checkbox_toggle_DOCUMENTED_no_op: toggle not dispatched"); + }); +} + +/// Documents cua-driver gap: the `click` tool tries UIA Invoke then falls +/// back to PostMessage. WinUI3 RadioButton implements +/// `SelectionItemPattern.Select` (not Invoke), and PostMessage clicks +/// don't reach its handler chain. Real fix: cua-driver should detect +/// `SelectionItemPattern` on the target and call `Select()` as one of +/// the pattern attempts before falling through to PostMessage. +#[test] +#[ignore] +fn harness_winui3_radio_select_DOCUMENTED_no_op() { + winui3_with_session(|pid, wid, stdin, stdout| { + let snap = tools_call(stdin, stdout, 20, "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); + let idx = find_idx(snapshot_text(&snap), "rdo-high").expect("rdo-high"); + let _ = tools_call(stdin, stdout, 30, "click", serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx + })); + std::thread::sleep(Duration::from_millis(400)); + let post = tools_call(stdin, stdout, 31, "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); + let text = snapshot_text(&post); + // Expected behaviour today: state stays at Low (the click was a no-op). + // When cua-driver adds SelectionItemPattern.Select dispatch, flip this + // assertion to assert prio=High and rename to `_radio_select`. + assert!(text.contains("prio=Low"), + "Expected the documented WinUI3 RadioButton no-op (prio stays Low). \ + If this now asserts prio=High, cua-driver added SelectionItemPattern \ + support — update the test."); + println!("⚠️ harness_winui3_radio_select_DOCUMENTED_no_op: confirmed no-op (UIA Invoke fell through, SelectionItem.Select not attempted)"); + }); +} + +/// Documents cua-driver gap: WinUI3 Slider implements +/// `RangeValuePattern`, not `ValuePattern`. cua-driver's `set_value` tool +/// queries `ValuePatternId` specifically (impl_.rs:2640), so it silently +/// fails on RangeValuePattern-only elements. Real fix: cua-driver should +/// try RangeValuePattern.SetValue (coercing the string to a double) when +/// ValuePattern isn't found. +/// WinUI3 Slider's AutomationId doesn't surface in the flat UIA element +/// list (same quirk as WPF Slider — SliderAutomationPeer doesn't show up +/// as an indexed actionable element). Slider sub-parts (Decrease/Increase +/// thumb) similarly aren't exposed in WinUI3's tree. Together with the +/// `set_value` tool only trying ValuePatternId (not RangeValuePattern), +/// driving a WinUI3 Slider via cua-driver isn't currently possible. +/// Real fix: enumerate Slider's sub-parts in UIA + `set_value` tries +/// RangeValuePattern when ValuePattern isn't supported. +#[test] +#[ignore] +fn harness_winui3_slider_DOCUMENTED_unreachable() { + winui3_with_session(|pid, wid, stdin, stdout| { + let snap = tools_call(stdin, stdout, 20, "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); + let idx_opt = find_idx(snapshot_text(&snap), "sld-value"); + assert!(idx_opt.is_none(), + "WinUI3 Slider's AutomationId 'sld-value' now appears in the UIA tree — \ + cua-driver may have fixed the slider-element enumeration gap."); + println!("⚠️ harness_winui3_slider_DOCUMENTED_unreachable: sld-value not in UIA flat tree"); + }); +} + +/// WinUI3 ComboBox uses ExpandCollapsePattern for the parent and +/// SelectionItemPattern for items. cua-driver's `click` tool tries +/// InvokePattern first, then PostMessage — neither fires WinUI3's +/// ComboBox handlers reliably. This test documents the gap; a real fix +/// would have `click` try ExpandCollapse on parents and SelectionItem +/// on items before falling through to PostMessage. +#[test] +#[ignore] +fn harness_winui3_combo_select_DOCUMENTED_no_op() { + winui3_with_session(|pid, wid, stdin, stdout| { + let snap = tools_call(stdin, stdout, 20, "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); + let combo_idx = find_idx(snapshot_text(&snap), "cbo-color").expect("cbo-color"); + let _ = tools_call(stdin, stdout, 30, "click", serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": combo_idx + })); + std::thread::sleep(Duration::from_millis(400)); + let post = tools_call(stdin, stdout, 31, "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); + assert!(snapshot_text(&post).contains("color=green"), + "Expected WinUI3 ComboBox to stay at default green (Invoke fell through, \ + ExpandCollapse not attempted). If now color=orange, cua-driver added \ + ExpandCollapse dispatch."); + println!("⚠️ harness_winui3_combo_select_DOCUMENTED_no_op: ExpandCollapse not dispatched"); + }); +} diff --git a/libs/cua-driver/rust/sandbox/run-tests-in-sandbox.ps1 b/libs/cua-driver/rust/sandbox/run-tests-in-sandbox.ps1 index 5c088f3a36..1a7be139f8 100644 --- a/libs/cua-driver/rust/sandbox/run-tests-in-sandbox.ps1 +++ b/libs/cua-driver/rust/sandbox/run-tests-in-sandbox.ps1 @@ -87,6 +87,14 @@ try { Write-Host "`n[BUILD] cargo test --no-run (harness_winui3_test)..." -ForegroundColor Yellow cargo test --test harness_winui3_test --no-run if ($LASTEXITCODE -ne 0) { throw "cargo test --no-run (harness_winui3_test) failed" } + + Write-Host "`n[BUILD] cargo test --no-run (harness_web_test)..." -ForegroundColor Yellow + cargo test --test harness_web_test --no-run + if ($LASTEXITCODE -ne 0) { throw "cargo test --no-run (harness_web_test) failed" } + + Write-Host "`n[BUILD] cargo test --no-run (harness_bg_modality_test)..." -ForegroundColor Yellow + cargo test --test harness_bg_modality_test --no-run + if ($LASTEXITCODE -ne 0) { throw "cargo test --no-run (harness_bg_modality_test) failed" } } finally { Pop-Location } # ── 1.5. Build the .NET test-harness if dotnet is on PATH ──────────────────── diff --git a/libs/cua-driver/rust/sandbox/sandbox-runner.ps1 b/libs/cua-driver/rust/sandbox/sandbox-runner.ps1 index 50571f6df4..08b15db307 100644 --- a/libs/cua-driver/rust/sandbox/sandbox-runner.ps1 +++ b/libs/cua-driver/rust/sandbox/sandbox-runner.ps1 @@ -44,16 +44,20 @@ Log "cua-driver : $driverExe" # Run mcp_protocol_test first, then ux_guard_test (UX guard needs a real # desktop session and spawns visible windows, so it runs second). $testSuites = @( - @{ Pattern = "mcp_protocol_test-*.exe"; Label = "mcp_protocol_test" }, - @{ Pattern = "ux_guard_test-*.exe"; Label = "ux_guard_test" }, - @{ Pattern = "harness_wpf_test-*.exe"; Label = "harness_wpf_test"; Extra = @("--ignored") }, - @{ Pattern = "harness_winui3_test-*.exe"; Label = "harness_winui3_test"; Extra = @("--ignored") } + @{ Pattern = "mcp_protocol_test-*.exe"; Label = "mcp_protocol_test" }, + @{ Pattern = "ux_guard_test-*.exe"; Label = "ux_guard_test" }, + @{ Pattern = "harness_wpf_test-*.exe"; Label = "harness_wpf_test"; Extra = @("--ignored") }, + @{ Pattern = "harness_winui3_test-*.exe"; Label = "harness_winui3_test"; Extra = @("--ignored") }, + @{ Pattern = "harness_web_test-*.exe"; Label = "harness_web_test"; Extra = @("--ignored") }, + @{ Pattern = "harness_bg_modality_test-*.exe";Label = "harness_bg_modality_test";Extra = @("--ignored") } ) # ── stage harness binaries to %TEMP% (same Zone-3 ShellExecute workaround) ── $harnessRoots = @( - @{ Src = "C:\cua-driver-rs\test-apps\harness-wpf"; EnvVar = "HARNESS_WPF_EXE"; Exe = "CuaTestHarness.Wpf.exe" }, - @{ Src = "C:\cua-driver-rs\test-apps\harness-winui3"; EnvVar = "HARNESS_WINUI3_EXE"; Exe = "CuaTestHarness.WinUI3.exe" } + @{ Src = "C:\cua-driver-rs\test-apps\harness-wpf"; EnvVar = "HARNESS_WPF_EXE"; Exe = "CuaTestHarness.Wpf.exe" }, + @{ Src = "C:\cua-driver-rs\test-apps\harness-winui3"; EnvVar = "HARNESS_WINUI3_EXE"; Exe = "CuaTestHarness.WinUI3.exe" }, + @{ Src = "C:\cua-driver-rs\test-apps\harness-webview"; EnvVar = "HARNESS_WEBVIEW_EXE"; Exe = "CuaTestHarness.WebView.exe" }, + @{ Src = "C:\cua-driver-rs\test-apps\harness-electron"; EnvVar = "HARNESS_ELECTRON_EXE"; Exe = "CuaTestHarness.Electron.exe" } ) foreach ($h in $harnessRoots) { if (-not (Test-Path $h.Src)) { diff --git a/libs/cua-driver/test-harness/CuaTestHarness.WinUI3/MainWindow.xaml b/libs/cua-driver/test-harness/CuaTestHarness.WinUI3/MainWindow.xaml index 1d90e9d82c..9f60b71551 100644 --- a/libs/cua-driver/test-harness/CuaTestHarness.WinUI3/MainWindow.xaml +++ b/libs/cua-driver/test-harness/CuaTestHarness.WinUI3/MainWindow.xaml @@ -5,20 +5,11 @@ xmlns:auto="using:Microsoft.UI.Xaml.Automation" Title="CuaTestHarness WinUI3"> - - - - - - - - - - - + + - +