From d73828679116fbda621141c1f04485a605c6cb47 Mon Sep 17 00:00:00 2001 From: Francesco Bonacci Date: Tue, 26 May 2026 10:13:05 +0000 Subject: [PATCH] fix(cua-driver-rs)(platform-windows): close 4 functional gaps from PR #1699 harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four real cua-driver bugs the phase 2 harness exposed and documented as inverted-assertion regression guards — all now actually fixed end-to-end. 1) page.click_element probe double-decode (page.rs) The CDP runtime.evaluate response wraps the probe's stringified JSON in another JSON-string. The previous parser tried `serde_json::from_str(&probe_json).or_else(|_| ...)` — but `from_str` happily parses a quoted JSON-string into a `Value::String`, so the inner-decode branch never ran and parsed.get("vx") returned None. Match on Value::String and re-decode explicitly. 2) drag dispatch:foreground via SendInput (mouse.rs + impl_.rs) New `send_drag_synthesized` helper modelled on `send_click_synthesized`. PostMessage drag doesn't update the per-thread keyboard state that GetKeyState(VK_LBUTTON) reads, so frameworks polling Mouse.LeftButton during their drag handler (WPF Thumb.IsDragging) never see the button as held and the drag no-ops. SendInput goes through the system input queue and DOES update GetKeyState — WPF Slider thumbs now track. Same UIAccess foreground-lock caveat as send_click_synthesized. 3) Slider parent AID in UIA flat tree (uia/mod.rs) Added UIA_RangeValuePatternId to the cache pre-fetch list and to `detect_cached_actions`. Without it, Slider/ProgressBar parents reported `actions=[]` -> marked non-actionable -> no `[N]` index in the rendered tree -> unaddressable by AutomationId. Now they surface with `actions=[set_value]` like ValuePattern targets, and the set_value tool already falls through to RangeValuePattern. 4) WebView2 CDP listener actually works (test fix only) No cua-driver change — earlier "WebView2 filters --remote-debugging-port" hypothesis was wrong. The real reason the page-tool test failed against WebView2 was the same `/json` read_to_end bug fixed in PR #1699's commit be1581e5. The flag IS honoured; CDP listens on the configured port; the earlier discovery hang was the shared underlying bug. Upgraded harness_webview_window_discoverable -> harness_webview_page_tool with full execute_javascript + click_element coverage. Verification: cargo test --test harness_wpf_test -> 18/18 cargo test --test harness_winui3_test -> 7/7 cargo test --test harness_web_test -> 5/5 (+1 from upgrade) cargo test --test harness_bg_modality_test -> 8/8 The remaining open ship-blockers documented by PR #1699 are the two WPF UIA focus-steal cases (Invoke + SetValue trigger UIElement.Focus() in the target process before cua-driver gets control back). Those require the cua-driver-uia.exe UIAccess worker to fix and remain inverted-assertion regression guards. Co-Authored-By: Claude Opus 4.7 --- .../cua-driver/tests/harness_web_test.rs | 100 +++++++++++--- .../cua-driver/tests/harness_winui3_test.rs | 36 +++-- .../cua-driver/tests/harness_wpf_test.rs | 58 +++++--- .../platform-windows/src/input/mouse.rs | 130 ++++++++++++++++++ .../platform-windows/src/tools/impl_.rs | 41 ++++-- .../crates/platform-windows/src/tools/page.rs | 36 +++-- .../crates/platform-windows/src/uia/mod.rs | 10 +- 7 files changed, 333 insertions(+), 78 deletions(-) diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs index e37f8d4848..eecd7609de 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs @@ -154,20 +154,65 @@ where F: FnOnce(u32, u64, &mut ChildStdin, &mut BufReader<&mut ChildStdout>) { drop(session); } -// ── WebView2 structural ────────────────────────────────────────────────────── +// ── WebView2 structural + page tool ───────────────────────────────────────── #[test] #[ignore] fn harness_webview_window_discoverable() { - // Smoke test: WebView2 harness launches, window appears via list_windows. - // Behavioural page-tool tests are deferred until WebView2 actually opens - // its CDP listener — see the module docstring TODO. run_with_session("webview", webview_exe(), "CuaTestHarness WebView", 9222, |pid, wid, _stdin, _stdout| { println!("✅ harness_webview_window_discoverable: pid={pid} wid={wid}"); }); } +#[test] +#[ignore] +fn harness_webview_page_tool() { + // Regression guard for WebView2 CDP exposure via + // CoreWebView2EnvironmentOptions.AdditionalBrowserArguments. + // Combined with the `/json` Content-Length fix in mcp-server/src/cdp.rs, + // the page tool now reaches WebView2's DOM via CDP just like Electron. + run_with_session("webview", webview_exe(), "CuaTestHarness WebView", 9222, + |pid, wid, stdin, stdout| { + + let marker_resp = tools_call(stdin, stdout, 30, "page", serde_json::json!({ + "pid": pid as i64, "window_id": wid, "action": "execute_javascript", + "javascript": "document.querySelector('[data-cua-id=\"page-marker\"]').textContent" + })); + let marker = marker_resp.get("result") + .and_then(|r| r.get("content")) + .and_then(|c| c.as_array()) + .and_then(|arr| arr.get(0)) + .and_then(|item| item.get("text")) + .and_then(|t| t.as_str()) + .unwrap_or(""); + assert!(marker.contains("WEB_HARNESS_MARKER_v1"), + "WebView2 CDP execute_javascript marker fetch: {marker:?}"); + + // click_element via DOM selector + counter readback. + let _ = tools_call(stdin, stdout, 31, "page", serde_json::json!({ + "pid": pid as i64, "window_id": wid, "action": "click_element", + "selector": "#btn-increment" + })); + std::thread::sleep(Duration::from_millis(500)); + + let counter_resp = tools_call(stdin, stdout, 32, "page", serde_json::json!({ + "pid": pid as i64, "window_id": wid, "action": "execute_javascript", + "javascript": "document.getElementById('lbl-counter').textContent" + })); + let post = counter_resp.get("result") + .and_then(|r| r.get("content")) + .and_then(|c| c.as_array()) + .and_then(|arr| arr.get(0)) + .and_then(|item| item.get("text")) + .and_then(|t| t.as_str()) + .unwrap_or(""); + assert!(post.contains("counter=1"), + "WebView2 counter didn't advance via page.click_element: {post:?}"); + println!("✅ harness_webview_page_tool: CDP+execute_javascript+click_element green"); + }); +} + // ── Electron structural + page tool ────────────────────────────────────────── #[test] @@ -217,25 +262,24 @@ fn harness_electron_page_tool() { }); } -/// Documents a separate gap in page.click_element: its CDP probe runs JS -/// that returns a JSON object (vx/vy/sx/sy/dpr) but the result is wrapped -/// as a CDP `runtime.evaluate.user_gesture` string by the time the page -/// tool's JSON parser sees it — causing "probe JSON missing required -/// field 'vx'". Fix: unwrap the CDP result's `.value` before parsing, -/// or send the probe with `returnByValue:true` in the Runtime.evaluate -/// params. Manually unwrapping the harness's verified JSON string works. +/// Regression guard for the page.click_element double-encode fix. +/// +/// Originally the CDP runtime.evaluate response for the probe JS came +/// back as a JSON-encoded string containing the actual `{vx,vy,...}` +/// object. The page tool's `serde_json::from_str(&probe_json).or_else(...)` +/// only fell into the inner-decode branch on a hard parse error, but +/// `from_str` happily parses a JSON-string into a `Value::String`, so the +/// inner-decode branch never ran and `parsed.get("vx")` returned None. +/// Fix: match on Value::String and re-decode explicitly. #[test] #[ignore] -fn harness_electron_click_element_DOCUMENTED_wrapper_bug() { +fn harness_electron_click_element() { run_with_session("electron", electron_exe(), "CuaTestHarness Electron", 9223, |pid, wid, stdin, stdout| { let resp = tools_call(stdin, stdout, 30, "page", serde_json::json!({ "pid": pid as i64, "window_id": wid, "action": "click_element", "selector": "#btn-increment" })); - // Safe traversal — error responses from `page` can vary in shape - // (sometimes `result.isError: true` + `content[0].text`, sometimes - // `error.message`). Don't blow up on a missing array entry. let text = resp.get("result") .and_then(|r| r.get("content")) .and_then(|c| c.as_array()) @@ -244,12 +288,24 @@ fn harness_electron_click_element_DOCUMENTED_wrapper_bug() { .and_then(|t| t.as_str()) .or_else(|| resp.get("error").and_then(|e| e.get("message")).and_then(|m| m.as_str())) .unwrap_or(""); - // Expect the gap: parser fails on the CDP-wrapped probe response. - let expected_gap = text.contains("probe JSON missing") || text.contains("required field"); - assert!(expected_gap, - "Expected click_element probe-wrapper gap. Got: {text:?}. \ - If this asserts on a success message, the wrapper-unwrap fix is in — \ - flip this assertion to assert success."); - println!("⚠️ harness_electron_click_element_DOCUMENTED_wrapper_bug: probe JSON parse gap present"); + assert!(!text.contains("probe JSON missing") && !text.contains("required field"), + "click_element probe parse regressed: {text:?}"); + std::thread::sleep(Duration::from_millis(400)); + + // Verify the click actually fired in the DOM. + let counter_resp = tools_call(stdin, stdout, 31, "page", serde_json::json!({ + "pid": pid as i64, "window_id": wid, "action": "execute_javascript", + "javascript": "document.getElementById('lbl-counter').textContent" + })); + let post = counter_resp.get("result") + .and_then(|r| r.get("content")) + .and_then(|c| c.as_array()) + .and_then(|arr| arr.get(0)) + .and_then(|item| item.get("text")) + .and_then(|t| t.as_str()) + .unwrap_or(""); + assert!(post.contains("counter=1"), + "Counter didn't advance after page.click_element: {post:?}"); + println!("✅ harness_electron_click_element: probe parsed, click fired, counter=1"); }); } diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs index 6f3feb1e5a..4f26d1273e 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs @@ -326,22 +326,36 @@ fn harness_winui3_radio_select() { /// fails on RangeValuePattern-only elements. Real fix: cua-driver should /// try RangeValuePattern.SetValue (coercing the string to a double) when /// ValuePattern isn't found. -/// WinUI3 Slider's parent AutomationId doesn't surface in the flat UIA -/// element list (SliderAutomationPeer's children are the actionable -/// elements, not the slider itself). Still document the gap — fixing -/// the UIA enumeration to expose the slider as an indexed element is -/// orthogonal to this branch's set_value fix. +/// Regression guard for Slider element enumeration + RangeValuePattern +/// set_value. cua-driver's UIA cache now pre-fetches RangeValuePattern, +/// and `detect_cached_actions` reports `set_value` when present — so +/// Slider parents get an `[N]` flat-tree index. The `set_value` tool +/// already falls back to RangeValuePattern, so writing the value works +/// end-to-end against a WinUI3 Slider. #[test] #[ignore] -fn harness_winui3_slider_DOCUMENTED_unreachable() { +fn harness_winui3_slider_set_value() { winui3_with_session(|pid, wid, stdin, stdout| { let snap = tools_call(stdin, stdout, 20, "get_window_state", serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); - let idx_opt = find_idx(snapshot_text(&snap), "sld-value"); - assert!(idx_opt.is_none(), - "WinUI3 Slider 'sld-value' now appears in the UIA flat tree — \ - cua-driver fixed the slider-element enumeration gap. Update this test."); - println!("⚠️ harness_winui3_slider_DOCUMENTED_unreachable: sld-value not in UIA flat tree"); + let idx = find_idx(snapshot_text(&snap), "sld-value") + .expect("sld-value should now be in the UIA flat tree after RangeValuePattern detection fix"); + let resp = tools_call(stdin, stdout, 30, "set_value", serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "value": "42" + })); + println!("set_value sld-value=42: {}", resp["result"]["content"][0]["text"]); + std::thread::sleep(Duration::from_millis(400)); + let post = tools_call(stdin, stdout, 31, "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); + let text = snapshot_text(&post); + let advanced = text.lines().any(|l| + l.contains("slider_value=") && !l.contains("slider_value=0\"")); + assert!(advanced, + "WinUI3 Slider didn't move via RangeValuePattern.SetValue. Lines: {}", + text.lines().filter(|l| l.contains("slider_value")) + .collect::>().join(" / ")); + println!("✅ harness_winui3_slider_set_value: value moved via UIA RangeValuePattern.SetValue"); }); } diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs index 7bb1e40e92..3f22376e03 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs @@ -676,37 +676,55 @@ fn harness_wpf_layered_popup_capture() { #[test] #[ignore] -fn harness_wpf_slider_drag_tool_returns() { - // Coverage note for the drag tool against a WPF Slider thumb. +fn harness_wpf_slider_drag() { + // Regression guard for the SendInput drag path. PostMessage drag + // doesn't update GetKeyState, so WPF's Thumb-drag handler (which + // polls Mouse.LeftButton via GetKeyState) never sees the button + // held — the thumb stays put. dispatch:"foreground" routes through + // send_drag_synthesized which goes via the system input queue and + // DOES update GetKeyState, so the thumb actually tracks. // - // PostMessage WM_LBUTTONDOWN / WM_MOUSEMOVE / WM_LBUTTONUP doesn't - // update the OS keyboard/mouse state visible to GetKeyState. WPF's - // Slider Thumb relies on Mouse.LeftButton (which polls GetKeyState) - // to recognise an in-progress drag — so a PostMessage drag never - // moves a WPF thumb, even when from/to are correctly on the thumb - // in client coords. The companion `harness_wpf_slider_increase_large` - // test covers the slider via UIA Invoke on its internal IncreaseLarge - // sub-button, which is what an agent SHOULD use for slider - // manipulation on a backgrounded window. - // - // We still exercise the drag tool against the slider so its codepath - // (coord translation, dispatch policy, message synthesis) is on the - // critical-path test list — we just don't assert on the value moving. - // TODO: add a SendInput-based drag path so dispatch:"foreground" can - // drive the thumb, then enable a behavioral assertion here. + // Foreground-lock caveat: SetForegroundWindow can be rejected from + // non-UIAccess processes during the daemon-process foreground swap. + // bring_to_front first to make the harness foreground (via + // AttachThreadInput), then SendInput's own SetForegroundWindow is a + // no-op success. with_session(|pid, wid, stdin, stdout| { focus_harness(stdin, stdout, pid, wid); + let pre = snapshot_elements(stdin, stdout, pid, wid); + assert!(snapshot_text(&pre).contains("slider_value=0"), + "initial slider_value=0 missing"); + let resp = tools_call(stdin, stdout, 30, "drag", serde_json::json!({ "pid": pid as i64, "window_id": wid, + // drag screen-coords path: send_drag_synthesized takes screen + // coords directly. The harness window is centered at + // (517, 66) with the slider track at client (50-330, 275); + // convert to screen via ClientToScreen approximation by + // offsetting by window position + non-client chrome + // (title bar + border ~30,8). screen coords here are + // re-derived in window-local form by the tool's existing + // ClientToScreen step. "from_x": 50.0, "from_y": 275.0, "to_x": 330.0, "to_y": 275.0, - "duration_ms": 600, "steps": 30 + "duration_ms": 700, "steps": 40, + "dispatch": "foreground" })); let msg = resp["result"]["content"][0]["text"].as_str().unwrap_or(""); - println!("drag slider: {msg}"); + println!("drag slider (foreground): {msg}"); assert!(msg.starts_with("✅"), "drag tool returned non-success: {msg}"); - println!("✅ harness_wpf_slider_drag_tool_returns: PostMessage drag emitted (known no-op vs WPF Thumb)"); + std::thread::sleep(Duration::from_millis(500)); + + let post = snapshot_elements(stdin, stdout, pid, wid); + let text = snapshot_text(&post); + let advanced = text.lines().any(|l| + l.contains("slider_value=") && !l.contains("slider_value=0\"")); + assert!(advanced, + "Slider value did not advance via SendInput drag. Lines: {}", + text.lines().filter(|l| l.contains("slider_value")) + .collect::>().join(" / ")); + println!("✅ harness_wpf_slider_drag: thumb tracked via SendInput drag"); }); } diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs index d42f29430c..8a7cbc2f47 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs @@ -377,3 +377,133 @@ pub fn send_click_synthesized( Ok(()) } + +/// Press-hold-move-release drag via `SendInput`. Companion to +/// [`send_click_synthesized`] for the `drag` tool's `dispatch:"foreground"` +/// path. +/// +/// Why a SendInput drag is needed at all: the PostMessage drag path posts +/// `WM_LBUTTONDOWN` + `WM_MOUSEMOVE`s + `WM_LBUTTONUP` to the target HWND. +/// PostMessage does NOT update the per-thread keyboard state that +/// `GetKeyState(VK_LBUTTON)` reads, so frameworks that poll the button-held +/// state during their drag handler (WPF's Thumb.IsDragging logic does this +/// via Mouse.LeftButton, which polls GetKeyState) never observe the button +/// as down and the drag is a no-op. SendInput goes through the system +/// input queue and DOES update GetKeyState, so a WPF Slider thumb actually +/// tracks the drag. +/// +/// Same UIAccess constraints as [`send_click_synthesized`] — the +/// `SetForegroundWindow` swap is rejected from non-UIAccess processes +/// when foreground-lock is active; route through `cua-driver-uia.exe` +/// for reliable operation. +pub fn send_drag_synthesized( + target: u64, + sx_from: i32, sy_from: i32, + sx_to: i32, sy_to: i32, + duration_ms: u64, + steps: usize, + button: &str, +) -> Result<()> { + let target = HWND(target as *mut _); + if target.0.is_null() { + bail!("invalid target hwnd"); + } + if let Some(msg) = crate::input::post_message_blocked_by_uipi(target.0 as u64) { + bail!(msg); + } + + let (down_flag, up_flag) = match button { + "right" => (MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP), + "middle" => (MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP), + _ => (MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP), + }; + + let (vd_x, vd_y, vd_w, vd_h) = unsafe { + ( + GetSystemMetrics(SM_XVIRTUALSCREEN), + GetSystemMetrics(SM_YVIRTUALSCREEN), + GetSystemMetrics(SM_CXVIRTUALSCREEN).max(1), + GetSystemMetrics(SM_CYVIRTUALSCREEN).max(1), + ) + }; + let norm = |sx: i32, sy: i32| -> (i32, i32) { + let nx = ((sx - vd_x) as i64 * 65535 / vd_w as i64).clamp(0, 65535) as i32; + let ny = ((sy - vd_y) as i64 * 65535 / vd_h as i64).clamp(0, 65535) as i32; + (nx, ny) + }; + let make_input = |dx: i32, dy: i32, flags| INPUT { + r#type: INPUT_MOUSE, + Anonymous: INPUT_0 { + mi: MOUSEINPUT { + dx, dy, mouseData: 0, + dwFlags: flags | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK, + time: 0, dwExtraInfo: 0, + }, + }, + }; + + let steps = steps.max(1); + let step_delay_ms = if steps > 1 { duration_ms / steps as u64 } else { 0 }; + + unsafe { + let prev_fg = GetForegroundWindow(); + let mut prev_cursor = POINT::default(); + let _ = GetCursorPos(&mut prev_cursor); + + let _ = SetForegroundWindow(target); + sleep(Duration::from_millis(8)); + let actual_fg = GetForegroundWindow(); + if actual_fg != target { + bail!( + "Foreground swap to target HWND {:?} was rejected by Windows \ + (actual foreground is HWND {:?}). Non-UIAccess processes can't \ + reliably change foreground under the foreground-lock. Route the \ + drag through cua-driver-uia.exe.", + target.0, actual_fg.0 + ); + } + + // 1. Move + press at the start of the drag. + let (nfx, nfy) = norm(sx_from, sy_from); + let _ = SetCursorPos(sx_from, sy_from); + let prelude = [ + make_input(nfx, nfy, MOUSEEVENTF_MOVE), + make_input(nfx, nfy, down_flag), + ]; + let sent = SendInput(&prelude, std::mem::size_of::() as i32); + if sent as usize != prelude.len() { + let _ = SetCursorPos(prev_cursor.x, prev_cursor.y); + let _ = SetForegroundWindow(prev_fg); + bail!("SendInput drag-prelude inserted {sent}/{} events", prelude.len()); + } + + // 2. Interpolate the path. SetCursorPos + MOUSEEVENTF_MOVE in lockstep + // so both the visible cursor and the system input queue track the + // same path — WPF's drag-handler watches GetKeyState during each + // move event. + for i in 1..=steps { + let t = i as f64 / steps as f64; + let x = sx_from + ((sx_to - sx_from) as f64 * t).round() as i32; + let y = sy_from + ((sy_to - sy_from) as f64 * t).round() as i32; + let (nx, ny) = norm(x, y); + let _ = SetCursorPos(x, y); + let mv = [make_input(nx, ny, MOUSEEVENTF_MOVE)]; + let _ = SendInput(&mv, std::mem::size_of::() as i32); + if step_delay_ms > 0 { + sleep(Duration::from_millis(step_delay_ms)); + } + } + + // 3. Release at the end. + let (ntx, nty) = norm(sx_to, sy_to); + let release = [make_input(ntx, nty, up_flag)]; + let _ = SendInput(&release, std::mem::size_of::() as i32); + + // Brief settle, then restore previous state. + sleep(Duration::from_millis(40)); + let _ = SetCursorPos(prev_cursor.x, prev_cursor.y); + let _ = SetForegroundWindow(prev_fg); + } + + Ok(()) +} diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs index dd26e7caba..7c9d6094b7 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs @@ -3414,19 +3414,36 @@ impl Tool for DragTool { { return background_unavailable_error(hwnd, EventKind::MouseClick); } - // dispatch:"foreground" — no SendInput-based drag helper yet. The - // PostMessage drag works for the GTK-canvas + plain-Win32 cases we - // care about today; Chromium DOM dragstart will need a real - // SendInput-MOUSE-MOVE sequence, tracked as TODO. + // dispatch:"foreground" — SendInput-based drag. Required for WPF + // Slider thumbs (and any framework that polls GetKeyState during + // its drag handler — PostMessage doesn't update per-thread input + // state, so those targets see a button-up world during the drag + // and never start tracking). Subject to the same UIAccess + // foreground-lock constraint as send_click_synthesized. if dispatch == DispatchMode::Foreground { - return ToolResult::error( - "dispatch:\"foreground\" is not yet implemented for the drag tool. \ - Use bring_to_front to activate the target first, then retry with \ - dispatch:\"auto\" (PostMessage WM_MOUSEMOVE works against an \ - already-foreground window in most cases). TODO: add a \ - send_drag_synthesized helper analogous to send_click_synthesized." - .to_string(), - ); + let btn_fg = button.clone(); + let prev_fg_addr = unsafe { + windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as usize + }; + let send_result = tokio::task::spawn_blocking(move || { + crate::input::mouse::send_drag_synthesized( + hwnd, + sx_from, sy_from, + sx_to, sy_to, + duration_ms, steps, &btn_fg, + ) + }).await; + tokio::spawn(restore_foreground_polling_best_effort(prev_fg_addr, pid)); + let button_suffix = if button == "left" { String::new() } else { format!(" ({} button)", button) }; + return match send_result { + Ok(Ok(())) => ToolResult::text(format!( + "✅ Sent drag{button_suffix} via SendInput on pid {raw_pid} \ + from screen ({sx_from},{sy_from}) → ({sx_to},{sy_to}) \ + in {duration_ms}ms / {steps} steps (dispatch:foreground)." + )), + Ok(Err(e)) => ToolResult::error(e.to_string()), + Err(e) => ToolResult::error(format!("Task error: {e}")), + }; } // Pin the agent-cursor overlay above the drag target so the synthetic diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/page.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/page.rs index 120d6999d3..0481dd8b44 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/tools/page.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/page.rs @@ -166,18 +166,30 @@ impl PageBackend for WindowsPageBackend { // The wrapper paths return the result JSON-stringified. The bookmark // wrapper wraps the user expression's return in JSON.stringify(), - // which means the inner string itself is JSON-encoded — we need to - // parse twice to get the actual coord object. - let parsed: serde_json::Value = serde_json::from_str(&probe_json) - .or_else(|_| { - // Bookmark wrapper double-encoded the string — decode the - // outer JSON-string, then parse the inner JSON. - let inner: String = serde_json::from_str(&probe_json)?; - serde_json::from_str::(&inner) - }) - .map_err(|e| anyhow::anyhow!( - "click_element: could not parse coord JSON from probe (raw: {probe_raw:?}): {e}" - ))?; + // and CDP's runtime.evaluate also serialises the JS return as a + // JSON-string when it's not a primitive — either way the inner + // string itself is JSON-encoded. We need to parse twice to get + // the actual coord object. + // + // Crucially, the first parse can SUCCEED as a Value::String (when + // the outer is a quoted JSON-string), so we can't rely on + // `.or_else(|_| ...)` to trigger the second decode — that branch + // only fires on a hard parse error. Inspect the parsed value and + // re-decode when it's a String. + let parsed: serde_json::Value = { + let first = serde_json::from_str::(&probe_json) + .map_err(|e| anyhow::anyhow!( + "click_element: could not parse coord JSON from probe (raw: {probe_raw:?}): {e}" + ))?; + match first { + serde_json::Value::String(inner) => serde_json::from_str(&inner) + .map_err(|e| anyhow::anyhow!( + "click_element: inner JSON parse failed for double-encoded probe \ + response (raw: {probe_raw:?}): {e}" + ))?, + other => other, + } + }; // Required-field validation. The previous version defaulted any // missing field to 0.0, which silently animated the visible cursor diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs index 39cb9fb9e9..76e9dacf21 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/mod.rs @@ -19,7 +19,7 @@ use windows::Win32::UI::Accessibility::{ UIA_ProcessIdPropertyId, UIA_ValueValuePropertyId, UIA_InvokePatternId, UIA_SelectionItemPatternId, UIA_TogglePatternId, UIA_ExpandCollapsePatternId, UIA_TextPatternId, - UIA_ValuePatternId, UIA_ScrollPatternId, + UIA_ValuePatternId, UIA_RangeValuePatternId, UIA_ScrollPatternId, TreeScope_Children, TreeScope_Subtree, }; @@ -100,6 +100,7 @@ unsafe fn walk_tree_unsafe(hwnd: u64, query: Option<&str>) -> UiaTreeResult { UIA_SelectionItemPatternId, UIA_ExpandCollapsePatternId, UIA_ValuePatternId, + UIA_RangeValuePatternId, UIA_TextPatternId, UIA_ScrollPatternId, ] { @@ -522,6 +523,13 @@ fn detect_cached_actions(element: &IUIAutomationElement, is_enabled: bool) -> Ve if element.GetCachedPattern(UIA_ValuePatternId).is_ok() { actions.push("set_value".into()); } + // RangeValuePattern is exposed by Sliders, ProgressBars, and other + // numeric-range controls. Without this entry the slider parent + // gets actions=[] → marked non-actionable → no `[N]` index in the + // flat tree, making the slider unaddressable by AutomationId. + if element.GetCachedPattern(UIA_RangeValuePatternId).is_ok() { + actions.push("set_value".into()); + } if element.GetCachedPattern(UIA_TextPatternId).is_ok() { actions.push("text".into()); }