diff --git a/.github/workflows/ci-rust-windows.yml b/.github/workflows/ci-rust-windows.yml index facc3f2fa2..cbd7af1138 100644 --- a/.github/workflows/ci-rust-windows.yml +++ b/.github/workflows/ci-rust-windows.yml @@ -10,7 +10,6 @@ on: - "libs/cua-driver/rust/crates/cua-driver-testkit/**" - "libs/cua-driver/rust/crates/platform-windows/**" - "libs/cua-driver/rust/crates/cua-driver-uia/**" - - "libs/cua-driver/rust/crates/focus-monitor-win/**" - "libs/cua-driver/tests/fixtures/**" - ".github/workflows/ci-rust-windows.yml" push: @@ -43,4 +42,4 @@ jobs: working-directory: libs/cua-driver/rust # Compile every Rust target without executing desktop-dependent integration # tests; interactive behavior belongs in e2e-rust-windows.yml. - run: cargo test -p cua-driver -p cua-driver-core -p cua-driver-testkit -p platform-windows -p cua-driver-uia -p focus-monitor-win --all-targets --no-run --locked + run: cargo test -p cua-driver -p cua-driver-core -p cua-driver-testkit -p platform-windows -p cua-driver-uia --all-targets --no-run --locked diff --git a/.github/workflows/e2e-rust-windows.yml b/.github/workflows/e2e-rust-windows.yml index f567bde6de..0047999813 100644 --- a/.github/workflows/e2e-rust-windows.yml +++ b/.github/workflows/e2e-rust-windows.yml @@ -4,15 +4,9 @@ on: workflow_dispatch: inputs: ref: - description: "Commit, branch, or tag to test" - required: true - default: "main" - suite: - description: "Rust desktop suite" - required: true - type: choice - options: [default, guard, shared, native, modality, all] - default: shared + description: "Optional full 40-character commit SHA; defaults to dispatch SHA" + required: false + default: "" runner: description: "Runner label; use the Azure RDP runner label for VM e2e" required: true @@ -23,83 +17,37 @@ permissions: actions: read jobs: - default: - if: inputs.suite == 'default' || inputs.suite == 'all' - name: "Windows / default Rust tests" - runs-on: ${{ inputs.runner }} - timeout-minutes: 90 - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - ref: ${{ inputs.ref }} - - name: Ensure FFmpeg for trajectory video - shell: pwsh - run: | - if (-not (Get-Command ffmpeg.exe -ErrorAction SilentlyContinue)) { - choco install ffmpeg -y --no-progress - } - ffmpeg -version - ffprobe -version - - name: Run default Rust tests - shell: pwsh - run: .\scripts\ci\windows\run-rust-e2e.ps1 -Suite default -RequireGui - - name: Collect logs - if: always() - shell: pwsh - run: .\scripts\ci\windows\collect-artifacts.ps1 - - name: Upload default results - if: always() - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 - with: - name: rust-windows-default - path: artifacts/cua-driver/windows - if-no-files-found: ignore - compression-level: 0 - retention-days: 14 - - guard: - if: inputs.suite == 'guard' || inputs.suite == 'all' - name: "Windows / UX guards" - runs-on: ${{ inputs.runner }} - timeout-minutes: 90 + source: + name: "Resolve exact source" + runs-on: ubuntu-latest + outputs: + sha: ${{ steps.resolve.outputs.sha }} steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - ref: ${{ inputs.ref }} - - name: Ensure FFmpeg for trajectory video - shell: pwsh + - id: resolve + name: Validate source SHA + shell: bash + env: + REQUESTED_SHA: ${{ inputs.ref }} + DISPATCH_SHA: ${{ github.sha }} run: | - if (-not (Get-Command ffmpeg.exe -ErrorAction SilentlyContinue)) { - choco install ffmpeg -y --no-progress - } - ffmpeg -version - ffprobe -version - - name: Run UX guard tests - shell: pwsh - run: .\scripts\ci\windows\run-rust-e2e.ps1 -Suite guard -RequireGui - - name: Collect logs - if: always() - shell: pwsh - run: .\scripts\ci\windows\collect-artifacts.ps1 - - name: Upload guard results - if: always() - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 - with: - name: rust-windows-guard - path: artifacts/cua-driver/windows - if-no-files-found: ignore - compression-level: 0 - retention-days: 14 + sha="${REQUESTED_SHA:-$DISPATCH_SHA}" + if [[ ! "$sha" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "ref must be a full 40-character commit SHA" >&2 + exit 2 + fi + echo "sha=${sha,,}" >> "$GITHUB_OUTPUT" shared: - if: inputs.suite == 'shared' || inputs.suite == 'all' name: "Windows / shared Electron + Tauri" + needs: source runs-on: ${{ inputs.runner }} timeout-minutes: 90 + env: + CUA_E2E_SOURCE_SHA: ${{ needs.source.outputs.sha }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: - ref: ${{ inputs.ref }} + ref: ${{ needs.source.outputs.sha }} - name: Ensure FFmpeg for trajectory video shell: pwsh run: | @@ -110,7 +58,9 @@ jobs: ffprobe -version - name: Run shared Rust behavior matrix shell: pwsh - run: .\scripts\ci\windows\run-rust-e2e.ps1 -Suite shared -RequireGui + env: + CUA_E2E_INTERNAL_LANE: shared + run: .\scripts\ci\windows\run-rust-e2e.ps1 -RequireGui - name: Collect logs if: always() shell: pwsh @@ -126,14 +76,16 @@ jobs: retention-days: 14 native: - if: inputs.suite == 'native' || inputs.suite == 'all' - name: "Windows / native WPF + WebView2" + name: "Windows / native WPF + WinUI3 + WebView2" + needs: source runs-on: ${{ inputs.runner }} timeout-minutes: 90 + env: + CUA_E2E_SOURCE_SHA: ${{ needs.source.outputs.sha }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: - ref: ${{ inputs.ref }} + ref: ${{ needs.source.outputs.sha }} - name: Ensure FFmpeg for trajectory video shell: pwsh run: | @@ -144,7 +96,9 @@ jobs: ffprobe -version - name: Run native Rust harnesses shell: pwsh - run: .\scripts\ci\windows\run-rust-e2e.ps1 -Suite native -RequireGui + env: + CUA_E2E_INTERNAL_LANE: native + run: .\scripts\ci\windows\run-rust-e2e.ps1 -RequireGui - name: Collect logs if: always() shell: pwsh @@ -159,15 +113,17 @@ jobs: compression-level: 0 retention-days: 14 - modality: - if: inputs.suite == 'modality' || inputs.suite == 'all' - name: "Windows / modality input E2E" + capture: + name: "Windows / capture and desktop scope" + needs: source runs-on: ${{ inputs.runner }} timeout-minutes: 90 + env: + CUA_E2E_SOURCE_SHA: ${{ needs.source.outputs.sha }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: - ref: ${{ inputs.ref }} + ref: ${{ needs.source.outputs.sha }} - name: Ensure FFmpeg for trajectory video shell: pwsh run: | @@ -176,18 +132,20 @@ jobs: } ffmpeg -version ffprobe -version - - name: Run modality input E2E + - name: Run capture and desktop-scope contracts shell: pwsh - run: .\scripts\ci\windows\run-rust-e2e.ps1 -Suite modality -RequireGui + env: + CUA_E2E_INTERNAL_LANE: capture + run: .\scripts\ci\windows\run-rust-e2e.ps1 -RequireGui - name: Collect logs if: always() shell: pwsh run: .\scripts\ci\windows\collect-artifacts.ps1 - - name: Upload modality results + - name: Upload capture results if: always() uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 with: - name: rust-windows-modality + name: rust-windows-capture path: artifacts/cua-driver/windows if-no-files-found: ignore compression-level: 0 @@ -195,11 +153,14 @@ jobs: summary: if: always() - needs: [default, guard, shared, native, modality] + needs: [source, shared, native, capture] name: "Windows / matrix summary" runs-on: ubuntu-latest timeout-minutes: 10 steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: ${{ needs.source.outputs.sha }} - name: Download lane results uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 with: @@ -216,19 +177,27 @@ jobs: echo "The lane jobs above are independent; a failure in one lane does not hide the others." echo } > "$summary_path" + artifacts_json=$(gh api \ + "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts?per_page=100") found=0 while IFS= read -r summary; do found=1 - echo "## $(basename "$(dirname "$summary")")" >> "$summary_path" - cat "$summary" >> "$summary_path" + artifact=$(basename "$(dirname "$summary")") + echo "## $artifact" >> "$summary_path" + artifact_id=$(jq -r --arg name "$artifact" \ + '.artifacts[] | select(.name == $name) | .id' <<< "$artifacts_json" | head -n 1) + if [[ -n "$artifact_id" && "$artifact_id" != "null" ]]; then + artifact_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts/$artifact_id" + scripts/ci/link-e2e-evidence.sh "$summary" "$artifact_url" >> "$summary_path" + else + cat "$summary" >> "$summary_path" + fi echo >> "$summary_path" done < <(find artifacts -type f -name summary.md -print | sort) if [[ "$found" == 0 ]]; then echo "No lane summary artifact was produced." >> "$summary_path" fi - artifacts_json=$(gh api \ - "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts?per_page=100") { echo echo "## Trajectory videos" @@ -255,15 +224,13 @@ jobs: fi echo "| $lane | $video_count | $artifact_link |" >> "$summary_path" done <<'EOF' - Default Rust|rust-windows-default - UX guards|rust-windows-guard Electron + Tauri|rust-windows-shared WPF + WinUI3 + WebView2|rust-windows-native - Modality input|rust-windows-modality + Capture + desktop scope|rust-windows-capture EOF { echo - echo "Each artifact stores videos under \`recordings//recording.mp4\` with a matching \`trajectory.json\`." + echo "Each evidence link opens its owning lane artifact; the row text is the exact \`recordings/-pid-/recording.mp4\` path, with an adjacent \`trajectory.json\`." } >> "$summary_path" cat "$summary_path" >> "$GITHUB_STEP_SUMMARY" - name: Upload matrix summary diff --git a/libs/cua-driver/rust/Cargo.lock b/libs/cua-driver/rust/Cargo.lock index 3fa53c2bcc..fda7abab4f 100644 --- a/libs/cua-driver/rust/Cargo.lock +++ b/libs/cua-driver/rust/Cargo.lock @@ -942,13 +942,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "focus-monitor-win" -version = "0.7.1" -dependencies = [ - "windows 0.58.0", -] - [[package]] name = "foldhash" version = "0.1.5" diff --git a/libs/cua-driver/rust/Cargo.toml b/libs/cua-driver/rust/Cargo.toml index 72419af9f5..a1aead5b1b 100644 --- a/libs/cua-driver/rust/Cargo.toml +++ b/libs/cua-driver/rust/Cargo.toml @@ -9,7 +9,6 @@ members = [ "crates/platform-windows", "crates/platform-linux", "crates/cursor-overlay", - "crates/focus-monitor-win", "crates/pip-preview", ] diff --git a/libs/cua-driver/rust/README.md b/libs/cua-driver/rust/README.md index 11570f7a52..96b339b73b 100644 --- a/libs/cua-driver/rust/README.md +++ b/libs/cua-driver/rust/README.md @@ -14,7 +14,6 @@ implementations, testkit, and helper crates. | `platform-linux` | Linux AT-SPI, X11/Wayland, capture, and input support | | `cua-driver-testkit` | Test-only helpers for spawning the daemon and parsing responses | | `cua-driver-uia` | Windows UIAccess worker | -| `focus-monitor-win` | Windows focus sentinel used by UX guard tests | | `cursor-overlay` | Cursor overlay support | | `pip-preview` | Packaging preview helper | diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs index f6d47c7c10..d47fdc136d 100644 --- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs +++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs @@ -1387,8 +1387,29 @@ fn validate_one_turn(turn: &Path, cell_id: &str, errors: &mut Vec) { .as_ref() .and_then(|value| value[phase][kind]["classification"].as_str()) }; + let tool = action.as_ref().and_then(|value| value["tool"].as_str()); + let successful_restore = action.as_ref().is_some_and(|value| { + value["result_summary"] + .as_str() + .is_some_and(|summary| summary.starts_with("✅ bring_to_front:")) + }); + let restored_state_captured = manifest.as_ref().is_some_and(|value| { + value["after"]["state"]["status"].as_str() == Some("captured") + }); + let expected_unavailable_screenshot = |phase: &str| { + tool == Some("bring_to_front") + && ((phase == "before" + && classification(phase, "screenshot") == Some("target_minimized")) + || (phase == "after" + && successful_restore + && restored_state_captured + && classification(phase, "screenshot") == Some("capture_failed"))) + }; for (phase, kind) in [("before", "screenshot"), ("after", "screenshot")] { + if expected_unavailable_screenshot(phase) { + continue; + } validate_capture_status( manifest.as_ref(), &[phase, kind], @@ -1403,6 +1424,9 @@ fn validate_one_turn(turn: &Path, cell_id: &str, errors: &mut Vec) { ("after.png", "after", "screenshot"), ("screenshot.png", "after", "screenshot"), ] { + if expected_unavailable_screenshot(phase) { + continue; + } validate_nonempty_file( &turn.join(file), cell_id, @@ -1912,6 +1936,99 @@ mod tests { })); } + #[test] + fn validator_accepts_minimized_preimage_for_restore_action() { + let (root, case, result, turn) = complete_turn_fixture(); + std::fs::write( + turn.join("action.json"), + br#"{ + "tool":"bring_to_front", + "arguments":{"pid":1,"window_id":2} + }"#, + ) + .expect("write restore action"); + std::fs::write( + turn.join("evidence.json"), + br#"{ + "schema":"cua-turn-evidence/v1", + "before":{"state":{"status":"captured"},"screenshot":{"status":"unavailable","classification":"target_minimized"}}, + "after":{"state":{"status":"captured"},"screenshot":{"status":"captured"}}, + "click":{"status":"not_applicable","classification":"no_target_pid"} + }"#, + ) + .expect("write restore evidence manifest"); + std::fs::remove_file(turn.join("before.png")).expect("remove unavailable preimage"); + + validate_catalog(&[case], &[result], Some(root.path()), true) + .expect("a minimized target cannot provide a pre-restore screenshot"); + } + + #[test] + fn validator_accepts_restored_state_when_host_capture_remains_unavailable() { + let (root, case, result, turn) = complete_turn_fixture(); + std::fs::write( + turn.join("action.json"), + r#"{ + "tool":"bring_to_front", + "arguments":{"pid":1,"window_id":2}, + "result_summary":"✅ bring_to_front: pid 1 hwnd 0x2 is now foreground (was 0x1)." + }"# + .as_bytes(), + ) + .expect("write successful restore action"); + std::fs::write( + turn.join("evidence.json"), + br#"{ + "schema":"cua-turn-evidence/v1", + "before":{"state":{"status":"captured"},"screenshot":{"status":"unavailable","classification":"target_minimized"}}, + "after":{"state":{"status":"captured"},"screenshot":{"status":"unavailable","classification":"capture_failed"}}, + "click":{"status":"not_applicable","classification":"no_target_pid"} + }"#, + ) + .expect("write restored-state evidence manifest"); + std::fs::remove_file(turn.join("before.png")).expect("remove unavailable preimage"); + std::fs::remove_file(turn.join("after.png")).expect("remove unavailable after-image"); + std::fs::remove_file(turn.join("screenshot.png")) + .expect("remove unavailable screenshot alias"); + + validate_catalog(&[case], &[result], Some(root.path()), true) + .expect("a successful restore remains valid with captured UIA state and video"); + } + + #[test] + fn validator_rejects_missing_after_image_when_restore_did_not_succeed() { + let (root, case, result, turn) = complete_turn_fixture(); + std::fs::write( + turn.join("action.json"), + br#"{ + "tool":"bring_to_front", + "arguments":{"pid":1,"window_id":2}, + "result_summary":"bring_to_front: restore request failed" + }"#, + ) + .expect("write failed restore action"); + std::fs::write( + turn.join("evidence.json"), + br#"{ + "schema":"cua-turn-evidence/v1", + "before":{"state":{"status":"captured"},"screenshot":{"status":"unavailable","classification":"target_minimized"}}, + "after":{"state":{"status":"captured"},"screenshot":{"status":"unavailable","classification":"capture_failed"}}, + "click":{"status":"not_applicable","classification":"no_target_pid"} + }"#, + ) + .expect("write failed restore evidence manifest"); + std::fs::remove_file(turn.join("before.png")).expect("remove unavailable preimage"); + std::fs::remove_file(turn.join("after.png")).expect("remove unavailable after-image"); + std::fs::remove_file(turn.join("screenshot.png")) + .expect("remove unavailable screenshot alias"); + + let errors = validate_catalog(&[case], &[result], Some(root.path()), true) + .expect_err("an unsuccessful restore must still fail closed"); + assert!(errors + .iter() + .any(|error| error.contains("turn-00001/after.png"))); + } + #[test] fn non_strict_validation_keeps_legacy_results_compatible() { let case = delivered_case("legacy"); diff --git a/libs/cua-driver/rust/crates/cua-driver/src/main.rs b/libs/cua-driver/rust/crates/cua-driver/src/main.rs index 54d5c56329..b44946646e 100644 --- a/libs/cua-driver/rust/crates/cua-driver/src/main.rs +++ b/libs/cua-driver/rust/crates/cua-driver/src/main.rs @@ -753,17 +753,8 @@ fn build_registry(cursor_cfg: cursor_overlay::CursorConfig) -> cua_driver_core:: let compat = CLAUDE_CODE_COMPAT.load(Ordering::SeqCst); #[cfg(target_os = "windows")] { - cua_driver_core::recording::set_screenshot_fn(|window_id, pid| { - if let Some(hwnd) = window_id { - platform_windows::capture::screenshot_window_bytes(hwnd).ok() - } else if let Some(p) = pid { - let wins = platform_windows::win32::list_windows(Some(p as u32)); - wins.first().and_then(|w| { - platform_windows::capture::screenshot_window_bytes(w.hwnd).ok() - }) - } else { - platform_windows::capture::screenshot_display_bytes().ok() - } + cua_driver_core::recording::set_classified_screenshot_fn(|window_id, pid| { + platform_windows::recording_hooks::screenshot_for_recording(window_id, pid) }); cua_driver_core::recording::set_click_marker_fn(|png_bytes, cx, cy| { platform_windows::capture::crosshair_png_bytes(png_bytes, cx, cy).ok() @@ -834,17 +825,8 @@ fn build_registry_no_cursor() -> cua_driver_core::tool::ToolRegistry { let compat = CLAUDE_CODE_COMPAT.load(Ordering::SeqCst); #[cfg(target_os = "windows")] { - cua_driver_core::recording::set_screenshot_fn(|window_id, pid| { - if let Some(hwnd) = window_id { - platform_windows::capture::screenshot_window_bytes(hwnd).ok() - } else if let Some(p) = pid { - let wins = platform_windows::win32::list_windows(Some(p as u32)); - wins.first().and_then(|w| { - platform_windows::capture::screenshot_window_bytes(w.hwnd).ok() - }) - } else { - platform_windows::capture::screenshot_display_bytes().ok() - } + cua_driver_core::recording::set_classified_screenshot_fn(|window_id, pid| { + platform_windows::recording_hooks::screenshot_for_recording(window_id, pid) }); cua_driver_core::recording::set_click_marker_fn(|png_bytes, cx, cy| { platform_windows::capture::crosshair_png_bytes(png_bytes, cx, cy).ok() diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/agent_cursor_windows_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/agent_cursor_windows_test.rs new file mode 100644 index 0000000000..1e68e456cb --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver/tests/agent_cursor_windows_test.rs @@ -0,0 +1,143 @@ +//! Windows agent-cursor rendering and desktop-side-effect contract. + +#![cfg(target_os = "windows")] + +use std::time::Duration; + +use cua_driver_testkit::e2e::{ + execute_case, recording_evidence, CaseSpec, Delivery, DriverRoute, Evidence, Observation, + OracleKind, Scope, Targeting, +}; +use cua_driver_testkit::sentinel::ForegroundSentinel; +use cua_driver_testkit::{Driver, McpDriver}; + +#[test] +#[ignore] +fn agent_cursor_overlay_is_visible_without_moving_real_cursor() { + let case = CaseSpec::delivered( + "windows-desktop-agent-cursor-px", + "desktop", + "win32", + "agent_cursor", + Targeting::Px, + Delivery::NotApplicable, + Scope::Desktop, + DriverRoute::WindowsOverlay, + vec![ + OracleKind::Pixels, + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::Cursor, + OracleKind::NoLeakedInput, + ], + ); + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_named("windows-desktop-agent-cursor-px") + .expect("required source-built driver did not start"); + *evidence = recording_evidence(driver.recording_dir()); + let sentinel = ForegroundSentinel::launch(&mut driver); + driver.start_behavior_recording(); + let screen = driver.call("get_screen_size", serde_json::json!({})); + assert!( + !screen.is_error(), + "get_screen_size failed: {}", + screen.text() + ); + let width = screen.structured()["width"].as_f64().unwrap_or(0.0); + let height = screen.structured()["height"].as_f64().unwrap_or(0.0); + assert!( + width >= 80.0 && height >= 80.0, + "invalid screen size {width}x{height}" + ); + let (x, y) = (width / 2.0, height / 2.0); + let cursor_id = "windows-agent-cursor-e2e"; + + let (_, mut passed) = sentinel + .observe_desktop(|| { + for (tool, arguments) in [ + ( + "set_agent_cursor_enabled", + serde_json::json!({"enabled": true, "cursor_id": cursor_id}), + ), + ( + "set_agent_cursor_motion", + serde_json::json!({ + "cursor_id": cursor_id, + "glide_duration_ms": 100, + "idle_hide_ms": 0 + }), + ), + ( + "move_cursor", + serde_json::json!({"x": x, "y": y, "cursor_id": cursor_id}), + ), + ] { + let response = driver.call(tool, arguments); + assert!(!response.is_error(), "{tool} failed: {}", response.text()); + } + std::thread::sleep(Duration::from_millis(350)); + }) + .unwrap_or_else(|error| panic!("agent cursor disturbed the real desktop: {error}")); + assert_required_background_oracles(&passed); + + let png = platform_windows::capture::screenshot_display_bytes() + .expect("screenshot_display_bytes failed"); + let image = image::load_from_memory(&png) + .expect("decode display screenshot") + .to_rgba8(); + let (image_width, image_height) = image.dimensions(); + let half = 20u32; + let center_x = x + .round() + .clamp(0.0, f64::from(image_width.saturating_sub(1))) as u32; + let center_y = y + .round() + .clamp(0.0, f64::from(image_height.saturating_sub(1))) as u32; + let x0 = center_x.saturating_sub(half); + let x1 = center_x.saturating_add(half).min(image_width); + let y0 = center_y.saturating_sub(half); + let y1 = center_y.saturating_add(half).min(image_height); + let visible_pixels = (y0..y1) + .flat_map(|pixel_y| (x0..x1).map(move |pixel_x| (pixel_x, pixel_y))) + .filter(|(pixel_x, pixel_y)| { + let [red, green, blue, alpha] = image.get_pixel(*pixel_x, *pixel_y).0; + if alpha < 10 { + return false; + } + let brightness = u32::from(red) + u32::from(green) + u32::from(blue); + let saturation = u32::from(red.max(green).max(blue) - red.min(green).min(blue)); + brightness > 60 && (saturation > 30 || brightness > 600) + }) + .count(); + assert!( + visible_pixels >= 5, + "agent cursor not visible at ({x:.0},{y:.0}): only {visible_pixels} qualifying pixels" + ); + + let disabled = driver.call( + "set_agent_cursor_enabled", + serde_json::json!({"enabled": false, "cursor_id": cursor_id}), + ); + assert!( + !disabled.is_error(), + "failed to disable agent cursor: {}", + disabled.text() + ); + passed.push(OracleKind::Pixels); + Observation::delivered(passed, Evidence::default()) + }); +} + +fn assert_required_background_oracles(passed: &[OracleKind]) { + for required in [ + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::Cursor, + OracleKind::NoLeakedInput, + ] { + assert!( + passed.contains(&required), + "agent cursor test omitted required {required:?} oracle" + ); + } +} diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_windows_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_windows_test.rs new file mode 100644 index 0000000000..07d9d7de9f --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_windows_test.rs @@ -0,0 +1,305 @@ +//! Harness integration test for the **desktop-scope** modality (#1968 / #2019). +//! +//! Desktop-scope is cua-driver's *foreground*, vision-only, **screen-absolute** +//! loop (the "Computer-Use 1.0" mode), the complement to the default per-window +//! background model that `harness_bg_modality_test` / `e2e_windows_bg_input_test` +//! cover. This test exercises the Windows Phase-1 actuator end-to-end against a +//! real harness app: +//! +//! 1. `set_config capture_scope=desktop` → `get_desktop_state` returns a +//! full-display capture with true `screen_width/height` (no downscale). +//! 2. A **window-less** screen-absolute `click` / `scroll` (no pid/window_id) +//! lands via `WindowFromPoint` while in desktop scope. +//! 3. Negative gate: the same window-less `click` under `capture_scope=window` +//! is rejected with the structured `desktop_scope_disabled` error. +//! +//! Note: `set_config` is a *session* override — it persists for the lifetime of +//! the one MCP server we spawn here (not across separate `cua-driver call` +//! processes), which is exactly why this test drives a single long-lived server. +//! +//! All tests are `#[ignore]` (need a real desktop session). Run explicitly: +//! cargo test -p cua-driver --test desktop_scope_windows_test -- --ignored --nocapture --test-threads=1 + +#![cfg(target_os = "windows")] + +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +use cua_driver_testkit::ax; +use cua_driver_testkit::e2e::{ + execute_case, recording_evidence, CaseSpec, Delivery, DriverRoute, Evidence, Observation, + OracleKind, Scope, Targeting, +}; +use cua_driver_testkit::{harness_app, Driver, McpDriver}; + +/// WPF harness app (built by `tests/fixtures/build/windows.ps1`). Path mirrors +/// `shared/scenarios.json`'s `wpf.exe_relative_path`. +fn harness_wpf_exe() -> std::path::PathBuf { + harness_app("harness-wpf", "CuaTestHarness.Wpf.exe") +} + +/// Launch the WPF harness app and return its pid and native window id. +/// Skips (returns None) if the harness app isn't built. +fn launch_wpf(driver: &mut McpDriver) -> Option<(u32, u64)> { + let exe = harness_wpf_exe(); + if !exe.exists() { + if std::env::var_os("CUA_TEST_REQUIRE_FIXTURES").is_some() { + panic!("required WPF harness is missing at {exe:?}"); + } + eprintln!("[desktop-scope] WPF harness not built ({exe:?}) — skipping window-target tests"); + return None; + } + let launched = driver.reaper().spawn( + Command::new(&exe) + .stdout(Stdio::null()) + .stderr(Stdio::null()), + ); + if let Err(error) = launched { + if std::env::var_os("CUA_TEST_REQUIRE_FIXTURES").is_some() { + panic!("failed to launch required WPF harness {exe:?}: {error}"); + } + eprintln!("[desktop-scope] WPF harness launch failed: {error}; skipping"); + return None; + } + + let deadline = Instant::now() + Duration::from_secs(15); + while Instant::now() < deadline { + let r = driver.call("list_windows", serde_json::json!({})); + if let Some(arr) = r.structured()["windows"].as_array() { + for w in arr { + let title = w["title"].as_str().unwrap_or(""); + if !title.contains("CuaTestHarness") { + continue; + } + let pid = w["pid"].as_u64().unwrap_or(0) as u32; + let wid = w["window_id"].as_u64().unwrap_or(0); + if pid != 0 && wid != 0 { + driver.reaper().track_pid(pid); + return Some((pid, wid)); + } + } + } + std::thread::sleep(Duration::from_millis(500)); + } + if std::env::var_os("CUA_TEST_REQUIRE_FIXTURES").is_some() { + panic!("required WPF harness window never appeared"); + } + eprintln!("[desktop-scope] WPF harness window never appeared — skipping"); + None +} + +fn snapshot(driver: &mut McpDriver, pid: u32, wid: u64) -> cua_driver_testkit::ToolResponse { + driver.call( + "get_window_state", + serde_json::json!({ "pid": pid as i64, "window_id": wid, "capture_mode": "ax" }), + ) +} + +fn element_center(state: &cua_driver_testkit::ToolResponse, id: &str) -> (i64, i64) { + let index = ax::element_index_by_id(state.tree_text(), id) + .unwrap_or_else(|| panic!("missing WPF element {id:?}: {}", state.tree_text())); + let element = state.structured()["elements"] + .as_array() + .and_then(|elements| { + elements + .iter() + .find(|element| element["element_index"].as_u64() == Some(index)) + }) + .unwrap_or_else(|| panic!("WPF element {id:?} has no structured frame")); + let frame = &element["frame"]; + ( + (frame["x"].as_f64().expect("frame x") + frame["w"].as_f64().expect("frame w") / 2.0) + as i64, + (frame["y"].as_f64().expect("frame y") + frame["h"].as_f64().expect("frame h") / 2.0) + as i64, + ) +} + +fn set_scope(driver: &mut McpDriver, scope: &str) { + let r = driver.call( + "set_config", + serde_json::json!({ "key": "capture_scope", "value": scope }), + ); + assert!( + !r.is_error(), + "set_config capture_scope={scope} failed: {}", + r.text() + ); + assert_eq!( + r.structured()["capture_scope"].as_str(), + Some(scope), + "set_config did not report capture_scope={scope}: {}", + r.text() + ); +} + +fn run_desktop_fixture_case( + action: &str, + route: DriverRoute, + test: impl FnOnce(u32, u64, &mut McpDriver), +) { + let cell_id = format!("windows-wpf-desktop-{action}-px-foreground").replace('_', "-"); + let case = CaseSpec::delivered( + cell_id.clone(), + "wpf", + "wpf", + action, + Targeting::Px, + Delivery::Foreground, + Scope::Desktop, + route, + vec![OracleKind::FixtureState], + ); + execute_case(case, |evidence| { + let mut driver = + McpDriver::spawn_named(&cell_id).expect("required source-built driver did not start"); + *evidence = recording_evidence(driver.recording_dir()); + let (pid, wid) = launch_wpf(&mut driver).expect("required WPF harness did not launch"); + set_scope(&mut driver, "desktop"); + let posture = driver.call( + "bring_to_front", + serde_json::json!({"pid": pid as i64, "window_id": wid}), + ); + assert!(!posture.is_error(), "could not foreground WPF fixture: {}", posture.text()); + std::thread::sleep(Duration::from_millis(300)); + driver.start_behavior_recording(); + test(pid, wid, &mut driver); + Observation::delivered_with_fixture_state(Vec::new()) + }); +} + +// ── tests ───────────────────────────────────────────────────────────────────── + +/// `get_desktop_state` in desktop scope returns a full-display capture with +/// real screen dimensions (the Session-0 `handle is invalid` case is only the +/// service-session wall; this needs a real interactive desktop). +#[test] +#[ignore] +fn desktop_scope_capture_returns_screen_dims() { + let case = CaseSpec::delivered( + "windows-desktop-state-px-not-applicable", + "desktop", + "win32", + "get_desktop_state", + Targeting::Px, + Delivery::NotApplicable, + Scope::Desktop, + DriverRoute::WindowState, + vec![OracleKind::Pixels], + ); + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_named("windows-desktop-state-px-not-applicable") + .expect("required source-built driver did not start"); + *evidence = recording_evidence(driver.recording_dir()); + set_scope(&mut driver, "desktop"); + driver.start_behavior_recording(); + let response = driver.call("get_desktop_state", serde_json::json!({})); + assert!( + !response.is_error(), + "get_desktop_state errored: {}", + response.text() + ); + let width = response.structured()["screen_width"].as_u64().unwrap_or(0); + let height = response.structured()["screen_height"].as_u64().unwrap_or(0); + assert!( + width > 0 && height > 0, + "get_desktop_state returned no screen size" + ); + Observation::delivered(vec![OracleKind::Pixels], Evidence::default()) + }); +} + +/// In desktop scope, a window-less screen-absolute click + scroll succeed and +/// resolve a real window via WindowFromPoint (no pid/window_id supplied). +#[test] +#[ignore] +fn desktop_scope_windowless_click_lands_on_control() { + run_desktop_fixture_case( + "left_click", + DriverRoute::WindowsSendInput, + |pid, wid, driver| { + let pre = snapshot(driver, pid, wid); + let (x, y) = element_center(&pre, "border-click-target"); + let response = driver.call("click", serde_json::json!({ "x": x, "y": y })); + assert!( + !response.is_error(), + "desktop click failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(400)); + let after = snapshot(driver, pid, wid); + assert!( + after.tree_text().contains("last_action=left_click"), + "desktop click did not update the WPF click oracle: {}", + after.tree_text() + ); + }, + ); +} + +#[test] +#[ignore] +fn desktop_scope_windowless_scroll_lands_on_control() { + run_desktop_fixture_case( + "scroll", + DriverRoute::WindowsSendInput, + |pid, wid, driver| { + let pre = snapshot(driver, pid, wid); + // The fixture's outer ScrollViewer is visible while the nested + // scroll-tall region begins below a 768px CI desktop. Wheel over a + // visible child and verify the outer viewport moved by observing a + // lower AX element's fresh screen coordinate. + let (x, y) = element_center(&pre, "border-click-target"); + let (_, before_y) = element_center(&pre, "btn-increment"); + let response = driver.call( + "scroll", + serde_json::json!({ "x": x, "y": y, "direction": "down", "amount": 5 }), + ); + assert!( + !response.is_error(), + "desktop scroll failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(500)); + let after = snapshot(driver, pid, wid); + let (_, after_y) = element_center(&after, "btn-increment"); + assert!( + after_y < before_y, + "desktop scroll did not move the WPF outer viewport: before_y={before_y}, after_y={after_y}" + ); + }, + ); +} + +/// Negative gate: a window-less screen-absolute click under `capture_scope=window` +/// must be rejected (the `desktop_scope_disabled` contract), not silently retargeted. +#[test] +#[ignore] +fn window_scope_rejects_windowless_click() { + let case = CaseSpec::delivered( + "windows-window-scope-gate-px-not-applicable", + "desktop", + "win32", + "window_scope_gate", + Targeting::Px, + Delivery::NotApplicable, + Scope::Window, + DriverRoute::Composite, + vec![OracleKind::Protocol], + ); + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_named("windows-window-scope-gate-px-not-applicable") + .expect("required source-built driver did not start"); + *evidence = recording_evidence(driver.recording_dir()); + set_scope(&mut driver, "window"); + driver.start_behavior_recording(); + let response = driver.call("click", serde_json::json!({ "x": 100, "y": 100 })); + assert!( + response.is_error() + && response.structured()["code"].as_str() == Some("desktop_scope_disabled"), + "window-scope window-less click was not rejected: {}", + response.text() + ); + Observation::delivered(vec![OracleKind::Protocol], Evidence::default()) + }); +} diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/guard_ux_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/guard_ux_test.rs deleted file mode 100644 index c2264a9590..0000000000 --- a/libs/cua-driver/rust/crates/cua-driver/tests/guard_ux_test.rs +++ /dev/null @@ -1,730 +0,0 @@ -//! UX-guard integration tests for Windows. -//! -//! These cover the UX-guard scenarios previously exercised by the legacy -//! Python suite: -//! - background focus preservation -//! - new-window click delivery -//! - visible app launch -//! - background menu shortcuts -//! -//! Invariant under test (the "UX guard"): -//! The agent must be able to click, type, and launch apps in background -//! windows WITHOUT stealing focus from the user's foreground window. -//! -//! Background target: the repo-local Electron harness staged at -//! test-apps/harness-electron/CuaTestHarness.Electron.exe. Notepad is used -//! only as a secondary fallback check. -//! -//! Background-action tests launch the target first, then foreground -//! focus-monitor-win (the "user's foreground window") before measuring focus -//! loss. The launch_app test starts the monitor first because launch behavior is -//! the action under test. -//! -//! Run in sandbox via: -//! ..\tests\runners\windows-sandbox\run-tests-in-sandbox.ps1 guard_ux - -#![cfg(target_os = "windows")] - -use std::collections::HashSet; -use std::path::PathBuf; -use std::process::{Child, Command, Stdio}; -use std::time::{Duration, Instant}; - -use cua_driver_testkit::{ax, driver_binary, spawn_in_job, workspace_root, Driver, McpDriver}; - -// ── focus-monitor + test-app fixtures ──────────────────────────────────────── - -fn gui_required() -> bool { - std::env::var("CUA_REQUIRE_GUI") - .ok() - .map(|v| matches!(v.to_ascii_lowercase().as_str(), "1" | "true" | "yes" | "on")) - .unwrap_or(false) -} - -fn skip_desktop(context: &str, reason: String) -> bool { - let msg = format!("{context}: {reason}; skipping GUI UX guard"); - if gui_required() { - panic!("{msg}"); - } - eprintln!("{msg}"); - false -} - -fn require_seedable_desktop(context: &str) -> bool { - let state = platform_windows::diagnostics::desktop_state(); - if state.session_id == Some(0) { - return skip_desktop( - context, - format!( - "running in Windows Session 0 ({}) - re-run from RDP/console/scheduled task in user session", - state.summary() - ), - ); - } - if !state.has_process_window_station { - return skip_desktop( - context, - format!("no attached process window station ({})", state.summary()), - ); - } - if !state.input_desktop_is_default() { - return skip_desktop( - context, - format!( - "input desktop is not the user Default desktop ({})", - state.summary() - ), - ); - } - true -} - -fn require_focus_monitor_foreground(context: &str, expected_hwnd: u64) -> bool { - let deadline = std::time::Instant::now() + Duration::from_secs(2); - loop { - let state = platform_windows::diagnostics::desktop_state(); - if state.foreground_hwnd == Some(expected_hwnd as usize) { - return true; - } - if std::time::Instant::now() >= deadline { - return skip_desktop( - context, - format!( - "focus monitor did not become foreground (expected HWND 0x{expected_hwnd:x}; {})", - state.summary() - ), - ); - } - std::thread::sleep(Duration::from_millis(100)); - } -} - -fn focus_monitor_path() -> PathBuf { - workspace_root().join("target/debug/focus-monitor-win.exe") -} - -fn test_app_path() -> PathBuf { - // In sandbox, sandbox-runner.ps1 copies the exe to %TEMP% to avoid the - // ShellExecuteW zone-security dialog that blocks on mapped-folder exes. - if let Ok(p) = std::env::var("HARNESS_ELECTRON_EXE") { - let pb = PathBuf::from(p); - if pb.exists() { - return pb; - } - } - workspace_root().join("test-apps/harness-electron/CuaTestHarness.Electron.exe") -} - -/// Launch the Electron harness in the background (tied to the driver's reaper) -/// and return its pid. Returns None if the binary doesn't exist or the app -/// fails to start. -fn launch_test_app(driver: &mut McpDriver) -> Option { - let exe = test_app_path(); - if !exe.exists() { - eprintln!("CuaTestHarness.Electron not found at {exe:?} - skipping"); - return None; - } - let child = spawn_in_job( - Command::new(&exe) - .stdout(Stdio::null()) - .stderr(Stdio::null()), - ) - .ok()?; - let pid = child.id(); - driver.reaper().push(child); - // Cold-start in sandbox can take a few seconds. - std::thread::sleep(Duration::from_secs(3)); - Some(pid) -} - -fn launch_driver_and_test_app() -> Option<(McpDriver, u32, u64)> { - let Some(mut driver) = McpDriver::spawn() else { - return None; - }; - - let Some(app_pid) = launch_test_app(&mut driver) else { - eprintln!("test app not available — skipping"); - return None; - }; - let Some((window_pid, app_wid)) = find_window_for_pid(&mut driver, app_pid as i64) else { - eprintln!("test app window not found — skipping"); - return None; - }; - - // Electron may create the visible window in a Chromium child process. - // Actions must use the PID that owns the HWND, not the launcher parent. - Some((driver, window_pid, app_wid)) -} - -fn kill_process_tree_by_image(image: &str) { - Command::new("taskkill") - .args(["/F", "/T", "/IM", image]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .ok(); -} - -struct KillProcessTreeOnDrop(&'static str); - -impl Drop for KillProcessTreeOnDrop { - fn drop(&mut self) { - kill_process_tree_by_image(self.0); - } -} - -fn loss_file() -> PathBuf { - std::env::temp_dir().join("focus_monitor_losses.txt") -} -fn key_loss_file() -> PathBuf { - std::env::temp_dir().join("focus_monitor_key_losses.txt") -} - -fn read_losses(path: &std::path::Path) -> u32 { - std::fs::read_to_string(path) - .ok() - .and_then(|s| s.trim().parse().ok()) - .unwrap_or(0) -} - -fn focus_pid_file() -> PathBuf { - std::env::temp_dir().join("focus_monitor_pid.txt") -} -fn focus_hwnd_file() -> PathBuf { - std::env::temp_dir().join("focus_monitor_hwnd.txt") -} - -/// Launch focus-monitor-win and return (process, hwnd, pid). -/// Reads FOCUS_PID and FOCUS_HWND from temp files written by the monitor -/// (avoids blocking on the stdout pipe if the sandbox redirects I/O). -fn launch_focus_monitor() -> Option<(Child, u64, u32)> { - if !require_seedable_desktop("focus-monitor-win") { - return None; - } - - let exe = focus_monitor_path(); - if !exe.exists() { - eprintln!("focus-monitor-win.exe not built at {exe:?} — skipping"); - return None; - } - // Reset all sentinel files so stale values are not mistaken for new ones. - let _ = std::fs::write(loss_file(), "0"); - let _ = std::fs::write(key_loss_file(), "0"); - let _ = std::fs::remove_file(focus_pid_file()); - let _ = std::fs::remove_file(focus_hwnd_file()); - - let mut child = spawn_in_job( - Command::new(&exe) - .stdout(Stdio::null()) - .stderr(Stdio::null()), - ) - .expect("spawn focus-monitor-win"); - - // Poll temp files until both PID and HWND are written (max 15s). - let deadline = std::time::Instant::now() + Duration::from_secs(15); - let (mut pid_val, mut hwnd_val) = (0u32, 0u64); - loop { - if std::time::Instant::now() > deadline { - panic!("focus-monitor-win did not write PID/HWND temp files within 15s"); - } - if pid_val == 0 { - pid_val = std::fs::read_to_string(focus_pid_file()) - .ok() - .and_then(|s| s.trim().parse().ok()) - .unwrap_or(0); - } - if hwnd_val == 0 { - hwnd_val = std::fs::read_to_string(focus_hwnd_file()) - .ok() - .and_then(|s| s.trim().parse().ok()) - .unwrap_or(0); - } - if pid_val != 0 && hwnd_val != 0 { - break; - } - std::thread::sleep(Duration::from_millis(100)); - } - if !require_focus_monitor_foreground("focus-monitor-win", hwnd_val) { - child.kill().ok(); - return None; - } - Some((child, hwnd_val, pid_val)) -} - -/// Find the first on-screen window belonging to the given pid. -fn find_window_for_pid(driver: &mut McpDriver, pid: i64) -> Option<(u32, u64)> { - let resp = driver.call( - "list_windows", - serde_json::json!({"pid": pid, "on_screen_only": true}), - ); - resp.structured()["windows"] - .as_array()? - .iter() - .find_map(|w| Some((w["pid"].as_u64()? as u32, w["window_id"].as_u64()?))) -} - -fn window_ids(driver: &mut McpDriver) -> HashSet { - let resp = driver.call("list_windows", serde_json::json!({})); - resp.structured()["windows"] - .as_array() - .map(|a| a.iter().filter_map(|w| w["window_id"].as_u64()).collect()) - .unwrap_or_default() -} - -fn wait_for_new_window(driver: &mut McpDriver, before: &HashSet) -> bool { - let deadline = Instant::now() + Duration::from_secs(3); - while Instant::now() < deadline { - let after = window_ids(driver); - if after.iter().any(|id| !before.contains(id)) { - return true; - } - std::thread::sleep(Duration::from_millis(100)); - } - false -} - -// ── UX guard assertion ──────────────────────────────────────────────────────── - -/// Assert act_losses stayed at `max_allowed` (usually 0) since `before`. -fn assert_ux_guard(before: u32, max_allowed: u32, context: &str) { - let after = read_losses(&loss_file()); - let delta = after.saturating_sub(before); - assert!( - delta <= max_allowed, - "UX guard violated: act_losses went from {before} to {after} \ - (delta={delta}, max_allowed={max_allowed}) during: {context}" - ); -} - -// ───────────────────────────────────────────────────────────────────────────── -// Test 1: background click + type do not steal focus -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn test_background_click_and_type_no_focus_steal() { - //! Background focus-preservation coverage. - //! - //! 1. Launch the Electron harness. - //! 2. Foreground FocusMonitorWin (simulates the user's active window). - //! 3. Click inside the app and type text via cua-driver. - //! 4. Assert act_losses on FocusMonitorWin stayed at 0. - - if !driver_binary().exists() { - eprintln!("Binary not found — skipping"); - return; - } - - let Some((mut driver, app_pid, app_wid)) = launch_driver_and_test_app() else { - return; - }; - - let Some((mut fm_proc, _fm_hwnd, _fm_pid)) = launch_focus_monitor() else { - return; - }; - let losses_before = read_losses(&loss_file()); - - // Click inside the app (background, via PostMessage). - let r = driver.call( - "click", - serde_json::json!({"pid": app_pid, "window_id": app_wid, "x": 200.0, "y": 200.0}), - ); - assert!( - r.raw["error"].is_null(), - "Protocol error from click: {:?}", - r.raw - ); - - // Type text into the app (background, via PostMessage). - let r = driver.call( - "type_text", - serde_json::json!({"pid": app_pid, "window_id": app_wid, "text": "ux-guard-test"}), - ); - assert!( - r.raw["error"].is_null(), - "Protocol error from type_text: {:?}", - r.raw - ); - assert_eq!( - r.verified(), - Some(false), - "background type_text without an element read-back must not report confirmed success: {}", - r.text() - ); - assert_ne!( - r.structured()["verify"].as_str(), - Some("confirmed"), - "background type_text reported a confirmed read-back on an unreadable path: {}", - r.text() - ); - - // Plain background key dispatch is likewise an unverified PostMessage send, - // not a confirmed keypress. - let r = driver.call( - "press_key", - serde_json::json!({"pid": app_pid, "window_id": app_wid, "key": "F24"}), - ); - assert!( - r.raw["error"].is_null(), - "Protocol error from press_key: {:?}", - r.raw - ); - assert_eq!( - r.verified(), - Some(false), - "background press_key must not report confirmed success: {}", - r.text() - ); - - // ux_guard: FocusMonitorWin must not have lost activation. - assert_ux_guard( - losses_before, - 0, - "background click + type_text into CuaTestHarness.Electron", - ); - - fm_proc.kill().ok(); -} - -// ───────────────────────────────────────────────────────────────────────────── -// Test 2: launch_app minimized mode does not steal focus -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn test_launch_app_minimized_no_focus_steal() { - //! Visible app-launch coverage. - //! - //! launch_app with start_minimized=true is the strict Windows background - //! launch mode: the app starts without displacing FocusMonitorWin. - - if !driver_binary().exists() { - return; - } - - let exe = test_app_path(); - if !exe.exists() { - eprintln!("test app not available — skipping"); - return; - } - let _cleanup = KillProcessTreeOnDrop("CuaTestHarness.Electron.exe"); - - let Some((mut fm_proc, _fm_hwnd, _fm_pid)) = launch_focus_monitor() else { - return; - }; - let losses_before = read_losses(&loss_file()); - - let Some(mut driver) = McpDriver::spawn() else { - fm_proc.kill().ok(); - return; - }; - - // Launch the test app via cua-driver launch_app in strict background mode. - let path_str = exe.to_string_lossy().into_owned(); - let r = driver.call( - "launch_app", - serde_json::json!({"path": path_str, "start_minimized": true}), - ); - if r.is_error() { - eprintln!("launch_app failed — skipping: {:?}", r.raw); - fm_proc.kill().ok(); - return; - } - - // Wait for the app window to appear (Electron startup ~2-3s). - let mut app_pid: Option = None; - for _ in 0..20 { - std::thread::sleep(Duration::from_millis(500)); - let r2 = driver.call("list_apps", serde_json::json!({})); - if let Some(procs) = r2.structured()["processes"].as_array() { - if let Some(p) = procs.iter().find(|p| { - p["name"] - .as_str() - .map(|n| { - let n = n.to_lowercase(); - n.contains("cuatestharness.electron") || n.contains("electron") - }) - .unwrap_or(false) - }) { - app_pid = p["pid"].as_i64(); - break; - } - } - } - if app_pid.is_none() { - eprintln!("CuaTestHarness.Electron not found in process list after launch_app - skipping"); - fm_proc.kill().ok(); - return; - } - - // ux_guard: FocusMonitorWin must not have lost activation. - assert_ux_guard( - losses_before, - 0, - "launch_app CuaTestHarness.Electron with start_minimized=true", - ); - - // Kill the launched app by exe name. - kill_process_tree_by_image("CuaTestHarness.Electron.exe"); - fm_proc.kill().ok(); -} - -// ───────────────────────────────────────────────────────────────────────────── -// Test 3: background hotkey does not steal focus -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn test_background_hotkey_no_focus_steal() { - //! Background menu-shortcut coverage. - //! - //! Send Ctrl+A to a background Electron harness window. - //! FocusMonitorWin must never lose activation. - - if !driver_binary().exists() { - return; - } - - let Some((mut driver, app_pid, app_wid)) = launch_driver_and_test_app() else { - return; - }; - - let Some((mut fm_proc, _fm_hwnd, _fm_pid)) = launch_focus_monitor() else { - return; - }; - let losses_before = read_losses(&loss_file()); - - // Send Ctrl+A hotkey to background app (PostMessage, no focus steal). - let r = driver.call( - "hotkey", - serde_json::json!({"pid": app_pid, "window_id": app_wid, "keys": ["ctrl", "a"]}), - ); - assert!( - r.raw["error"].is_null(), - "Protocol error from hotkey: {:?}", - r.raw - ); - - // ux_guard: FocusMonitorWin must not have lost activation. - assert_ux_guard( - losses_before, - 0, - "background hotkey ctrl+a to CuaTestHarness.Electron", - ); - - fm_proc.kill().ok(); -} - -// ───────────────────────────────────────────────────────────────────────────── -// Test 4: background click that opens a new window (e.g. File→New dialog) -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn test_background_click_opens_new_window_focus_preserved() { - //! New-window click-delivery coverage. - //! - //! 1. FocusMonitorWin is foreground. - //! 2. Click the CuaTestHarness.Electron child-window button. - //! 3. FocusMonitorWin must remain active throughout (UX guard). - //! - //! Verifies that a background click can cause a child window to appear - //! without activating either the original target or the new window. - - if !driver_binary().exists() { - return; - } - - let Some((mut driver, app_pid, app_wid)) = launch_driver_and_test_app() else { - return; - }; - let snap = driver.call( - "get_window_state", - serde_json::json!({"pid": app_pid, "window_id": app_wid, "capture_mode": "ax"}), - ); - let Some(open_idx) = ax::element_index_containing(snap.text(), "Open child window") else { - eprintln!("child-window button not found in test app — skipping"); - return; - }; - - let Some((mut fm_proc, _fm_hwnd, _fm_pid)) = launch_focus_monitor() else { - return; - }; - let windows_before = window_ids(&mut driver); - let losses_before = read_losses(&loss_file()); - - // Click the explicit child-window button. - let r = driver.call( - "click", - serde_json::json!({"pid": app_pid, "window_id": app_wid, "element_index": open_idx}), - ); - assert!( - r.raw["error"].is_null(), - "Protocol error from click: {:?}", - r.raw - ); - - assert!( - wait_for_new_window(&mut driver, &windows_before), - "background click did not open a new harness window" - ); - - // ux_guard: FocusMonitorWin must not have lost activation. - assert_ux_guard( - losses_before, - 0, - "background click in CuaTestHarness.Electron (may open new window)", - ); - - // Verify FocusMonitorWin is still alive. - assert!( - fm_proc.try_wait().expect("try_wait").is_none(), - "FocusMonitorWin crashed during the test" - ); - - fm_proc.kill().ok(); -} - -// ───────────────────────────────────────────────────────────────────────────── -// Test 5: screenshot of background window doesn't steal focus -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn test_background_screenshot_no_focus_steal() { - //! PrintWindow captures a background window without activating it. - - if !driver_binary().exists() { - return; - } - - let Some((mut driver, _app_pid, app_wid)) = launch_driver_and_test_app() else { - return; - }; - - let Some((mut fm_proc, _fm_hwnd, _fm_pid)) = launch_focus_monitor() else { - return; - }; - let losses_before = read_losses(&loss_file()); - - // Screenshot via PrintWindow — must not activate the window. - let r = driver.call("screenshot", serde_json::json!({"window_id": app_wid})); - assert!( - r.raw["error"].is_null(), - "Protocol error from screenshot: {:?}", - r.raw - ); - - // ux_guard - assert_ux_guard( - losses_before, - 0, - "screenshot of background CuaTestHarness.Electron", - ); - - fm_proc.kill().ok(); -} - -// ───────────────────────────────────────────────────────────────────────────── -// Test 6: agent cursor is visually present on screen after move_cursor -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn test_agent_cursor_visible_on_screen() { - //! Computer-vision check: after move_cursor the overlay must be visible. - //! - //! Steps: - //! 1. Enable the agent cursor and move it to a known screen position. - //! 2. Wait for the glide animation to settle (default 750ms). - //! 3. Capture the screen at the cursor position using screenshot_display_bytes - //! (BitBlt from display DC — captures layered/overlay windows). - //! 4. Decode the PNG and sample a 40×40 px patch centred on the cursor. - //! 5. Assert the patch contains cursor-like pixels (bright or saturated). - - if !driver_binary().exists() { - eprintln!("Binary not found — skipping"); - return; - } - if !require_seedable_desktop("agent cursor visibility") { - return; - } - - let Some(mut driver) = McpDriver::spawn() else { - return; - }; - - // Safe centre-ish position on primary monitor. - let cx = 640.0_f64; - let cy = 400.0_f64; - let cursor_id = "guard-ux-cursor"; - - // Enable cursor overlay and glide to target. - let r = driver.call( - "set_agent_cursor_enabled", - serde_json::json!({"enabled": true, "cursor_id": cursor_id}), - ); - assert!( - r.raw["error"].is_null(), - "set_agent_cursor_enabled failed: {:?}", - r.raw - ); - - let r = driver.call( - "set_agent_cursor_motion", - serde_json::json!({ - "cursor_id": cursor_id, - "glide_duration_ms": 100, - "idle_hide_ms": 0 - }), - ); - assert!( - r.raw["error"].is_null(), - "set_agent_cursor_motion failed: {:?}", - r.raw - ); - - let r = driver.call( - "move_cursor", - serde_json::json!({"x": cx, "y": cy, "cursor_id": cursor_id}), - ); - assert!(r.raw["error"].is_null(), "move_cursor failed: {:?}", r.raw); - - // Wait for the fixed 100ms glide plus a few render frames. - std::thread::sleep(Duration::from_millis(350)); - - // Capture the screen directly (includes layered windows like the overlay). - let png_bytes = platform_windows::capture::screenshot_display_bytes() - .expect("screenshot_display_bytes failed"); - - drop(driver); - - // Decode PNG. - let img = image::load_from_memory(&png_bytes).expect("decode PNG"); - let rgba = img.to_rgba8(); - let (iw, ih) = rgba.dimensions(); - - // Sample 40×40 patch centred on (cx, cy). - let half = 20u32; - let x0 = (cx as u32).saturating_sub(half).min(iw.saturating_sub(1)); - let x1 = (cx as u32 + half).min(iw); - let y0 = (cy as u32).saturating_sub(half).min(ih.saturating_sub(1)); - let y1 = (cy as u32 + half).min(ih); - - let mut colourful_pixels = 0u32; - for py in y0..y1 { - for px in x0..x1 { - let [r, g, b, a] = rgba.get_pixel(px, py).0; - if a < 10 { - continue; - } - let brightness = r as u32 + g as u32 + b as u32; - let saturation = r.max(g).max(b) as u32 - r.min(g).min(b) as u32; - // Accept bright-white stroke pixels OR coloured gradient pixels. - if brightness > 60 && (saturation > 30 || brightness > 600) { - colourful_pixels += 1; - } - } - } - - assert!( - colourful_pixels >= 5, - "Agent cursor not visible at ({cx},{cy}): only {colourful_pixels} qualifying pixels \ - in 40×40 patch (x={x0}..{x1}, y={y0}..{y1}, image={iw}x{ih}). \ - Overlay may not be rendering or is positioned off-screen." - ); -} diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs index 89b1bfc1b7..3e85411c51 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs @@ -6,116 +6,380 @@ //! Run via: //! cargo test --test harness_web_test -- --ignored --nocapture //! -//! ## Known cua-driver gaps these tests document -//! -//! - **CDP `/json` HTTP read uses `read_to_end`** — `mcp-server/src/cdp.rs` -//! sends `Connection: close` and then calls `stream.read_to_end()`, but -//! Chromium's CDP HTTP server ignores `Connection: close` and keeps the -//! socket alive, so `read_to_end` hangs until the 10 s discovery timeout. -//! Confirmed against Electron 31 on port 9223 (verified manually via -//! curl: instant 200, JSON body present). Fix: parse `Content-Length` -//! and `read_exact` that many bytes, or honour `Transfer-Encoding: -//! chunked`. Tracked in this test as a structural assertion (window -//! discoverable) rather than a behavioural one (page tool round-trip). -//! -//! - **WebView2 `--remote-debugging-port` ignored** — passing -//! `AdditionalBrowserArguments = "--remote-debugging-port=9222"` via -//! `CoreWebView2EnvironmentOptions` does not open a CDP listener on the -//! WebView2 helper processes. WebView2 may be filtering the flag. -//! Tracked here as a TODO for the harness rather than a cua-driver -//! issue (since this is a WebView2 configuration concern). +//! The page-tool tests cover CDP discovery and a DOM round-trip together. +//! WebView2 can expose its listener before its first page target is ready, so +//! the driver must tolerate a briefly empty `/json` response. #![cfg(target_os = "windows")] +use std::cell::Cell; +use std::io::Read; use std::path::PathBuf; use std::process::{Command, Stdio}; -use std::time::Duration; +use std::time::{Duration, Instant}; -use cua_driver_testkit::{harness_app, spawn_in_job, Driver, McpDriver}; +use cua_driver_testkit::e2e::{ + execute_case, native_background_case, recording_evidence, DriverRoute, Observation, Targeting, +}; +use cua_driver_testkit::observer::TargetWindow; +use cua_driver_testkit::sentinel::run_with_background_oracles; +use cua_driver_testkit::{harness_app, spawn_in_job, Driver, FixtureJournal, McpDriver, ToolResponse}; // ── workspace paths ────────────────────────────────────────────────────────── fn webview_exe() -> PathBuf { if let Ok(p) = std::env::var("HARNESS_WEBVIEW_EXE") { let pb = PathBuf::from(p); - if pb.exists() { return pb; } + if pb.exists() { + return pb; + } } harness_app("harness-webview", "CuaTestHarness.WebView.exe") } fn electron_exe() -> PathBuf { if let Ok(p) = std::env::var("HARNESS_ELECTRON_EXE") { let pb = PathBuf::from(p); - if pb.exists() { return pb; } + if pb.exists() { + return pb; + } } harness_app("harness-electron", "CuaTestHarness.Electron.exe") } // ── shared session helper ──────────────────────────────────────────────────── -/// Wait (up to ~5s) for `port` to become free. These web tests use FIXED CDP -/// ports (9222/9223) and a process-global `CUA_DRIVER_CDP_PORT`, so they must -/// run serially (`--test-threads=1`). A previous test's host can still be -/// releasing its port when the next launches; reusing it before then makes the -/// daemon discover the OLD host's page (`pages[0]`), so the click lands on a -/// stale window and the counter check fails. This guard closes that teardown -/// overlap — belt-and-braces on top of serial execution. -fn wait_port_free(port: u16) { - for _ in 0..50 { - if std::net::TcpStream::connect(("127.0.0.1", port)).is_err() { - return; +fn allocate_loopback_port() -> u16 { + let listener = + std::net::TcpListener::bind(("127.0.0.1", 0)).expect("allocate an ephemeral CDP port"); + listener.local_addr().expect("read CDP port").port() +} + +fn wait_for_page_text( + driver: &mut McpDriver, + pid: i64, + wid: u64, + javascript: &str, + expected: &str, +) -> String { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + let response = driver.call( + "page", + serde_json::json!({ + "pid": pid, + "window_id": wid, + "action": "execute_javascript", + "javascript": javascript, + }), + ); + let text = response.text().to_owned(); + if text.contains(expected) { + return text; } + assert!( + std::time::Instant::now() < deadline, + "page state did not reach {expected:?}: {text:?}" + ); std::thread::sleep(Duration::from_millis(100)); } - eprintln!("warning: CDP port {port} still bound after 5s — prior host may not have released it"); } /// Launch the harness exe + a cua-driver child with `CUA_DRIVER_CDP_PORT` /// pointing at the harness's CDP endpoint. Polls list_windows until the /// host's window appears. -fn run_with_session(label: &str, host_exe: PathBuf, title_substr: &str, cdp_port: u16, f: F) +fn run_web_case(toolkit: &str, action: &str, host_exe: PathBuf, title_substr: &str, f: F) where F: FnOnce(i64, u64, &mut McpDriver), { - if !host_exe.exists() { - eprintln!("{label} host exe not found at {host_exe:?} — run tests/fixtures/build/windows.ps1"); - return; + let case = native_background_case(toolkit, action, Targeting::Page, DriverRoute::Cdp); + run_web_case_with_preparation( + case, + toolkit, + host_exe, + title_substr, + |_, _, _, _| {}, + |pid, wid, driver, _| f(pid, wid, driver), + ); +} + +fn run_web_case_with_preparation( + case: cua_driver_testkit::e2e::CaseSpec, + toolkit: &str, + host_exe: PathBuf, + title_substr: &str, + prepare: P, + f: F, +) where + P: FnOnce(i64, u64, &mut McpDriver, &FixtureJournal), + F: FnOnce(i64, u64, &mut McpDriver, &FixtureJournal), +{ + let cell_id = case.cell_id.clone(); + execute_case(case, |evidence| { + assert!( + host_exe.exists(), + "required {toolkit} host is missing at {host_exe:?}" + ); + let cdp_port = allocate_loopback_port(); + let cdp_port_string = cdp_port.to_string(); + let journal = FixtureJournal::start(); + let mut driver = McpDriver::spawn_named_with_env( + &cell_id, + &[("CUA_DRIVER_CDP_PORT", cdp_port_string.as_str())], + ) + .expect("required source-built Windows driver did not start"); + *evidence = recording_evidence(driver.recording_dir()); + + let env_var = if toolkit == "webview2" { + "CUA_WEBVIEW_CDP_PORT" + } else { + "CUA_ELECTRON_CDP_PORT" + }; + let mut cmd = Command::new(&host_exe); + cmd.env(env_var, &cdp_port_string) + .env("CUA_E2E_FIXTURE_JOURNAL_URL", journal.url()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()); + let mut app = spawn_in_job(&mut cmd).expect("spawn web harness"); + let pid = app.id() as i64; + // WebView2's first CoreWebView2Environment creation can exceed 12s on a + // cold hosted runner. Keep polling the externally visible ready title; + // process exit and the final deadline still fail closed. + let deadline = std::time::Instant::now() + Duration::from_secs(30); + let mut observed_titles = Vec::new(); + let (wid, _) = 'ready: loop { + if let Some(status) = app.try_wait().expect("poll web harness process") { + let mut stderr = String::new(); + if let Some(mut pipe) = app.stderr.take() { + let _ = pipe.read_to_string(&mut stderr); + } + panic!( + "{toolkit} fixture exited before readiness with {status}: {}", + stderr.trim() + ); + } + let response = driver.call("list_windows", serde_json::json!({ "pid": pid })); + observed_titles.clear(); + if let Some(windows) = response.structured()["windows"].as_array() { + for window in windows { + let title = window["title"].as_str().unwrap_or(""); + observed_titles.push(title.to_owned()); + if title.contains(title_substr) { + if let Some(wid) = window["window_id"].as_u64() { + break 'ready (wid, title.to_owned()); + } + } + } + } + if std::time::Instant::now() >= deadline { + let _ = app.kill(); + let _ = app.wait(); + let mut stderr = String::new(); + if let Some(mut pipe) = app.stderr.take() { + let _ = pipe.read_to_string(&mut stderr); + } + panic!( + "{toolkit} window with title containing {title_substr:?} did not become ready; \ + observed titles={observed_titles:?}; stderr={:?}", + stderr.trim() + ); + } + std::thread::sleep(Duration::from_millis(100)); + }; + driver.reaper().push(app); + let journal_deadline = Instant::now() + Duration::from_secs(5); + while !journal.contains("WEB_HARNESS_MARKER_v1") { + assert!( + Instant::now() < journal_deadline, + "{toolkit} fixture journal did not become ready: {}", + journal.snapshot() + ); + std::thread::sleep(Duration::from_millis(50)); + } + prepare(pid, wid, &mut driver, &journal); + let (_, passed) = run_with_background_oracles( + &mut driver, + TargetWindow { + pid: pid as u32, + native_id: wid, + }, + |driver| f(pid, wid, driver, &journal), + ) + .unwrap_or_else(|error| panic!("background desktop contract failed: {error}")); + Observation::delivered_with_fixture_state(passed) + }); +} + +fn snapshot(driver: &mut McpDriver, pid: i64, wid: u64) -> ToolResponse { + driver.call( + "get_window_state", + serde_json::json!({ + "pid": pid, + "window_id": wid, + "capture_mode": "ax" + }), + ) +} + +fn window_bounds(driver: &mut McpDriver, pid: i64, wid: u64) -> (f64, f64, f64, f64) { + let response = driver.call("list_windows", serde_json::json!({ "pid": pid })); + let window = response.structured()["windows"] + .as_array() + .and_then(|windows| { + windows + .iter() + .find(|window| window["window_id"].as_u64() == Some(wid)) + }) + .unwrap_or_else(|| { + panic!( + "WebView2 window {wid} is missing from list_windows: {}", + response.text() + ) + }); + let bounds = &window["bounds"]; + ( + bounds["x"].as_f64().expect("WebView2 bounds need x"), + bounds["y"].as_f64().expect("WebView2 bounds need y"), + bounds["width"] + .as_f64() + .expect("WebView2 bounds need width"), + bounds["height"] + .as_f64() + .expect("WebView2 bounds need height"), + ) +} + +fn pixel_from_screen( + state: &ToolResponse, + screen_x: f64, + screen_y: f64, + window: (f64, f64, f64, f64), +) -> (f64, f64) { + let (window_x, window_y, window_w, window_h) = window; + assert!( + window_w > 0.0 && window_h > 0.0, + "WebView2 window needs positive geometry: {window:?}" + ); + let screenshot_w = state.structured()["screenshot_width"] + .as_f64() + .expect("PX targeting requires screenshot_width"); + let screenshot_h = state.structured()["screenshot_height"] + .as_f64() + .expect("PX targeting requires screenshot_height"); + let scale_x = screenshot_w / window_w; + let scale_y = screenshot_h / window_h; + let x = (screen_x - window_x) * scale_x; + let y = (screen_y - window_y) * scale_y; + assert!( + x >= 0.0 && x < screenshot_w && y >= 0.0 && y < screenshot_h, + "WebView2 PX target center ({x:.1}, {y:.1}) is outside the capture ({screenshot_w:.1}x{screenshot_h:.1})" + ); + (x, y) +} + +fn wait_for_journal_text(journal: &FixtureJournal, id: &str, expected: &str) { + let deadline = Instant::now() + Duration::from_secs(3); + loop { + if journal.text(id).as_deref() == Some(expected) { + return; + } + assert!( + Instant::now() < deadline, + "WebView2 fixture journal {id:?} did not reach {expected:?}: {}", + journal.snapshot() + ); + std::thread::sleep(Duration::from_millis(50)); } - // A prior test's host may still hold this fixed CDP port — wait for it to - // free so the daemon doesn't discover the stale host's page. - wait_port_free(cdp_port); - // Set the CDP port the daemon should probe; the spawned cua-driver child - // inherits it from this process's environment. - std::env::set_var("CUA_DRIVER_CDP_PORT", cdp_port.to_string()); - let Some(mut driver) = McpDriver::spawn() else { return }; - - // Set the CDP port the host should use so the daemon can find it. - let env_var = if label == "webview" { "CUA_WEBVIEW_CDP_PORT" } else { "CUA_ELECTRON_CDP_PORT" }; - let mut cmd = Command::new(&host_exe); - cmd.env(env_var, cdp_port.to_string()) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let app = spawn_in_job(&mut cmd).expect("spawn host"); - let pid = app.id() as i64; - driver.reaper().push(app); - println!("{label} pid={pid} cdp_port={cdp_port}"); - std::thread::sleep(Duration::from_secs(2)); // small cold-start for runtime spin-up - - let (wid, _title) = driver - .find_window(pid, title_substr) - .unwrap_or_else(|| panic!("{label} window with title containing {title_substr:?} not found")); - - f(pid, wid, &mut driver); } -// ── WebView2 structural + page tool ───────────────────────────────────────── +// ── WebView2 page tool ────────────────────────────────────────────────────── #[test] #[ignore] -fn harness_webview_window_discoverable() { - run_with_session("webview", webview_exe(), "CuaTestHarness WebView", 9222, - |pid, wid, _driver| { - println!("✅ harness_webview_window_discoverable: pid={pid} wid={wid}"); - }); +fn harness_webview_left_click_px_background() { + let case = native_background_case( + "webview2", + "left_click", + Targeting::Px, + DriverRoute::UiaInvoke, + ); + let point = Cell::new(None); + run_web_case_with_preparation( + case, + "webview2", + webview_exe(), + "CuaTestHarness WebView [ready", + |pid, wid, driver, journal| { + wait_for_journal_text(journal, "lbl-counter", "counter=0"); + let bounds = window_bounds(driver, pid, wid); + let ready_state = snapshot(driver, pid, wid); + let dom_probe = driver.call( + "page", + serde_json::json!({ + "pid": pid, + "window_id": wid, + "action": "click_element", + "selector": "#btn-increment" + }), + ); + assert!( + !dom_probe.is_error(), + "WebView2 DOM geometry probe failed: {}", + dom_probe.text() + ); + let screen_x = dom_probe.structured()["screen_x"] + .as_f64() + .expect("WebView2 DOM probe needs screen_x"); + let screen_y = dom_probe.structured()["screen_y"] + .as_f64() + .expect("WebView2 DOM probe needs screen_y"); + wait_for_journal_text(journal, "lbl-counter", "counter=1"); + let (x, y) = pixel_from_screen(&ready_state, screen_x, screen_y, bounds); + let geometry_probe = driver.call( + "click", + serde_json::json!({ + "pid": pid, + "window_id": wid, + "x": x, + "y": y, + "delivery_mode": "foreground" + }), + ); + assert!( + !geometry_probe.is_error(), + "WebView2 foreground PX geometry probe failed: {}", + geometry_probe.text() + ); + wait_for_journal_text(journal, "lbl-counter", "counter=2"); + point.set(Some((x, y))); + }, + |pid, wid, driver, journal| { + let (x, y) = point + .get() + .expect("foreground geometry probe did not set a PX target"); + let click = driver.call( + "click", + serde_json::json!({ + "pid": pid, + "window_id": wid, + "x": x, + "y": y, + "delivery_mode": "background" + }), + ); + assert!( + !click.is_error(), + "WebView2 PX background click failed: {}", + click.text() + ); + assert_eq!( + click.structured()["path"].as_str(), + Some("ax"), + "WebView2 PX background click used an unexpected driver route: {}", + click.text() + ); + wait_for_journal_text(journal, "lbl-counter", "counter=3"); + }, + ); } #[test] @@ -125,43 +389,42 @@ fn harness_webview_page_tool() { // CoreWebView2EnvironmentOptions.AdditionalBrowserArguments. // Combined with the `/json` Content-Length fix in mcp-server/src/cdp.rs, // the page tool now reaches WebView2's DOM via CDP just like Electron. - run_with_session("webview", webview_exe(), "CuaTestHarness WebView", 9222, + run_web_case( + "webview2", + "page_roundtrip", + webview_exe(), + "CuaTestHarness WebView [ready", |pid, wid, driver| { - - let marker = driver.call("page", serde_json::json!({ + let marker = driver.call("page", serde_json::json!({ "pid": pid, "window_id": wid, "action": "execute_javascript", "javascript": "document.querySelector('[data-cua-id=\"page-marker\"]').textContent" })).text().to_string(); - assert!(marker.contains("WEB_HARNESS_MARKER_v1"), - "WebView2 CDP execute_javascript marker fetch: {marker:?}"); + assert!( + marker.contains("WEB_HARNESS_MARKER_v1"), + "WebView2 CDP execute_javascript marker fetch: {marker:?}" + ); - // click_element via DOM selector + counter readback. - let _ = driver.call("page", serde_json::json!({ - "pid": pid, "window_id": wid, "action": "click_element", - "selector": "#btn-increment" - })); - std::thread::sleep(Duration::from_millis(500)); - - let post = driver.call("page", serde_json::json!({ - "pid": pid, "window_id": wid, "action": "execute_javascript", - "javascript": "document.getElementById('lbl-counter').textContent" - })).text().to_string(); - assert!(post.contains("counter=1"), - "WebView2 counter didn't advance via page.click_element: {post:?}"); - println!("✅ harness_webview_page_tool: CDP+execute_javascript+click_element green"); - }); + // click_element via DOM selector + counter readback. + let _ = driver.call( + "page", + serde_json::json!({ + "pid": pid, "window_id": wid, "action": "click_element", + "selector": "#btn-increment" + }), + ); + wait_for_page_text( + driver, + pid, + wid, + "document.getElementById('lbl-counter').textContent", + "counter=1", + ); + println!("✅ harness_webview_page_tool: CDP+execute_javascript+click_element green"); + }, + ); } -// ── Electron structural + page tool ────────────────────────────────────────── - -#[test] -#[ignore] -fn harness_electron_window_discoverable() { - run_with_session("electron", electron_exe(), "CuaTestHarness Electron", 9223, - |pid, wid, _driver| { - println!("✅ harness_electron_window_discoverable: pid={pid} wid={wid}"); - }); -} +// ── Electron page tool ─────────────────────────────────────────────────────── #[test] #[ignore] @@ -169,34 +432,47 @@ fn harness_electron_page_tool() { // Regression guard for the CDP /json discovery fix (parse // Content-Length / Transfer-Encoding instead of read_to_end). // cua-driver's page tool now reaches Electron's CDP successfully. - run_with_session("electron", electron_exe(), "CuaTestHarness Electron", 9223, + run_web_case( + "electron", + "page_execute", + electron_exe(), + "CuaTestHarness Electron", |pid, wid, driver| { - - // 1. execute_javascript via CDP. - let marker = driver.call("page", serde_json::json!({ + // 1. execute_javascript via CDP. + let marker = driver.call("page", serde_json::json!({ "pid": pid, "window_id": wid, "action": "execute_javascript", "javascript": "document.querySelector('[data-cua-id=\"page-marker\"]').textContent" })).text().to_string(); - assert!(marker.contains("WEB_HARNESS_MARKER_v1"), - "Electron CDP execute_javascript marker fetch: {marker:?}"); - - // 2. Increment counter via direct execute_javascript (the - // click_element path has a separate probe-JSON-parsing gap - // documented below — track separately). - let _ = driver.call("page", serde_json::json!({ - "pid": pid, "window_id": wid, "action": "execute_javascript", - "javascript": "document.getElementById('btn-increment').click()" - })); - std::thread::sleep(Duration::from_millis(300)); + assert!( + marker.contains("WEB_HARNESS_MARKER_v1"), + "Electron CDP execute_javascript marker fetch: {marker:?}" + ); - let post = driver.call("page", serde_json::json!({ - "pid": pid, "window_id": wid, "action": "execute_javascript", - "javascript": "document.getElementById('lbl-counter').textContent" - })).text().to_string(); - assert!(post.contains("counter=1"), - "Electron counter did not advance via execute_javascript: {post:?}"); - println!("✅ harness_electron_page_tool: CDP+execute_javascript green"); - }); + // 2. Increment counter via direct execute_javascript (the + // click_element path has a separate probe-JSON-parsing gap + // documented below — track separately). + let click = driver.call( + "page", + serde_json::json!({ + "pid": pid, "window_id": wid, "action": "execute_javascript", + "javascript": "document.getElementById('btn-increment').click()" + }), + ); + assert!( + !click.is_error(), + "Electron execute_javascript click failed: {}", + click.text() + ); + wait_for_page_text( + driver, + pid, + wid, + "document.getElementById('lbl-counter').textContent", + "counter=1", + ); + println!("✅ harness_electron_page_tool: CDP+execute_javascript green"); + }, + ); } /// Regression guard for the page.click_element double-encode fix. @@ -211,29 +487,41 @@ fn harness_electron_page_tool() { #[test] #[ignore] fn harness_electron_click_element() { - run_with_session("electron", electron_exe(), "CuaTestHarness Electron", 9223, + run_web_case( + "electron", + "click_element_probe", + electron_exe(), + "CuaTestHarness Electron", |pid, wid, driver| { - let resp = driver.call("page", serde_json::json!({ - "pid": pid, "window_id": wid, "action": "click_element", - "selector": "#btn-increment" - })); - // Prefer the tool text; fall back to a JSON-RPC error message. - let text = if resp.text().is_empty() { - resp.raw["error"]["message"].as_str().unwrap_or("").to_string() - } else { - resp.text().to_string() - }; - assert!(!text.contains("probe JSON missing") && !text.contains("required field"), - "click_element probe parse regressed: {text:?}"); - std::thread::sleep(Duration::from_millis(400)); - - // Verify the click actually fired in the DOM. - let post = driver.call("page", serde_json::json!({ - "pid": pid, "window_id": wid, "action": "execute_javascript", - "javascript": "document.getElementById('lbl-counter').textContent" - })).text().to_string(); - assert!(post.contains("counter=1"), - "Counter didn't advance after page.click_element: {post:?}"); - println!("✅ harness_electron_click_element: probe parsed, click fired, counter=1"); - }); + let resp = driver.call( + "page", + serde_json::json!({ + "pid": pid, "window_id": wid, "action": "click_element", + "selector": "#btn-increment" + }), + ); + // Prefer the tool text; fall back to a JSON-RPC error message. + let text = if resp.text().is_empty() { + resp.raw["error"]["message"] + .as_str() + .unwrap_or("") + .to_string() + } else { + resp.text().to_string() + }; + assert!( + !text.contains("probe JSON missing") && !text.contains("required field"), + "click_element probe parse regressed: {text:?}" + ); + // Verify the click actually fired in the DOM. + wait_for_page_text( + driver, + pid, + wid, + "document.getElementById('lbl-counter').textContent", + "counter=1", + ); + println!("✅ harness_electron_click_element: probe parsed, click fired, counter=1"); + }, + ); } diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs index 2731fb9dc7..827a7de0c9 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs @@ -22,9 +22,15 @@ use std::path::PathBuf; use std::process::{Command, Stdio}; -use std::time::Duration; +use std::time::{Duration, Instant}; use cua_driver_testkit::ax::element_index_by_id; +use cua_driver_testkit::e2e::{ + execute_case, native_background_case, native_readonly_case, recording_evidence, DriverRoute, + Evidence, Observation, OracleKind, Targeting, +}; +use cua_driver_testkit::observer::TargetWindow; +use cua_driver_testkit::sentinel::run_with_background_oracles; use cua_driver_testkit::{harness_app, spawn_in_job, Driver, McpDriver}; /// Resolve the WinUI3 harness exe — the `HARNESS_WINUI3_EXE` override wins (if it @@ -47,132 +53,198 @@ fn launch_winui3(driver: &mut McpDriver) -> Option { eprintln!("WinUI3 harness exe not found at {exe:?} — run tests/fixtures/build/windows.ps1"); return None; } - let child = spawn_in_job(Command::new(&exe).stdout(Stdio::null()).stderr(Stdio::null())).ok()?; + let child = spawn_in_job( + Command::new(&exe) + .stdout(Stdio::null()) + .stderr(Stdio::null()), + ) + .ok()?; let pid = child.id(); driver.reaper().push(child); - // Short fixed cold-start settle (window creation + foreground - // establishment after spawn). `find_window`'s polling handles the - // variable tail (WinUI3 first-run cold-start under sandbox load). - std::thread::sleep(Duration::from_millis(1500)); Some(pid) } -#[test] -#[ignore] -fn harness_winui3_smoke() { - let Some(mut driver) = McpDriver::spawn() else { return }; - let Some(pid) = launch_winui3(&mut driver) else { return }; - println!("WinUI3 harness pid={pid}"); - - let (wid, _) = driver - .find_window(pid as i64, "CuaTestHarness WinUI3") - .expect("WinUI3 main window not found"); - - let snap = driver.call( - "get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode":"ax"}), - ); - let text = snap.text(); - - // Button-class controls surface AutomationIds in the UIA tree. - for aid in [ - "btn-increment", "btn-reset", - "btn-open-flyout", - "btn-open-popup", - "btn-exit", - ] { - assert!(text.contains(&format!("id={aid}")), - "missing AutomationId {aid} in WinUI3 UIA snapshot"); +fn wait_for_winui3_ready(driver: &mut McpDriver, pid: u32, window_id: u64) { + let deadline = Instant::now() + Duration::from_secs(20); + loop { + let state = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": window_id}), + ); + let last_state = state.text(); + if !state.is_error() + && last_state.contains("HARNESS_TEXT_MARKER_v1") + && last_state.contains("id=chk-agreed") + { + return; + } + assert!( + Instant::now() < deadline, + "WinUI3 UIA tree did not become ready: {last_state}" + ); + std::thread::sleep(Duration::from_millis(100)); } +} - // TextBlock content (no AutomationId surfaces) — assert markers. - assert!(text.contains("HARNESS_TEXT_MARKER_v1"), "WinUI3 text_body marker not in snapshot"); - assert!(text.contains("counter=0"), "WinUI3 initial counter label not in snapshot"); +fn run_case( + case: cua_driver_testkit::e2e::CaseSpec, + test: impl FnOnce(u32, u64, &mut McpDriver) -> Observation, +) { + let cell_id = case.cell_id.clone(); + let delivery = case.delivery; + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_named(&cell_id) + .expect("required source-built Windows driver did not start"); + *evidence = recording_evidence(driver.recording_dir()); + let pid = launch_winui3(&mut driver).expect("required WinUI3 harness did not launch"); + let (wid, _) = driver + .find_window(pid as i64, "CuaTestHarness WinUI3") + .expect("WinUI3 main window not found"); + wait_for_winui3_ready(&mut driver, pid, wid); + if delivery != cua_driver_testkit::e2e::Delivery::Background { + driver.start_behavior_recording(); + } + test(pid, wid, &mut driver) + }); +} - println!("✅ harness_winui3_smoke: all expected scenarios present in UIA tree"); +fn run_background_case( + action: &str, + route: DriverRoute, + test: impl FnOnce(u32, u64, &mut McpDriver), +) { + run_case( + native_background_case("winui3", action, Targeting::Ax, route), + |pid, wid, driver| { + let (_, passed) = run_with_background_oracles( + driver, + TargetWindow { + pid, + native_id: wid, + }, + |driver| test(pid, wid, driver), + ) + .unwrap_or_else(|error| panic!("background desktop contract failed: {error}")); + Observation::delivered_with_fixture_state(passed) + }, + ); } #[test] #[ignore] -fn harness_winui3_type_text() { - let Some(mut driver) = McpDriver::spawn() else { return }; - let Some(pid) = launch_winui3(&mut driver) else { return }; - - let (wid, _) = driver - .find_window(pid as i64, "CuaTestHarness WinUI3") - .expect("WinUI3 main window"); - - let snap = driver.call( - "get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode":"ax"}), +fn harness_winui3_smoke() { + run_case( + native_readonly_case( + "winui3", + "ax_tree", + Targeting::Ax, + DriverRoute::AxRead, + vec![OracleKind::AxState], + ), + |pid, wid, driver| { + println!("WinUI3 harness pid={pid}"); + let snap = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode":"ax"}), + ); + assert!( + !snap.is_error(), + "WinUI3 AX snapshot failed: {}", + snap.text() + ); + let text = snap.text(); + for aid in [ + "btn-increment", + "btn-reset", + "btn-open-flyout", + "btn-open-popup", + "btn-exit", + ] { + assert!( + text.contains(&format!("id={aid}")), + "missing AutomationId {aid} in WinUI3 UIA snapshot" + ); + } + assert!( + text.contains("HARNESS_TEXT_MARKER_v1"), + "WinUI3 text_body marker not in snapshot" + ); + assert!( + text.contains("counter=0"), + "WinUI3 initial counter label not in snapshot" + ); + Observation::delivered(vec![OracleKind::AxState], Evidence::default()) + }, ); - let idx = element_index_by_id(snap.text(), "txt-input").expect("txt-input not in WinUI3 snapshot"); - - // WinUI3 is a XAML host — type_text requires element_index + window_id - // (routes through UIA ValuePattern.SetValue, see Windows backend docs). - let resp = driver.call("type_text", serde_json::json!({ - "pid": pid as i64, - "window_id": wid, - "element_index": idx, - "text": "winui3-typed" - })); - println!("type_text (WinUI3): {}", resp.text()); - std::thread::sleep(Duration::from_millis(500)); +} - let post = driver.call( - "get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode":"ax"}), - ); - assert!(post.text().contains("mirror=winui3-typed"), - "WinUI3 TextBox mirror did not advance. Snapshot excerpt: {}", - post.text().chars().take(600).collect::()); - println!("✅ harness_winui3_type_text: WinUI3 TextBox mirror reflects 'winui3-typed'"); +#[test] +#[ignore] +fn harness_winui3_type_text() { + run_background_case("type_text", DriverRoute::UiaValue, |pid, wid, driver| { + let snap = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode":"ax"}), + ); + let idx = element_index_by_id(snap.text(), "txt-input") + .expect("txt-input not in WinUI3 snapshot"); + let resp = driver.call( + "type_text", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "text": "winui3-typed", "delivery_mode": "background" + }), + ); + assert!(!resp.is_error(), "WinUI3 type_text failed: {}", resp.text()); + std::thread::sleep(Duration::from_millis(500)); + let post = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode":"ax"}), + ); + assert!( + post.text().contains("mirror=winui3-typed"), + "WinUI3 TextBox mirror did not advance. Snapshot excerpt: {}", + post.text().chars().take(600).collect::() + ); + }); } #[test] #[ignore] fn harness_winui3_xaml_popup_open() { - let Some(mut driver) = McpDriver::spawn() else { return }; - let Some(pid) = launch_winui3(&mut driver) else { return }; - - let (wid, _) = driver - .find_window(pid as i64, "CuaTestHarness WinUI3") - .expect("WinUI3 main window"); - - let snap = driver.call( - "get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode":"ax"}), + run_background_case( + "xaml_popup_open", + DriverRoute::UiaExpandCollapse, + |pid, wid, driver| { + let snap = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode":"ax"}), + ); + let idx = element_index_by_id(snap.text(), "btn-open-popup").expect("btn-open-popup"); + let response = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "background" + }), + ); + assert!( + !response.is_error(), + "open popup failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(500)); + let post = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode":"ax"}), + ); + assert!( + post.text().contains("XAML_POPUP_MARKER_v1"), + "XAML popup body did not appear in tree after click. Excerpt: {}", + post.text().chars().take(600).collect::() + ); + }, ); - let idx = element_index_by_id(snap.text(), "btn-open-popup").expect("btn-open-popup"); - - let _ = driver.call("click", serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx - })); - std::thread::sleep(Duration::from_millis(500)); - - let post = driver.call( - "get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode":"ax"}), - ); - let text = post.text(); - assert!(text.contains("XAML_POPUP_MARKER_v1"), - "XAML popup body did not appear in tree after click. Excerpt: {}", - text.chars().take(600).collect::()); - println!("✅ harness_winui3_xaml_popup_open: popup body visible in UIA tree"); -} - -// ── Session helper for the additional control tests ────────────────────────── - -fn winui3_with_session(f: F) -where - F: FnOnce(u32, u64, &mut McpDriver), -{ - let Some(mut driver) = McpDriver::spawn() else { return }; - let Some(pid) = launch_winui3(&mut driver) else { return }; - let (wid, _) = driver - .find_window(pid as i64, "CuaTestHarness WinUI3") - .expect("WinUI3 main window"); - f(pid, wid, &mut driver); } /// Regression guard for the click → TogglePattern dispatch fix. @@ -182,40 +254,78 @@ where #[test] #[ignore] fn harness_winui3_checkbox_toggle() { - winui3_with_session(|pid, wid, driver| { - let snap = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); - let idx = element_index_by_id(snap.text(), "chk-agreed").expect("chk-agreed"); - let _ = driver.call("click", serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx - })); - std::thread::sleep(Duration::from_millis(400)); - let post = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); - assert!(post.text().contains("agreed=True"), - "WinUI3 CheckBox didn't toggle: TogglePattern dispatch may have regressed."); - println!("✅ harness_winui3_checkbox_toggle: agreed=True via UIA Toggle"); - }); + run_background_case( + "checkbox_toggle", + DriverRoute::UiaToggle, + |pid, wid, driver| { + let snap = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}), + ); + let idx = element_index_by_id(snap.text(), "chk-agreed").expect("chk-agreed"); + let response = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "background" + }), + ); + assert!( + !response.is_error(), + "checkbox toggle failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(400)); + let post = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}), + ); + assert!( + post.text().contains("agreed=True"), + "WinUI3 CheckBox didn't toggle: TogglePattern dispatch may have regressed." + ); + println!("✅ harness_winui3_checkbox_toggle: agreed=True via UIA Toggle"); + }, + ); } /// Regression guard for SelectionItem.Select dispatch on RadioButton. #[test] #[ignore] fn harness_winui3_radio_select() { - winui3_with_session(|pid, wid, driver| { - let snap = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); - let idx = element_index_by_id(snap.text(), "rdo-high").expect("rdo-high"); - let _ = driver.call("click", serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx - })); - std::thread::sleep(Duration::from_millis(400)); - let post = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); - assert!(post.text().contains("prio=High"), - "WinUI3 radio didn't select High via SelectionItem.Select."); - println!("✅ harness_winui3_radio_select: prio=High via UIA SelectionItem"); - }); + run_background_case( + "radio_select", + DriverRoute::UiaSelection, + |pid, wid, driver| { + let snap = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}), + ); + let idx = element_index_by_id(snap.text(), "rdo-high").expect("rdo-high"); + let response = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "background" + }), + ); + assert!( + !response.is_error(), + "radio select failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(400)); + let post = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}), + ); + assert!( + post.text().contains("prio=High"), + "WinUI3 radio didn't select High via SelectionItem.Select." + ); + println!("✅ harness_winui3_radio_select: prio=High via UIA SelectionItem"); + }, + ); } /// Documents cua-driver gap: WinUI3 Slider implements @@ -233,28 +343,45 @@ fn harness_winui3_radio_select() { #[test] #[ignore] fn harness_winui3_slider_set_value() { - winui3_with_session(|pid, wid, driver| { - let snap = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); - let idx = element_index_by_id(snap.text(), "sld-value") + run_background_case( + "slider_set_value", + DriverRoute::UiaRangeValue, + |pid, wid, driver| { + let snap = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}), + ); + let idx = element_index_by_id(snap.text(), "sld-value") .expect("sld-value should now be in the UIA flat tree after RangeValuePattern detection fix"); - let resp = driver.call("set_value", serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx, - "value": "42" - })); - println!("set_value sld-value=42: {}", resp.text()); - std::thread::sleep(Duration::from_millis(400)); - let post = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); - let text = post.text(); - let advanced = text.lines().any(|l| - l.contains("slider_value=") && !l.contains("slider_value=0\"")); - assert!(advanced, - "WinUI3 Slider didn't move via RangeValuePattern.SetValue. Lines: {}", - text.lines().filter(|l| l.contains("slider_value")) - .collect::>().join(" / ")); - println!("✅ harness_winui3_slider_set_value: value moved via UIA RangeValuePattern.SetValue"); - }); + let resp = driver.call( + "set_value", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "value": "42" + }), + ); + println!("set_value sld-value=42: {}", resp.text()); + assert!(!resp.is_error(), "slider set_value failed: {}", resp.text()); + std::thread::sleep(Duration::from_millis(400)); + let post = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}), + ); + let text = post.text(); + let advanced = text + .lines() + .any(|l| l.contains("slider_value=") && !l.contains("slider_value=0\"")); + assert!( + advanced, + "WinUI3 Slider didn't move via RangeValuePattern.SetValue. Lines: {}", + text.lines() + .filter(|l| l.contains("slider_value")) + .collect::>() + .join(" / ") + ); + println!("✅ harness_winui3_slider_set_value: value moved via UIA RangeValuePattern.SetValue"); + }, + ); } /// Regression guard for ExpandCollapse.Expand + SelectionItem.Select on @@ -262,29 +389,51 @@ fn harness_winui3_slider_set_value() { #[test] #[ignore] fn harness_winui3_combo_select() { - winui3_with_session(|pid, wid, driver| { - let snap = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); - let combo_idx = element_index_by_id(snap.text(), "cbo-color").expect("cbo-color"); - // Expand the dropdown via ExpandCollapse. - let _ = driver.call("click", serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": combo_idx - })); - std::thread::sleep(Duration::from_millis(400)); - // Re-snapshot — items materialize after expand. - let snap2 = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); - let item_idx = element_index_by_id(snap2.text(), "cbo-item-orange") - .expect("cbo-item-orange after expand"); - // Select the item via SelectionItem.Select. - let _ = driver.call("click", serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": item_idx - })); - std::thread::sleep(Duration::from_millis(400)); - let post = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); - assert!(post.text().contains("color=orange"), - "WinUI3 combo didn't switch to orange via ExpandCollapse + SelectionItem.Select."); - println!("✅ harness_winui3_combo_select: color=orange via UIA Expand + Select"); - }); + run_background_case( + "combo_select", + DriverRoute::Composite, + |pid, wid, driver| { + let snap = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}), + ); + let combo_idx = element_index_by_id(snap.text(), "cbo-color").expect("cbo-color"); + // Expand the dropdown via ExpandCollapse. + let expand = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": combo_idx, + "delivery_mode": "background" + }), + ); + assert!(!expand.is_error(), "combo expand failed: {}", expand.text()); + std::thread::sleep(Duration::from_millis(400)); + // Re-snapshot — items materialize after expand. + let snap2 = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}), + ); + let item_idx = element_index_by_id(snap2.text(), "cbo-item-orange") + .expect("cbo-item-orange after expand"); + // Select the item via SelectionItem.Select. + let select = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": item_idx, + "delivery_mode": "background" + }), + ); + assert!(!select.is_error(), "combo select failed: {}", select.text()); + std::thread::sleep(Duration::from_millis(400)); + let post = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}), + ); + assert!( + post.text().contains("color=orange"), + "WinUI3 combo didn't switch to orange via ExpandCollapse + SelectionItem.Select." + ); + println!("✅ harness_winui3_combo_select: color=orange via UIA Expand + Select"); + }, + ); } diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs index ede8d31e8b..97987e4910 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs @@ -39,8 +39,15 @@ use std::path::PathBuf; use std::process::{Command, Stdio}; -use std::time::Duration; - +use std::time::{Duration, Instant}; + +use cua_driver_testkit::e2e::{ + execute_case, native_background_case, native_foreground_case, native_readonly_case, + recording_evidence, CaseSpec, Delivery, DriverRoute, Evidence, Observation, OracleKind, + RefusalCode, Scope, Targeting, +}; +use cua_driver_testkit::observer::TargetWindow; +use cua_driver_testkit::sentinel::{run_with_background_oracles, ForegroundSentinel}; use cua_driver_testkit::{ax, harness_app, spawn_in_job, Driver, McpDriver, ToolResponse}; // ── harness launcher ───────────────────────────────────────────────────────── @@ -64,17 +71,24 @@ fn harness_exe() -> PathBuf { /// hasn't been fully reaped and there are briefly two CuaTestHarness.Wpf /// windows on the desktop. fn launch_harness(driver: &mut McpDriver) -> Option { + launch_harness_with_state_file(driver, None) +} + +fn launch_harness_with_state_file( + driver: &mut McpDriver, + state_path: Option<&std::path::Path>, +) -> Option { let exe = harness_exe(); if !exe.exists() { eprintln!("harness exe not found at {exe:?} — run tests/fixtures/build/windows.ps1 first"); return None; } - let app = spawn_in_job( - Command::new(&exe) - .stdout(Stdio::null()) - .stderr(Stdio::null()), - ) - .ok()?; + let mut command = Command::new(&exe); + command.stdout(Stdio::null()).stderr(Stdio::null()); + if let Some(path) = state_path { + command.env("CUA_E2E_FIXTURE_STATE_PATH", path); + } + let app = spawn_in_job(&mut command).ok()?; let pid = app.id(); driver.reaper().push(app); // Short fixed settle for cold-start (window-creation + initial @@ -120,66 +134,136 @@ fn snapshot_lines_containing(text: &str, needles: &[&str]) -> String { } } -// ── tests ──────────────────────────────────────────────────────────────────── +fn fixture_state_line<'a>(text: &'a str, marker: &str) -> &'a str { + text.lines() + .find(|line| line.contains(marker)) + .unwrap_or_else(|| panic!("fixture state marker {marker:?} missing from snapshot")) +} -#[test] -#[ignore] -fn harness_wpf_smoke() { - let Some(mut driver) = McpDriver::spawn() else { - return; - }; - let Some(pid) = launch_harness(&mut driver) else { - return; - }; - println!("harness pid={}", pid); - - let (wid, title) = driver - .find_window(pid as i64, "CuaTestHarness WPF") - .expect("main window not found via list_windows"); - println!("main window: id={} title={:?}", wid, title); - - let snap = snapshot(&mut driver, pid, wid); - let text = snap.text(); - - // Buttons appear with explicit id= tags in the UIA markdown. - for aid in [ - "btn-increment", - "btn-reset", - "btn-open-msgbox", - "btn-save", - "btn-cancel", // regression guard for #1696 - "btn-open-owned", - "btn-open-layered", - "btn-exit", - ] { - assert!( - ax::has_id(text, aid), - "missing AutomationId {aid} in WPF UIA snapshot" - ); - } +fn window_bounds(driver: &mut McpDriver, pid: u32, wid: u64) -> (f64, f64, f64, f64) { + let response = driver.call("list_windows", serde_json::json!({ "pid": pid as i64 })); + let window = response.structured()["windows"] + .as_array() + .and_then(|windows| { + windows + .iter() + .find(|window| window["window_id"].as_u64() == Some(wid)) + }) + .unwrap_or_else(|| { + panic!( + "WPF window {wid} is missing from list_windows: {}", + response.text() + ) + }); + let bounds = &window["bounds"]; + ( + bounds["x"].as_f64().expect("WPF window bounds need x"), + bounds["y"].as_f64().expect("WPF window bounds need y"), + bounds["width"] + .as_f64() + .expect("WPF window bounds need width"), + bounds["height"] + .as_f64() + .expect("WPF window bounds need height"), + ) +} - // TextBlocks are reported as bare Text nodes (no UIA Invoke/Value pattern, - // no AutomationId in the rendered tree). Assert on their content instead. +fn pixel_center(state: &ToolResponse, target_id: &str, window: (f64, f64, f64, f64)) -> (f64, f64) { + let target_index = ax::element_index_by_id(state.text(), target_id) + .unwrap_or_else(|| panic!("missing PX target {target_id:?}: {}", state.text())); + let elements = state.structured()["elements"] + .as_array() + .expect("PX targeting requires structured elements"); + let target = elements + .iter() + .find(|element| element["element_index"].as_u64() == Some(target_index)) + .and_then(|element| element["frame"].as_object()) + .unwrap_or_else(|| panic!("element [{target_index}] has no structured frame")); + let target_w = target["w"].as_f64().unwrap_or(0.0); + let target_h = target["h"].as_f64().unwrap_or(0.0); + let (window_x, window_y, window_w, window_h) = window; assert!( - text.contains("HARNESS_TEXT_MARKER_v1"), - "text_body marker not in snapshot" + target_w > 0.0 && target_h > 0.0 && window_w > 0.0 && window_h > 0.0, + "WPF PX target and window need positive geometry: target={target:?}, window={window:?}" ); + let screenshot_w = state.structured()["screenshot_width"] + .as_f64() + .expect("PX targeting requires screenshot_width"); + let screenshot_h = state.structured()["screenshot_height"] + .as_f64() + .expect("PX targeting requires screenshot_height"); + let scale_x = screenshot_w / window_w; + let scale_y = screenshot_h / window_h; + let x = (target["x"].as_f64().unwrap_or(0.0) + target_w / 2.0 - window_x) * scale_x; + let y = (target["y"].as_f64().unwrap_or(0.0) + target_h / 2.0 - window_y) * scale_y; assert!( - text.contains("counter=0"), - "initial counter label not in snapshot" - ); - assert!( - text.contains("accel_fired=0"), - "initial accel label not in snapshot" + x >= 0.0 && x < screenshot_w && y >= 0.0 && y < screenshot_h, + "WPF PX target center ({x:.1}, {y:.1}) is outside the capture ({screenshot_w:.1}x{screenshot_h:.1})" ); + (x, y) +} - // HwndHost child should surface the native Win32 BUTTON as a UIA Button. - assert!( - text.contains("\"Native Win32 Child\""), - "native HWND child button not in snapshot" - ); +fn wait_for_fixture_file_text(path: &std::path::Path, id: &str, expected: &str) { + let deadline = Instant::now() + Duration::from_secs(3); + loop { + if let Ok(body) = std::fs::read(path) { + if let Ok(state) = serde_json::from_slice::(&body) { + if state[id]["text"].as_str() == Some(expected) { + return; + } + } + } + assert!( + Instant::now() < deadline, + "WPF fixture state {id:?} did not reach {expected:?}: {}", + std::fs::read_to_string(path).unwrap_or_else(|_| "".to_owned()) + ); + std::thread::sleep(Duration::from_millis(50)); + } +} - println!("✅ harness_wpf_smoke: all expected scenarios present in UIA tree"); +// ── tests ──────────────────────────────────────────────────────────────────── + +#[test] +#[ignore] +fn harness_wpf_smoke() { + run_case( + native_readonly_case( + "wpf", + "ax_tree", + Targeting::Ax, + DriverRoute::AxRead, + vec![OracleKind::AxState], + ), + |pid, wid, driver| { + let snap = snapshot(driver, pid, wid); + assert!(!snap.is_error(), "WPF AX snapshot failed: {}", snap.text()); + let text = snap.text(); + for aid in [ + "btn-increment", + "btn-reset", + "btn-open-msgbox", + "btn-save", + "btn-cancel", + "btn-open-owned", + "btn-open-layered", + "btn-exit", + ] { + assert!( + ax::has_id(text, aid), + "missing AutomationId {aid} in WPF UIA snapshot" + ); + } + for marker in ["HARNESS_TEXT_MARKER_v1", "counter=0", "accel_fired=0"] { + assert!(text.contains(marker), "missing WPF AX marker {marker}"); + } + assert!( + text.contains("\"Native Win32 Child\""), + "native HWND child button not in snapshot" + ); + Observation::delivered(vec![OracleKind::AxState], Evidence::default()) + }, + ); } // ── shared driver session helper ───────────────────────────────────────────── @@ -188,11 +272,11 @@ fn harness_wpf_smoke() { /// everything down (the harness app is reaped with the driver via the Job /// Object). Returns whatever the closure returns. The closure receives the /// harness pid, a pre-resolved main window_id, and the driver. -fn with_session(f: F) -> Option +fn with_named_session(label: &str, f: F) -> Option where F: FnOnce(u32, u64, &mut McpDriver) -> R, { - let mut driver = McpDriver::spawn()?; + let mut driver = McpDriver::spawn_named(label)?; let pid = launch_harness(&mut driver)?; let (wid, _) = driver .find_window(pid as i64, "CuaTestHarness WPF") @@ -200,142 +284,343 @@ where Some(f(pid, wid, &mut driver)) } -#[test] -#[ignore] -fn harness_wpf_counter_invoke() { - let Some(mut driver) = McpDriver::spawn() else { - return; - }; - let Some(pid) = launch_harness(&mut driver) else { - return; - }; +fn run_case(case: CaseSpec, test: impl FnOnce(u32, u64, &mut McpDriver) -> Observation) { + let cell_id = case.cell_id.clone(); + let delivery = case.delivery; + execute_case(case, |evidence| { + with_named_session(&cell_id, |pid, wid, driver| { + *evidence = recording_evidence(driver.recording_dir()); + if delivery == Delivery::NotApplicable { + driver.start_behavior_recording(); + } + test(pid, wid, driver) + }) + .expect("required WPF session did not start") + }); +} - let (wid, _) = driver - .find_window(pid as i64, "CuaTestHarness WPF") - .expect("main window"); - // Pre-snapshot so element_cache has indices we can address. - let pre = snapshot(&mut driver, pid, wid); - let idx = ax::element_index_by_id(pre.text(), "btn-increment") - .expect("btn-increment not in pre-snapshot"); +fn run_foreground_case( + action: &str, + targeting: Targeting, + route: DriverRoute, + extra_oracles: Vec, + test: impl FnOnce(u32, u64, &mut McpDriver) -> Vec, +) { + let mut case = native_foreground_case("wpf", action, targeting, route); + case.oracles.extend(extra_oracles); + case.oracles.sort(); + case.oracles.dedup(); + run_case(case, |pid, wid, driver| { + let mut passed = test(pid, wid, driver); + passed.push(OracleKind::FixtureState); + Observation::delivered(passed, Evidence::default()) + }); +} - let click = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, - "window_id": wid, - "element_index": idx - }), +fn run_background_case( + action: &str, + route: DriverRoute, + test: impl FnOnce(u32, u64, &mut McpDriver), +) { + run_case( + native_background_case("wpf", action, Targeting::Ax, route), + |pid, wid, driver| { + let (_, passed) = run_with_background_oracles( + driver, + TargetWindow { + pid, + native_id: wid, + }, + |driver| test(pid, wid, driver), + ) + .unwrap_or_else(|error| panic!("background desktop contract failed: {error}")); + Observation::delivered_with_fixture_state(passed) + }, ); - println!("click [{idx}] btn-increment: {}", click.text()); +} - std::thread::sleep(Duration::from_millis(300)); +fn observe_background( + driver: &mut McpDriver, + pid: u32, + wid: u64, + action: impl FnOnce(&mut McpDriver) -> R, +) -> (R, Vec) { + let sentinel = ForegroundSentinel::launch(driver); + sentinel + .assert_background_posture(TargetWindow { + pid, + native_id: wid, + }) + .expect("establish WPF background posture before recording"); + driver.start_behavior_recording(); + let (result, passed) = sentinel + .observe_background( + TargetWindow { + pid, + native_id: wid, + }, + || action(driver), + ) + .unwrap_or_else(|error| panic!("background desktop contract failed: {error}")); + for required in [ + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::Cursor, + OracleKind::NoLeakedInput, + ] { + assert!( + passed.contains(&required), + "background observer omitted required {required:?} oracle" + ); + } + (result, passed) +} - let post = snapshot(&mut driver, pid, wid); - let text = post.text(); - assert!( - text.contains("counter=1"), - "counter label did not advance after click — snapshot text: {}", - text.chars().take(400).collect::() - ); - println!("✅ harness_wpf_counter_invoke: counter advanced to 1"); +fn background_case(action: &str, route: DriverRoute) -> CaseSpec { + CaseSpec::delivered( + format!("windows-wpf-{action}-ax-background").replace('_', "-"), + "wpf", + "wpf", + action, + Targeting::Ax, + Delivery::Background, + Scope::Window, + route, + vec![ + OracleKind::FixtureState, + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::Cursor, + OracleKind::NoLeakedInput, + ], + ) +} + +fn delivered_with_fixture_state(mut passed: Vec) -> Observation { + passed.push(OracleKind::FixtureState); + passed.sort(); + passed.dedup(); + Observation::delivered(passed, Evidence::default()) } #[test] #[ignore] -fn harness_wpf_type_text() { - with_session(|pid, wid, driver| { - let snap = snapshot(driver, pid, wid); - let idx = - ax::element_index_by_id(snap.text(), "txt-input").expect("txt-input not in snapshot"); - - // WPF's TextBox needs *keyboard focus* for WM_CHAR delivery — and - // PostMessage(WM_LBUTTONDOWN) doesn't reliably transfer keyboard - // focus (WPF's input system treats posted events differently from - // real ones). Use dispatch:"foreground" → SendInput synthesizes - // an OS-level click that WPF treats identically to a user mouse, - // landing actual keyboard focus on the TextBox. - let _ = driver.call( - "bring_to_front", - serde_json::json!({ - "pid": pid as i64, "window_id": wid - }), - ); - std::thread::sleep(Duration::from_millis(300)); +fn harness_wpf_counter_invoke() { + execute_case( + background_case("left_click", DriverRoute::UiaInvoke), + |evidence| { + let mut driver = McpDriver::spawn_named("windows-wpf-left-click-ax-background") + .expect("required source-built driver did not start"); + *evidence = recording_evidence(driver.recording_dir()); + let pid = launch_harness(&mut driver).expect("required WPF harness did not launch"); + let (wid, _) = driver + .find_window(pid as i64, "CuaTestHarness WPF") + .expect("main window"); + let pre = snapshot(&mut driver, pid, wid); + let idx = ax::element_index_by_id(pre.text(), "btn-increment") + .expect("btn-increment not in pre-snapshot"); + let (click, passed) = observe_background(&mut driver, pid, wid, |driver| { + driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "background" + }), + ) + }); + assert!(!click.is_error(), "counter click failed: {}", click.text()); + std::thread::sleep(Duration::from_millis(300)); + let post = snapshot(&mut driver, pid, wid); + assert!( + post.text().contains("counter=1"), + "counter label did not advance after click: {}", + post.text().chars().take(400).collect::() + ); + delivered_with_fixture_state(passed) + }, + ); +} - let _ = driver.call( +#[test] +#[ignore] +fn harness_wpf_left_click_px_background() { + let case = native_background_case("wpf", "left_click", Targeting::Px, DriverRoute::UiaInvoke); + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_named("windows-wpf-left-click-px-background") + .expect("required source-built driver did not start"); + *evidence = recording_evidence(driver.recording_dir()); + let state_dir = tempfile::tempdir().expect("create WPF fixture state directory"); + let state_path = state_dir.path().join("state.json"); + let pid = launch_harness_with_state_file(&mut driver, Some(&state_path)) + .expect("required WPF harness did not launch"); + let (wid, _) = driver + .find_window(pid as i64, "CuaTestHarness WPF") + .expect("main window"); + wait_for_fixture_file_text(&state_path, "lbl-click-count", "clicks=0"); + + let bounds = window_bounds(&mut driver, pid, wid); + let ready_state = snapshot(&mut driver, pid, wid); + let (x, y) = pixel_center(&ready_state, "border-click-target", bounds); + let geometry_probe = driver.call( "click", serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx, + "pid": pid as i64, + "window_id": wid, + "x": x, + "y": y, "delivery_mode": "foreground" }), ); - std::thread::sleep(Duration::from_millis(400)); - - // SendInput's restore_foreground_polling_best_effort may yank - // foreground back from the harness window between click and - // type_text. Re-assert foreground so PostMessage WM_CHAR finds - // the TextBox with keyboard focus. - let _ = driver.call( - "bring_to_front", - serde_json::json!({ - "pid": pid as i64, "window_id": wid - }), + assert!( + !geometry_probe.is_error(), + "WPF foreground PX geometry probe failed: {}", + geometry_probe.text() ); - std::thread::sleep(Duration::from_millis(300)); + wait_for_fixture_file_text(&state_path, "lbl-click-count", "clicks=1"); - let resp = driver.call( - "type_text", - serde_json::json!({ - "pid": pid as i64, - "text": "harness-typed", - "delivery_mode": "foreground" - }), - ); - println!("type_text: {}", resp.text()); - std::thread::sleep(Duration::from_millis(700)); - - let post = snapshot(driver, pid, wid); - let text = post.text(); - let mirror_lines: Vec<&str> = text - .lines() - .filter(|l| l.contains("mirror=") || l.contains("txt-input")) - .collect(); + let (click, passed) = observe_background(&mut driver, pid, wid, |driver| { + driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "x": x, + "y": y, + "delivery_mode": "background" + }), + ) + }); assert!( - text.contains("mirror=harness-typed"), - "TextBox mirror did not reflect typed text. Mirror/input lines: {:?}", - mirror_lines + !click.is_error(), + "WPF PX background click failed: {}", + click.text() + ); + assert_eq!( + click.structured()["path"].as_str(), + Some("ax"), + "WPF PX background click used an unexpected driver route: {}", + click.text() ); - println!("✅ harness_wpf_type_text: TextBox mirror advanced to 'harness-typed'"); + wait_for_fixture_file_text(&state_path, "lbl-click-count", "clicks=2"); + wait_for_fixture_file_text(&state_path, "lbl-last-action", "last_action=left_click"); + delivered_with_fixture_state(passed) }); } #[test] #[ignore] -fn harness_wpf_set_value() { - // Companion to harness_wpf_type_text: exercises the UIA ValuePattern - // write path via the `set_value` tool. No focus needed — purely UIA. - with_session(|pid, wid, driver| { - let snap = snapshot(driver, pid, wid); - let idx = - ax::element_index_by_id(snap.text(), "txt-input").expect("txt-input not in snapshot"); - let _ = driver.call( - "set_value", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx, - "value": "via-uia-setvalue" - }), - ); - std::thread::sleep(Duration::from_millis(400)); +fn harness_wpf_type_text() { + run_foreground_case( + "type_text", + Targeting::Ax, + DriverRoute::WindowsSendInput, + Vec::new(), + |pid, wid, driver| { + let snap = snapshot(driver, pid, wid); + let idx = ax::element_index_by_id(snap.text(), "txt-input") + .expect("txt-input not in snapshot"); + + // WPF's TextBox needs *keyboard focus* for WM_CHAR delivery — and + // PostMessage(WM_LBUTTONDOWN) doesn't reliably transfer keyboard + // focus (WPF's input system treats posted events differently from + // real ones). Use dispatch:"foreground" → SendInput synthesizes + // an OS-level click that WPF treats identically to a user mouse, + // landing actual keyboard focus on the TextBox. + let _ = driver.call( + "bring_to_front", + serde_json::json!({ + "pid": pid as i64, "window_id": wid + }), + ); + std::thread::sleep(Duration::from_millis(300)); + driver.start_behavior_recording(); - let post = snapshot(driver, pid, wid); - let text = post.text(); - assert!( - text.contains("mirror=via-uia-setvalue"), - "set_value did not update TextBox. Excerpt: {}", - text.chars().take(500).collect::() - ); - println!("✅ harness_wpf_set_value: ValuePattern.SetValue wrote to TextBox"); - }); + let _ = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "foreground" + }), + ); + std::thread::sleep(Duration::from_millis(400)); + + // SendInput's restore_foreground_polling_best_effort may yank + // foreground back from the harness window between click and + // type_text. Re-assert foreground so PostMessage WM_CHAR finds + // the TextBox with keyboard focus. + let _ = driver.call( + "bring_to_front", + serde_json::json!({ + "pid": pid as i64, "window_id": wid + }), + ); + std::thread::sleep(Duration::from_millis(300)); + + let resp = driver.call( + "type_text", + serde_json::json!({ + "pid": pid as i64, + "text": "harness-typed", + "delivery_mode": "foreground" + }), + ); + println!("type_text: {}", resp.text()); + std::thread::sleep(Duration::from_millis(700)); + + let post = snapshot(driver, pid, wid); + let text = post.text(); + let mirror_lines: Vec<&str> = text + .lines() + .filter(|l| l.contains("mirror=") || l.contains("txt-input")) + .collect(); + assert!( + text.contains("mirror=harness-typed"), + "TextBox mirror did not reflect typed text. Mirror/input lines: {:?}", + mirror_lines + ); + println!("✅ harness_wpf_type_text: TextBox mirror advanced to 'harness-typed'"); + Vec::new() + }, + ); +} + +#[test] +#[ignore] +fn harness_wpf_set_value() { + execute_case( + background_case("set_value", DriverRoute::UiaValue), + |evidence| { + with_named_session("windows-wpf-set-value-ax-background", |pid, wid, driver| { + *evidence = recording_evidence(driver.recording_dir()); + let snap = snapshot(driver, pid, wid); + let idx = ax::element_index_by_id(snap.text(), "txt-input") + .expect("txt-input not in snapshot"); + let (response, passed) = observe_background(driver, pid, wid, |driver| { + driver.call( + "set_value", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "value": "via-uia-setvalue" + }), + ) + }); + assert!( + !response.is_error(), + "set_value failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(400)); + let post = snapshot(driver, pid, wid); + assert!( + post.text().contains("mirror=via-uia-setvalue"), + "set_value did not update TextBox: {}", + post.text().chars().take(500).collect::() + ); + delivered_with_fixture_state(passed) + }) + .expect("required WPF session did not start") + }, + ); } // In test-batch mode (many harnesses launched/killed in sequence) the WPF @@ -353,374 +638,453 @@ fn focus_harness(driver: &mut McpDriver, pid: u32, wid: u64) { }), ); std::thread::sleep(Duration::from_millis(300)); + driver.start_behavior_recording(); } #[test] #[ignore] fn harness_wpf_right_click() { - with_session(|pid, wid, driver| { - focus_harness(driver, pid, wid); - let snap = snapshot(driver, pid, wid); - let idx = ax::element_index_by_id(snap.text(), "border-click-target") - .expect("border-click-target not in snapshot"); - // Same dispatch:foreground rationale as type_text — PostMessage - // WM_RBUTTONDOWN doesn't always reach WPF's MouseRightButtonDown - // routed-event chain (intermittent in batch runs). - let resp = driver.call( - "right_click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx, - "delivery_mode": "foreground" - }), - ); - println!("right_click: {}", resp.text()); - std::thread::sleep(Duration::from_millis(400)); - - let post = snapshot(driver, pid, wid); - let text = post.text(); - let action_lines: Vec<&str> = text - .lines() - .filter(|l| l.contains("last_action=") || l.contains("clicks=")) - .collect(); - assert!( - text.contains("last_action=right_click"), - "right_click handler did not fire. Action/click lines: {:?}", - action_lines - ); - println!("✅ harness_wpf_right_click: last_action=right_click"); - }); + run_foreground_case( + "right_click", + Targeting::Ax, + DriverRoute::WindowsSendInput, + Vec::new(), + |pid, wid, driver| { + focus_harness(driver, pid, wid); + let snap = snapshot(driver, pid, wid); + let idx = ax::element_index_by_id(snap.text(), "border-click-target") + .expect("border-click-target not in snapshot"); + // Same dispatch:foreground rationale as type_text — PostMessage + // WM_RBUTTONDOWN doesn't always reach WPF's MouseRightButtonDown + // routed-event chain (intermittent in batch runs). + let resp = driver.call( + "right_click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "foreground" + }), + ); + println!("right_click: {}", resp.text()); + std::thread::sleep(Duration::from_millis(400)); + + let post = snapshot(driver, pid, wid); + let text = post.text(); + let action_lines: Vec<&str> = text + .lines() + .filter(|l| l.contains("last_action=") || l.contains("clicks=")) + .collect(); + assert!( + text.contains("last_action=right_click"), + "right_click handler did not fire. Action/click lines: {:?}", + action_lines + ); + println!("✅ harness_wpf_right_click: last_action=right_click"); + Vec::new() + }, + ); } #[test] #[ignore] fn harness_wpf_double_click() { - with_session(|pid, wid, driver| { - focus_harness(driver, pid, wid); - let snap = snapshot(driver, pid, wid); - let idx = ax::element_index_by_id(snap.text(), "border-click-target") - .expect("border-click-target not in snapshot"); - // dispatch:foreground for the same reason as right_click — - // PostMessage WM_LBUTTONDOWN ×2 doesn't always reach WPF's - // MouseDoubleClick / ClickCount=2 path under test-batch load. - let resp = driver.call( - "double_click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx, - "delivery_mode": "foreground" - }), - ); - println!("double_click: {}", resp.text()); - std::thread::sleep(Duration::from_millis(400)); - - let post = snapshot(driver, pid, wid); - let text = post.text(); - let action_lines: Vec<&str> = text - .lines() - .filter(|l| l.contains("last_action=") || l.contains("clicks=")) - .collect(); - assert!( - text.contains("last_action=double_click"), - "double_click handler did not register a 2nd click. \ + run_foreground_case( + "double_click", + Targeting::Ax, + DriverRoute::WindowsSendInput, + Vec::new(), + |pid, wid, driver| { + focus_harness(driver, pid, wid); + let snap = snapshot(driver, pid, wid); + let idx = ax::element_index_by_id(snap.text(), "border-click-target") + .expect("border-click-target not in snapshot"); + // dispatch:foreground for the same reason as right_click — + // PostMessage WM_LBUTTONDOWN ×2 doesn't always reach WPF's + // MouseDoubleClick / ClickCount=2 path under test-batch load. + let resp = driver.call( + "double_click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "foreground" + }), + ); + println!("double_click: {}", resp.text()); + std::thread::sleep(Duration::from_millis(400)); + + let post = snapshot(driver, pid, wid); + let text = post.text(); + let action_lines: Vec<&str> = text + .lines() + .filter(|l| l.contains("last_action=") || l.contains("clicks=")) + .collect(); + assert!( + text.contains("last_action=double_click"), + "double_click handler did not register a 2nd click. \ Action/click lines: {:?}", - action_lines - ); - println!("✅ harness_wpf_double_click: last_action=double_click"); - }); + action_lines + ); + println!("✅ harness_wpf_double_click: last_action=double_click"); + Vec::new() + }, + ); } #[test] #[ignore] fn harness_wpf_press_key_accelerator() { - // F5 binding rather than the Ctrl+Shift+H one: cua-driver's hotkey - // PostMessage path doesn't update OS modifier-key state (GetKeyState - // returns "not pressed" for VK_CONTROL), so WPF's KeyBinding with - // Modifiers=Control+Shift never matches. The UIA-worker SendInput - // path would handle modifiers but requires the cua-driver-uia.exe - // helper that isn't in our test config. F5 has no modifier and works - // on the PostMessage path. - with_session(|pid, wid, driver| { - let resp = driver.call( - "press_key", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "key": "f5" - }), - ); - println!("press_key f5: {}", resp.text()); - std::thread::sleep(Duration::from_millis(400)); - - let post = snapshot(driver, pid, wid); - let text = post.text(); - assert!( - text.contains("accel_fired=1"), - "F5 KeyBinding did not fire. Snapshot excerpt: {}", - text.chars().take(500).collect::() - ); - println!("✅ harness_wpf_press_key_accelerator: accel_fired=1 (F5 via PostMessage)"); - }); + // WPF's InputManager ignores posted key messages while another native + // window owns foreground, even for an unmodified F5 binding. The driver + // must refuse before posting instead of returning an unverifiable success. + execute_case( + background_case("keyboard", DriverRoute::PostMessage) + .expecting_refusal(vec![RefusalCode::BackgroundUnavailable]), + |evidence| { + with_named_session("windows-wpf-keyboard-ax-background", |pid, wid, driver| { + *evidence = recording_evidence(driver.recording_dir()); + let (response, mut passed) = observe_background(driver, pid, wid, |driver| { + driver.call( + "press_key", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "key": "f5", + "delivery_mode": "background" + }), + ) + }); + assert!( + response.is_error(), + "WPF background press_key unexpectedly reported delivery: {}", + response.text() + ); + let code = response.structured()["code"] + .as_str() + .and_then(RefusalCode::from_driver_code) + .expect("WPF background key refusal needs a structured code"); + let post = snapshot(driver, pid, wid); + assert!( + post.text().contains("accel_fired=0"), + "refused WPF key mutated fixture state: {}", + post.text().chars().take(500).collect::() + ); + passed.push(OracleKind::FixtureState); + Observation::refused(code, passed, response.text(), Evidence::default()) + }) + .expect("required WPF session did not start") + }, + ); } #[test] #[ignore] fn harness_wpf_scroll() { - with_session(|pid, wid, driver| { - // Pre-snapshot to populate the cache + read initial offset. - let pre = snapshot(driver, pid, wid); - let pre_text = pre.text(); - assert!( - pre_text.contains("scroll_offset=0"), - "expected initial scroll_offset=0, got: {}", - pre_text - .lines() - .filter(|l| l.contains("scroll_offset")) - .collect::>() - .join(" / ") - ); - - // Click into the ScrollViewer so it gets focus / its descendants - // become the WM_VSCROLL target. - let idx = ax::element_index_by_id(pre.text(), "scroll-tall") - .expect("scroll-tall not in snapshot"); - let _ = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, - "window_id": wid, - "element_index": idx, - "delivery_mode": "foreground" - }), - ); - std::thread::sleep(Duration::from_millis(200)); - - // Scroll down 5 lines. Keep this on the default background rung: the - // WPF harness translates the driver's WM_VSCROLL messages into the - // ScrollViewer movement we assert below. - let resp = driver.call( - "scroll", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, - "direction": "down", "by": "line", "amount": 5, - }), - ); - println!("scroll down: {}", resp.text()); - std::thread::sleep(Duration::from_millis(400)); - - let post = snapshot(driver, pid, wid); - let text = post.text(); - let advanced = text - .lines() - .any(|l| l.contains("scroll_offset=") && !l.contains("scroll_offset=0\"")); - assert!( - advanced, - "scroll offset did not advance after WM_VSCROLL. Lines: {}", - text.lines() - .filter(|l| l.contains("scroll_offset")) - .collect::>() - .join(" / ") - ); - println!("✅ harness_wpf_scroll: scroll_offset advanced past 0"); - }); + execute_case( + background_case("scroll", DriverRoute::UiaScroll), + |evidence| { + with_named_session("windows-wpf-scroll-ax-background", |pid, wid, driver| { + *evidence = recording_evidence(driver.recording_dir()); + // Pre-snapshot to populate the cache + read initial offset. + let pre = snapshot(driver, pid, wid); + let pre_text = pre.text(); + assert!( + pre_text.contains("scroll_offset=0"), + "expected initial scroll_offset=0, got: {}", + pre_text + .lines() + .filter(|l| l.contains("scroll_offset")) + .collect::>() + .join(" / ") + ); + + // Click into the ScrollViewer so it gets focus / its descendants + // become the WM_VSCROLL target. + let idx = ax::element_index_by_id(pre.text(), "scroll-tall") + .expect("scroll-tall not in snapshot"); + let _ = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "element_index": idx, + "delivery_mode": "foreground" + }), + ); + std::thread::sleep(Duration::from_millis(200)); + + // Scroll down 5 lines. Keep this on the default background rung: the + // WPF harness translates the driver's WM_VSCROLL messages into the + // ScrollViewer movement we assert below. + let (resp, passed) = observe_background(driver, pid, wid, |driver| { + driver.call( + "scroll", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, + "direction": "down", "by": "line", "amount": 5, + "delivery_mode": "background" + }), + ) + }); + assert!(!resp.is_error(), "scroll failed: {}", resp.text()); + println!("scroll down: {}", resp.text()); + std::thread::sleep(Duration::from_millis(400)); + + let post = snapshot(driver, pid, wid); + let text = post.text(); + let advanced = text + .lines() + .any(|l| l.contains("scroll_offset=") && !l.contains("scroll_offset=0\"")); + assert!( + advanced, + "scroll offset did not advance after WM_VSCROLL. Lines: {}", + text.lines() + .filter(|l| l.contains("scroll_offset")) + .collect::>() + .join(" / ") + ); + delivered_with_fixture_state(passed) + }) + .expect("required WPF session did not start") + }, + ); } #[test] #[ignore] fn harness_wpf_modal_messagebox() { - with_session(|pid, wid, driver| { - let snap = snapshot(driver, pid, wid); - let idx = ax::element_index_by_id(snap.text(), "btn-open-msgbox") - .expect("btn-open-msgbox not in snapshot"); - let _ = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx - }), - ); - std::thread::sleep(Duration::from_millis(600)); + run_foreground_case( + "modal_messagebox", + Targeting::Ax, + DriverRoute::WindowsSendInput, + vec![OracleKind::AxState], + |pid, wid, driver| { + focus_harness(driver, pid, wid); + let snap = snapshot(driver, pid, wid); + let idx = ax::element_index_by_id(snap.text(), "btn-open-msgbox") + .expect("btn-open-msgbox not in snapshot"); + let open = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "foreground" + }), + ); + assert!(!open.is_error(), "open message box failed: {}", open.text()); + std::thread::sleep(Duration::from_millis(600)); - // List windows — the modal MessageBox should be a new top-level window - // owned by the same pid. - let resp = driver.call( - "list_windows", - serde_json::json!({ - "pid": pid as i64 - }), - ); - let windows = resp.structured()["windows"] - .as_array() - .expect("windows array"); - let modal = windows - .iter() - .find(|w| { - w["title"] - .as_str() - .map(|t| t.contains("Harness MessageBox")) - .unwrap_or(false) - }) - .expect("Harness MessageBox modal window not found"); - let modal_wid = modal["window_id"].as_u64().unwrap(); - println!("modal window_id={}", modal_wid); + // List windows — the modal MessageBox should be a new top-level window + // owned by the same pid. + let resp = driver.call( + "list_windows", + serde_json::json!({ + "pid": pid as i64 + }), + ); + let windows = resp.structured()["windows"] + .as_array() + .expect("windows array"); + let modal = windows + .iter() + .find(|w| { + w["title"] + .as_str() + .map(|t| t.contains("Harness MessageBox")) + .unwrap_or(false) + }) + .expect("Harness MessageBox modal window not found"); + let modal_wid = modal["window_id"].as_u64().unwrap(); + println!("modal window_id={}", modal_wid); - // Walk the modal's UIA tree — expect OK and Cancel buttons. - let modal_snap = driver.call( - "get_window_state", - serde_json::json!({ - "pid": pid as i64, "window_id": modal_wid, "capture_mode": "ax" - }), - ); - let modal_text = modal_snap.text(); - assert!( - modal_text.contains("\"OK\""), - "MessageBox UIA tree missing OK button. Tree: {}", - modal_text.chars().take(800).collect::() - ); - assert!( - modal_text.contains("\"Cancel\""), - "MessageBox UIA tree missing Cancel button" - ); + // Walk the modal's UIA tree — expect OK and Cancel buttons. + let modal_snap = driver.call( + "get_window_state", + serde_json::json!({ + "pid": pid as i64, "window_id": modal_wid, "capture_mode": "ax" + }), + ); + let modal_text = modal_snap.text(); + assert!( + modal_text.contains("\"OK\""), + "MessageBox UIA tree missing OK button. Tree: {}", + modal_text.chars().take(800).collect::() + ); + assert!( + modal_text.contains("\"Cancel\""), + "MessageBox UIA tree missing Cancel button" + ); - // Dismiss by clicking Cancel in the modal. - let cancel_idx = modal_text - .lines() - .find(|l| l.contains("\"Cancel\"") && l.contains('[')) - .and_then(|l| { - let s = l.find('[')? + 1; - let e = l[s..].find(']')? + s; - l[s..e].trim().parse::().ok() - }) - .expect("Cancel button element_index not parseable"); - let _ = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": modal_wid, "element_index": cancel_idx - }), - ); - std::thread::sleep(Duration::from_millis(400)); - println!("✅ harness_wpf_modal_messagebox: opened + parsed + dismissed"); - }); + // Dismiss by clicking Cancel in the modal. + let cancel_idx = modal_text + .lines() + .find(|l| l.contains("\"Cancel\"") && l.contains('[')) + .and_then(|l| { + let s = l.find('[')? + 1; + let e = l[s..].find(']')? + s; + l[s..e].trim().parse::().ok() + }) + .expect("Cancel button element_index not parseable"); + let dismiss = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": modal_wid, "element_index": cancel_idx, + "delivery_mode": "foreground" + }), + ); + assert!( + !dismiss.is_error(), + "dismiss message box failed: {}", + dismiss.text() + ); + std::thread::sleep(Duration::from_millis(400)); + println!("✅ harness_wpf_modal_messagebox: opened + parsed + dismissed"); + vec![OracleKind::AxState] + }, + ); } #[test] #[ignore] fn harness_wpf_owned_popup() { - with_session(|pid, wid, driver| { - let snap = snapshot(driver, pid, wid); - let idx = ax::element_index_by_id(snap.text(), "btn-open-owned") - .expect("btn-open-owned not in snapshot"); - let _ = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx - }), - ); - std::thread::sleep(Duration::from_millis(500)); + run_foreground_case( + "owned_popup", + Targeting::Ax, + DriverRoute::WindowsSendInput, + vec![OracleKind::AxState], + |pid, wid, driver| { + focus_harness(driver, pid, wid); + let snap = snapshot(driver, pid, wid); + let idx = ax::element_index_by_id(snap.text(), "btn-open-owned") + .expect("btn-open-owned not in snapshot"); + let open = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "foreground" + }), + ); + assert!(!open.is_error(), "open owned popup failed: {}", open.text()); + std::thread::sleep(Duration::from_millis(500)); - let resp = driver.call( - "list_windows", - serde_json::json!({ - "pid": pid as i64 - }), - ); - let windows = resp.structured()["windows"].as_array().unwrap(); - let owned = windows - .iter() - .find(|w| { - w["title"] - .as_str() - .map(|t| t.contains("Harness Owned Popup")) - .unwrap_or(false) - }) - .expect("Harness Owned Popup window not found in list_windows"); - let owned_wid = owned["window_id"].as_u64().unwrap(); + let resp = driver.call( + "list_windows", + serde_json::json!({ + "pid": pid as i64 + }), + ); + let windows = resp.structured()["windows"].as_array().unwrap(); + let owned = windows + .iter() + .find(|w| { + w["title"] + .as_str() + .map(|t| t.contains("Harness Owned Popup")) + .unwrap_or(false) + }) + .expect("Harness Owned Popup window not found in list_windows"); + let owned_wid = owned["window_id"].as_u64().unwrap(); - let owned_snap = driver.call( - "get_window_state", - serde_json::json!({ - "pid": pid as i64, "window_id": owned_wid, "capture_mode": "ax" - }), - ); - let owned_text = owned_snap.text(); - assert!( - owned_text.contains("OWNED_POPUP_MARKER_v1"), - "owned popup body marker missing. Tree: {}", - owned_text.chars().take(600).collect::() - ); - println!("✅ harness_wpf_owned_popup: opened + parsed"); - }); + let owned_snap = driver.call( + "get_window_state", + serde_json::json!({ + "pid": pid as i64, "window_id": owned_wid, "capture_mode": "ax" + }), + ); + let owned_text = owned_snap.text(); + assert!( + owned_text.contains("OWNED_POPUP_MARKER_v1"), + "owned popup body marker missing. Tree: {}", + owned_text.chars().take(600).collect::() + ); + println!("✅ harness_wpf_owned_popup: opened + parsed"); + vec![OracleKind::AxState] + }, + ); } #[test] #[ignore] fn harness_wpf_layered_popup_capture() { - with_session(|pid, wid, driver| { - let snap = snapshot(driver, pid, wid); - let idx = ax::element_index_by_id(snap.text(), "btn-open-layered") - .expect("btn-open-layered not in snapshot"); - let _ = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx - }), - ); - std::thread::sleep(Duration::from_millis(600)); + run_foreground_case( + "layered_popup_capture", + Targeting::Ax, + DriverRoute::Composite, + vec![OracleKind::Pixels], + |pid, wid, driver| { + focus_harness(driver, pid, wid); + let snap = snapshot(driver, pid, wid); + let idx = ax::element_index_by_id(snap.text(), "btn-open-layered") + .expect("btn-open-layered not in snapshot"); + let open = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "foreground" + }), + ); + assert!( + !open.is_error(), + "open layered popup failed: {}", + open.text() + ); + std::thread::sleep(Duration::from_millis(600)); - let resp = driver.call( - "list_windows", - serde_json::json!({ - "pid": pid as i64 - }), - ); - let windows = resp.structured()["windows"].as_array().unwrap(); - let layered = windows - .iter() - .find(|w| { - w["title"] - .as_str() - .map(|t| t.contains("Harness Layered Popup")) - .unwrap_or(false) - }) - .expect("Harness Layered Popup window not found"); - let layered_wid = layered["window_id"].as_u64().unwrap(); - - // Capture-only path — assert the screenshot is not all-black, which - // is the failure mode for PrintWindow against layered windows - // without the WGC fallback. - let cap = driver.call( - "get_window_state", - serde_json::json!({ - "pid": pid as i64, "window_id": layered_wid, "capture_mode": "vision" - }), - ); - let img_b64 = cap.raw["result"]["content"] - .as_array() - .and_then(|arr| { - arr.iter().find_map(|c| { - if c["type"] == "image" { - c["data"].as_str() - } else { - None - } + let resp = driver.call( + "list_windows", + serde_json::json!({ + "pid": pid as i64 + }), + ); + let windows = resp.structured()["windows"].as_array().unwrap(); + let layered = windows + .iter() + .find(|w| { + w["title"] + .as_str() + .map(|t| t.contains("Harness Layered Popup")) + .unwrap_or(false) }) - }) - .expect("layered window capture returned no image"); - // Decode the PNG and look for any non-black pixel. - let bytes = base64::Engine::decode(&base64::engine::general_purpose::STANDARD, img_b64) - .expect("base64"); - let img = image::load_from_memory(&bytes).expect("png decode"); - let rgb = img.to_rgb8(); - let any_color = rgb - .pixels() - .any(|p| p.0[0] > 12 || p.0[1] > 12 || p.0[2] > 12); - assert!( + .expect("Harness Layered Popup window not found"); + let layered_wid = layered["window_id"].as_u64().unwrap(); + + // Capture-only path — assert the screenshot is not all-black, which + // is the failure mode for PrintWindow against layered windows + // without the WGC fallback. + let cap = driver.call( + "get_window_state", + serde_json::json!({ + "pid": pid as i64, "window_id": layered_wid, "capture_mode": "vision" + }), + ); + let img_b64 = cap.raw["result"]["content"] + .as_array() + .and_then(|arr| { + arr.iter().find_map(|c| { + if c["type"] == "image" { + c["data"].as_str() + } else { + None + } + }) + }) + .expect("layered window capture returned no image"); + // Decode the PNG and look for any non-black pixel. + let bytes = base64::Engine::decode(&base64::engine::general_purpose::STANDARD, img_b64) + .expect("base64"); + let img = image::load_from_memory(&bytes).expect("png decode"); + let rgb = img.to_rgb8(); + let any_color = rgb + .pixels() + .any(|p| p.0[0] > 12 || p.0[1] > 12 || p.0[2] > 12); + assert!( any_color, "layered window capture is all-black ({}x{}). PrintWindow likely needs WGC fallback.", rgb.width(), rgb.height() ); - println!( - "✅ harness_wpf_layered_popup_capture: capture has non-black pixels ({}x{})", - rgb.width(), - rgb.height() - ); - }); + println!( + "✅ harness_wpf_layered_popup_capture: capture has non-black pixels ({}x{})", + rgb.width(), + rgb.height() + ); + vec![OracleKind::Pixels] + }, + ); } // ── slider / checkable / combo / list / menu coverage ──────────────────────── @@ -740,52 +1104,112 @@ fn harness_wpf_slider_drag() { // bring_to_front first to make the harness foreground (via // AttachThreadInput), then SendInput's own SetForegroundWindow is a // no-op success. - with_session(|pid, wid, driver| { - focus_harness(driver, pid, wid); - let pre = snapshot(driver, pid, wid); - assert!( - pre.text().contains("slider_value=0"), - "initial slider_value=0 missing" - ); + run_foreground_case( + "slider_drag", + Targeting::Px, + DriverRoute::WindowsSendInput, + Vec::new(), + |pid, wid, driver| { + focus_harness(driver, pid, wid); + let pre = snapshot(driver, pid, wid); + assert!( + pre.text().contains("slider_value=0"), + "initial slider_value=0 missing" + ); - let resp = driver.call( - "drag", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, - // Window-local coords along the slider TRACK. The track row sits at - // window-local y≈304 (verified on the VM: y=275 landed ~29px above - // it, on empty GroupBox space, so the thumb never moved); the thumb - // rests at the left (x≈44) at value=0. Dragging left→right advances - // the value. (TODO: derive these from the `sld-value` element frame - // in get_window_state for DPI/placement independence.) - "from_x": 44.0, "from_y": 304.0, - "to_x": 330.0, "to_y": 304.0, - "duration_ms": 700, "steps": 40, - "delivery_mode": "foreground" - }), - ); - let msg = resp.text(); - println!("drag slider (foreground): {msg}"); - assert!( - msg.starts_with("✅"), - "drag tool returned non-success: {msg}" - ); - std::thread::sleep(Duration::from_millis(500)); + let resp = driver.call( + "drag", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, + // Window-local coords along the slider TRACK. The track row sits at + // window-local y≈304 (verified on the VM: y=275 landed ~29px above + // it, on empty GroupBox space, so the thumb never moved); the thumb + // rests at the left (x≈44) at value=0. Dragging left→right advances + // the value. (TODO: derive these from the `sld-value` element frame + // in get_window_state for DPI/placement independence.) + "from_x": 44.0, "from_y": 304.0, + "to_x": 330.0, "to_y": 304.0, + "duration_ms": 700, "steps": 40, + "delivery_mode": "foreground" + }), + ); + let msg = resp.text(); + println!("drag slider (foreground): {msg}"); + assert!( + msg.starts_with("✅"), + "drag tool returned non-success: {msg}" + ); + std::thread::sleep(Duration::from_millis(500)); + + let post = snapshot(driver, pid, wid); + let text = post.text(); + let advanced = text + .lines() + .any(|l| l.contains("slider_value=") && !l.contains("slider_value=0\"")); + assert!( + advanced, + "Slider value did not advance via SendInput drag. Lines: {}", + text.lines() + .filter(|l| l.contains("slider_value")) + .collect::>() + .join(" / ") + ); + println!("✅ harness_wpf_slider_drag: thumb tracked via SendInput drag"); + Vec::new() + }, + ); +} - let post = snapshot(driver, pid, wid); - let text = post.text(); - let advanced = text - .lines() - .any(|l| l.contains("slider_value=") && !l.contains("slider_value=0\"")); +#[test] +#[ignore] +fn harness_wpf_slider_drag_background_refusal() { + let case = native_background_case( + "wpf", + "slider_drag", + Targeting::Px, + DriverRoute::WindowsTargetedInjection, + ) + .expecting_refusal(vec![RefusalCode::BackgroundUnavailable]); + run_case(case, |pid, wid, driver| { + let before = snapshot(driver, pid, wid); + let before_value = fixture_state_line(before.text(), "slider_value="); + let (response, mut passed) = observe_background(driver, pid, wid, |driver| { + driver.call( + "drag", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, + "from_x": 44.0, "from_y": 304.0, + "to_x": 330.0, "to_y": 304.0, + "duration_ms": 700, "steps": 40, + "delivery_mode": "background" + }), + ) + }); assert!( - advanced, - "Slider value did not advance via SendInput drag. Lines: {}", - text.lines() - .filter(|l| l.contains("slider_value")) - .collect::>() - .join(" / ") + response.is_error(), + "WPF background drag unexpectedly reported delivery: {}", + response.text() + ); + assert_eq!( + response.structured()["code"].as_str(), + Some("background_unavailable"), + "WPF background drag returned the wrong refusal: {}", + response.text() ); - println!("✅ harness_wpf_slider_drag: thumb tracked via SendInput drag"); + std::thread::sleep(Duration::from_millis(200)); + let after = snapshot(driver, pid, wid); + assert_eq!( + fixture_state_line(after.text(), "slider_value="), + before_value, + "refused WPF background drag changed the slider value" + ); + passed.push(OracleKind::FixtureState); + Observation::refused( + RefusalCode::BackgroundUnavailable, + passed, + response.text(), + Evidence::default(), + ) }); } @@ -795,228 +1219,294 @@ fn harness_wpf_slider_increase_large() { // Companion to slider_drag — exercises UIA Invoke on the Slider's // internal IncreaseLarge "page-up" button. Doesn't depend on screen // coords, so it's the more robust slider integration test. - with_session(|pid, wid, driver| { - focus_harness(driver, pid, wid); - let snap = snapshot(driver, pid, wid); - let idx = ax::element_index_by_id(snap.text(), "IncreaseLarge") - .expect("slider IncreaseLarge button not in snapshot"); - for i in 0..3 { - let resp = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx - }), + run_background_case( + "slider_increase_large", + DriverRoute::UiaInvoke, + |pid, wid, driver| { + let snap = snapshot(driver, pid, wid); + let idx = ax::element_index_by_id(snap.text(), "IncreaseLarge") + .expect("slider IncreaseLarge button not in snapshot"); + for i in 0..3 { + let resp = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx + }), + ); + println!("invoke IncreaseLarge #{i}: {}", resp.text()); + assert!( + !resp.is_error(), + "IncreaseLarge invoke failed: {}", + resp.text() + ); + std::thread::sleep(Duration::from_millis(150)); + } + std::thread::sleep(Duration::from_millis(300)); + let post = snapshot(driver, pid, wid); + let text = post.text(); + let advanced = text + .lines() + .any(|l| l.contains("slider_value=") && !l.contains("slider_value=0\"")); + assert!( + advanced, + "slider IncreaseLarge invokes did not advance value. Lines: {}", + text.lines() + .filter(|l| l.contains("slider_value")) + .collect::>() + .join(" / ") ); - println!("invoke IncreaseLarge #{i}: {}", resp.text()); - std::thread::sleep(Duration::from_millis(150)); - } - std::thread::sleep(Duration::from_millis(300)); - let post = snapshot(driver, pid, wid); - let text = post.text(); - let advanced = text - .lines() - .any(|l| l.contains("slider_value=") && !l.contains("slider_value=0\"")); - assert!( - advanced, - "slider IncreaseLarge invokes did not advance value. Lines: {}", - text.lines() - .filter(|l| l.contains("slider_value")) - .collect::>() - .join(" / ") - ); - println!("✅ harness_wpf_slider_increase_large: advanced via UIA Invoke"); - }); + println!("✅ harness_wpf_slider_increase_large: advanced via UIA Invoke"); + }, + ); } #[test] #[ignore] fn harness_wpf_checkbox_toggle() { - with_session(|pid, wid, driver| { - focus_harness(driver, pid, wid); - let snap = snapshot(driver, pid, wid); - let idx = ax::element_index_by_id(snap.text(), "chk-agreed").expect("chk-agreed missing"); - // CheckBox exposes UIA TogglePattern (actions=[toggle]), not Invoke. - // cua-driver's click tool tries UIA Invoke first; for elements that - // don't support it the PostMessage fallback path runs. Use - // dispatch:"foreground" to land a SendInput click that WPF - // recognises as a real user click and processes through Toggle. - let resp = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx, - "delivery_mode": "foreground" - }), - ); - println!("click chk-agreed: {}", resp.text()); - std::thread::sleep(Duration::from_millis(400)); - let post = snapshot(driver, pid, wid); - assert!( - post.text().contains("agreed=True"), - "checkbox didn't toggle: {}", - post.text() - .lines() - .filter(|l| l.contains("agreed=")) - .collect::>() - .join(" / ") - ); - println!("✅ harness_wpf_checkbox_toggle: agreed=True"); - }); + run_foreground_case( + "checkbox_toggle", + Targeting::Ax, + DriverRoute::WindowsSendInput, + Vec::new(), + |pid, wid, driver| { + focus_harness(driver, pid, wid); + let snap = snapshot(driver, pid, wid); + let idx = + ax::element_index_by_id(snap.text(), "chk-agreed").expect("chk-agreed missing"); + // CheckBox exposes UIA TogglePattern (actions=[toggle]), not Invoke. + // cua-driver's click tool tries UIA Invoke first; for elements that + // don't support it the PostMessage fallback path runs. Use + // dispatch:"foreground" to land a SendInput click that WPF + // recognises as a real user click and processes through Toggle. + let resp = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "foreground" + }), + ); + println!("click chk-agreed: {}", resp.text()); + std::thread::sleep(Duration::from_millis(400)); + let post = snapshot(driver, pid, wid); + assert!( + post.text().contains("agreed=True"), + "checkbox didn't toggle: {}", + post.text() + .lines() + .filter(|l| l.contains("agreed=")) + .collect::>() + .join(" / ") + ); + println!("✅ harness_wpf_checkbox_toggle: agreed=True"); + Vec::new() + }, + ); } #[test] #[ignore] fn harness_wpf_radio_select() { - with_session(|pid, wid, driver| { - focus_harness(driver, pid, wid); - let snap = snapshot(driver, pid, wid); - let idx = ax::element_index_by_id(snap.text(), "rdo-high").expect("rdo-high missing"); - // RadioButton exposes SelectionItem pattern (actions=[select]). - // Same dispatch:foreground rationale as the checkbox test. - let _ = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx, - "delivery_mode": "foreground" - }), - ); - std::thread::sleep(Duration::from_millis(400)); - let post = snapshot(driver, pid, wid); - assert!( - post.text().contains("prio=High"), - "radio didn't switch to High" - ); - println!("✅ harness_wpf_radio_select: prio=High"); - }); + run_foreground_case( + "radio_select", + Targeting::Ax, + DriverRoute::WindowsSendInput, + Vec::new(), + |pid, wid, driver| { + focus_harness(driver, pid, wid); + let snap = snapshot(driver, pid, wid); + let idx = ax::element_index_by_id(snap.text(), "rdo-high").expect("rdo-high missing"); + // RadioButton exposes SelectionItem pattern (actions=[select]). + // Same dispatch:foreground rationale as the checkbox test. + let response = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "foreground" + }), + ); + assert!( + !response.is_error(), + "radio select failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(400)); + let post = snapshot(driver, pid, wid); + assert!( + post.text().contains("prio=High"), + "radio didn't switch to High" + ); + println!("✅ harness_wpf_radio_select: prio=High"); + Vec::new() + }, + ); } #[test] #[ignore] fn harness_wpf_combo_select() { - with_session(|pid, wid, driver| { - focus_harness(driver, pid, wid); - let snap = snapshot(driver, pid, wid); - let combo_idx = - ax::element_index_by_id(snap.text(), "cbo-color").expect("cbo-color missing"); - // WPF ComboBox UIA peer surfaces ExpandCollapsePattern (actions=[expand]) - // but not ValuePattern — set_value at the parent is a no-op. Standard - // recipe: invoke the combo to expand the dropdown, re-snapshot so the - // item AIDs land in the element cache, then click the target item. - let _ = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": combo_idx, - "delivery_mode": "foreground" - }), - ); - std::thread::sleep(Duration::from_millis(500)); - - let snap2 = snapshot(driver, pid, wid); - let item_idx = ax::element_index_by_id(snap2.text(), "cbo-item-orange") - .expect("cbo-item-orange missing after expand"); - let _ = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": item_idx, - "delivery_mode": "foreground" - }), - ); - std::thread::sleep(Duration::from_millis(500)); + run_background_case( + "combo_select", + DriverRoute::Composite, + |pid, wid, driver| { + let snap = snapshot(driver, pid, wid); + let combo_idx = + ax::element_index_by_id(snap.text(), "cbo-color").expect("cbo-color missing"); + // WPF ComboBox UIA peer surfaces ExpandCollapsePattern (actions=[expand]) + // but not ValuePattern — set_value at the parent is a no-op. Standard + // recipe: invoke the combo to expand the dropdown, re-snapshot so the + // item AIDs land in the element cache, then click the target item. + let expand = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": combo_idx, + "action": "expand", "delivery_mode": "background" + }), + ); + assert!(!expand.is_error(), "combo expand failed: {}", expand.text()); + std::thread::sleep(Duration::from_millis(500)); - let post = snapshot(driver, pid, wid); - assert!( - post.text().contains("color=orange"), - "combo didn't switch to orange: {}", - post.text() - .lines() - .filter(|l| l.contains("color=")) - .collect::>() - .join(" / ") - ); - println!("✅ harness_wpf_combo_select: color=orange"); - }); + let snap2 = snapshot(driver, pid, wid); + let item_idx = ax::element_index_by_id(snap2.text(), "cbo-item-orange") + .expect("cbo-item-orange missing after expand"); + let select = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": item_idx, + "delivery_mode": "background" + }), + ); + assert!(!select.is_error(), "combo select failed: {}", select.text()); + std::thread::sleep(Duration::from_millis(500)); + + let post = snapshot(driver, pid, wid); + assert!( + post.text().contains("color=orange"), + "combo didn't switch to orange: {}", + post.text() + .lines() + .filter(|l| l.contains("color=")) + .collect::>() + .join(" / ") + ); + println!("✅ harness_wpf_combo_select: color=orange"); + }, + ); } #[test] #[ignore] fn harness_wpf_listbox_select() { - with_session(|pid, wid, driver| { - focus_harness(driver, pid, wid); - let snap = snapshot(driver, pid, wid); - let idx = ax::element_index_by_id(snap.text(), "lst-cherry").expect("lst-cherry missing"); - let _ = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx, - "delivery_mode": "foreground" - }), - ); - std::thread::sleep(Duration::from_millis(400)); - let post = snapshot(driver, pid, wid); - assert!( - post.text().contains("selected=cherry"), - "list didn't select cherry: {}", - post.text() - .lines() - .filter(|l| l.contains("selected=")) - .collect::>() - .join(" / ") - ); - println!("✅ harness_wpf_listbox_select: selected=cherry"); - }); + run_foreground_case( + "listbox_select", + Targeting::Ax, + DriverRoute::WindowsSendInput, + Vec::new(), + |pid, wid, driver| { + focus_harness(driver, pid, wid); + let snap = snapshot(driver, pid, wid); + let idx = + ax::element_index_by_id(snap.text(), "lst-cherry").expect("lst-cherry missing"); + let response = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "foreground" + }), + ); + assert!( + !response.is_error(), + "listbox select failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(400)); + let post = snapshot(driver, pid, wid); + assert!( + post.text().contains("selected=cherry"), + "list didn't select cherry: {}", + post.text() + .lines() + .filter(|l| l.contains("selected=")) + .collect::>() + .join(" / ") + ); + println!("✅ harness_wpf_listbox_select: selected=cherry"); + Vec::new() + }, + ); } #[test] #[ignore] fn harness_wpf_menu_invoke() { - with_session(|pid, wid, driver| { - focus_harness(driver, pid, wid); - // Expand File menu first (UIA expand pattern on MenuItem) - let snap = snapshot(driver, pid, wid); - let file_idx = ax::element_index_by_id(snap.text(), "menu-file") - .or_else(|| ax::element_index_containing(snap.text(), "File")) - .unwrap_or_else(|| { - panic!( - "menu-file missing. Menu-related snapshot lines: {}", - snapshot_lines_containing(snap.text(), &["menu", "file", "new", "open"]) - ) - }); - let _ = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": file_idx, - }), - ); - std::thread::sleep(Duration::from_millis(400)); - - // Re-snapshot so menu-file-new is in the cache (it materialized - // when the menu expanded). - let snap2 = snapshot(driver, pid, wid); - let new_idx = ax::element_index_by_id(snap2.text(), "menu-file-new") - .or_else(|| ax::element_index_containing(snap2.text(), "New")) - .unwrap_or_else(|| { - panic!( - "menu-file-new missing after expand. Menu-related snapshot lines: {}", - snapshot_lines_containing(snap2.text(), &["menu", "file", "new", "open"]) - ) - }); - let _ = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": new_idx, - }), - ); - std::thread::sleep(Duration::from_millis(500)); - - let post = snapshot(driver, pid, wid); - assert!( - post.text().contains("menu_action=file_new"), - "File>New didn't invoke: {}", - post.text() - .lines() - .filter(|l| l.contains("menu_action=")) - .collect::>() - .join(" / ") - ); - println!("✅ harness_wpf_menu_invoke: menu_action=file_new"); - }); + run_foreground_case( + "menu_invoke", + Targeting::Ax, + DriverRoute::UiaExpandCollapse, + Vec::new(), + |pid, wid, driver| { + focus_harness(driver, pid, wid); + // Expand File menu first (UIA expand pattern on MenuItem) + let snap = snapshot(driver, pid, wid); + let file_idx = ax::element_index_by_id(snap.text(), "menu-file") + .or_else(|| ax::element_index_containing(snap.text(), "File")) + .unwrap_or_else(|| { + panic!( + "menu-file missing. Menu-related snapshot lines: {}", + snapshot_lines_containing(snap.text(), &["menu", "file", "new", "open"]) + ) + }); + let expand = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": file_idx, + "action": "expand", + "delivery_mode": "foreground" + }), + ); + assert!( + !expand.is_error(), + "File menu expand failed: {}", + expand.text() + ); + std::thread::sleep(Duration::from_millis(400)); + + // Re-snapshot so menu-file-new is in the cache (it materialized + // when the menu expanded). + let snap2 = snapshot(driver, pid, wid); + let new_idx = ax::element_index_by_id(snap2.text(), "menu-file-new") + .or_else(|| ax::element_index_containing(snap2.text(), "New")) + .unwrap_or_else(|| { + panic!( + "menu-file-new missing after expand. Menu-related snapshot lines: {}", + snapshot_lines_containing(snap2.text(), &["menu", "file", "new", "open"]) + ) + }); + let invoke = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": new_idx, + "delivery_mode": "foreground" + }), + ); + assert!( + !invoke.is_error(), + "File > New invoke failed: {}", + invoke.text() + ); + std::thread::sleep(Duration::from_millis(500)); + + let post = snapshot(driver, pid, wid); + assert!( + post.text().contains("menu_action=file_new"), + "File>New didn't invoke: {}", + post.text() + .lines() + .filter(|l| l.contains("menu_action=")) + .collect::>() + .join(" / ") + ); + println!("✅ harness_wpf_menu_invoke: menu_action=file_new"); + Vec::new() + }, + ); } diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/launch_windows_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/launch_windows_test.rs new file mode 100644 index 0000000000..e56113e721 --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver/tests/launch_windows_test.rs @@ -0,0 +1,265 @@ +//! Windows launch behavior against the repo-local Electron harness. + +#![cfg(target_os = "windows")] + +use std::collections::HashSet; +use std::process::{Command, Stdio}; +use std::time::Duration; + +use cua_driver_testkit::e2e::{ + execute_case, recording_evidence, CaseSpec, Delivery, DriverRoute, Evidence, Observation, + OracleKind, RefusalCode, Scope, Targeting, +}; +use cua_driver_testkit::sentinel::ForegroundSentinel; +use cua_driver_testkit::{harness_app, spawn_in_job, Driver, McpDriver}; +use windows::core::BOOL; +use windows::Win32::Foundation::{HWND, LPARAM, TRUE}; +use windows::Win32::UI::WindowsAndMessaging::{ + EnumWindows, GetWindowTextLengthW, GetWindowTextW, GetWindowThreadProcessId, IsIconic, + ShowWindow, SW_MINIMIZE, +}; + +#[test] +#[ignore] +fn minimized_window_is_listed_and_bring_to_front_restores_it() { + let cell_id = "windows-electron-minimized-window-restore"; + let case = CaseSpec::delivered( + cell_id, + "electron", + "chromium", + "minimized_window_restore", + Targeting::NotApplicable, + Delivery::Foreground, + Scope::Window, + DriverRoute::Composite, + vec![OracleKind::FixtureState, OracleKind::Protocol], + ); + execute_case(case, |evidence| { + let executable = harness_app("harness-electron", "CuaTestHarness.Electron.exe"); + assert!( + executable.exists(), + "required Electron launch harness is missing: {}", + executable.display() + ); + let mut driver = + McpDriver::spawn_named(cell_id).expect("required source-built driver did not start"); + *evidence = recording_evidence(driver.recording_dir()); + let mut command = Command::new(&executable); + command.stdout(Stdio::null()).stderr(Stdio::null()); + let app = spawn_in_job(&mut command).expect("Electron harness did not start"); + let pid = app.id(); + driver.reaper().push(app); + + let (window_id, _) = wait_for_window(pid); + let hwnd = HWND(window_id as *mut _); + unsafe { + let _ = ShowWindow(hwnd, SW_MINIMIZE); + } + wait_until(Duration::from_secs(2), || { + unsafe { IsIconic(hwnd) }.as_bool() + }); + driver.start_behavior_recording(); + + let listed = driver.call("list_windows", serde_json::json!({"pid": pid as i64})); + let window = listed.structured()["windows"] + .as_array() + .and_then(|windows| { + windows + .iter() + .find(|window| window["window_id"].as_u64() == Some(window_id)) + }) + .expect("minimized target window missing from list_windows"); + assert_eq!(window["minimized"].as_bool(), Some(true)); + assert_eq!(window["is_on_screen"].as_bool(), Some(false)); + + let on_screen = driver.call( + "list_windows", + serde_json::json!({"pid": pid as i64, "on_screen_only": true}), + ); + assert!( + on_screen.structured()["windows"] + .as_array() + .is_some_and(|windows| windows + .iter() + .all(|window| { window["window_id"].as_u64() != Some(window_id) })), + "on_screen_only retained minimized target" + ); + + let restored = driver.call( + "bring_to_front", + serde_json::json!({"pid": pid as i64, "window_id": window_id}), + ); + assert!(!restored.is_error(), "restore failed: {}", restored.text()); + assert_eq!(restored.structured()["restored"].as_bool(), Some(true)); + wait_until(Duration::from_secs(2), || { + !unsafe { IsIconic(hwnd) }.as_bool() + }); + + let listed = driver.call("list_windows", serde_json::json!({"pid": pid as i64})); + let window = listed.structured()["windows"] + .as_array() + .and_then(|windows| { + windows + .iter() + .find(|window| window["window_id"].as_u64() == Some(window_id)) + }) + .expect("restored target window missing from list_windows"); + assert_eq!(window["minimized"].as_bool(), Some(false)); + assert_eq!(window["is_on_screen"].as_bool(), Some(true)); + Observation::delivered( + vec![OracleKind::FixtureState, OracleKind::Protocol], + Evidence::default(), + ) + }); +} + +#[test] +#[ignore] +fn launch_app_minimized_preserves_foreground() { + let case = CaseSpec::delivered( + "windows-electron-launch-app-background", + "electron", + "chromium", + "launch_app", + Targeting::NotApplicable, + Delivery::Background, + Scope::Window, + DriverRoute::WindowsShellExecute, + vec![ + OracleKind::FixtureState, + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::Cursor, + OracleKind::NoLeakedInput, + ], + ) + .expecting_refusal(vec![RefusalCode::BackgroundUnavailable]); + execute_case(case, |evidence| { + let executable = harness_app("harness-electron", "CuaTestHarness.Electron.exe"); + assert!( + executable.exists(), + "required Electron launch harness is missing: {}", + executable.display() + ); + let mut driver = McpDriver::spawn_named("windows-electron-launch-app-background") + .expect("required source-built driver did not start"); + *evidence = recording_evidence(driver.recording_dir()); + let sentinel = ForegroundSentinel::launch(&mut driver); + let before = window_ids(); + driver.start_behavior_recording(); + + let (response, mut passed) = sentinel + .observe_desktop(|| { + driver.call( + "launch_app", + serde_json::json!({ + "path": executable.to_string_lossy(), + "start_minimized": true + }), + ) + }) + .unwrap_or_else(|error| panic!("minimized launch disturbed the desktop: {error}")); + assert_required_background_oracles(&passed); + assert!( + response.is_error(), + "minimized launch unexpectedly proceeded" + ); + assert_eq!( + response.structured()["code"].as_str(), + Some("background_unavailable"), + "minimized launch returned the wrong refusal: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(500)); + assert!( + window_ids().is_subset(&before), + "refused minimized launch created a new desktop window" + ); + passed.push(OracleKind::FixtureState); + Observation::refused( + RefusalCode::BackgroundUnavailable, + passed, + response.text(), + Evidence::default(), + ) + }); +} + +fn native_windows() -> Vec<(u32, u64, String)> { + unsafe extern "system" fn callback(hwnd: HWND, lparam: LPARAM) -> BOOL { + let windows = &mut *(lparam.0 as *mut Vec<(u32, u64, String)>); + let title_len = GetWindowTextLengthW(hwnd); + if title_len > 0 { + let mut title = vec![0u16; title_len as usize + 1]; + let copied = GetWindowTextW(hwnd, &mut title); + if copied > 0 { + let mut pid = 0u32; + GetWindowThreadProcessId(hwnd, Some(&mut pid)); + windows.push(( + pid, + hwnd.0 as u64, + String::from_utf16_lossy(&title[..copied as usize]), + )); + } + } + TRUE + } + + let mut windows = Vec::new(); + unsafe { + let _ = EnumWindows( + Some(callback), + LPARAM(&mut windows as *mut Vec<(u32, u64, String)> as isize), + ); + } + windows +} + +fn wait_for_window(pid: u32) -> (u64, String) { + let deadline = std::time::Instant::now() + Duration::from_secs(10); + loop { + if let Some((_, window_id, title)) = native_windows() + .into_iter() + .find(|(window_pid, _, _)| *window_pid == pid) + { + return (window_id, title); + } + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for pid {pid} to create a window" + ); + std::thread::sleep(Duration::from_millis(100)); + } +} + +fn wait_until(timeout: Duration, predicate: impl Fn() -> bool) { + let deadline = std::time::Instant::now() + timeout; + while !predicate() { + assert!( + std::time::Instant::now() < deadline, + "condition did not become true within {timeout:?}" + ); + std::thread::sleep(Duration::from_millis(25)); + } +} + +fn window_ids() -> HashSet { + native_windows() + .into_iter() + .map(|(_, window_id, _)| window_id) + .collect() +} + +fn assert_required_background_oracles(passed: &[OracleKind]) { + for required in [ + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::Cursor, + OracleKind::NoLeakedInput, + ] { + assert!( + passed.contains(&required), + "minimized launch omitted required {required:?} oracle" + ); + } +} diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs index 84c506b597..08fe9e59d0 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs @@ -25,12 +25,51 @@ fn tools_list_schema_shape() { let list_resp = d.recv(); let tools = list_resp["result"]["tools"].as_array().expect("tools array"); + let properties = |name: &str| { + &tools + .iter() + .find(|tool| tool["name"] == name) + .unwrap_or_else(|| panic!("{name} not found in tools/list"))["inputSchema"] + ["properties"] + }; + let enum_contains = |schema: &serde_json::Value, expected: &str| { + schema["enum"] + .as_array() + .map(|values| values.iter().any(|value| value.as_str() == Some(expected))) + .unwrap_or(false) + }; + // Deprecated alias is hidden from tools/list (accepted at invoke time only). assert!( tools.iter().all(|t| t["name"] != "type_text_chars"), "type_text_chars should be hidden from tools/list" ); + #[cfg(target_os = "windows")] + { + for tool in [ + "click", + "double_click", + "right_click", + "type_text", + "press_key", + "hotkey", + "scroll", + ] { + let delivery = &properties(tool)["delivery_mode"]; + assert!( + enum_contains(delivery, "background") && enum_contains(delivery, "foreground"), + "{tool}.delivery_mode should advertise background and foreground: {delivery:?}" + ); + } + + let capture_scope = &properties("set_config")["capture_scope"]; + assert!( + enum_contains(capture_scope, "window") && enum_contains(capture_scope, "desktop"), + "set_config.capture_scope should advertise window and desktop: {capture_scope:?}" + ); + } + // list_windows schema has on_screen_only. let lw = tools.iter().find(|t| t["name"] == "list_windows") .expect("list_windows not found in tools/list"); diff --git a/libs/cua-driver/rust/crates/focus-monitor-win/Cargo.toml b/libs/cua-driver/rust/crates/focus-monitor-win/Cargo.toml deleted file mode 100644 index fbe73e08ef..0000000000 --- a/libs/cua-driver/rust/crates/focus-monitor-win/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "focus-monitor-win" -version.workspace = true -edition.workspace = true -# Windows-only binary used by integration tests to verify the UX guard: -# clicking/typing into background windows must not steal focus from this window. - -[[bin]] -name = "focus-monitor-win" -path = "src/main.rs" - -[target.'cfg(target_os = "windows")'.dependencies] -windows = { version = "0.58", features = [ - "Win32_Foundation", - "Win32_UI_WindowsAndMessaging", - "Win32_System_Threading", - "Win32_Graphics_Gdi", -] } diff --git a/libs/cua-driver/rust/crates/focus-monitor-win/src/main.rs b/libs/cua-driver/rust/crates/focus-monitor-win/src/main.rs deleted file mode 100644 index bc1deb68c5..0000000000 --- a/libs/cua-driver/rust/crates/focus-monitor-win/src/main.rs +++ /dev/null @@ -1,184 +0,0 @@ -/// focus-monitor-win — Windows equivalent of macOS FocusMonitorApp. -/// -/// Creates a visible Win32 window and tracks three kinds of focus loss: -/// -/// 1. WM_ACTIVATE (wParam==WA_INACTIVE): the window loses activation. -/// Written to %TEMP%\focus_monitor_losses.txt -/// -/// 2. WM_KILLFOCUS: the window loses keyboard focus. -/// Written to %TEMP%\focus_monitor_key_losses.txt -/// -/// Prints FOCUS_PID= on stdout at startup so the test harness can -/// discover the process, then prints FOCUS_HWND= so tests -/// can target it with cua-driver tools. -/// -/// Exits cleanly on WM_DESTROY. - -#[cfg(not(target_os = "windows"))] -fn main() { - eprintln!("focus-monitor-win is Windows-only."); - std::process::exit(1); -} - -#[cfg(target_os = "windows")] -mod win { - use std::ffi::OsStr; - use std::os::windows::ffi::OsStrExt; - use std::sync::atomic::{AtomicU32, Ordering}; - use windows::Win32::Foundation::*; - use windows::Win32::Graphics::Gdi::*; - use windows::Win32::System::Threading::GetCurrentProcessId; - use windows::Win32::UI::WindowsAndMessaging::*; - - // ── global loss counters ───────────────────────────────────────────────── - static ACTIVATE_LOSSES: AtomicU32 = AtomicU32::new(0); - static ACTIVATE_GAINS: AtomicU32 = AtomicU32::new(0); - static KEY_LOSSES: AtomicU32 = AtomicU32::new(0); - static KEY_GAINS: AtomicU32 = AtomicU32::new(0); - - fn loss_file() -> std::path::PathBuf { - loss_path("focus_monitor_losses.txt") - } - fn gain_file() -> std::path::PathBuf { - loss_path("focus_monitor_gains.txt") - } - fn key_loss_file() -> std::path::PathBuf { - loss_path("focus_monitor_key_losses.txt") - } - fn key_gain_file() -> std::path::PathBuf { - loss_path("focus_monitor_key_gains.txt") - } - - fn loss_path(name: &str) -> std::path::PathBuf { - let mut p = std::env::temp_dir(); - p.push(name); - p - } - - fn write_count(path: &std::path::Path, n: u32) { - let _ = std::fs::write(path, n.to_string()); - } - - fn wide(s: &str) -> Vec { - OsStr::new(s) - .encode_wide() - .chain(std::iter::once(0)) - .collect() - } - - unsafe extern "system" fn wnd_proc( - hwnd: HWND, - msg: u32, - wparam: WPARAM, - lparam: LPARAM, - ) -> LRESULT { - match msg { - WM_ACTIVATE => { - // WA_INACTIVE == 0 in the low word of wParam; WA_ACTIVE == 1, WA_CLICKACTIVE == 2 - if (wparam.0 & 0xFFFF) == 0 { - let n = ACTIVATE_LOSSES.fetch_add(1, Ordering::SeqCst) + 1; - write_count(&loss_file(), n); - } else { - let n = ACTIVATE_GAINS.fetch_add(1, Ordering::SeqCst) + 1; - write_count(&gain_file(), n); - } - let _ = InvalidateRect(hwnd, None, true); - } - WM_KILLFOCUS => { - let n = KEY_LOSSES.fetch_add(1, Ordering::SeqCst) + 1; - write_count(&key_loss_file(), n); - let _ = InvalidateRect(hwnd, None, true); - } - WM_SETFOCUS => { - let n = KEY_GAINS.fetch_add(1, Ordering::SeqCst) + 1; - write_count(&key_gain_file(), n); - let _ = InvalidateRect(hwnd, None, true); - } - WM_PAINT => { - let mut ps = PAINTSTRUCT::default(); - let hdc = BeginPaint(hwnd, &mut ps); - let act_l = ACTIVATE_LOSSES.load(Ordering::SeqCst); - let act_g = ACTIVATE_GAINS.load(Ordering::SeqCst); - let key_l = KEY_LOSSES.load(Ordering::SeqCst); - let key_g = KEY_GAINS.load(Ordering::SeqCst); - let text = wide(&format!( - "act: {act_l}L / {act_g}G key: {key_l}L / {key_g}G (should stay net 0)" - )); - let _ = TextOutW(hdc, 10, 10, &text); - let _ = EndPaint(hwnd, &ps); - } - WM_DESTROY => { - PostQuitMessage(0); - } - _ => return DefWindowProcW(hwnd, msg, wparam, lparam), - } - LRESULT(0) - } - - pub fn run() { - unsafe { - let class_name = wide("FocusMonitorWin"); - - let wc = WNDCLASSW { - lpfnWndProc: Some(wnd_proc), - hInstance: HINSTANCE(std::ptr::null_mut()), - lpszClassName: windows::core::PCWSTR(class_name.as_ptr()), - hbrBackground: HBRUSH(COLOR_WINDOW.0 as *mut _), - ..Default::default() - }; - RegisterClassW(&wc); - - let title = wide("Focus Monitor (cua-driver UX guard)"); - let hwnd = CreateWindowExW( - WINDOW_EX_STYLE(0), - windows::core::PCWSTR(class_name.as_ptr()), - windows::core::PCWSTR(title.as_ptr()), - WS_OVERLAPPEDWINDOW, - 100, - 100, - 600, - 140, - None, - None, - HINSTANCE(std::ptr::null_mut()), - None, - ) - .expect("CreateWindowExW failed"); - - let _ = ShowWindow(hwnd, SW_SHOWNORMAL); - let _ = UpdateWindow(hwnd).ok(); - - // Write initial zeros so tests can read even before any event. - write_count(&loss_file(), 0); - write_count(&gain_file(), 0); - write_count(&key_loss_file(), 0); - write_count(&key_gain_file(), 0); - - // Signal the test harness via temp files (avoids pipe-blocking issues - // when stdout is captured by the test runner in sandbox environments). - let pid = GetCurrentProcessId(); - let hwnd_val = hwnd.0 as usize; - let pid_file = std::env::temp_dir().join("focus_monitor_pid.txt"); - let hwnd_file = std::env::temp_dir().join("focus_monitor_hwnd.txt"); - let _ = std::fs::write(&pid_file, pid.to_string()); - let _ = std::fs::write(&hwnd_file, hwnd_val.to_string()); - // Also print to stdout as a secondary signal. - println!("FOCUS_PID={pid}"); - println!("FOCUS_HWND={hwnd_val}"); - use std::io::Write; - std::io::stdout().flush().ok(); - - // Message loop. - let mut msg = MSG::default(); - while GetMessageW(&mut msg, None, 0, 0).as_bool() { - let _ = TranslateMessage(&msg); - DispatchMessageW(&msg); - } - } - } -} - -#[cfg(target_os = "windows")] -fn main() { - win::run(); -} diff --git a/libs/cua-driver/rust/crates/platform-windows/src/capture.rs b/libs/cua-driver/rust/crates/platform-windows/src/capture.rs index 6237436ed2..95cb281913 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/capture.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/capture.rs @@ -204,7 +204,48 @@ pub fn screenshot_window_bytes(hwnd: u64) -> Result> { /// user / LLM should attach an explicit warning. See `target_is_obscured` /// for the sampling heuristic. pub fn screenshot_window_bytes_with_occlusion(hwnd: u64) -> Result<(Vec, bool)> { - unsafe { screenshot_window_bytes_with_occlusion_unsafe(hwnd) } + match unsafe { screenshot_window_bytes_with_occlusion_unsafe(hwnd) } { + Ok(capture) => Ok(capture), + Err(primary_error) => { + if primary_error.to_string().contains("minimized window") { + return Err(primary_error); + } + // A freshly restored DirectComposition window can temporarily have + // no usable GDI surface even though DWM is already rendering it. + // WGC reads the compositor-owned frame and is therefore the right + // first fallback for this class of capture failure. + match crate::wgc::screenshot_window_via_wgc(hwnd) { + Ok((pixels, width, height)) => Ok(( + cua_driver_core::image_utils::encode_bgra_to_png( + &pixels, width, height, + )?, + false, + )), + Err(wgc_error) => { + // Headless/virtualized Windows sessions can expose DWM but + // no WGC-compatible hardware device. Once a window is + // visible, a desktop-region crop remains a truthful final + // fallback; report whether another window covered it. + let target = HWND(hwnd as *mut _); + let occluded = unsafe { target_is_obscured(target) }; + match unsafe { screenshot_via_screen_region(target) } { + Ok((pixels, width, height)) => Ok(( + cua_driver_core::image_utils::encode_bgra_to_png( + &pixels, + width as u32, + height as u32, + )?, + occluded, + )), + Err(screen_error) => Err(primary_error.context(format!( + "Windows.Graphics.Capture fallback failed: {wgc_error}; \ + desktop-region fallback failed: {screen_error}" + ))), + } + } + } + } + } } /// Capture a window by HWND, returning (base64_png, width, height). @@ -238,13 +279,13 @@ unsafe fn screenshot_window_bytes_with_occlusion_unsafe(hwnd: u64) -> Result<(Ve // The WGC sibling path at `wgc.rs:58` already short-circuits this case; // the GDI/PrintWindow fallback below + the screen-region BitBlt fallback // both happily produced the degenerate PNG. Guarding here covers both - // and matches the WGC error shape so callers can `list_windows` or - // raise the window before retrying. + // and matches the WGC error shape so callers can `list_windows` and + // restore the window before retrying. if IsIconic(hwnd).as_bool() { bail!( "cannot capture minimized window 0x{hwnd_raw:x}: it has no \ - rendered content. Restore the window first via list_windows \ - / raise_window. The PrintWindow GDI path and the screen-region \ + rendered content. Call bring_to_front with this window_id to \ + restore it first. The PrintWindow GDI path and the screen-region \ BitBlt fallback both return an all-black bitmap for iconic \ windows." ); @@ -565,4 +606,3 @@ pub fn crosshair_png_bytes(png_bytes: &[u8], cx: f64, cy: f64) -> Result pub fn png_dimensions_pub(data: &[u8]) -> Result<(u32, u32)> { cua_driver_core::image_utils::png_dimensions(data) } - diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/delivery.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/delivery.rs index 89b11c86ec..93aca9a094 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/input/delivery.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/input/delivery.rs @@ -140,21 +140,38 @@ pub fn would_be_silently_dropped(hwnd: u64, kind: EventKind) -> bool { use EventKind::*; if crate::input::is_chromium_target_window(hwnd) { // Chromium's input thread architecture requires SendInput-queue - // origin for mouse + key-combo events (#1623). Plain keystrokes and - // text input via WM_CHAR still work because they go through - // Chromium's IME path, which DOES consume Win32 messages. - return matches!(kind, MouseClick | MouseMove | MouseScroll | KeyCombo); + // origin for pointer and keyboard events (#1623). Posted WM_CHAR and + // plain key messages can return success while a background renderer + // receives nothing, so they must be refused as honestly as chords. + return matches!( + kind, + MouseClick | MouseMove | MouseScroll | Keystroke | KeyCombo | TextInput + ); + } + if crate::input::has_chromium_descendant(hwnd) { + // Embedded WebView2 hosts retain useful UIA/top-level routes for + // clicks and ValuePattern text. Their drag, wheel and modifier-chord + // paths still depend on the renderer's system input queue. + return matches!(kind, MouseMove | MouseScroll | KeyCombo); } if is_wpf_target_window(hwnd) { - // WPF ignores posted pointer messages (its input manager drops - // WM_MOUSE* unless the live system cursor is over the window). It must - // be driven by coordinate-routed system-queue input for clicks/moves. + // WPF ignores posted pointer messages unless the live system cursor is + // over the window. Its InputManager also ignores posted key messages + // while another native window owns foreground; PostMessage still + // returns success, so both routes need an honest refusal. // // Do not classify WM_VSCROLL/WM_HSCROLL here: the scroll tool posts the // scrollbar messages directly to the top-level HWND, and WPF hosts that // explicitly handle those messages (including our harness hook) can // consume them without a foreground swap. - return matches!(kind, MouseClick | MouseMove); + return wpf_drops_event(kind, target_is_foreground(hwnd)); + } + if is_tk_target_window(hwnd) { + // Tk's Windows event loop does not treat posted WM_CHAR/WM_KEYDOWN as + // genuine keyboard input for the focused widget. The messages can be + // accepted by PostMessage while the Entry receives nothing, so refuse + // instead of reporting a false background success. + return matches!(kind, Keystroke | KeyCombo | TextInput); } // NB: WinUI3 (`WinUIDesktopWin32WindowClass`) is deliberately NOT flagged // here. It looks WPF-like, but its composition input-site does NOT consume @@ -190,6 +207,25 @@ pub fn would_be_silently_dropped(hwnd: u64, kind: EventKind) -> bool { false } +fn wpf_drops_event(kind: EventKind, target_is_foreground: bool) -> bool { + matches!(kind, EventKind::MouseClick | EventKind::MouseMove) + || (!target_is_foreground && matches!(kind, EventKind::Keystroke | EventKind::KeyCombo)) +} + +fn target_is_foreground(hwnd: u64) -> bool { + use windows::Win32::Foundation::HWND; + use windows::Win32::UI::WindowsAndMessaging::{GetAncestor, GetForegroundWindow, GA_ROOT}; + if hwnd == 0 { + return false; + } + unsafe { + let target = GetAncestor(HWND(hwnd as *mut _), GA_ROOT); + let foreground = GetForegroundWindow(); + let foreground_root = GetAncestor(foreground, GA_ROOT); + !target.0.is_null() && target == foreground_root + } +} + /// Detect LibreOffice / OpenOffice (VCL framework) windows. /// /// VCL on Windows registers window classes with a `SAL` prefix (StarOffice's @@ -222,6 +258,16 @@ pub fn is_wpf_target_window(hwnd: u64) -> bool { read_class_name(hwnd).starts_with("HwndWrapper") } +/// Detect Tk/Tkinter top-level windows. Tk registers this stable class name +/// for its root and child toplevels on Windows. +pub fn is_tk_target_window(hwnd: u64) -> bool { + is_tk_class_name(&read_class_name(hwnd)) +} + +fn is_tk_class_name(class: &str) -> bool { + class == "TkTopLevel" || class.starts_with("TkTopLevel.") +} + /// Detect WinUI3 / Windows-App-SDK desktop top-level windows. The frame is a /// Win32 HWND of class `WinUIDesktopWin32WindowClass`, but — unlike WPF — that /// frame does NOT host the visual tree or consume pointer input. The XAML @@ -351,6 +397,25 @@ pub fn background_unavailable_error_with_cause( mod tests { use super::*; + #[test] + fn detects_tk_toplevel_classes_without_matching_unrelated_windows() { + assert!(is_tk_class_name("TkTopLevel")); + assert!(is_tk_class_name("TkTopLevel.1")); + assert!(!is_tk_class_name("TkChild")); + assert!(!is_tk_class_name("Chrome_WidgetWin_1")); + } + + #[test] + fn wpf_refuses_posted_pointer_and_keyboard_events() { + assert!(wpf_drops_event(EventKind::MouseClick, true)); + assert!(wpf_drops_event(EventKind::MouseMove, true)); + assert!(wpf_drops_event(EventKind::Keystroke, false)); + assert!(wpf_drops_event(EventKind::KeyCombo, false)); + assert!(!wpf_drops_event(EventKind::Keystroke, true)); + assert!(!wpf_drops_event(EventKind::KeyCombo, true)); + assert!(!wpf_drops_event(EventKind::TextInput, false)); + assert!(!wpf_drops_event(EventKind::MouseScroll, false)); + } #[test] fn delivery_mode_parses_known_values() { let j = |s: &str| serde_json::json!({"delivery_mode": s}); @@ -401,17 +466,12 @@ mod tests { ); let structured = result.structured_content.as_ref().expect("structured"); assert_eq!(result.is_error, Some(true)); - assert_eq!( - structured["code"].as_str(), - Some("background_occluded") - ); + assert_eq!(structured["code"].as_str(), Some("background_occluded")); assert_eq!(structured["event_kind"].as_str(), Some("mouse_click")); - assert!( - structured["cause"] - .as_str() - .unwrap_or_default() - .contains("occluded") - ); + assert!(structured["cause"] + .as_str() + .unwrap_or_default() + .contains("occluded")); let text = match &result.content[0] { cua_driver_core::protocol::Content::Text { text, .. } => text, diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs index 4abc077dce..ce8157fc58 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs @@ -34,6 +34,7 @@ use std::thread::sleep; use std::time::Duration; use windows::Win32::Foundation::{HANDLE, HWND, POINT, RECT}; +use windows::Win32::System::Threading::{AttachThreadInput, GetCurrentThreadId}; use windows::Win32::UI::Controls::{ CreateSyntheticPointerDevice, DestroySyntheticPointerDevice, HSYNTHETICPOINTERDEVICE, POINTER_FEEDBACK_DEFAULT, POINTER_TYPE_INFO, POINTER_TYPE_INFO_0, @@ -42,13 +43,68 @@ use windows::Win32::UI::Input::Pointer::{ InjectSyntheticPointerInput, POINTER_FLAG_DOWN, POINTER_FLAG_INCONTACT, POINTER_FLAG_INRANGE, POINTER_FLAG_UP, POINTER_FLAG_UPDATE, POINTER_INFO, POINTER_PEN_INFO, POINTER_TOUCH_INFO, }; -use windows::Win32::System::Threading::{AttachThreadInput, GetCurrentThreadId}; use windows::Win32::UI::WindowsAndMessaging::{ GetAncestor, GetCursorPos, GetForegroundWindow, GetWindowLongPtrW, GetWindowThreadProcessId, - IsWindow, SetCursorPos, SetForegroundWindow, SetWindowLongPtrW, WindowFromPoint, GA_ROOT, - GWL_EXSTYLE, PT_PEN, PT_TOUCH, WS_EX_NOACTIVATE, + IsWindow, LockSetForegroundWindow, SetCursorPos, SetForegroundWindow, SetWindowLongPtrW, + WindowFromPoint, GA_ROOT, GWL_EXSTYLE, LSFW_LOCK, LSFW_UNLOCK, PT_PEN, PT_TOUCH, + WS_EX_NOACTIVATE, }; +#[derive(Default)] +struct ForegroundLockState { + holders: usize, + locked: bool, +} + +static FOREGROUND_LOCK_STATE: Mutex = Mutex::new(ForegroundLockState { + holders: 0, + locked: false, +}); + +/// Prevent other processes from taking the foreground during a background +/// launch. Windows automatically clears this lock on genuine user input; Drop +/// still balances the documented unlock call and overlapping driver launches. +pub struct ForegroundLockGuard { + held: bool, +} + +impl ForegroundLockGuard { + pub fn acquire() -> Self { + let mut state = FOREGROUND_LOCK_STATE.lock().unwrap(); + if state.holders == 0 { + state.locked = unsafe { LockSetForegroundWindow(LSFW_LOCK) }.is_ok(); + if state.locked { + tracing::debug!(target: "launch_app.focus_lock", "locked foreground changes during background launch"); + } else { + tracing::warn!(target: "launch_app.focus_lock", "could not lock foreground changes during background launch"); + } + } + if state.locked { + state.holders += 1; + } + Self { held: state.locked } + } + + pub fn acquired(&self) -> bool { + self.held + } +} + +impl Drop for ForegroundLockGuard { + fn drop(&mut self) { + if !self.held { + return; + } + let mut state = FOREGROUND_LOCK_STATE.lock().unwrap(); + state.holders = state.holders.saturating_sub(1); + if state.holders == 0 { + let _ = unsafe { LockSetForegroundWindow(LSFW_UNLOCK) }; + state.locked = false; + tracing::debug!(target: "launch_app.focus_lock", "unlocked foreground changes after background launch"); + } + } +} + /// Bring `target` to the foreground using the AttachThreadInput trick, which /// inherits the current foreground thread's FG-lock token so the swap is /// honored even on a foreground-locked session without UIAccess (mirrors the @@ -89,7 +145,6 @@ pub struct NoActivateGuard { // Store the handle as an integer so the guard is `Send` and can be held // across `.await` in the async tools. root_addr: isize, - prev_exstyle: isize, applied: bool, } @@ -111,7 +166,10 @@ impl NoActivateGuard { // by UIPI on higher-integrity targets). (GetWindowLongPtrW(root, GWL_EXSTYLE) & want) != 0 }; - Self { root_addr: root.0 as isize, prev_exstyle: prev, applied } + Self { + root_addr: root.0 as isize, + applied, + } } } } @@ -120,7 +178,13 @@ impl Drop for NoActivateGuard { fn drop(&mut self) { if self.applied { unsafe { - let _ = SetWindowLongPtrW(HWND(self.root_addr as *mut _), GWL_EXSTYLE, self.prev_exstyle); + let root = HWND(self.root_addr as *mut _); + let current = GetWindowLongPtrW(root, GWL_EXSTYLE); + let noactivate = WS_EX_NOACTIVATE.0 as isize; + // Clear only the bit this guard added. Restoring the full + // captured value can clobber unrelated style changes the app + // made while the background action was in flight. + SetWindowLongPtrW(root, GWL_EXSTYLE, current & !noactivate); } } } @@ -537,4 +601,3 @@ pub fn inject_drag_screen( // WPF/terminal text) is now reported as `background_unavailable`; the agent // escalates to `delivery_mode:"foreground"`, which uses the explicit // SetForegroundWindow path (send_key_synthesized / send_text_synthesized). - diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/keyboard.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/keyboard.rs index ffb23cb153..75e00bcb6e 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/input/keyboard.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/input/keyboard.rs @@ -17,18 +17,17 @@ use anyhow::{bail, Result}; use std::thread::sleep; -use std::time::Duration; -use windows::Win32::Foundation::{HWND, LPARAM, WPARAM}; -use windows::Win32::System::Threading::{AttachThreadInput, GetCurrentThreadId}; -use windows::Win32::UI::Input::KeyboardAndMouse::GetFocus; +use std::time::{Duration, Instant}; +use windows::Win32::Foundation::{BOOL, HWND, LPARAM, TRUE, WPARAM}; use windows::Win32::UI::Input::KeyboardAndMouse::{ MapVirtualKeyW, SendInput, INPUT, INPUT_0, INPUT_KEYBOARD, KEYBDINPUT, KEYBD_EVENT_FLAGS, KEYEVENTF_EXTENDEDKEY, KEYEVENTF_KEYUP, KEYEVENTF_SCANCODE, KEYEVENTF_UNICODE, MAPVK_VK_TO_VSC, VIRTUAL_KEY, }; use windows::Win32::UI::WindowsAndMessaging::{ - GetClassNameW, GetWindowThreadProcessId, IsChild, PostMessageW, WM_CHAR, WM_KEYDOWN, WM_KEYUP, - WM_SYSKEYDOWN, WM_SYSKEYUP, + EnumChildWindows, GetClassNameW, GetGUIThreadInfo, GetParent, GetWindowThreadProcessId, + IsChild, PostMessageW, GUITHREADINFO, WM_CHAR, WM_KEYDOWN, WM_KEYUP, WM_SYSKEYDOWN, + WM_SYSKEYUP, }; use windows::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, SetForegroundWindow}; @@ -127,50 +126,97 @@ pub fn is_xaml_host_hwnd(hwnd: u64) -> bool { const KEY_DELAY_MS: u64 = 4; -/// If the target's UI thread has a focused child window that's a descendant -/// of `parent`, return that child. Otherwise `None`. Used to retarget +/// If any UI thread under the target has a focused child window that's a +/// descendant of `parent`, return that child. Otherwise `None`. Used to retarget /// `PostMessage(WM_CHAR/WM_KEYDOWN)` from the top-level frame to the actual /// editor control (Scintilla in Notepad++, RichEdit in WordPad, etc.) — /// top-level WindowProcs don't forward keyboard messages to embedded editors /// automatically, so without this drill-down `type_text` silently no-ops /// against any app that puts its text surface in a child HWND. /// -/// Uses `AttachThreadInput` to read the target thread's focus state, which -/// is the standard cross-thread way to read another thread's `GetFocus()`. -/// We detach immediately after — attaching for the duration of the post -/// would change input-state visibility for the duration. +/// Embedded renderers such as WebView2 may put their focused child on a +/// different UI thread from the native top-level frame. Enumerating descendant +/// thread ids is therefore required; checking only the frame thread queues the +/// message successfully but leaves the renderer untouched. More than one of +/// those threads can retain a focused HWND, so choose the deepest focused +/// descendant rather than whichever thread happens to enumerate first. fn focused_descendant(parent: HWND) -> Option { if parent.0.is_null() { return None; } - let mut target_pid: u32 = 0; - let target_thread = unsafe { GetWindowThreadProcessId(parent, Some(&mut target_pid)) }; - if target_thread == 0 { + let parent_thread = unsafe { GetWindowThreadProcessId(parent, None) }; + if parent_thread == 0 { return None; } - let our_thread = unsafe { GetCurrentThreadId() }; - let focused = if our_thread == target_thread { - unsafe { GetFocus() } - } else { - let _ = unsafe { AttachThreadInput(our_thread, target_thread, true) }; - let f = unsafe { GetFocus() }; - let _ = unsafe { AttachThreadInput(our_thread, target_thread, false) }; - f - }; - if focused.0.is_null() { - return None; + unsafe extern "system" fn collect_thread(child: HWND, lparam: LPARAM) -> BOOL { + let threads = &mut *(lparam.0 as *mut Vec); + let thread = GetWindowThreadProcessId(child, None); + if thread != 0 && !threads.contains(&thread) { + threads.push(thread); + } + TRUE } - if focused == parent { - return None; + + let mut target_threads = vec![parent_thread]; + unsafe { + let _ = EnumChildWindows( + parent, + Some(collect_thread), + LPARAM(&mut target_threads as *mut Vec as isize), + ); + } + let mut best: Option<(usize, HWND)> = None; + for target_thread in target_threads { + let mut info = GUITHREADINFO { + cbSize: std::mem::size_of::() as u32, + ..Default::default() + }; + if unsafe { GetGUIThreadInfo(target_thread, &mut info) }.is_err() { + continue; + } + let focused = info.hwndFocus; + if focused.0.is_null() + || focused == parent + || !unsafe { IsChild(parent, focused) }.as_bool() + { + continue; + } + + let mut depth = 0usize; + let mut current = focused; + while current != parent && depth < 64 { + let Ok(next) = (unsafe { GetParent(current) }) else { + break; + }; + if next.0.is_null() { + break; + } + depth += 1; + current = next; + } + if current == parent && best.as_ref().map_or(true, |(d, _)| depth > *d) { + best = Some((depth, focused)); + } } - // Only retarget if focus is genuinely a descendant of `parent` — protects - // against accidentally posting to an unrelated window if the target is - // not the foreground app at the moment. - if unsafe { IsChild(parent, focused) }.as_bool() { - Some(focused) - } else { - None + best.map(|(_, focused)| focused) +} + +/// Wait for an element-focused embedded renderer to expose its child HWND. +/// UIA SetFocus can complete before WebView2 updates GUITHREADINFO; polling the +/// observable focus target avoids posting the key to the native frame in that +/// short interval. +pub fn wait_for_focused_descendant(hwnd: u64, timeout: Duration) -> Option { + let parent = HWND(hwnd as *mut _); + let deadline = Instant::now() + timeout; + loop { + if let Some(target) = focused_descendant(parent) { + return Some(target.0 as usize as u64); + } + if Instant::now() >= deadline { + return None; + } + sleep(Duration::from_millis(10)); } } diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs index e45be446fd..0501db47a1 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs @@ -14,11 +14,18 @@ pub mod keyboard; pub mod delivery; pub mod inject; -pub use inject::{inject_click_screen, point_in_window_bounds, NoActivateGuard}; -pub use mouse::{is_chromium_target_window, post_click, post_click_screen, send_click_synthesized, send_click_synthesized_mods, send_wheel_synthesized}; +pub use inject::{ + inject_click_screen, point_in_window_bounds, ForegroundLockGuard, NoActivateGuard, +}; +pub(crate) use inject::force_foreground_attached; +pub use mouse::{ + has_chromium_descendant, is_chromium_target_window, post_click, post_click_screen, + send_click_synthesized, send_click_synthesized_active_mods, send_click_synthesized_mods, + send_wheel_synthesized, +}; pub use keyboard::{ is_xaml_host_hwnd, post_char, post_key, post_type_text, post_type_text_with_delay, - send_key_synthesized, send_text_synthesized, + send_key_synthesized, send_text_synthesized, wait_for_focused_descendant, }; use windows::Win32::Foundation::{CloseHandle, HANDLE, HWND}; diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs index f2fb50d3c9..d370855657 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs @@ -16,13 +16,14 @@ use windows::Win32::UI::Input::KeyboardAndMouse::{ MOUSEINPUT, SendInput, }; use windows::Win32::UI::WindowsAndMessaging::{ - ChildWindowFromPointEx, CWP_SKIPDISABLED, CWP_SKIPINVISIBLE, CWP_SKIPTRANSPARENT, - GetCursorPos, GetForegroundWindow, GetSystemMetrics, GetWindowLongPtrW, PostMessageW, - SetCursorPos, SetWindowPos, GWL_EXSTYLE, HWND_NOTOPMOST, HWND_TOP, HWND_TOPMOST, - SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN, SM_XVIRTUALSCREEN, SM_YVIRTUALSCREEN, - SWP_NOACTIVATE, SWP_NOMOVE, SWP_NOSIZE, WS_EX_TOPMOST, - WM_LBUTTONDOWN, WM_LBUTTONUP, WM_MBUTTONDOWN, WM_MBUTTONUP, - WM_MOUSEMOVE, WM_RBUTTONDOWN, WM_RBUTTONUP, + ChildWindowFromPointEx, GetAncestor, GetClassLongPtrW, GetCursorPos, GetForegroundWindow, + GetSystemMetrics, GetWindowLongPtrW, PostMessageW, SetCursorPos, SetWindowPos, CS_DBLCLKS, + CWP_SKIPDISABLED, CWP_SKIPINVISIBLE, GA_ROOT, GCL_STYLE, + CWP_SKIPTRANSPARENT, GWL_EXSTYLE, HWND_NOTOPMOST, HWND_TOP, HWND_TOPMOST, + SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN, SM_XVIRTUALSCREEN, SM_YVIRTUALSCREEN, SWP_NOACTIVATE, + SWP_NOMOVE, SWP_NOSIZE, WM_LBUTTONDBLCLK, WM_LBUTTONDOWN, WM_LBUTTONUP, + WM_MBUTTONDBLCLK, WM_MBUTTONDOWN, WM_MBUTTONUP, WM_MOUSEMOVE, WM_RBUTTONDBLCLK, + WM_RBUTTONDOWN, WM_RBUTTONUP, WS_EX_TOPMOST, }; const MK_LBUTTON: u32 = 0x0001; @@ -31,6 +32,14 @@ const MK_RBUTTON: u32 = 0x0002; const CLICK_DELAY_MS: u64 = 35; +fn posted_press_message(down: u32, double: u32, click_index: usize, wants_double: bool) -> u32 { + if wants_double && click_index % 2 == 1 { + double + } else { + down + } +} + /// Walk from `root` down to the deepest visible child that contains /// `screen_pt`, mirroring trope-cua's DeepestChildFromScreenPoint. /// @@ -99,20 +108,39 @@ fn post_click_on(hwnd: HWND, x: i32, y: i32, count: usize, button: &str) -> Resu anyhow::bail!(msg); } - let (down_msg, up_msg, mk_flag) = match button { - "right" => (WM_RBUTTONDOWN, WM_RBUTTONUP, MK_RBUTTON), - "middle" => (WM_MBUTTONDOWN, WM_MBUTTONUP, MK_MBUTTON), - _ => (WM_LBUTTONDOWN, WM_LBUTTONUP, MK_LBUTTON), + let (down_msg, double_msg, up_msg, mk_flag) = match button { + "right" => (WM_RBUTTONDOWN, WM_RBUTTONDBLCLK, WM_RBUTTONUP, MK_RBUTTON), + "middle" => ( + WM_MBUTTONDOWN, + WM_MBUTTONDBLCLK, + WM_MBUTTONUP, + MK_MBUTTON, + ), + _ => (WM_LBUTTONDOWN, WM_LBUTTONDBLCLK, WM_LBUTTONUP, MK_LBUTTON), }; let lparam = make_lparam(x, y); let wdown = WPARAM(mk_flag as usize); let wup = WPARAM(0); + let wants_double = + unsafe { (GetClassLongPtrW(hwnd, GCL_STYLE) as u32 & CS_DBLCLKS.0) != 0 }; + let prev_fg = unsafe { GetForegroundWindow() }; + let target_root = unsafe { + let root = GetAncestor(hwnd, GA_ROOT); + if root.0.is_null() { hwnd } else { root } + }; + // Posted pointer messages are normally non-activating, but WebView hosts can + // call SetForegroundWindow from their event handlers. Keep the top-level + // categorically non-activatable until the complete burst has settled. + let _noact = crate::input::NoActivateGuard::arm(hwnd); for i in 0..count { + let press_msg = posted_press_message(down_msg, double_msg, i, wants_double); unsafe { // WM_MOUSEMOVE first so hover state is correct before the click. PostMessageW(hwnd, WM_MOUSEMOVE, WPARAM(0), lparam)?; - PostMessageW(hwnd, down_msg, wdown, lparam)?; + // Win32 controls do not infer a double-click from two posted DOWN + // messages. The second press must use WM_*BUTTONDBLCLK. + PostMessageW(hwnd, press_msg, wdown, lparam)?; sleep(Duration::from_millis(CLICK_DELAY_MS)); PostMessageW(hwnd, up_msg, wup, lparam)?; } @@ -120,6 +148,17 @@ fn post_click_on(hwnd: HWND, x: i32, y: i32, count: usize, button: &str) -> Resu sleep(Duration::from_millis(80)); } } + sleep(Duration::from_millis(50)); + unsafe { + if !prev_fg.0.is_null() + && prev_fg != target_root + && GetForegroundWindow() == target_root + { + crate::input::force_foreground_attached(prev_fg); + sleep(Duration::from_millis(12)); + crate::input::force_foreground_attached(prev_fg); + } + } Ok(()) } @@ -298,6 +337,46 @@ pub fn is_chromium_target_window(hwnd: u64) -> bool { is_chromium } +/// Return true when `hwnd` hosts a Chromium/WebView2 renderer child even if +/// its own top-level class is framework-specific (for example a Tauri host). +/// Keep this separate from [`is_chromium_target_window`]: embedded WebView2 +/// surfaces support some UIA/top-level background routes that direct Chromium +/// frames do not, so delivery policy needs to distinguish the two shapes. +pub fn has_chromium_descendant(hwnd: u64) -> bool { + use windows::Win32::Foundation::{BOOL, FALSE, LPARAM, TRUE}; + use windows::Win32::UI::WindowsAndMessaging::{EnumChildWindows, GetClassNameW}; + + if hwnd == 0 { + return false; + } + struct Scan { + found: bool, + } + unsafe extern "system" fn child_cb(child: HWND, lparam: LPARAM) -> BOOL { + let scan = &mut *(lparam.0 as *mut Scan); + let mut buf = [0u16; 64]; + let n = GetClassNameW(child, &mut buf); + if n > 0 { + let class = String::from_utf16_lossy(&buf[..n as usize]); + if class.starts_with("Chrome_WidgetWin_") || class.starts_with("CefBrowser") { + scan.found = true; + return FALSE; + } + } + TRUE + } + + let mut scan = Scan { found: false }; + unsafe { + let _ = EnumChildWindows( + HWND(hwnd as *mut _), + Some(child_cb), + LPARAM(&mut scan as *mut Scan as isize), + ); + } + scan.found +} + /// Click at **screen** coordinates `(sx, sy)` via `SendInput` against the /// system input queue, briefly focusing `target` so the click lands there. /// @@ -344,6 +423,33 @@ pub fn send_click_synthesized_mods( count: usize, button: &str, modifiers: &[&str], +) -> Result<()> { + send_click_synthesized_mods_impl(target, sx, sy, count, button, modifiers, false) +} + +/// SendInput click for an explicit foreground request. Unlike the historical +/// z-order-assisted path, this activates the target and does not add +/// `WS_EX_NOACTIVATE`, so retained-mode frameworks such as WPF process the +/// system-queue pointer event. The caller owns any later foreground restore. +pub fn send_click_synthesized_active_mods( + target: u64, + sx: i32, + sy: i32, + count: usize, + button: &str, + modifiers: &[&str], +) -> Result<()> { + send_click_synthesized_mods_impl(target, sx, sy, count, button, modifiers, true) +} + +fn send_click_synthesized_mods_impl( + target: u64, + sx: i32, + sy: i32, + count: usize, + button: &str, + modifiers: &[&str], + activate: bool, ) -> Result<()> { let target = HWND(target as *mut _); if target.0.is_null() { @@ -404,8 +510,8 @@ pub fn send_click_synthesized_mods( r#type: INPUT_MOUSE, Anonymous: INPUT_0 { mi: MOUSEINPUT { - dx: norm_x, dy: norm_y, mouseData: 0, - dwFlags: down_flag | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK, + dx: 0, dy: 0, mouseData: 0, + dwFlags: down_flag, time: 0, dwExtraInfo: 0, }, }, @@ -414,8 +520,8 @@ pub fn send_click_synthesized_mods( r#type: INPUT_MOUSE, Anonymous: INPUT_0 { mi: MOUSEINPUT { - dx: norm_x, dy: norm_y, mouseData: 0, - dwFlags: up_flag | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK, + dx: 0, dy: 0, mouseData: 0, + dwFlags: up_flag, time: 0, dwExtraInfo: 0, }, }, @@ -437,12 +543,20 @@ pub fn send_click_synthesized_mods( // restore" for pointer input, done the one Windows way that doesn't // need UIAccess — the technique the OG GTK path used. (Keyboard // foreground still needs *real* focus; only pointer can be z-routed.) - let _noact = crate::input::NoActivateGuard::arm(target); // Capture whether the target was ALREADY always-on-top so we don't strip // that state on restore — only demote below if WE promoted it. let was_topmost = (GetWindowLongPtrW(target, GWL_EXSTYLE) as u32) & WS_EX_TOPMOST.0 != 0; - let _ = SetWindowPos(target, HWND_TOPMOST, 0, 0, 0, 0, SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE); + let foreground_attach_failed = activate && !crate::input::force_foreground_attached(target); + let noactivate = (!activate).then(|| crate::input::NoActivateGuard::arm(target)); + if !activate || foreground_attach_failed { + let flags = if activate { + SWP_NOMOVE | SWP_NOSIZE + } else { + SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE + }; + let _ = SetWindowPos(target, HWND_TOPMOST, 0, 0, 0, 0, flags); + } // Move the cursor so the OS hover state matches before the click; the // MOUSEEVENTF_MOVE input ensures Chromium's input filter sees a @@ -462,6 +576,9 @@ pub fn send_click_synthesized_mods( let count = count.max(1); let mut sent_ok = true; for i in 0..count { + // Only the move record carries absolute coordinates. Button-only + // records act at the current pointer position; adding ABSOLUTE to + // them can prevent retained-mode controls from seeing the press. let events = [move_input, down_input, up_input]; let sent = SendInput(&events, std::mem::size_of::() as i32); if sent as usize != events.len() { @@ -478,21 +595,31 @@ pub fn send_click_synthesized_mods( SendInput(&mod_ups, std::mem::size_of::() as i32); } - // Brief settle so the target processes the click, then restore z-order: - // demote the target out of the topmost band and restack the user's - // window on top (no activation), and restore the cursor. - sleep(Duration::from_millis(40)); - if !was_topmost { + // Let the target process mouse-up before any background-route restore. + // Retained-mode frameworks establish capture/focus on mouse-down and can + // lose the click if the real cursor is warped away while those queued + // messages are still being dispatched. + sleep(Duration::from_millis(if activate { 120 } else { 40 })); + if !was_topmost && (!activate || foreground_attach_failed) { let _ = SetWindowPos(target, HWND_NOTOPMOST, 0, 0, 0, 0, SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE); } - if !prev_fg.0.is_null() && prev_fg != target { - let _ = SetWindowPos(prev_fg, HWND_TOP, 0, 0, 0, 0, SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE); + if !activate { + if !prev_fg.0.is_null() && prev_fg != target { + let _ = SetWindowPos(prev_fg, HWND_TOP, 0, 0, 0, 0, SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE); + } + let _ = SetCursorPos(prev_cursor.x, prev_cursor.y); } - let _ = SetCursorPos(prev_cursor.x, prev_cursor.y); - drop(_noact); + drop(noactivate); if !sent_ok { bail!("SendInput inserted fewer mouse events than expected for the foreground click."); } + if activate { + let foreground_root = GetAncestor(GetForegroundWindow(), GA_ROOT); + let target_root = GetAncestor(target, GA_ROOT); + if foreground_root != target_root { + bail!("The foreground click did not activate its target window."); + } + } } Ok(()) @@ -709,7 +836,28 @@ pub fn send_wheel_synthesized(sx: i32, sy: i32, ticks: i32, horizontal: bool) -> #[cfg(test)] mod wheel_tests { - use super::{wheel_mouse_data, WHEEL_DELTA}; + use super::{posted_press_message, wheel_mouse_data, WHEEL_DELTA}; + use windows::Win32::UI::WindowsAndMessaging::{WM_LBUTTONDBLCLK, WM_LBUTTONDOWN}; + + #[test] + fn posted_double_click_uses_the_win32_double_click_message() { + assert_eq!( + posted_press_message(WM_LBUTTONDOWN, WM_LBUTTONDBLCLK, 0, true), + WM_LBUTTONDOWN + ); + assert_eq!( + posted_press_message(WM_LBUTTONDOWN, WM_LBUTTONDBLCLK, 1, true), + WM_LBUTTONDBLCLK + ); + assert_eq!( + posted_press_message(WM_LBUTTONDOWN, WM_LBUTTONDBLCLK, 2, true), + WM_LBUTTONDOWN + ); + assert_eq!( + posted_press_message(WM_LBUTTONDOWN, WM_LBUTTONDBLCLK, 1, false), + WM_LBUTTONDOWN + ); + } #[test] fn wheel_data_up_is_positive_one_notch() { diff --git a/libs/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs b/libs/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs index f587d311b8..4a94156bad 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs @@ -12,6 +12,19 @@ use std::sync::{Arc, OnceLock}; #[cfg(target_os = "windows")] use crate::uia::ElementCache; +use cua_driver_core::recording::ScreenshotCapture; + +#[cfg(target_os = "windows")] +use windows::Win32::Foundation::HWND; + +#[cfg(target_os = "windows")] +use windows::Win32::UI::Input::KeyboardAndMouse::IsWindowEnabled; + +#[cfg(target_os = "windows")] +use windows::Win32::UI::WindowsAndMessaging::{ + GetLastActivePopup, GetWindowThreadProcessId, IsWindow, IsWindowVisible, +}; + #[cfg(target_os = "windows")] static ELEMENT_CACHE: OnceLock> = OnceLock::new(); @@ -20,15 +33,69 @@ pub fn set_element_cache(cache: Arc) { let _ = ELEMENT_CACHE.set(cache); } +/// Resolve the window whose application evidence should be captured. Keep a +/// live explicit HWND so occluded/background turns capture the exact target. +/// When an action closes a modal HWND, fall back to another top-level window +/// owned by the same pid for the post-action application state. #[cfg(target_os = "windows")] -pub fn app_state_json_for(window_id: Option, pid: Option) -> Option> { +pub fn resolve_window_for_recording(window_id: Option, pid: Option) -> Option { + if let Some(window_id) = window_id { + let hwnd = HWND(window_id as *mut _); + if unsafe { IsWindow(hwnd) }.as_bool() { + let popup = unsafe { GetLastActivePopup(hwnd) }; + if !unsafe { IsWindowEnabled(hwnd) }.as_bool() + && popup != hwnd + && unsafe { IsWindow(popup) }.as_bool() + && unsafe { IsWindowVisible(popup) }.as_bool() + { + let mut popup_pid = 0; + unsafe { GetWindowThreadProcessId(popup, Some(&mut popup_pid)) }; + if pid.and_then(|value| u32::try_from(value).ok()) == Some(popup_pid) { + return Some(popup.0 as u64); + } + } + return Some(window_id); + } + } let pid = u32::try_from(pid?).ok()?; - let hwnd = match window_id { - Some(w) => w, - None => crate::win32::list_windows(Some(pid)).first().map(|w| w.hwnd)?, + crate::win32::list_windows(Some(pid)) + .first() + .map(|window| window.hwnd) +} + +#[cfg(target_os = "windows")] +pub fn screenshot_for_recording( + window_id: Option, + pid: Option, +) -> ScreenshotCapture { + if window_id.is_none() && pid.is_none() { + return crate::capture::screenshot_display_bytes() + .map(ScreenshotCapture::captured) + .unwrap_or_else(|_| ScreenshotCapture::unavailable("capture_failed")); + } + let Some(hwnd) = resolve_window_for_recording(window_id, pid) else { + return ScreenshotCapture::unavailable("target_unavailable"); }; + match crate::capture::screenshot_window_bytes_with_occlusion(hwnd) { + Ok((_, true)) => ScreenshotCapture::unavailable("background_occluded"), + Ok((png, false)) => ScreenshotCapture::captured(png), + Err(error) if error.to_string().contains("minimized window") => { + ScreenshotCapture::unavailable("target_minimized") + } + Err(_) => ScreenshotCapture::unavailable("capture_failed"), + } +} + +#[cfg(target_os = "windows")] +pub fn app_state_json_for(window_id: Option, pid: Option) -> Option> { + let pid = u32::try_from(pid?).ok()?; + let hwnd = resolve_window_for_recording(window_id, Some(pid.into()))?; let result = crate::uia::walk_tree(hwnd, None); - let element_count = result.nodes.iter().filter(|n| n.element_index.is_some()).count(); + let element_count = result + .nodes + .iter() + .filter(|n| n.element_index.is_some()) + .count(); let payload = serde_json::json!({ "pid": pid, "window_id": hwnd, @@ -52,6 +119,25 @@ pub fn element_window_local_xy(window_id: u64, pid: i64, element_index: u32) -> } #[cfg(not(target_os = "windows"))] -pub fn app_state_json_for(_window_id: Option, _pid: Option) -> Option> { None } +pub fn app_state_json_for(_window_id: Option, _pid: Option) -> Option> { + None +} #[cfg(not(target_os = "windows"))] -pub fn element_window_local_xy(_window_id: u64, _pid: i64, _element_index: u32) -> Option<(f64, f64)> { None } +pub fn resolve_window_for_recording(_window_id: Option, _pid: Option) -> Option { + None +} +#[cfg(not(target_os = "windows"))] +pub fn screenshot_for_recording( + _window_id: Option, + _pid: Option, +) -> ScreenshotCapture { + ScreenshotCapture::unavailable("unsupported_platform") +} +#[cfg(not(target_os = "windows"))] +pub fn element_window_local_xy( + _window_id: u64, + _pid: i64, + _element_index: u32, +) -> Option<(f64, f64)> { + None +} diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs index 610ca60791..ad0b54dda6 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs @@ -134,6 +134,11 @@ fn bitmap_to_screen(hwnd: u64, px: i32, py: i32) -> (i32, i32) { } } +fn screen_to_bitmap(hwnd: u64, sx: i32, sy: i32) -> (i32, i32) { + let (origin_x, origin_y) = bitmap_to_screen(hwnd, 0, 0); + (sx - origin_x, sy - origin_y) +} + /// Animate the agent cursor to (sx, sy) in screen coordinates and wait for the /// glide to finish before returning. No-op when the overlay is not enabled. /// @@ -633,11 +638,9 @@ impl Tool for ListWindowsTool { one?\".\n\n\ Per-record fields: window_id (HWND), pid + app_name, title, \ bounds {x, y, width, height}, layer (always 0), z_index (stacking order), \ - is_on_screen. The macOS-specific on_current_space / space_ids fields are \ + is_on_screen, minimized. The macOS-specific on_current_space / space_ids fields are \ omitted on Windows; current_space_id is null.\n\n\ - Inputs: pid (optional pid filter), on_screen_only (bool, default false — \ - Windows currently only enumerates visible non-minimized windows; this flag \ - is accepted but has no effect on the result set yet).".into(), + Inputs: pid (optional pid filter), on_screen_only (bool, default false).".into(), input_schema: json!({"type":"object","properties":{ "pid":{"type":"integer","description":"Optional pid filter. When set, only this pid's windows are returned."}, "on_screen_only":{"type":"boolean","description":"When true, drop windows that aren't currently on-screen. Default false."} @@ -649,8 +652,8 @@ impl Tool for ListWindowsTool { async fn invoke(&self, args: Value) -> ToolResult { use cua_driver_core::tool_args::ArgsExt; let filter_pid = args.opt_u64("pid").map(|v| v as u32); - let _on_screen_only = args.bool_or("on_screen_only", false); - let (windows, pid_to_name) = tokio::task::spawn_blocking(move || { + let on_screen_only = args.bool_or("on_screen_only", false); + let (mut windows, pid_to_name) = tokio::task::spawn_blocking(move || { let wins = crate::win32::list_windows(filter_pid); let procs = crate::win32::list_processes(); let map: std::collections::HashMap = @@ -659,9 +662,12 @@ impl Tool for ListWindowsTool { }) .await .unwrap_or_default(); + if on_screen_only { + windows.retain(|w| w.is_on_screen); + } // Swift surfaces a warning when a pid filter matches nothing. - if let Some(fp) = filter_pid { + let missing_pid_warning = if let Some(fp) = filter_pid { if windows.is_empty() { use windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow; use windows::Win32::UI::WindowsAndMessaging::GetWindowThreadProcessId; @@ -672,14 +678,17 @@ impl Tool for ListWindowsTool { p }; let fg_name = pid_to_name.get(&fg_pid).map(|s| s.as_str()).unwrap_or("?"); - let msg = format!( + Some(format!( "⚠️ No windows found for pid {fp}. The pid may be wrong or the app may not \ have created a window yet. The current frontmost app appears to be \ \"{fg_name}\" (pid {fg_pid})." - ); - return ToolResult::text(msg); + )) + } else { + None } - } + } else { + None + }; // z_index: list_windows merges EnumWindows first (which the Win32 // window manager returns in canonical top-to-bottom z-order), then @@ -704,7 +713,8 @@ impl Tool for ListWindowsTool { "bounds": { "x": w.x, "y": w.y, "width": w.width, "height": w.height }, "layer": 0, "z_index": z_index, - "is_on_screen": true, + "is_on_screen": w.is_on_screen, + "minimized": w.minimized, }) }) .collect(); @@ -726,6 +736,9 @@ impl Tool for ListWindowsTool { (SkyLight Space SPIs unavailable — on_current_space / space_ids omitted.)" ); let mut lines = vec![header]; + if let Some(warning) = missing_pid_warning { + lines.push(warning); + } for r in &records { let app = r["app_name"].as_str().unwrap_or("?"); let pid = r["pid"].as_u64().unwrap_or(0); @@ -754,6 +767,7 @@ impl Tool for ListWindowsTool { json!({ "window_id": w.hwnd, "pid": w.pid, "title": w.title, "x": w.x, "y": w.y, "width": w.width, "height": w.height, + "is_on_screen": w.is_on_screen, "minimized": w.minimized, }) }) .collect(); @@ -928,7 +942,7 @@ impl Tool for GetWindowStateTool { // surface *why* there's no image (the iconic-window guard from // #1973 / PR #1974 is the load-bearing case: minimized windows // legitimately can't be captured, and the caller needs to know - // to call `raise_window` / `list_windows` instead of retrying). + // to call `bring_to_front` instead of retrying). // The previous `Err(_) => None` silently dropped the error and // upstream agents saw an empty response with no signal. let (screenshot, screenshot_err) = if do_shot { @@ -1549,7 +1563,7 @@ impl Tool for LaunchAppTool { "cdp_debugging_port":{"type":"integer","description":"Accepted for cross-platform parity; currently no-op on Windows."}, "webkit_inspector_port":{"type":"integer","description":"Accepted for cross-platform parity; no-op on Windows."}, "creates_new_application_instance":{"type":"boolean","description":"Accepted for parity; no-op on Windows (ShellExecuteEx always creates a new process)."}, - "start_minimized":{"type":"boolean","description":"When true, launch the app's window minimized to the taskbar instead of restored-but-not-activated. Use this when the agent wants to drive the app entirely in the background — the user's previously-frontmost window (e.g. terminal) stays visually on top. Implementation uses SW_SHOWMINNOACTIVE for the ShellExecuteEx path and a follow-up ShowWindow(SW_MINIMIZE) on the AUMID path. UIA / background dispatch still work on a minimized window; only `screenshot` and `delivery_mode:\"foreground\"` need it restored."} + "start_minimized":{"type":"boolean","description":"When true, launch the app's window minimized to the taskbar instead of restored-but-not-activated. Use this when the agent wants to drive the app entirely in the background — the user's previously-frontmost window (e.g. terminal) stays visually on top. Desktop launches hold the foreground lock through startup and use SW_SHOWMINNOACTIVE; packaged-app activation remains broker-controlled and receives a best-effort SW_SHOWMINNOACTIVE post-pass. UIA / background dispatch still work on a minimized window; only `screenshot` and `delivery_mode:\"foreground\"` need it restored."} },"additionalProperties":false}), read_only: false, destructive: false, idempotent: true, open_world: true, }) @@ -1773,6 +1787,31 @@ impl Tool for LaunchAppTool { } }; + // Strict no-activation is available only for the desktop launch path. + // UWP activation is broker-controlled and retains its existing + // restore-after-activation behavior. + let mut foreground_lock = if start_minimized && aumid_for_uwp.is_none() { + Some(crate::input::ForegroundLockGuard::acquire()) + } else { + None + }; + if foreground_lock + .as_ref() + .is_some_and(|guard| !guard.acquired()) + { + return ToolResult::error( + "Background minimized launch is unavailable because Windows did not grant the \ + foreground lock required to prevent the new process from activating. No process \ + was started. Launch without start_minimized only when a foreground change is \ + acceptable.", + ) + .with_structured(json!({ + "code": "background_unavailable", + "delivery_mode": "background", + "event_kind": "app_launch", + })); + } + // Branch: AUMID activation if we resolved one; else legacy // ShellExecuteExW. Both branches still need to handle `urls` // (additional URLs always go through ShellExecuteExW since the @@ -1801,6 +1840,13 @@ impl Tool for LaunchAppTool { let target_for_shell = target_file_opt.clone(); let extra_for_shell = extra_joined.clone(); let n_show_for_shell = n_show; + let direct_minimized_exe = start_minimized + && target_for_shell.as_deref().is_some_and(|target| { + std::path::Path::new(target).is_file() + && std::path::Path::new(target) + .extension() + .is_some_and(|extension| extension.eq_ignore_ascii_case("exe")) + }); // Bound the shell launch with a timeout. An unregistered protocol // or file association makes `ShellExecuteExW` block on a modal shell // dialog ("you'll need a new app to open this …") on the *session* @@ -1811,9 +1857,12 @@ impl Tool for LaunchAppTool { // backstop for any *other* blocking broker dialog (SmartScreen, an // elevation/consent surface) so a bad target can't hang the daemon. let launch = tokio::task::spawn_blocking(move || -> anyhow::Result { - use windows::core::PCWSTR; + use windows::core::{PCWSTR, PWSTR}; use windows::Win32::Foundation::CloseHandle; - use windows::Win32::System::Threading::GetProcessId; + use windows::Win32::System::Threading::{ + CreateProcessW, GetProcessId, PROCESS_CREATION_FLAGS, PROCESS_INFORMATION, + STARTF_USESHOWWINDOW, STARTUPINFOW, + }; use windows::Win32::UI::Shell::{ ShellExecuteExW, SEE_MASK_FLAG_NO_UI, SEE_MASK_NOCLOSEPROCESS, SHELLEXECUTEINFOW, @@ -1830,30 +1879,63 @@ impl Tool for LaunchAppTool { }); let args_w = to_wide(&extra_for_shell); - let mut info = SHELLEXECUTEINFOW { - cbSize: std::mem::size_of::() as u32, - fMask: SEE_MASK_NOCLOSEPROCESS | SEE_MASK_FLAG_NO_UI, - lpVerb: PCWSTR(op_w.as_ptr()), - lpFile: PCWSTR(file_w.as_ptr()), - lpParameters: if extra_for_shell.is_empty() { - PCWSTR::null() + let pid = if direct_minimized_exe { + let target = target_for_shell.as_deref().expect("checked executable path"); + let mut command_line = to_wide(&if extra_for_shell.is_empty() { + format!(r#""{target}""#) } else { - PCWSTR(args_w.as_ptr()) - }, - nShow: n_show_for_shell, - ..Default::default() - }; - unsafe { - ShellExecuteExW(&mut info)?; - } - let pid = if !info.hProcess.is_invalid() { - let p = unsafe { GetProcessId(info.hProcess) }; + format!(r#""{target}" {extra_for_shell}"#) + }); + let startup = STARTUPINFOW { + cb: std::mem::size_of::() as u32, + dwFlags: STARTF_USESHOWWINDOW, + wShowWindow: n_show_for_shell as u16, + ..Default::default() + }; + let mut process = PROCESS_INFORMATION::default(); unsafe { - let _ = CloseHandle(info.hProcess); + CreateProcessW( + PCWSTR(file_w.as_ptr()), + PWSTR(command_line.as_mut_ptr()), + None, + None, + false, + PROCESS_CREATION_FLAGS(0), + None, + PCWSTR::null(), + &startup, + &mut process, + )?; + let _ = CloseHandle(process.hThread); + let _ = CloseHandle(process.hProcess); } - p + process.dwProcessId } else { - 0 + let mut info = SHELLEXECUTEINFOW { + cbSize: std::mem::size_of::() as u32, + fMask: SEE_MASK_NOCLOSEPROCESS | SEE_MASK_FLAG_NO_UI, + lpVerb: PCWSTR(op_w.as_ptr()), + lpFile: PCWSTR(file_w.as_ptr()), + lpParameters: if extra_for_shell.is_empty() { + PCWSTR::null() + } else { + PCWSTR(args_w.as_ptr()) + }, + nShow: n_show_for_shell, + ..Default::default() + }; + unsafe { + ShellExecuteExW(&mut info)?; + } + if !info.hProcess.is_invalid() { + let p = unsafe { GetProcessId(info.hProcess) }; + unsafe { + let _ = CloseHandle(info.hProcess); + } + p + } else { + 0 + } }; // Open any additional URLs in the default browser (no focus @@ -2130,12 +2212,9 @@ impl Tool for LaunchAppTool { // and minimizes anything that materializes. The task runs detached // so the launch_app response isn't held up by it. // - // SW_MINIMIZE itself activates "the next top-level window in z-order" - // which would shift the user's focus, but the foreground-restore - // polling task (spawned earlier in this method via - // `restore_foreground_polling_best_effort`) flips foreground back to - // the pre-launch window, so the net effect is "minimize and leave - // the user's window where it was". + // SW_SHOWMINNOACTIVE preserves the foreground while minimizing. Using + // SW_MINIMIZE here would itself activate the next z-order window and + // force a visible restore-after-steal cycle. if start_minimized { // First, minimize anything already resolved (covers the common // single-process path where windows_json was populated). @@ -2156,17 +2235,21 @@ impl Tool for LaunchAppTool { // minimize a user's unrelated app that started during the 5 s // poll window. let parent_pid = pid; + let immediate_hwnds_for_poll = immediate_hwnds.clone(); let _ = tokio::task::spawn_blocking(move || { use windows::Win32::Foundation::HWND; - use windows::Win32::UI::WindowsAndMessaging::{ShowWindow, SW_MINIMIZE}; + use windows::Win32::UI::WindowsAndMessaging::{ShowWindow, SW_SHOWMINNOACTIVE}; for h in immediate_hwnds { unsafe { - let _ = ShowWindow(HWND(h as *mut _), SW_MINIMIZE); + let _ = ShowWindow(HWND(h as *mut _), SW_SHOWMINNOACTIVE); } } }) .await; - // Detached polling for launcher-stub late-window cases. + // Poll launcher-stub late-window cases before returning. The + // start_minimized contract is observable at response time; a + // detached task allowed callers to see a transient restored + // window immediately after a successful launch. // Strategy: every 200 ms for 5 s, find pids that // (a) weren't in the pre-launch snapshot, AND // (b) are part of the launched app's family — either a @@ -2181,17 +2264,21 @@ impl Tool for LaunchAppTool { // the regression CodeRabbit flagged. // // Loop ends early once we've minimized the first set of - // windows AND remained idle for one tick — the typical + // windows AND they remain minimized for three ticks — the typical // app has its main window up within ~2 s of launch. let pre_pids = pre_launch_pids.clone(); let basename_for_poll = stub_basename.clone(); - tokio::spawn(async move { + let launch_foreground_lock = foreground_lock.take(); + (async move { use std::collections::HashSet; use windows::Win32::Foundation::HWND; - use windows::Win32::UI::WindowsAndMessaging::{ShowWindow, SW_MINIMIZE}; - let mut minimized: HashSet = HashSet::new(); + use windows::Win32::UI::WindowsAndMessaging::{ + IsIconic, SW_SHOWMINNOACTIVE, ShowWindow, + }; + let _foreground_lock = launch_foreground_lock; + let mut minimized: HashSet = immediate_hwnds_for_poll.into_iter().collect(); let mut idle_ticks_after_any_hit: u8 = 0; - let mut hit_count_total: usize = 0; + let mut hit_count_total = minimized.len(); for _ in 0..25 { let pre_pids_clone = pre_pids.clone(); let basename_clone = basename_for_poll.clone(); @@ -2222,22 +2309,32 @@ impl Tool for LaunchAppTool { .await .unwrap_or_default(); for w in wins { - if minimized.insert(w.hwnd) { - tick_hits += 1; + let is_new = minimized.insert(w.hwnd); + if is_new { hit_count_total += 1; - let hwnd_iso = w.hwnd as usize; - let _ = tokio::task::spawn_blocking(move || unsafe { - let _ = ShowWindow(HWND(hwnd_iso as *mut _), SW_MINIMIZE); - }) - .await; + } + let hwnd_iso = w.hwnd as usize; + let restored = tokio::task::spawn_blocking(move || unsafe { + let hwnd = HWND(hwnd_iso as *mut _); + if IsIconic(hwnd).as_bool() { + false + } else { + let _ = ShowWindow(hwnd, SW_SHOWMINNOACTIVE); + true + } + }) + .await + .unwrap_or(false); + if is_new || restored { + tick_hits += 1; } } } if tick_hits == 0 { idle_ticks_after_any_hit += 1; if hit_count_total > 0 && idle_ticks_after_any_hit >= 3 { - // Stable: had hits, then 600 ms of nothing new. - // Done. + // Stable: known windows remained minimized and no + // new window appeared for 600 ms. break; } } else { @@ -2245,7 +2342,8 @@ impl Tool for LaunchAppTool { } tokio::time::sleep(std::time::Duration::from_millis(200)).await; } - }); + }) + .await; } // Match Swift text format 1:1. @@ -2413,6 +2511,25 @@ impl Tool for ClickTool { let sx = args.f64_or("x", 0.0) as i32; let sy = args.f64_or("y", 0.0) as i32; + // Resolve the application window before moving the agent cursor. + // WindowFromPoint can return a transparent layered overlay, and the + // cursor overlay is about to occupy this exact screen point. + let root = unsafe { + use windows::Win32::Foundation::POINT; + use windows::Win32::UI::WindowsAndMessaging::{ + GetAncestor, WindowFromPoint, GA_ROOT, + }; + let target = WindowFromPoint(POINT { x: sx, y: sy }); + if target.0.is_null() { + return ToolResult::error(format!( + "No window under screen point ({sx},{sy})." + )); + } + let root = GetAncestor(target, GA_ROOT); + if root.0.is_null() { target } else { root } + }; + let hwnd_u = root.0 as u64; + // Animate the agent cursor to the screen point, then click. overlay_glide_to(&cursor_key, sx as f64, sy as f64).await; crate::overlay::send_command( @@ -2423,22 +2540,12 @@ impl Tool for ClickTool { }, ); - // Resolve the HWND that owns this screen pixel and click it via - // send_click_synthesized — it does the foreground-swap + UIPI checks - // on whatever owns the pixel, which is what lands Chromium-content - // clicks. WindowFromPoint walks to the leaf window at the point. - // (send_click_synthesized restores the previous foreground + cursor - // itself ~40ms after the click, so no extra restore guard here.) + // Click the HWND that owned the pixel before the driver overlay + // moved there. The active SendInput path performs the foreground + // swap and UIPI checks needed for Chromium and retained-mode apps. let send_result = tokio::task::spawn_blocking(move || -> anyhow::Result { - use windows::Win32::Foundation::POINT; - use windows::Win32::UI::WindowsAndMessaging::WindowFromPoint; - let target = unsafe { WindowFromPoint(POINT { x: sx, y: sy }) }; - if target.0.is_null() { - anyhow::bail!("No window under screen point ({sx},{sy})."); - } - let hwnd_u = target.0 as u64; let mod_refs: Vec<&str> = modifiers.iter().map(String::as_str).collect(); - crate::input::send_click_synthesized_mods( + crate::input::send_click_synthesized_active_mods( hwnd_u, sx, sy, count, &button, &mod_refs, )?; Ok(hwnd_u) @@ -2682,6 +2789,57 @@ impl Tool for ClickTool { ); let btn = button.clone(); + // An explicit accessibility action is a semantic request, not a + // pixel gesture hint. Route expand through ExpandCollapsePattern + // even when foreground delivery was allowed; this reliably opens + // WPF/WinUI menus and tree nodes whose visual click target is + // transient or scroll-adjusted. + if action_req.as_deref() == Some("expand") { + let state = self.state.clone(); + let expand = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { + use windows::core::Interface; + use windows::Win32::UI::Accessibility::{ + IUIAutomationElement, IUIAutomationExpandCollapsePattern, + UIA_ExpandCollapsePatternId, + }; + + let retained = state + .element_cache + .get_element_retained(pid, hwnd, idx) + .ok_or_else(|| { + anyhow::anyhow!("element [{idx}] is not in the UIA cache") + })?; + if !retained.is_uia() { + anyhow::bail!("element [{idx}] is not a UIA element"); + } + let element = + unsafe { IUIAutomationElement::from_raw(retained.as_ptr() as *mut _) }; + let pattern = unsafe { + element + .GetCurrentPattern(UIA_ExpandCollapsePatternId) + .and_then(|value| value.cast::()) + } + .map_err(|error| { + anyhow::anyhow!("ExpandCollapsePattern unavailable: {error}") + })?; + let result = + crate::uia::fg_bypass::run_with_uwp_bypass(hwnd as isize, || unsafe { + pattern.Expand() + }); + std::mem::forget(element); + result.map_err(|error| anyhow::anyhow!("ExpandCollapse.Expand failed: {error}")) + }) + .await; + return match expand { + Ok(Ok(())) => ToolResult::text(format!( + "✅ Expanded UIA element [{idx}] via ExpandCollapsePattern." + )) + .with_structured(json!({ "path": "uia_expand_collapse", "verified": false, "effect": "unverifiable" })), + Ok(Err(error)) => ToolResult::error(error.to_string()), + Err(error) => ToolResult::error(format!("Task error: {error}")), + }; + } + // delivery_mode:"foreground" — skip UIA Invoke and use SendInput at the // cached element center. The caller explicitly chose foreground // delivery; UIA Invoke would be background-safe (which they @@ -2734,6 +2892,34 @@ impl Tool for ClickTool { return r; } } + // Chromium's UIA Invoke raises a fully occluded renderer, while a + // targeted PostMessage left click reaches the renderer without + // changing foreground, z-order, or the real cursor. Keep AX for + // target resolution and use the posted-message transport only for + // the empirically verified single-left-click shape. + if delivery == DeliveryMode::Background + && btn == "left" + && count == 1 + && crate::input::is_chromium_target_window(hwnd) + { + let posted = tokio::task::spawn_blocking(move || { + crate::input::post_click_screen(hwnd, cx, cy, count, &btn) + }) + .await; + return match posted { + Ok(Ok(())) => ToolResult::text(format!( + "✅ Posted click on Chromium element [{idx}] at screen ({cx},{cy}) \ + (background, no foreground swap)." + )) + .with_structured(json!({ + "path": "post_message", + "verified": false, + "effect": "unverifiable" + })), + Ok(Err(error)) => ToolResult::error(error.to_string()), + Err(error) => ToolResult::error(format!("Task error: {error}")), + }; + } // Try UIA Invoke first (it works for UWP / modern XAML / web // content where PostMessage(WM_LBUTTONDOWN) hits the outer // HWND but never reaches the inner XAML/composition element). @@ -2747,6 +2933,25 @@ impl Tool for ClickTool { let state_clone = self.state.clone(); let use_uia_invoke = (btn == "left" || btn == "middle") && count == 1; let result = tokio::task::spawn_blocking(move || -> anyhow::Result { + // Direct Chromium UIA Invoke can return S_OK without firing a + // DOM event while occluded. Try the honest coordinate actuator + // first: it lands while visible and reports occlusion without + // raising the window when hidden. + if delivery == DeliveryMode::Background + && crate::input::is_chromium_target_window(hwnd) + { + let (cx, cy) = resolve_onscreen_point_with_scroll( + &state_clone.element_cache, pid, hwnd, idx, cx, cy, "clicking", + ) + .map_err(|message| anyhow::anyhow!(message))?; + return crate::input::inject_click_screen(hwnd, cx, cy, count, &btn) + .map(|()| format!( + "✅ Injected click on [{idx}] (screen ({cx},{cy}), background, no foreground swap)." + )) + .map_err(|error| anyhow::anyhow!( + "__CUA_BG_UNAVAILABLE_CLICK__{error}" + )); + } if use_uia_invoke { // Retain the element out of the cache (AddRef under the // cache lock) so it can't be freed by a concurrent @@ -2838,30 +3043,31 @@ impl Tool for ClickTool { std::mem::forget(elem); } } - // PostMessage fallback (legacy Win32 + non-Invokable elements). - // delivery_mode:"background" on targets that silently drop PostMessage - // clicks (Chromium content, GTK buttons): route through the - // universal coordinate-injection actuator (touch injection, no - // foreground swap, z-order preserved) so the caller never needs - // to know the target is Chromium/GTK and never sees a raise. - // Only the structured error remains as a last resort (e.g. a - // right-click, which has no clean touch mapping). if delivery == DeliveryMode::Background - && crate::input::delivery::would_be_silently_dropped(hwnd, EventKind::MouseClick) + && crate::input::delivery::would_be_silently_dropped( + hwnd, + EventKind::MouseClick, + ) { - // Coordinate injection lands at (cx,cy); scroll the element - // into view if it's off-screen, else preserve the clean - // off-screen failure. let (cx, cy) = resolve_onscreen_point_with_scroll( - &state_clone.element_cache, pid, hwnd, idx, cx, cy, "clicking", + &state_clone.element_cache, + pid, + hwnd, + idx, + cx, + cy, + "clicking", ) - .map_err(|m| anyhow::anyhow!(m))?; - match crate::input::inject_click_screen(hwnd, cx, cy, count, &btn) { - Ok(()) => return Ok(format!( - "✅ Injected click on [{idx}] (screen ({cx},{cy}), background, no foreground swap)." - )), - Err(e) => anyhow::bail!("__CUA_BG_UNAVAILABLE_CLICK__{e}"), - } + .map_err(|message| anyhow::anyhow!(message))?; + return crate::input::inject_click_screen(hwnd, cx, cy, count, &btn) + .map(|()| { + format!( + "✅ Injected click on [{idx}] (screen ({cx},{cy}), background, no foreground swap)." + ) + }) + .map_err(|error| { + anyhow::anyhow!("__CUA_BG_UNAVAILABLE_CLICK__{error}") + }); } crate::input::post_click_screen(hwnd, cx, cy, count, &btn)?; let action_name = match btn.as_str() { @@ -2965,7 +3171,7 @@ impl Tool for ClickTool { let mods_owned = modifiers.clone(); let send_result = tokio::task::spawn_blocking(move || { let mod_refs: Vec<&str> = mods_owned.iter().map(String::as_str).collect(); - crate::input::send_click_synthesized_mods( + crate::input::send_click_synthesized_active_mods( hwnd, sx as i32, sy as i32, count, &btn, &mod_refs, ) }) @@ -3001,6 +3207,56 @@ impl Tool for ClickTool { return r; } } + // Match the AX-addressed Chromium route above: the point remains + // PX-resolved, but transport uses the background-safe posted + // message path proven against the fully occluded fixture. + if delivery == DeliveryMode::Background + && btn == "left" + && count == 1 + && crate::input::is_chromium_target_window(hwnd) + { + let posted = tokio::task::spawn_blocking(move || { + crate::input::post_click_screen(hwnd, sx_i, sy_i, count, &btn) + }) + .await; + return match posted { + Ok(Ok(())) => ToolResult::text(format!( + "✅ Posted click to Chromium pid {pid} at ({sx},{sy}) \ + (background, no foreground swap)." + )) + .with_structured(json!({ + "path": "post_message", + "verified": false, + "effect": "unverifiable" + })), + Ok(Err(error)) => ToolResult::error(error.to_string()), + Err(error) => ToolResult::error(format!("Task error: {error}")), + }; + } + // As above, bypass Chromium's false-positive UIA Invoke and use + // the coordinate actuator before attempting any accessibility hit + // test. The actuator itself distinguishes visible delivery from a + // fully occluded structured refusal. + if delivery == DeliveryMode::Background && crate::input::is_chromium_target_window(hwnd) + { + let btn2 = btn.clone(); + let inj = tokio::task::spawn_blocking(move || { + crate::input::inject_click_screen(hwnd, sx as i32, sy as i32, count, &btn2) + }) + .await; + return match inj { + Ok(Ok(())) => ToolResult::text(format!( + "✅ Injected click to pid {pid} at ({sx},{sy}) (background, no foreground swap)." + )) + .with_structured(json!({ "path": "pixel", "verified": false, "effect": "unverifiable" })), + Ok(Err(error)) => crate::input::delivery::background_unavailable_error_with_cause( + hwnd, + EventKind::MouseClick, + error.to_string(), + ), + Err(error) => ToolResult::error(format!("Task error: {error}")), + }; + } let use_uia = (btn == "left" || btn == "middle") && count == 1; if use_uia { let invoked = tokio::task::spawn_blocking(move || { @@ -3022,20 +3278,8 @@ impl Tool for ClickTool { } } - // UIA hit-test didn't land. Decide between PostMessage / injection / - // SendInput based on dispatch mode. - // - // delivery_mode:"background" (the default) — never swap foreground. If the - // target silently drops PostMessage mouse events (Chromium DOM - // content, GTK button widgets), route through the universal - // coordinate-injection actuator: touch injection lands in the system - // input queue (so Chromium/Electron/WPF accept it; the OS promotes to - // WM_*BUTTON for legacy Win32) WITHOUT SetForegroundWindow, and a - // cloak+restore z-order guard keeps the target from visibly raising. - // This is what lets a caller "just target the app and play actions" - // without knowing whether it's Chromium/GTK/etc. The structured - // background_unavailable error only survives as a last resort for - // inputs injection can't express (e.g. right/middle clicks). + // UIA did not land. Known dropped surfaces other than direct + // Chromium (handled above) get one targeted injection attempt. if delivery == DeliveryMode::Background && crate::input::delivery::would_be_silently_dropped(hwnd, EventKind::MouseClick) { @@ -3045,23 +3289,16 @@ impl Tool for ClickTool { }) .await; return match inj { - Ok(Ok(())) => { - let click_word = match count { - 2 => "double-click", - 3 => "triple-click", - _ => "click", - }; - ToolResult::text(format!( - "✅ Injected {click_word} to pid {pid} at ({sx},{sy}) (background, no foreground swap)." - )) - .with_structured(json!({ "path": "pixel", "verified": false, "effect": "unverifiable" })) - } - Ok(Err(e)) => crate::input::delivery::background_unavailable_error_with_cause( + Ok(Ok(())) => ToolResult::text(format!( + "✅ Injected click to pid {pid} at ({sx},{sy}) (background, no foreground swap)." + )) + .with_structured(json!({ "path": "pixel", "verified": false, "effect": "unverifiable" })), + Ok(Err(error)) => crate::input::delivery::background_unavailable_error_with_cause( hwnd, EventKind::MouseClick, - e.to_string(), + error.to_string(), ), - Err(e) => ToolResult::error(format!("Task error: {e}")), + Err(error) => ToolResult::error(format!("Task error: {error}")), }; } @@ -3146,9 +3383,11 @@ async fn focus_by_pixel( .invoke(click_args) .await; if focus.is_error == Some(true) { - return Err(ToolResult::error(format!( - "focus pixel-click at ({x:.0},{y:.0}) failed." - ))); + // Preserve the click tool's structured background refusal (for example + // background_occluded / background_uipi_blocked). Re-wrapping it as a + // text-only error made keyboard-family PX calls lose the actionable + // capability result produced by the actual actuator. + return Err(focus); } // Brief settle so the renderer registers focus before the keystrokes. tokio::time::sleep(std::time::Duration::from_millis(120)).await; @@ -3330,12 +3569,12 @@ impl Tool for TypeTextTool { }; let text_len = text.chars().count(); - // delivery_mode:"background" — TextInput is currently never flagged as - // silently dropped (Chromium accepts WM_CHAR through its IME path), - // but call the helper so the policy stays centralised in delivery.rs - // and future targets can be added without touching this site. + // Refuse known background drops before the final WM_CHAR path. WPF is + // conditional: indexed text still has a working UIA ValuePattern route, + // while unindexed text would be posted to the top-level and disappear. if delivery == DeliveryMode::Background - && crate::input::delivery::would_be_silently_dropped(hwnd, EventKind::TextInput) + && (crate::input::delivery::would_be_silently_dropped(hwnd, EventKind::TextInput) + || (elem_idx.is_none() && crate::input::delivery::is_wpf_target_window(hwnd))) { return crate::input::delivery::background_unavailable_error( hwnd, @@ -3350,6 +3589,53 @@ impl Tool for TypeTextTool { // rejected (daemon not at UIAccess integrity), it returns an error // rather than a false success. if delivery == DeliveryMode::Foreground { + // An indexed foreground type targets that element, not whichever + // child happened to retain focus in the top-level window. UIA + // SetFocus is not sufficient for Chromium renderer controls, so + // establish real system focus with the same foreground coordinate + // actuator used by an indexed click before sending Unicode input. + if let Some(idx) = elem_idx { + let (cx, cy) = + match self + .state + .element_cache + .get_element_center(pid, hwnd, idx as usize) + { + Some(center) => center, + None => { + return ToolResult::error(format!( + "Element {idx} not in cache for hwnd={hwnd}. Call get_window_state first." + )) + } + }; + let (cx, cy) = match resolve_onscreen_point_with_scroll( + &self.state.element_cache, + pid, + hwnd, + idx as usize, + cx, + cy, + "foreground typing", + ) { + Ok(point) => point, + Err(message) => return ToolResult::error(message), + }; + let focus_result = tokio::task::spawn_blocking(move || { + crate::input::send_click_synthesized(hwnd, cx, cy, 1, "left") + }) + .await; + match focus_result { + Ok(Ok(())) => { + tokio::time::sleep(std::time::Duration::from_millis(120)).await; + } + Ok(Err(error)) => return ToolResult::error(error.to_string()), + Err(error) => { + return ToolResult::error(format!( + "foreground element-focus task failed: {error}" + )) + } + } + } let text_fg = text.clone(); let r = tokio::task::spawn_blocking(move || { crate::input::send_text_synthesized(hwnd, &text_fg) @@ -3892,50 +4178,104 @@ impl Tool for PressKeyTool { } }; + // Classify known background drops before touching UIA focus. Focusing + // first made an honest Chromium refusal transiently activate the target. + let event_kind = if mods.is_empty() { + EventKind::Keystroke + } else { + EventKind::KeyCombo + }; + if delivery == DeliveryMode::Background + && crate::input::delivery::would_be_silently_dropped(hwnd, event_kind) + { + return crate::input::delivery::background_unavailable_error(hwnd, event_kind); + } + // W1: an element-addressed key needs the control's actual focus - // target, not merely its owning top-level HWND. Keep background - // delivery non-activating while UIA establishes child focus. - let _noact = if elem_idx.is_some() && delivery == DeliveryMode::Background { + // target, not merely its owning top-level HWND. Embedded WebView hosts + // can activate their frame from UIA SetFocus even under + // WS_EX_NOACTIVATE. Their proven-safe pixel route establishes renderer + // focus with a posted click, so reuse that route at the AX element's + // cached center. + let background_webview_focus = elem_idx.is_some() + && delivery == DeliveryMode::Background + && crate::input::has_chromium_descendant(hwnd); + let mut noact = if elem_idx.is_some() && delivery == DeliveryMode::Background { Some(crate::input::NoActivateGuard::arm( windows::Win32::Foundation::HWND(hwnd as *mut _), )) } else { None }; - if let Some(idx) = elem_idx { + if let Some(idx) = elem_idx.filter(|_| background_webview_focus) { + let Some((cx, cy)) = self + .state + .element_cache + .get_element_center(pid, hwnd, idx as usize) + else { + return ToolResult::error(format!( + "Element {idx} not in cache for hwnd={hwnd}. Call get_window_state first." + )); + }; + let (cx, cy) = match resolve_onscreen_point_with_scroll( + &self.state.element_cache, + pid, + hwnd, + idx as usize, + cx, + cy, + "focusing for key delivery", + ) { + Ok(point) => point, + Err(message) => return ToolResult::error(message), + }; + let (mut px, mut py) = screen_to_bitmap(hwnd, cx, cy); + if let Some(ratio) = self.state.resize_registry.ratio(pid) { + px = (px as f64 / ratio).round() as i32; + py = (py as f64 / ratio).round() as i32; + } + // Release the outer guard before the shared pixel helper. ClickTool + // owns a guard around the click itself, then releases it before its + // renderer settle period. This is the exact route already proven by + // the PX background cell, including targeted injection fallback. + drop(noact.take()); + if let Err(error) = focus_by_pixel( + &self.state, + pid, + Some(hwnd), + px as f64, + py as f64, + false, + args.opt_str("session"), + args.opt_str("_session_id"), + false, + ) + .await + { + return error; + } + } else if let Some(idx) = elem_idx { let state = self.state.clone(); let focused = tokio::task::spawn_blocking(move || { - state.element_cache.focus_element(pid, hwnd, idx as usize) + crate::uia::fg_bypass::run_with_uwp_bypass(hwnd as isize, || { + state.element_cache.focus_element(pid, hwnd, idx as usize) + }) }) .await; match focused { - Ok(Ok(())) => {} + Ok(Ok(())) => { + if delivery == DeliveryMode::Background { + let _ = crate::input::wait_for_focused_descendant( + hwnd, + std::time::Duration::from_millis(500), + ); + } + } Ok(Err(e)) => return ToolResult::error(e.to_string()), Err(e) => return ToolResult::error(format!("UIA focus task failed: {e}")), } } let key_display = key.clone(); - // Background mode: plain keystrokes (no modifiers) go through Chromium - // and GTK fine — would_be_silently_dropped returns false for the - // Keystroke variant by design. KeyCombo (modifiers) on Chromium IS - // dropped, so check that when modifiers are present. - let event_kind = if mods.is_empty() { - EventKind::Keystroke - } else { - EventKind::KeyCombo - }; - if !px_focus - && delivery == DeliveryMode::Background - && crate::input::delivery::would_be_silently_dropped(hwnd, event_kind) - { - // macOS-aligned contract: a `background` actuation never fronts. This - // key would be silently dropped by the target's input stack - // (TranslateAccelerator-based VCL/classic Win32, or Chromium key- - // combos) and the only way to land it is a foreground/focus grab — - // which background must not do. Surface background_unavailable so the - // agent escalates to delivery_mode:"foreground" (which may front). - return crate::input::delivery::background_unavailable_error(hwnd, event_kind); - } // Foreground: send_key_synthesized takes the SetForegroundWindow path. // Skipped when px-focus already fronted/clicked the target — the key then // goes via the plain background post path below. @@ -4254,13 +4594,11 @@ impl Tool for HotkeyTool { // reaches here means the key combo is NOT silently dropped on this // target (the drop-check above returned early otherwise), so it stays // on PostMessage and the no-foreground contract holds. - // px-focus delivers the combo via PostMessage to the now-focused field, so - // it never takes the SendInput foreground swap. - let use_send_input = !px_focus - && match delivery { - DeliveryMode::Foreground => true, - DeliveryMode::Background => false, - }; + // Foreground is an explicit request for system-queue delivery. This is + // still required after a PX focus click: PostMessage does not update + // global modifier state, so Chromium never observes Ctrl+Shift+7 as a + // chord even though the renderer control is focused. + let use_send_input = delivery == DeliveryMode::Foreground; let result = tokio::task::spawn_blocking(move || { let m: Vec<&str> = mods.iter().map(String::as_str).collect(); if use_send_input { @@ -4513,8 +4851,8 @@ impl Tool for ScrollTool { "by":{"type":"string","enum":["line","page"],"description":"Scroll granularity. Default: line."}, "amount":{"type":"integer","minimum":1,"maximum":50, "description":"Number of scroll ticks. Default 3."}, - "x":{"type":"number","description":"Screen-absolute X (desktop scope only) — wheel routes to the window under (x,y). Must be paired with y and no pid/window_id."}, - "y":{"type":"number","description":"Screen-absolute Y (desktop scope only). Must be paired with x and no pid/window_id."}, + "x":{"type":"number","description":"With pid/window_id: window-local screenshot X used to target a nested scroll surface in foreground mode. Without pid/window_id: screen-absolute X for desktop scope. Must be paired with y."}, + "y":{"type":"number","description":"With pid/window_id: window-local screenshot Y used to target a nested scroll surface in foreground mode. Without pid/window_id: screen-absolute Y for desktop scope. Must be paired with x."}, "window_id":{"type":"integer","description":"HWND of the target window. Required when element_index is used; otherwise auto-resolves the pid's first visible window."}, "element_index":{"type":"integer","description":"Optional element_index. Accepted for parity; currently no-op on Windows."}, "element_token": cua_driver_core::tool_schema::element_token_schema(), @@ -4653,7 +4991,27 @@ impl Tool for ScrollTool { // UIA while their top-level HWND ignores WM_VSCROLL. Prefer the // accessibility channel for an indexed target; the message path below // remains the fallback for native Win32 scrollbars. + if delivery == DeliveryMode::Background && crate::input::is_chromium_target_window(hwnd) { + return crate::input::delivery::background_unavailable_error( + hwnd, + EventKind::MouseScroll, + ); + } if let Some(idx) = elem_idx { + let prev_fg_addr = if delivery == DeliveryMode::Background { + Some(unsafe { + windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as isize + }) + } else { + None + }; + let _noact = if delivery == DeliveryMode::Background { + Some(crate::input::NoActivateGuard::arm( + windows::Win32::Foundation::HWND(hwnd as *mut _), + )) + } else { + None + }; let state = self.state.clone(); let direction_for_uia = direction.clone(); let uia_result = tokio::task::spawn_blocking(move || { @@ -4674,6 +5032,32 @@ impl Tool for ScrollTool { }) .await; if matches!(uia_result, Ok(Ok(()))) { + if delivery == DeliveryMode::Background { + // Keep WS_EX_NOACTIVATE armed through any WebView handler + // queued by the UIA operation. + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + if let Some(previous_addr) = prev_fg_addr { + use windows::Win32::Foundation::HWND; + use windows::Win32::UI::WindowsAndMessaging::{ + GetAncestor, GetForegroundWindow, GA_ROOT, + }; + let previous = HWND(previous_addr as *mut _); + let current = unsafe { GetForegroundWindow() }; + let current_root = unsafe { GetAncestor(current, GA_ROOT) }; + let target_root = unsafe { GetAncestor(HWND(hwnd as *mut _), GA_ROOT) }; + if !previous.0.is_null() + && current != previous + && !target_root.0.is_null() + && current_root == target_root + { + unsafe { + crate::input::force_foreground_attached(previous); + std::thread::sleep(std::time::Duration::from_millis(12)); + crate::input::force_foreground_attached(previous); + } + } + } + } return ToolResult::text(format!( "Scrolled {direction} {amount} ticks via UIA (delivery_mode:background)." )) @@ -4685,6 +5069,16 @@ impl Tool for ScrollTool { } } + if delivery == DeliveryMode::Background + && args.get("x").is_some_and(serde_json::Value::is_number) + && args.get("y").is_some_and(serde_json::Value::is_number) + { + return crate::input::delivery::background_unavailable_error( + hwnd, + EventKind::MouseScroll, + ); + } + // delivery_mode:"background" — WM_VSCROLL/HSCROLL is silently dropped by // Chromium and may be by GTK. Surface the standard structured // background_unavailable error: its remediation (bring_to_front + @@ -4719,16 +5113,27 @@ impl Tool for ScrollTool { }; let per: i32 = if by == "page" { 3 } else { 1 }; let ticks = sign * (amount as i32) * per; - // Target the window's screen center so the wheel lands on it. - let center = tokio::task::spawn_blocking(move || { - crate::win32::list_windows(Some(pid)) - .into_iter() - .find(|w| w.hwnd == hwnd) - .map(|w| (w.x + w.width / 2, w.y + w.height / 2)) - }) - .await - .ok() - .flatten(); + // A supplied PX target is window-local in the get_window_state + // bitmap. Route the wheel there so nested web scrollers receive it; + // otherwise retain the whole-window center fallback. + let px = args.get("x").and_then(|value| value.as_f64()); + let py = args.get("y").and_then(|value| value.as_f64()); + if px.is_some() != py.is_some() { + return ToolResult::error("scroll requires x and y together."); + } + let center = if let (Some(x), Some(y)) = (px, py) { + Some(bitmap_to_screen(hwnd, x as i32, y as i32)) + } else { + tokio::task::spawn_blocking(move || { + crate::win32::list_windows(Some(pid)) + .into_iter() + .find(|w| w.hwnd == hwnd) + .map(|w| (w.x + w.width / 2, w.y + w.height / 2)) + }) + .await + .ok() + .flatten() + }; let (cx, cy) = match center { Some(c) => c, None => { @@ -5791,7 +6196,7 @@ impl Tool for DragTool { // no cursor move; the target is held non-activatable + cloaked for the // stroke (mirrors the click pen path). if delivery == DeliveryMode::Background - && crate::input::delivery::would_be_silently_dropped(hwnd, EventKind::MouseClick) + && crate::input::delivery::would_be_silently_dropped(hwnd, EventKind::MouseMove) { let target = hwnd; let btn = button.clone(); @@ -5832,7 +6237,11 @@ impl Tool for DragTool { (delivery_mode:background, PostMessage would have been dropped)." )) } - Ok(Err(e)) => ToolResult::error(e.to_string()), + Ok(Err(e)) => crate::input::delivery::background_unavailable_error_with_cause( + hwnd, + EventKind::MouseMove, + e.to_string(), + ), Err(e) => ToolResult::error(format!("Task error: {e}")), }; } @@ -7436,12 +7845,15 @@ impl Tool for BringToFrontTool { // trick mirrors `send_key_synthesized` (input/keyboard.rs:313-345) // and is validated by `flash-repro/16-edge-launch-fg.ps1` for the // Edge launch focus-steal recovery case. - let outcome = tokio::task::spawn_blocking(move || -> Result<(u64, u64, bool), String> { + let outcome = + tokio::task::spawn_blocking(move || -> Result<(u64, u64, bool, bool), String> { use windows::Win32::Foundation::HWND; + use windows::Win32::Graphics::Dwm::DwmFlush; use windows::Win32::System::Threading::{AttachThreadInput, GetCurrentThreadId}; use windows::Win32::UI::WindowsAndMessaging::{ - GetForegroundWindow, GetWindowThreadProcessId, IsWindow, SetForegroundWindow, - SetWindowPos, HWND_NOTOPMOST, HWND_TOPMOST, SWP_NOACTIVATE, SWP_NOMOVE, SWP_NOSIZE, + GetForegroundWindow, GetWindowThreadProcessId, IsIconic, IsWindow, + SetForegroundWindow, SetWindowPos, ShowWindowAsync, HWND_NOTOPMOST, HWND_TOPMOST, + SWP_NOACTIVATE, SWP_NOMOVE, SWP_NOSIZE, SW_RESTORE, }; let target = HWND(hwnd as *mut _); @@ -7452,6 +7864,25 @@ impl Tool for BringToFrontTool { let prev_fg = unsafe { GetForegroundWindow() }; let prev_fg_addr = prev_fg.0 as u64; + // Iconic windows have no rendered pixels and live at the sentinel + // (-32000, -32000) position. Restore before changing z-order so + // bring_to_front is also the advertised recovery path for capture. + let was_minimized = unsafe { IsIconic(target) }.as_bool(); + if was_minimized { + let _ = unsafe { ShowWindowAsync(target, SW_RESTORE) }; + for _ in 0..20 { + if !unsafe { IsIconic(target) }.as_bool() { + break; + } + std::thread::sleep(std::time::Duration::from_millis(25)); + } + if unsafe { IsIconic(target) }.as_bool() { + return Err(format!( + "restore request did not complete for minimized hwnd 0x{hwnd:x}" + )); + } + } + // Lock-free z-order raise FIRST: bring the window to the top of the // normal band (the HWND_TOPMOST→HWND_NOTOPMOST force-to-front trick) // so it's brought to the VISIBLE front even when the foreground-lock @@ -7499,12 +7930,20 @@ impl Tool for BringToFrontTool { let _ = unsafe { AttachThreadInput(my_tid, fg_tid, false) }; } let now_fg = unsafe { GetForegroundWindow() }; - Ok((prev_fg_addr, now_fg.0 as u64, raised)) + + // A restored HWND can stop reporting iconic before its compositor + // surface is painted. Flush DWM before returning, but do not make + // the restore operation depend on any particular capture backend. + // Capture has its own WGC fallback for freshly restored surfaces. + if was_minimized { + let _ = unsafe { DwmFlush() }; + } + Ok((prev_fg_addr, now_fg.0 as u64, raised, was_minimized)) }) .await; match outcome { - Ok(Ok((prev, now, raised))) => { + Ok(Ok((prev, now, raised, restored))) => { let focused = now == hwnd; let msg = if focused { format!("✅ bring_to_front: pid {pid} hwnd 0x{hwnd:x} is now foreground (was 0x{prev:x}).") @@ -7532,6 +7971,7 @@ impl Tool for BringToFrontTool { "target_hwnd": format!("0x{hwnd:x}"), "landed_on_target": focused, "raised": raised, + "restored": restored, })) } Ok(Err(e)) => ToolResult::error(format!("bring_to_front: {e}")), diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs index e5e7fb8504..e95278b7b0 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs @@ -262,6 +262,25 @@ impl ElementCache { result.map_err(|e| anyhow::anyhow!("UIA SetFocus failed: {e}")) } + pub fn element_has_keyboard_focus( + &self, + pid: u32, + hwnd: u64, + element_index: usize, + ) -> Option { + let retained = self.get_element_retained(pid, hwnd, element_index)?; + if !retained.is_uia() { + return None; + } + let element: IUIAutomationElement = + unsafe { IUIAutomationElement::from_raw(retained.as_ptr() as *mut _) }; + let focused = unsafe { element.CurrentHasKeyboardFocus() } + .ok() + .map(|value| value.as_bool()); + std::mem::forget(element); + focused + } + /// Cached screen rect for the element. Used by the click tool to /// compute the right-edge dispatch point for `action:"expand"` on /// MSAA BUTTONDROPDOWN. diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs index 5973f34789..b18c2317f7 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs @@ -671,6 +671,8 @@ unsafe fn window_info_from_uia_element(elem: &IUIAutomationElement) -> Option Result<(Vec, u32, u32)> { if IsIconic(hwnd).as_bool() { bail!( "WGC cannot capture a minimized window (no rendered content). \ - Restore the window first — `get_window_state` still returns the \ + Call bring_to_front with this window_id to restore it first. \ + `get_window_state` still returns the \ UIA tree for a minimized window (the screenshot is reported \ unavailable)." ); diff --git a/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs b/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs index 17b31d8e9d..8db8e571b5 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs @@ -14,9 +14,10 @@ //! `FindAll(TreeScope::Children, ...)` makes no z-order guarantee, so we //! deliberately do NOT let it reorder anything Win32 already reported. //! -//! Both sources apply the same filters (visible, non-iconic, non-empty -//! title). The `filter_pid` argument is applied to the merged list so the -//! union/dedupe pipeline runs unconditionally. +//! Both sources apply the same filters (visible, non-empty title). Minimized +//! windows remain addressable and are reported as off-screen so callers can +//! restore them explicitly. The `filter_pid` argument is applied to the merged +//! list so the union/dedupe pipeline runs unconditionally. use std::collections::HashSet; use std::sync::Mutex; @@ -40,6 +41,8 @@ pub struct WindowInfo { pub y: i32, pub width: i32, pub height: i32, + pub is_on_screen: bool, + pub minimized: bool, } struct EnumState { @@ -82,7 +85,7 @@ pub fn list_windows(filter_pid: Option) -> Vec { merged } -/// Walk `EnumWindows` and collect every visible, non-iconic, non-empty-titled +/// Walk `EnumWindows` and collect every visible, non-empty-titled /// top-level window. No pid filter is applied here — the caller does that on /// the merged list. fn enumerate_via_enum_windows() -> Vec { @@ -97,10 +100,12 @@ fn enumerate_via_enum_windows() -> Vec { unsafe extern "system" fn enum_windows_cb(hwnd: HWND, lparam: LPARAM) -> BOOL { let state = &*(lparam.0 as *const Mutex); - // Skip invisible or minimized windows. - if IsWindowVisible(hwnd).0 == 0 || IsIconic(hwnd).0 != 0 { + // Invisible helper windows are not user-addressable. Iconic windows are: + // retain them with explicit state so callers can restore them. + if IsWindowVisible(hwnd).0 == 0 { return TRUE; } + let minimized = IsIconic(hwnd).0 != 0; // Get pid. let mut pid: u32 = 0; @@ -128,6 +133,8 @@ unsafe extern "system" fn enum_windows_cb(hwnd: HWND, lparam: LPARAM) -> BOOL { y, width: w, height: h, + is_on_screen: !minimized, + minimized, }); TRUE @@ -301,5 +308,7 @@ pub fn resolve_uwp_host_window(app_pid: u32) -> Option { y, width: w, height: h, + is_on_screen: true, + minimized: false, }) } diff --git a/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/build.ps1 b/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/build.ps1 index 3f0c4e4d2c..62d9c4e0ae 100644 --- a/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/build.ps1 +++ b/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/build.ps1 @@ -48,6 +48,7 @@ try { $appDir = Join-Path $outDir "resources\app" if (-not (Test-Path $appDir)) { New-Item -ItemType Directory $appDir -Force | Out-Null } Copy-Item (Join-Path $elecDir "main.js") $appDir -Force + Copy-Item (Join-Path $elecDir "preload.js") $appDir -Force Copy-Item (Join-Path $elecDir "package.json") $appDir -Force Copy-Item $webDir (Join-Path $appDir "web") -Recurse -Force diff --git a/libs/cua-driver/tests/fixtures/apps/windows/webview2/MainWindow.xaml b/libs/cua-driver/tests/fixtures/apps/windows/webview2/MainWindow.xaml index 27025fbfb3..12b4acb0c2 100644 --- a/libs/cua-driver/tests/fixtures/apps/windows/webview2/MainWindow.xaml +++ b/libs/cua-driver/tests/fixtures/apps/windows/webview2/MainWindow.xaml @@ -2,7 +2,7 @@ xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:wv2="clr-namespace:Microsoft.Web.WebView2.Wpf;assembly=Microsoft.Web.WebView2.Wpf" - Title="CuaTestHarness WebView" + Title="Loading WebView2 fixture" AutomationProperties.AutomationId="wnd-main" Width="940" Height="780" WindowStartupLocation="CenterScreen"> diff --git a/libs/cua-driver/tests/fixtures/apps/windows/webview2/MainWindow.xaml.cs b/libs/cua-driver/tests/fixtures/apps/windows/webview2/MainWindow.xaml.cs index 22a8049495..65c6729126 100644 --- a/libs/cua-driver/tests/fixtures/apps/windows/webview2/MainWindow.xaml.cs +++ b/libs/cua-driver/tests/fixtures/apps/windows/webview2/MainWindow.xaml.cs @@ -1,5 +1,7 @@ using System; using System.IO; +using System.Text.Json; +using System.Threading.Tasks; using System.Windows; using Microsoft.Web.WebView2.Core; @@ -17,9 +19,6 @@ private async void OnLoaded(object sender, RoutedEventArgs e) { try { - var userData = Path.Combine(Path.GetTempPath(), "CuaTestHarness.WebView.UserData"); - Directory.CreateDirectory(userData); - // Read the CDP port from CUA_WEBVIEW_CDP_PORT (default 9222). // cua-driver's `page` tool routes JS execution through CDP when // `--remote-debugging-port` is exposed; this is the analogue of @@ -37,6 +36,14 @@ private async void OnLoaded(object sender, RoutedEventArgs e) throw new InvalidOperationException( $"Invalid CUA_WEBVIEW_CDP_PORT: '{portStr}'. Expected an integer in 1-65535."); } + // WebView2 requires every process sharing a user-data directory to + // use identical environment options. Each fixture gets a different + // CDP port, so isolate its browser environment by process and port. + var userData = Path.Combine( + Path.GetTempPath(), + "CuaTestHarness.WebView.UserData", + $"{Environment.ProcessId}-{cdpPort}"); + Directory.CreateDirectory(userData); var opts = new CoreWebView2EnvironmentOptions { AdditionalBrowserArguments = $"--remote-debugging-port={cdpPort}", @@ -44,6 +51,17 @@ private async void OnLoaded(object sender, RoutedEventArgs e) var env = await CoreWebView2Environment.CreateAsync(userDataFolder: userData, options: opts); await Wv.EnsureCoreWebView2Async(env); + // The Rust E2E harness owns the loopback receiver. Publish DOM state + // through the shared fixture script so click delivery is judged + // independently of cua-driver's UIA or CDP read-back channels. + var journalUrl = Environment.GetEnvironmentVariable("CUA_E2E_FIXTURE_JOURNAL_URL"); + if (!string.IsNullOrWhiteSpace(journalUrl)) + { + var encodedJournalUrl = JsonSerializer.Serialize(journalUrl); + await Wv.CoreWebView2.AddScriptToExecuteOnDocumentCreatedAsync( + $"window.__CUA_E2E_FIXTURE_JOURNAL_URL = {encodedJournalUrl};"); + } + var htmlPath = Path.Combine(AppContext.BaseDirectory, "web", "index.html"); if (!File.Exists(htmlPath)) { @@ -54,14 +72,30 @@ private async void OnLoaded(object sender, RoutedEventArgs e) htmlPath); } var fileUri = new Uri(htmlPath).AbsoluteUri; + var navigation = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + void OnNavigationCompleted( + object? navigationSender, + CoreWebView2NavigationCompletedEventArgs navigationArgs) + { + Wv.NavigationCompleted -= OnNavigationCompleted; + navigation.TrySetResult(navigationArgs); + } + Wv.NavigationCompleted += OnNavigationCompleted; Wv.Source = new Uri(fileUri); + var navigationResult = await navigation.Task; + if (!navigationResult.IsSuccess) + { + throw new InvalidOperationException( + $"Web fixture navigation failed: {navigationResult.WebErrorStatus}"); + } LblPageUrl.Text = fileUri; - Title = $"CuaTestHarness WebView [cdp={cdpPort}]"; + Title = $"CuaTestHarness WebView [ready cdp={cdpPort}]"; } catch (Exception ex) { - MessageBox.Show($"WebView2 init failed: {ex.Message}", "harness", MessageBoxButton.OK, MessageBoxImage.Error); - throw; + Console.Error.WriteLine($"WebView2 init failed: {ex}"); + Environment.Exit(1); } } diff --git a/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml b/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml index 10e65205ba..c500005cac 100644 --- a/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml +++ b/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml @@ -66,7 +66,8 @@ AutomationProperties.AutomationId="border-click-target" Content="Click target (left / right / double)" Width="260" Height="40" HorizontalAlignment="Left" - MouseLeftButtonDown="OnTargetLeftDown" + PreviewMouseLeftButtonDown="OnTargetLeftDown" + Click="OnTargetClick" MouseRightButtonDown="OnTargetRightDown" MouseDoubleClick="OnTargetDoubleClick"/> = 2) { @@ -155,9 +165,29 @@ private void OnTargetLeftDown(object sender, MouseButtonEventArgs e) LblLastAction.Text = "last_action=left_click"; } LblClickCount.Text = $"clicks={_clickCount}"; + PublishFixtureState(); // Don't mark handled — let the Button's own logic still run. } + private void OnTargetClick(object sender, RoutedEventArgs e) + { + // A real pointer click already passed through PreviewMouseLeftButtonDown. + // UIA Invoke raises Click directly, so count that path here. This gives + // the PX-background row observable fixture state for both its foreground + // geometry probe and its occluded delivery action. + if (_targetPointerSeen) + { + _targetPointerSeen = false; + } + else + { + _clickCount++; + LblLastAction.Text = "last_action=left_click"; + LblClickCount.Text = $"clicks={_clickCount}"; + } + PublishFixtureState(); + } + private void OnTargetDoubleClick(object sender, MouseButtonEventArgs e) { // Belt + suspenders: Button raises MouseDoubleClick separately from @@ -165,11 +195,36 @@ private void OnTargetDoubleClick(object sender, MouseButtonEventArgs e) // back-end implementations that fire only one path still register. LblLastAction.Text = "last_action=double_click"; LblClickCount.Text = $"clicks={_clickCount}"; + PublishFixtureState(); } private void OnTargetRightDown(object sender, MouseButtonEventArgs e) { LblLastAction.Text = "last_action=right_click"; + PublishFixtureState(); + } + + private void PublishFixtureState() + { + if (string.IsNullOrWhiteSpace(_fixtureStatePath)) return; + + var state = new Dictionary + { + ["page-marker"] = new { text = "WPF_HARNESS_MARKER_v1" }, + ["lbl-counter"] = new { text = LblCounter?.Text ?? "counter=0" }, + ["lbl-last-action"] = new { text = LblLastAction?.Text ?? "last_action=none" }, + ["lbl-click-count"] = new { text = LblClickCount?.Text ?? "clicks=0" }, + }; + try + { + var temporaryPath = $"{_fixtureStatePath}.{Environment.ProcessId}.tmp"; + File.WriteAllText(temporaryPath, JsonSerializer.Serialize(state)); + File.Move(temporaryPath, _fixtureStatePath, true); + } + catch (Exception ex) + { + Console.Error.WriteLine($"WPF fixture state publish failed: {ex.Message}"); + } } private void OnScrollChanged(object sender, ScrollChangedEventArgs e) diff --git a/libs/cua-driver/tests/fixtures/build/windows.ps1 b/libs/cua-driver/tests/fixtures/build/windows.ps1 index 7e3b150522..7e0f98b194 100644 --- a/libs/cua-driver/tests/fixtures/build/windows.ps1 +++ b/libs/cua-driver/tests/fixtures/build/windows.ps1 @@ -12,7 +12,9 @@ param( [ValidateSet("none","wpf","winui3","webview","electron","tauri")] - [string]$Skip = "none" + [string]$Skip = "none", + [ValidateSet("wpf","winui3","webview","electron","tauri")] + [string[]]$Targets = @("wpf","winui3","webview","electron","tauri") ) Set-StrictMode -Version Latest @@ -31,6 +33,11 @@ if (-not (Get-Command dotnet -ErrorAction SilentlyContinue)) { New-Item -ItemType Directory -Force $testAppsDir | Out-Null +function Should-Build { + param([string]$Name) + return $Skip -ne $Name -and $Targets -contains $Name +} + function Publish-Project { param([string]$ProjPath, [string]$OutDirName) $outDir = Join-Path $testAppsDir $OutDirName @@ -54,10 +61,10 @@ function Publish-Project { Write-Host "[OK] Published: $outDir" -ForegroundColor Green } -if ($Skip -ne "wpf") { +if (Should-Build "wpf") { Publish-Project (Join-Path $harnessDir "apps\windows\wpf\CuaTestHarness.Wpf.csproj") "harness-wpf" } -if ($Skip -ne "winui3") { +if (Should-Build "winui3") { $winuiProj = Join-Path $harnessDir "apps\windows\winui3\CuaTestHarness.WinUI3.csproj" if (Test-Path $winuiProj) { Publish-Project $winuiProj "harness-winui3" @@ -65,7 +72,7 @@ if ($Skip -ne "winui3") { Write-Host "[SKIP] WinUI3 project not present yet - skipping." -ForegroundColor Yellow } } -if ($Skip -ne "webview") { +if (Should-Build "webview") { $webProj = Join-Path $harnessDir "apps\windows\webview2\CuaTestHarness.WebView.csproj" if (Test-Path $webProj) { Publish-Project $webProj "harness-webview" @@ -73,41 +80,26 @@ if ($Skip -ne "webview") { Write-Host "[SKIP] WebView project not present yet - skipping." -ForegroundColor Yellow } } -if ($Skip -ne "electron") { +if (Should-Build "electron") { $elecBuild = Join-Path $harnessDir "apps\cross-platform\electron\build.ps1" if (Test-Path $elecBuild) { Write-Host "" Write-Host "[BUILD] electron -> $testAppsDir\harness-electron\" -ForegroundColor Cyan - # Electron build sets $ErrorActionPreference=Stop internally and - # throws on npm install / publish failure. Wrap so a throw degrades - # to a warning rather than aborting the whole harness build. - try { - & $elecBuild - if ($LASTEXITCODE -ne 0) { - Write-Host "[WARN] Electron build failed (exit $LASTEXITCODE)" -ForegroundColor Yellow - } - } catch { - Write-Host "[WARN] Electron build errored: $($_.Exception.Message)" -ForegroundColor Yellow - } + & $elecBuild + if ($LASTEXITCODE -ne 0) { throw "Electron harness build failed" } } else { - Write-Host "[SKIP] Electron project not present yet - skipping." -ForegroundColor Yellow + throw "Electron build script not found: $elecBuild" } } -if ($Skip -ne "tauri") { +if (Should-Build "tauri") { $tauriBuild = Join-Path $harnessDir "apps\cross-platform\tauri\build.ps1" if (Test-Path $tauriBuild) { Write-Host "" Write-Host "[BUILD] tauri -> $testAppsDir\harness-tauri\" -ForegroundColor Cyan - try { - & $tauriBuild - if ($LASTEXITCODE -ne 0) { - Write-Host "[WARN] Tauri build failed (exit $LASTEXITCODE)" -ForegroundColor Yellow - } - } catch { - Write-Host "[WARN] Tauri build errored: $($_.Exception.Message)" -ForegroundColor Yellow - } + & $tauriBuild + if ($LASTEXITCODE -ne 0) { throw "Tauri harness build failed" } } else { - Write-Host "[SKIP] Tauri project not present yet - skipping." -ForegroundColor Yellow + throw "Tauri build script not found: $tauriBuild" } } diff --git a/libs/cua-driver/tests/runners/windows-sandbox/run-tests-in-sandbox.ps1 b/libs/cua-driver/tests/runners/windows-sandbox/run-tests-in-sandbox.ps1 index b4ac011867..0f52b7abca 100644 --- a/libs/cua-driver/tests/runners/windows-sandbox/run-tests-in-sandbox.ps1 +++ b/libs/cua-driver/tests/runners/windows-sandbox/run-tests-in-sandbox.ps1 @@ -70,10 +70,6 @@ try { cargo test -p cua-driver --test protocol_handshake_test --no-run if ($LASTEXITCODE -ne 0) { throw "cargo test --no-run (protocol_handshake_test) failed" } - Write-Host "`n[BUILD] cargo test --no-run (guard_ux_test)..." -ForegroundColor Yellow - cargo test -p cua-driver --test guard_ux_test --no-run - if ($LASTEXITCODE -ne 0) { throw "cargo test --no-run (guard_ux_test) failed" } - Write-Host "`n[BUILD] cargo test --no-run (harness_wpf_test)..." -ForegroundColor Yellow cargo test -p cua-driver --test harness_wpf_test --no-run if ($LASTEXITCODE -ne 0) { throw "cargo test --no-run (harness_wpf_test) failed" } @@ -86,9 +82,14 @@ try { cargo test -p cua-driver --test harness_web_test --no-run if ($LASTEXITCODE -ne 0) { throw "cargo test --no-run (harness_web_test) failed" } - Write-Host "`n[BUILD] cargo test --no-run (modality_input_e2e_test)..." -ForegroundColor Yellow - cargo test -p cua-driver --test modality_input_e2e_test --no-run - if ($LASTEXITCODE -ne 0) { throw "cargo test --no-run (modality_input_e2e_test) failed" } + Write-Host "`n[BUILD] cargo test --no-run (launch_windows_test)..." -ForegroundColor Yellow + cargo test -p cua-driver --test launch_windows_test --no-run + if ($LASTEXITCODE -ne 0) { throw "cargo test --no-run (launch_windows_test) failed" } + + Write-Host "`n[BUILD] cargo test --no-run (agent_cursor_windows_test)..." -ForegroundColor Yellow + cargo test -p cua-driver --test agent_cursor_windows_test --no-run + if ($LASTEXITCODE -ne 0) { throw "cargo test --no-run (agent_cursor_windows_test) failed" } + } finally { Pop-Location } # -- 1.5. Build the test fixtures if dependencies are on PATH ------------------ @@ -123,10 +124,6 @@ $protocolBin = Get-ChildItem "$rustRoot\target\debug\deps\protocol_handshake_tes if (-not $protocolBin) { throw "protocol_handshake_test-*.exe not found" } Write-Host "protocol_handshake_test: $($protocolBin.Name)" -$guardBin = Get-ChildItem "$rustRoot\target\debug\deps\guard_ux_test-*.exe" | - Sort-Object LastWriteTime -Descending | Select-Object -First 1 -if ($guardBin) { Write-Host "guard_ux_test : $($guardBin.Name)" } else { Write-Host "guard_ux_test : (not found, will skip)" } - # -- 2. Prepare shared output folder ------------------------------------------ $outputDir = "$env:TEMP\cua-sandbox-output" New-Item -ItemType Directory -Force $outputDir | Out-Null diff --git a/libs/cua-driver/tests/runners/windows-sandbox/sandbox-runner.ps1 b/libs/cua-driver/tests/runners/windows-sandbox/sandbox-runner.ps1 index 30dc7f2593..3e27c7e21d 100644 --- a/libs/cua-driver/tests/runners/windows-sandbox/sandbox-runner.ps1 +++ b/libs/cua-driver/tests/runners/windows-sandbox/sandbox-runner.ps1 @@ -32,15 +32,14 @@ if (-not (Test-Path $driverExe)) { Log "cua-driver : $driverExe" # -- find test binaries ------------------------------------------------------- -# Run protocol_handshake_test first, then guard_ux_test (UX guard needs a real -# desktop session and spawns visible windows, so it runs second). +# Run the protocol test first, followed by the typed interactive harnesses. $testSuites = @( @{ Pattern = "protocol_handshake_test-*.exe"; Label = "protocol_handshake_test" }, - @{ Pattern = "guard_ux_test-*.exe"; Label = "guard_ux_test" }, @{ Pattern = "harness_wpf_test-*.exe"; Label = "harness_wpf_test"; Extra = @("--ignored") }, @{ Pattern = "harness_winui3_test-*.exe"; Label = "harness_winui3_test"; Extra = @("--ignored") }, @{ Pattern = "harness_web_test-*.exe"; Label = "harness_web_test"; Extra = @("--ignored") }, - @{ Pattern = "modality_input_e2e_test-*.exe"; Label = "modality_input_e2e_test"; Extra = @("--ignored") } + @{ Pattern = "launch_windows_test-*.exe"; Label = "launch_windows_test"; Extra = @("--ignored") }, + @{ Pattern = "agent_cursor_windows_test-*.exe"; Label = "agent_cursor_windows_test"; Extra = @("--ignored") } ) # -- stage harness binaries to %TEMP% (same Zone-3 ShellExecute workaround) -- diff --git a/libs/cua-driver/tests/runners/windows/README.md b/libs/cua-driver/tests/runners/windows/README.md index ffb2e409c0..a64ba79cf0 100644 --- a/libs/cua-driver/tests/runners/windows/README.md +++ b/libs/cua-driver/tests/runners/windows/README.md @@ -9,7 +9,6 @@ Run from `libs/cua-driver` in an RDP or console session: .\tests\runners\windows\run-all.ps1 -RequireGui ``` -The runner builds repo-local Windows fixtures and runs the Rust default, -guard, harness, and modality suites. It intentionally skips optional -external-app suites such as LibreOffice because those require extra software -on the VM image. +The runner builds repo-local Windows fixtures and runs the Rust unit and typed +harness matrix. It intentionally skips optional external-app suites such as +LibreOffice because those require extra software on the VM image. diff --git a/libs/cua-driver/tests/runners/windows/run-all.ps1 b/libs/cua-driver/tests/runners/windows/run-all.ps1 index 4661ec5494..2df101f53a 100644 --- a/libs/cua-driver/tests/runners/windows/run-all.ps1 +++ b/libs/cua-driver/tests/runners/windows/run-all.ps1 @@ -12,95 +12,12 @@ param( Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" -$runnerDir = Split-Path -Parent $MyInvocation.MyCommand.Definition -$runnersDir = Split-Path -Parent $runnerDir -$testsDir = Split-Path -Parent $runnersDir -$cuaDriverRoot = Split-Path -Parent $testsDir -$rustRoot = Join-Path $cuaDriverRoot "rust" -$fixtureBuild = Join-Path $cuaDriverRoot "tests\fixtures\build\windows.ps1" +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..\..\..\..") +$canonicalRunner = Join-Path $repoRoot "scripts\ci\windows\run-rust-e2e.ps1" -Write-Host "=== cua-driver Windows Rust run-all ===" -ForegroundColor Cyan -Write-Host "cua-driver root: $cuaDriverRoot" -Write-Host "Rust workspace : $rustRoot" - -if ($RequireGui) { - $env:CUA_REQUIRE_GUI = "1" - Write-Host "CUA_REQUIRE_GUI=1" -ForegroundColor Yellow -} - -$results = New-Object System.Collections.Generic.List[object] - -function Add-Result { - param([string]$Name, [int]$ExitCode) - $status = if ($ExitCode -eq 0) { "PASS" } else { "FAIL" } - $results.Add([pscustomobject]@{ - Name = $Name - Status = $status - ExitCode = $ExitCode - }) | Out-Null -} - -function Run-Step { - param( - [string]$Name, - [string]$WorkingDirectory, - [string[]]$CommandArgs - ) - - Write-Host "`n[RUN] $Name" -ForegroundColor Yellow - Push-Location $WorkingDirectory - try { - & cargo @CommandArgs - $code = if ($null -eq $LASTEXITCODE) { 0 } else { $LASTEXITCODE } - } finally { - Pop-Location - } - Add-Result $Name $code - if ($code -ne 0) { - Write-Host "[FAIL] $Name exited $code" -ForegroundColor Red - } +if (-not (Test-Path $canonicalRunner)) { + throw "Canonical Windows E2E runner not found: $canonicalRunner" } -if (-not $NoBuild) { - if (-not (Test-Path $fixtureBuild)) { - throw "Windows fixture build script not found: $fixtureBuild" - } - Write-Host "`n[BUILD] Windows fixtures" -ForegroundColor Yellow - & $fixtureBuild - if ($LASTEXITCODE -ne 0) { - Add-Result "windows fixtures" $LASTEXITCODE - throw "Windows fixture build failed with exit $LASTEXITCODE" - } - Add-Result "windows fixtures" 0 -} - -Run-Step "default Rust tests" $rustRoot @( - "test", "-p", "cua-driver", "-p", "platform-windows", "--", "--nocapture" -) -Run-Step "guard UX" $rustRoot @( - "test", "-p", "cua-driver", "--test", "guard_ux_test", "--", "--nocapture", "--test-threads=1" -) -Run-Step "WPF harness" $rustRoot @( - "test", "-p", "cua-driver", "--test", "harness_wpf_test", "--", "--ignored", "--nocapture", "--test-threads=1" -) -Run-Step "WinUI3 harness" $rustRoot @( - "test", "-p", "cua-driver", "--test", "harness_winui3_test", "--", "--ignored", "--nocapture", "--test-threads=1" -) -Run-Step "WebView2/Electron harness" $rustRoot @( - "test", "-p", "cua-driver", "--test", "harness_web_test", "--", "--ignored", "--nocapture", "--test-threads=1" -) -Run-Step "Windows modality input e2e" $rustRoot @( - "test", "-p", "cua-driver", "--test", "modality_input_e2e_test", "--", "--ignored", "--nocapture", "--test-threads=1" -) - -Write-Host "`n=== Summary ===" -ForegroundColor Cyan -$failed = 0 -foreach ($r in $results) { - $color = if ($r.Status -eq "PASS") { "Green" } else { "Red" } - Write-Host ("{0,-32} {1} ({2})" -f $r.Name, $r.Status, $r.ExitCode) -ForegroundColor $color - if ($r.ExitCode -ne 0) { $failed++ } -} - -if ($failed -gt 0) { - exit 1 -} +& $canonicalRunner -NoBuild:$NoBuild -RequireGui:$RequireGui +exit $LASTEXITCODE diff --git a/nix/cua-driver/package.nix b/nix/cua-driver/package.nix index 4a59f798ac..6bc154bb26 100644 --- a/nix/cua-driver/package.nix +++ b/nix/cua-driver/package.nix @@ -35,7 +35,7 @@ pkgs.rustPlatform.buildRustPackage { cargoLock.lockFile = "${src}/Cargo.lock"; # Build only the main binary crate. The workspace also contains - # platform-macos, platform-windows, cua-driver-uia, and focus-monitor-win + # platform-macos, platform-windows, and cua-driver-uia # which are gated behind cfg(target_os) and won't compile on Linux. # Using -p cua-driver ensures Cargo only resolves Linux dependencies. # diff --git a/scripts/ci/windows/build-harnesses.ps1 b/scripts/ci/windows/build-harnesses.ps1 index 0db8d1eb03..dcdbafef0e 100644 --- a/scripts/ci/windows/build-harnesses.ps1 +++ b/scripts/ci/windows/build-harnesses.ps1 @@ -1,7 +1,9 @@ # Build all repo-local Windows harness apps from source. param( [ValidateSet("none", "wpf", "winui3", "webview", "electron", "tauri")] - [string]$Skip = "none" + [string]$Skip = "none", + [ValidateSet("wpf", "winui3", "webview", "electron", "tauri")] + [string[]]$Targets = @("wpf", "winui3", "webview", "electron", "tauri") ) Set-StrictMode -Version Latest @@ -10,7 +12,7 @@ $scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Definition $repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..\..")).Path $fixtureBuild = Join-Path $repoRoot "libs\cua-driver\tests\fixtures\build\windows.ps1" -& $fixtureBuild -Skip $Skip +& $fixtureBuild -Skip $Skip -Targets $Targets if ($LASTEXITCODE -ne 0) { throw "Windows harness build failed with exit code $LASTEXITCODE" } diff --git a/scripts/ci/windows/run-rust-e2e.ps1 b/scripts/ci/windows/run-rust-e2e.ps1 index 11e8114d5b..e67533d3c8 100644 --- a/scripts/ci/windows/run-rust-e2e.ps1 +++ b/scripts/ci/windows/run-rust-e2e.ps1 @@ -2,8 +2,6 @@ # Scenario definitions and assertions stay in the Rust integration test. param( [switch]$NoBuild, - [ValidateSet("default", "guard", "shared", "native", "modality", "all")] - [string]$Suite = "shared", [switch]$RequireGui ) @@ -14,22 +12,27 @@ $scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Definition $repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..\..")).Path $driverRoot = Join-Path $repoRoot "libs\cua-driver" $rustRoot = Join-Path $driverRoot "rust" +$suite = if ([string]::IsNullOrWhiteSpace($env:CUA_E2E_INTERNAL_LANE)) { "all" } else { $env:CUA_E2E_INTERNAL_LANE } +if ($suite -notin @("shared", "native", "capture", "all")) { + throw "Unsupported internal lane: $suite" +} $artifactDir = Join-Path $repoRoot "artifacts\cua-driver\windows" New-Item -ItemType Directory -Force $artifactDir | Out-Null $recordingRoot = Join-Path $artifactDir "recordings" Remove-Item -Path $recordingRoot -Recurse -Force -ErrorAction SilentlyContinue New-Item -ItemType Directory -Force $recordingRoot | Out-Null $resultsPath = Join-Path $artifactDir "results.jsonl" +$casesPath = Join-Path $artifactDir "cases.jsonl" +$environmentPath = Join-Path $artifactDir "environment.jsonl" $summaryPath = Join-Path $artifactDir "summary.md" -@( - "# CUA Rust Windows E2E matrix", - "", - "| Platform | Host/lane | Scenario | Status | Duration | Details |", - "| --- | --- | --- | --- | --- | --- |" -) | Set-Content -Path $summaryPath -Remove-Item -Force -ErrorAction SilentlyContinue $resultsPath +foreach ($path in @($casesPath, $environmentPath, $resultsPath)) { + New-Item -ItemType File -Force $path | Out-Null + Clear-Content $path +} +Remove-Item -Force -ErrorAction SilentlyContinue $summaryPath +$env:CUA_E2E_DECLARATIONS_FILE = $casesPath +$env:CUA_E2E_ENVIRONMENT_FILE = $environmentPath $env:CUA_E2E_RESULTS_FILE = $resultsPath -$env:CUA_E2E_SUMMARY_FILE = $summaryPath $env:CUA_E2E_RECORDINGS_ROOT = $recordingRoot $ffmpeg = Get-Command ffmpeg.exe -ErrorAction SilentlyContinue @@ -50,24 +53,69 @@ $env:CUA_TEST_DRIVER_STDERR = "1" if (-not $NoBuild) { & cargo build --release -p cua-driver --manifest-path (Join-Path $rustRoot "Cargo.toml") if ($LASTEXITCODE -ne 0) { throw "Rust driver build failed" } - & (Join-Path $scriptDir "build-harnesses.ps1") -} - -if ($Suite -in @("default", "guard", "modality", "all")) { - & cargo build -p focus-monitor-win --manifest-path (Join-Path $rustRoot "Cargo.toml") - if ($LASTEXITCODE -ne 0) { throw "Focus monitor build failed" } + $fixtureTargets = switch ($suite) { + "shared" { @("electron", "tauri") } + "native" { @("wpf", "winui3", "webview", "electron") } + "capture" { @("wpf", "electron") } + default { @("wpf", "winui3", "webview", "electron", "tauri") } + } + & (Join-Path $scriptDir "build-harnesses.ps1") -Targets $fixtureTargets } if (-not (Test-Path $env:CUA_TEST_DRIVER_BIN)) { throw "Driver binary not found: $($env:CUA_TEST_DRIVER_BIN)" } -foreach ($fixture in @( - (Join-Path $env:CUA_TEST_APPS_ROOT "harness-electron\CuaTestHarness.Electron.exe"), - (Join-Path $env:CUA_TEST_APPS_ROOT "harness-tauri\CuaTestHarness.Tauri.exe") -)) { +$requiredFixtures = @() +$requiredFixtures += Join-Path $env:CUA_TEST_APPS_ROOT "harness-electron\CuaTestHarness.Electron.exe" +if ($suite -in @("shared", "all")) { + $requiredFixtures += Join-Path $env:CUA_TEST_APPS_ROOT "harness-tauri\CuaTestHarness.Tauri.exe" +} +if ($suite -in @("native", "capture", "all")) { + $requiredFixtures += Join-Path $env:CUA_TEST_APPS_ROOT "harness-wpf\CuaTestHarness.Wpf.exe" +} +if ($suite -in @("native", "all")) { + $requiredFixtures += @( + (Join-Path $env:CUA_TEST_APPS_ROOT "harness-winui3\CuaTestHarness.WinUI3.exe"), + (Join-Path $env:CUA_TEST_APPS_ROOT "harness-webview\CuaTestHarness.WebView.exe") + ) +} +foreach ($fixture in $requiredFixtures) { if (-not (Test-Path $fixture)) { throw "Required fixture was not built: $fixture" } } +function Invoke-E2eReport { + Push-Location $rustRoot + try { + & cargo run -p cua-driver-testkit --bin cua-e2e-report -- ` + --declarations $casesPath ` + --environment $environmentPath ` + --results $resultsPath ` + --artifact-root $artifactDir ` + --require-video ` + --output $summaryPath | Out-Host + $exitCode = $LASTEXITCODE + return $exitCode + } finally { + Pop-Location + } +} + +Write-Host "[PREFLIGHT] Windows desktop, fixture, UIA, capture, and video" -ForegroundColor Yellow +Push-Location $rustRoot +try { + $preflightLog = Join-Path $artifactDir "environment-preflight.log" + $preflightOutput = & cargo test -p cua-driver --test e2e_environment_preflight_test -- ` + --ignored --exact canonical_e2e_environment_is_ready --nocapture --test-threads=1 2>&1 + $preflightExit = $LASTEXITCODE + $preflightOutput | Tee-Object -FilePath $preflightLog +} finally { + Pop-Location +} +if ($preflightExit -ne 0) { + Invoke-E2eReport | Out-Null + throw "Windows E2E environment preflight failed" +} + function Invoke-CargoTest { param([string]$Name, [string[]]$Arguments) Write-Host "[RUN] $Name" -ForegroundColor Yellow @@ -77,44 +125,6 @@ function Invoke-CargoTest { $output = & cargo @Arguments 2>&1 $exitCode = $LASTEXITCODE $output | Tee-Object -FilePath $logPath - foreach ($line in $output) { - $match = [regex]::Match( - [string]$line, - '^\s*test\s+(?\S+)\s+\.\.\.\s+(?ok|FAILED|ignored)\s*$' - ) - if (-not $match.Success) { continue } - - $testStatus = switch ($match.Groups["status"].Value) { - "ok" { "PASS" } - "FAILED" { "FAIL" } - default { "SKIP" } - } - $testName = $match.Groups["name"].Value - $testMessage = if ($testStatus -eq "FAIL") { "test case failed; see lane log" } else { "" } - $testRecord = [ordered]@{ - schema = "cua-e2e-result/v1" - platform = "windows" - host = "cargo" - scenario = $testName - status = $testStatus - message = $testMessage - } | ConvertTo-Json -Compress - Add-Content -Path $resultsPath -Value $testRecord - $testDetails = if ($testMessage) { $testMessage } else { "-" } - Add-Content -Path $summaryPath -Value "| Windows | cargo | $testName | $testStatus | n/a | $testDetails |" - } - $status = if ($exitCode -eq 0) { "PASS" } else { "FAIL" } - $record = [ordered]@{ - schema = "cua-e2e-result/v1" - platform = "windows" - host = "lane" - scenario = $Name - status = $status - message = if ($exitCode -eq 0) { "" } else { "exit code $exitCode" } - } | ConvertTo-Json -Compress - Add-Content -Path $resultsPath -Value $record - $details = if ($exitCode -eq 0) { "-" } else { "exit code $exitCode" } - Add-Content -Path $summaryPath -Value "| Windows | lane | $Name | $status | n/a | $details |" if ($exitCode -ne 0) { $script:FailureCount++ } @@ -146,33 +156,38 @@ function Test-E2eRecordings { Write-Host "[VIDEO PASS] $($video.FullName)" -ForegroundColor Green } } + + $ownedVideos = @{} + Get-Content $resultsPath | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | ForEach-Object { + $result = $_ | ConvertFrom-Json + if ($null -ne $result.evidence.video) { + $ownedVideos[$result.evidence.video.Replace("\", "/")] = $true + } + } + foreach ($video in $videos) { + $relative = [System.IO.Path]::GetRelativePath($artifactDir, $video.FullName).Replace("\", "/") + if ($relative -like "recordings/environment-preflight-*/recording.mp4") { + continue + } + if (-not $ownedVideos.ContainsKey($relative)) { + Write-Host "[VIDEO FAIL] Orphan trajectory has no typed result row: $relative" -ForegroundColor Red + $failureCount++ + } + } return $failureCount } $script:FailureCount = 0 -if ($Suite -in @("shared", "all")) { +if ($suite -in @("shared", "all")) { Invoke-CargoTest "shared behavior matrix" @( "test", "-p", "cua-driver", "--test", "cross_platform_behavior_test", "--", - "--ignored", "--nocapture", "--test-threads=1" - ) -} - -if ($Suite -in @("default", "all")) { - Invoke-CargoTest "default Rust tests" @( - "test", "-p", "cua-driver", "-p", "platform-windows", "--", - "--nocapture", "--test-threads=1" - ) -} - -if ($Suite -in @("guard", "all")) { - Invoke-CargoTest "guard UX" @( - "test", "-p", "cua-driver", "--test", "guard_ux_test", "--", + "--ignored", "--exact", "shared_web_action_matrix_is_state_verified", "--nocapture", "--test-threads=1" ) } -if ($Suite -in @("native", "all")) { +if ($suite -in @("native", "all")) { Invoke-CargoTest "Windows native harnesses" @( "test", "-p", "cua-driver", "--test", "harness_wpf_test", "--", "--ignored", "--nocapture", "--test-threads=1" @@ -185,19 +200,37 @@ if ($Suite -in @("native", "all")) { "test", "-p", "cua-driver", "--test", "harness_web_test", "--", "--ignored", "--nocapture", "--test-threads=1" ) + Invoke-CargoTest "Windows minimized launch" @( + "test", "-p", "cua-driver", "--test", "launch_windows_test", "--", + "--ignored", "--nocapture", "--test-threads=1" + ) + Invoke-CargoTest "Windows agent cursor" @( + "test", "-p", "cua-driver", "--test", "agent_cursor_windows_test", "--", + "--ignored", "--nocapture", "--test-threads=1" + ) } -if ($Suite -in @("modality", "all")) { - Invoke-CargoTest "Windows modality input e2e" @( - "test", "-p", "cua-driver", "--test", "modality_input_e2e_test", "--", +if ($suite -in @("capture", "all")) { + Invoke-CargoTest "capture contract" @( + "test", "-p", "cua-driver", "--test", "capture_contract_test", "--", + "--ignored", "--nocapture", "--test-threads=1" + ) + Invoke-CargoTest "Windows desktop scope" @( + "test", "-p", "cua-driver", "--test", "desktop_scope_windows_test", "--", "--ignored", "--nocapture", "--test-threads=1" ) } $script:FailureCount += (Test-E2eRecordings) +$reportExit = Invoke-E2eReport +if ($reportExit -ne 0) { + Write-Host "Windows E2E result validation failed" -ForegroundColor Red + $script:FailureCount++ +} + if ($script:FailureCount -ne 0) { throw "Windows Rust e2e suite had $($script:FailureCount) failing lane(s)" } -Write-Host "Windows Rust e2e suite completed: $Suite" -ForegroundColor Green +Write-Host "Windows Rust e2e matrix completed: $suite" -ForegroundColor Green