From a9b40ea64d8be13f08e9b08abc274654d3bbd59e Mon Sep 17 00:00:00 2001 From: Francesco Bonacci Date: Thu, 21 May 2026 08:35:31 +0200 Subject: [PATCH 1/3] fix(cua-driver-rs/windows): click prefers ExpandCollapsePattern for menu items (extends #1611) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the UIA element under the click point exposes BOTH InvokePattern AND ExpandCollapsePattern (Qt top-level MenuItems advertise both), the intended behavior is "open the submenu" — Invoke alone is a no-op for menu-bar items. Prefer ExpandCollapse.Expand in that case, fall back to Invoke on failure. Also relaxes the element filter to accept elements that support EITHER pattern (was: InvokePattern only). Without this, ExpandCollapse-only elements (rare but exist, e.g. some tree-view nodes) were skipped entirely by `try_invoke_in_window_at_point`. Found while testing FreeCAD on the Windows VM — click on File menu returned ✅ but the dropdown never appeared because Invoke on Qt menubar items doesn't expand the submenu. Not opening as PR per the in-flight overnight-test directive — branch pushed for backup; user reviews + opens PR when ready. Co-Authored-By: Claude Opus 4.7 --- .../platform-windows/src/uia/windows_enum.rs | 53 ++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/libs/cua-driver-rs/crates/platform-windows/src/uia/windows_enum.rs b/libs/cua-driver-rs/crates/platform-windows/src/uia/windows_enum.rs index b0752839c1..94c8b9e47f 100644 --- a/libs/cua-driver-rs/crates/platform-windows/src/uia/windows_enum.rs +++ b/libs/cua-driver-rs/crates/platform-windows/src/uia/windows_enum.rs @@ -220,7 +220,18 @@ pub fn try_invoke_in_window_at_point(hwnd: isize, sx: i32, sy: i32) -> bool { if sx < rect.left || sx > rect.right || sy < rect.top || sy > rect.bottom { continue; } - if elem.GetCurrentPattern(UIA_InvokePatternId).is_err() { + // Accept elements that support EITHER InvokePattern OR + // ExpandCollapsePattern. Qt menu-bar items advertise both — + // Invoke does nothing on them, only Expand opens the submenu. + // (See FreeCAD finding 2026-05-21: clicking File menu via Invoke + // returned ✅ but the menu never opened.) + let has_invoke = elem.GetCurrentPattern(UIA_InvokePatternId).is_ok(); + let has_expand = elem + .GetCurrentPattern( + windows::Win32::UI::Accessibility::UIA_ExpandCollapsePatternId, + ) + .is_ok(); + if !has_invoke && !has_expand { continue; } let w = (rect.right - rect.left).max(0) as i64; @@ -237,11 +248,49 @@ pub fn try_invoke_in_window_at_point(hwnd: isize, sx: i32, sy: i32) -> bool { None => { tracing::debug!( target: "click", - "no InvokePattern descendant of 0x{hwnd:x} contains screen ({sx},{sy}) (scanned {n} elems)" + "no Invoke/ExpandCollapse descendant of 0x{hwnd:x} contains screen ({sx},{sy}) (scanned {n} elems)" ); return false; } }; + // Pattern preference for menu items: when both Invoke AND + // ExpandCollapse are advertised, the element is almost always a + // top-level MenuItem whose intended click behaviour is "open the + // submenu" — Invoke would be a no-op. Prefer ExpandCollapse.Expand + // in that case. Pure-Invoke leaves (buttons, links, etc.) go + // through Invoke as before. + let winner_has_expand = winner + .GetCurrentPattern( + windows::Win32::UI::Accessibility::UIA_ExpandCollapsePatternId, + ) + .is_ok(); + let winner_has_invoke = winner.GetCurrentPattern(UIA_InvokePatternId).is_ok(); + if winner_has_expand && winner_has_invoke { + // Try Expand first, fall back to Invoke if Expand fails. + if let Ok(pat) = winner.GetCurrentPattern( + windows::Win32::UI::Accessibility::UIA_ExpandCollapsePatternId, + ) { + if let Ok(ec) = pat + .cast::() + { + if ec.Expand().is_ok() { + return true; + } + } + } + // Expand failed — fall through to Invoke as best-effort. + } else if winner_has_expand && !winner_has_invoke { + if let Ok(pat) = winner.GetCurrentPattern( + windows::Win32::UI::Accessibility::UIA_ExpandCollapsePatternId, + ) { + if let Ok(ec) = pat + .cast::() + { + return ec.Expand().is_ok(); + } + } + return false; + } let pattern = match winner.GetCurrentPattern(UIA_InvokePatternId) { Ok(p) => p, Err(_) => return false, From e28e1d7c15eda4b774e30f11f491116882366ee0 Mon Sep 17 00:00:00 2001 From: Francesco Bonacci Date: Thu, 21 May 2026 08:39:28 +0200 Subject: [PATCH 2/3] feat(test-fixtures): shared cua-driver gesture playground SPA + fixtures dir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Creates `libs/cua-driver-fixtures/` as the new shared home for HTML test fixtures used by both Swift cua-driver and Rust cua-driver-rs integration test suites. Previously each port had its own duplicated copy of `interactive.html` etc. — the two `driver_client.py` files have drifted already and the HTML copies will too. This commit adds the *new* shared fixture only; the deprecation path for the duplicated fixtures under each port is documented in the README's Migration plan but not yet executed (would need both ports' integration tests adjusted to resolve from the shared path). ## What's in gesture-playground.html A single self-contained 461-line HTML page with embedded JS covering every cua-driver gesture: | Panel | Tool tested | State exposed | |---|---|---| | 1 click counter | `click(element_index)` | `state.counter` int | | 2 click types | `click`, `right_click`, `double_click` | `state.multi {type, at}` | | 3 type_text mirror | `type_text` | `state.text` | | 4 keyboard / hotkey | `press_key`, `hotkey` with modifier-state check | `state.key {key, code, ctrl, alt, shift, meta}` | | 5 pixel-coord click | `click(x, y)` with pixel-accuracy distance | `state.coord {x, y}`, `state.coord_dist` | | 6 drag-and-drop | `drag` (verifies dragstart → dragover → drop) | `state.drag {dropped_from, dropped_at}` | | 7 scroll | `scroll` | `state.scroll` (px) | | 8 canvas | mousedown/move/up on HTML5 canvas — proves SendInput-vs-PostMessage delivery to custom-drawn surfaces | `state.canvas {type, x, y}` | | 9 form-all-inputs | submit handler with every HTML input type (back-compat with existing fixtures) | `state.form` | | 10 cumulative state dump | reads everything back as JSON for test assertions | — | Each panel has stable `data-test=""` attributes for targeting and inner elements have stable `id`s. State is exposed in a single JSON dump at `#state-dump` so tests can assert end-state in one read. ## Why this matters This playground specifically exercises gaps that surfaced during the overnight Windows VM stress test: - **Modifier-state propagation** (panel 4) verifies SendInput-vs-PostMessage: after `hotkey ["ctrl", "s"]`, `state.key.ctrl === true` proves SendInput is correctly updating GetKeyState - **Pixel accuracy** (panel 5) reports distance from a known target point - **Canvas vs DOM events** (panel 8) catches the universal "PostMessage doesn't reach custom-drawn surfaces" pattern seen in Audacity/GIMP/Blender Co-Authored-By: Claude Opus 4.7 --- libs/cua-driver-fixtures/README.md | 98 ++++ .../gesture-playground.html | 461 ++++++++++++++++++ 2 files changed, 559 insertions(+) create mode 100644 libs/cua-driver-fixtures/README.md create mode 100644 libs/cua-driver-fixtures/gesture-playground.html diff --git a/libs/cua-driver-fixtures/README.md b/libs/cua-driver-fixtures/README.md new file mode 100644 index 0000000000..d764a9faa1 --- /dev/null +++ b/libs/cua-driver-fixtures/README.md @@ -0,0 +1,98 @@ +# cua-driver-fixtures + +Shared test fixtures for cua-driver (Swift / macOS) and cua-driver-rs (Rust / +Windows + Linux + macOS). One source of truth; both ports' integration tests +reference these files. + +## What's here + +- **`gesture-playground.html`** — single-page interactive harness covering + every cua-driver gesture: click (left/right/middle/double), type_text, + press_key, hotkey (with modifier-state propagation check), pixel-coord + clicks, drag-and-drop, scroll, custom canvas mousedown/move/up, and a form + with every HTML input type. State is exposed as a JSON dump that tests can + read back via `get_window_state` + DOM reads or via screenshot. + + Each panel has a `data-test=""` attribute for stable test targeting. + + Open in any browser: + ```bash + open libs/cua-driver-fixtures/gesture-playground.html # macOS + start libs\cua-driver-fixtures\gesture-playground.html # Windows + xdg-open libs/cua-driver-fixtures/gesture-playground.html # Linux + ``` + +- **`interactive.html`** (planned) — keep as a thin redirect to + `gesture-playground.html` for backward compatibility with existing + fixtures that hard-code the filename. Both ports' `Tests/integration/` + trees previously had their own copy; canonical source is now here. + +## Migration plan + +1. ✅ Add this dir with the new gesture-playground.html +2. Update `libs/cua-driver/Tests/integration/conftest.py` to resolve + fixtures from this shared dir (with a fallback to the local copy for + in-flight branches) +3. Same for `libs/cua-driver-rs/tests/integration/conftest.py` and the v2 + harness +4. After both ports compile + tests pass against the shared dir, remove the + duplicated copies under each port + +## State exposed by `gesture-playground.html` + +The page maintains a single `state` object updated on every interaction. +A `
` panel renders it as JSON; tests read back via
+either:
+
+- `cua-driver get_window_state` + filter on the JSON text inside `#state-dump`
+- DOM read via the browser's accessibility tree (each panel has a stable
+  `data-test` attribute and inner elements have stable `id`s)
+- `cua-driver screenshot` + OCR (last resort for visual regression)
+
+### State fields
+
+| Panel | State field | Updated on |
+|-------|-------------|------------|
+| `1 · click counter` | `state.counter` (int) | `click` on `#counter-btn` |
+| `2 · click types` | `state.multi` (`{type, at}`) | left/right/middle/dbl `click` |
+| `3 · type_text mirror` | `state.text` (str) | `input` event on `#textbox` |
+| `4 · keyboard` | `state.key` (`{key, code, ctrl, alt, shift, meta}`) | `keydown` on `#keyfocus` |
+| `5 · pixel-coord click` | `state.coord` (`[x, y]`), `state.coord_dist` (px from (60,60)) | `click` on `#coord-area` |
+| `6 · drag` | `state.drag` (`{dropped_from, dropped_at}`) | `drop` on `#drag-target` |
+| `7 · scroll` | `state.scroll` (int, px) | `scroll` on `#scroll-area` |
+| `8 · canvas` | `state.canvas` (`{type, x, y}`) | `mousedown/move/up` on `#canvas` |
+| `9 · form` | `state.form` (form data) | `submit` on `#big-form` |
+
+### Why this matters for cua-driver verification
+
+- **Modifier-state propagation**: panel 4 reports `ctrl=true/false` from the
+  keyboard event. After `hotkey ["ctrl", "s"]`, you can verify `state.key.ctrl
+  === true` — that proves SendInput is correctly updating GetKeyState (the
+  PostMessage-based path doesn't).
+- **Pixel accuracy**: panel 5 reports distance from the ring center (60, 60).
+  After `click(x=60, y=60)`, distance should be ~0. After `click(x=61, y=58)`
+  it should be ~2. Lets you assert pixel-level click precision.
+- **Custom canvas vs DOM events**: panel 8 captures `mousedown` events on a
+  canvas. PostMessage WM_LBUTTONDOWN typically does NOT fire `mousedown` on
+  HTML5 canvases; SendInput does. Same model as Audacity timeline / GIMP /
+  Blender viewport behavior.
+- **Drag**: panel 6 verifies the full HTML5 drag-and-drop sequence
+  (`dragstart` → `dragover` → `drop`). Useful for asserting cua-driver's
+  `drag` tool fires all three.
+- **Scroll**: panel 7's scroll position is a simple int the test can read
+  back to verify `scroll` direction + amount.
+
+## Adding new fixtures
+
+If you need a fixture for a specific bug or gesture not covered here:
+
+1. Add it under `libs/cua-driver-fixtures/` (NOT under
+   `libs/cua-driver/Tests/integration/fixtures/` or
+   `libs/cua-driver-rs/tests/integration/fixtures/` — those dirs are
+   deprecated, see Migration plan above)
+2. Document the state shape it exposes in this README
+3. Reference from both Swift + Rust integration test trees
+
+## License
+
+Same as the parent project (MIT, see `LICENSE.md`).
diff --git a/libs/cua-driver-fixtures/gesture-playground.html b/libs/cua-driver-fixtures/gesture-playground.html
new file mode 100644
index 0000000000..d52f0020b7
--- /dev/null
+++ b/libs/cua-driver-fixtures/gesture-playground.html
@@ -0,0 +1,461 @@
+
+
+
+
+cua-driver gesture playground
+
+
+
+
+
+

+  
cua-driver gesture playground

+  
A self-contained HTML page exercising every cua-driver Windows + macOS tool. Open this file in any browser; drive it via the cua-driver MCP. Every interaction updates a state display you can read back via get_window_state or screenshot.

+

+
+

+
+  
+  

+    
1 · click counter

+    
+    
clicks: 0

+    
Tool: click(element_index) via UIA — counter increments per click event.

+  

+
+  
+  

+    
2 · click types

+    
+    
last: —

+    
Tools: click, right_click, double_click. Triple-click via click(count=3).

+  

+
+  
+  

+    
3 · type_text mirror

+    
+    

+    
Tool: type_text. Type any Unicode; mirror updates on every char.

+  

+
+  
+  

+    
4 · press_key + hotkey detector

+    
+    
(focus the field above and press a key)

+    
Display shows ctrl=… alt=… shift=… key="…" for the most recent keydown. Use to verify modifier state of hotkey calls (should report all 4 fields, not just the bare key — that proves SendInput modifier-state propagation, not just PostMessage).

+  

+
+  
+  

+    
5 · pixel-coord click target

+    

+      

+      
+    

+    
last click @ 
+      —
+       · distance from target (60, 60):
+      —
+    

+    
Tools: click(x, y). The blue ring is centered at the panel-local (60, 60). Use the distance badge to verify pixel-accuracy.

+  

+
+  
+  

+    
6 · drag-and-drop

+    

+      
drag me

+      
drop here

+    

+    
last drop: —

+    
Tool: drag. Should fire dragstart on source, dragover on target, drop on release.

+  

+
+  
+  

+    
7 · scroll position

+    

+      
+    

+    
scrollTop: 0
+       of …

+    
Tool: scroll. Use to verify scroll-amount math + direction.

+  

+
+  
+  

+    
8 · canvas (custom-drawn) — tests pixel input on a non-DOM widget

+    
+    
last canvas event: —

+    
Captures mousedown/move/up events. Custom-drawn surfaces in apps like Audacity / GIMP / Blender behave similarly — PostMessage WM_LBUTTONDOWN typically does NOT fire mousedown here; SendInput-routed clicks do.

+  

+
+  
+  

+    
9 · form (all input types — backward-compat with existing fixtures)

+    

+      

+        
+        
+        
+        
+      

+      

+        
+        
+        
+        
+      

+      

+        
+        
+      

+      

+        
+        
+        
+        
+        
+      

+      

+        
+        not submitted
+      

+    

+  

+
+  
+  

+    
10 · cumulative state dump (read this back for end-to-end verification)

+    

+      state JSON (auto-updates)
+      

+    

+    
+  

+
+

+
+
+
+

From e72cb6b42a2a3eb4aaf75c09f3039677c19bf200 Mon Sep 17 00:00:00 2001
From: Francesco Bonacci 
Date: Thu, 21 May 2026 11:44:12 +0200
Subject: [PATCH 3/3] refactor(test-fixtures): consolidate canonical Swift
 fixtures into shared dir
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The previous commit on this branch added a separate `gesture-playground.html`
SPA with its own ID conventions (`#counter-state`, `#type-state`, etc.) — a
parallel harness, not a unified one. That was wrong: both ports already
share three canonical HTML fixtures (`interactive.html`, `form_all_inputs.html`,
`test_page.html`), and their integration tests target specific stable IDs
in those files. The right consolidation is to make those existing fixtures
the single source of truth, not to introduce a fourth fixture.

## What this commit does

1. Moves the three canonical fixtures into `libs/cua-driver-fixtures/`:
   - `interactive.html` (was `cua-driver/Tests/integration/fixtures/`)
   - `form_all_inputs.html` (was same)
   - `test_page.html` (was `cua-driver/Tests/integration/assets/`)

2. Replaces each port's copy with a relative symlink to the canonical:
   ```
   cua-driver/Tests/integration/fixtures/interactive.html       -> ../../../../cua-driver-fixtures/interactive.html
   cua-driver/Tests/integration/fixtures/form_all_inputs.html   -> ../../../../cua-driver-fixtures/form_all_inputs.html
   cua-driver/Tests/integration/assets/test_page.html           -> ../../../../cua-driver-fixtures/test_page.html
   cua-driver-rs/tests/integration/fixtures/interactive.html    -> ../../../../cua-driver-fixtures/interactive.html
   cua-driver-rs/tests/integration/v2/assets/test_page.html     -> ../../../../../cua-driver-fixtures/test_page.html
   ```
   Existing test files keep working unchanged — every `os.path.join(_THIS_DIR,
   "fixtures", "interactive.html")` and `f"{html_server}/test_page.html"`
   resolves through the symlink to the canonical copy.

3. Removes the misguided `gesture-playground.html` from this branch.

4. Adds `gesture_panels.html` — a small (140-line) extension fixture that
   follows the *same* ID-convention style as `test_page.html`, covering four
   gestures the v2 harness doesn't probe and that the May 2026 Windows
   stress test showed needed coverage:
   - **Hotkey + modifier-state propagation** (`#hotkey-status` prints
     `ctrl=true|false` so SendInput-vs-PostMessage routing is observable
     in one assertion — directly the architectural proof for #1614)
   - **Pixel-coord pinpoint accuracy** (`#coord-status` prints distance from
     a known target at `(60,60)`)
   - **Drag-and-drop event sequence** (`#drag-status` records the
     `dragstart → dragover → drop` chain)
   - **Scroll position** (`#scroll-status` prints live `scrollTop`)

   Each panel exposes its state via `window.getGesturePanelState()` for
   `browser_eval`-based readback when Chromium is launched with
   `--remote-debugging-port`.

## Result

Drift between Swift port and Rust port fixtures is impossible by construction
— edits propagate to both ports via the single canonical copy.

Net diff: +581 / -1045 lines (the deleted playground + 4 duplicate fixture
copies vs. one canonical of each).

## Validated

`python3 os.path.isfile` reports all five historic test paths resolve to
the canonical copies with correct byte counts; no test code needs to
change to consume the consolidated layout.

Co-Authored-By: Claude Opus 4.7 
---
 libs/cua-driver-fixtures/README.md            | 194 ++++----
 libs/cua-driver-fixtures/form_all_inputs.html | 164 +++++++
 .../gesture-playground.html                   | 461 ------------------
 libs/cua-driver-fixtures/gesture_panels.html  | 140 ++++++
 libs/cua-driver-fixtures/interactive.html     |  31 ++
 libs/cua-driver-fixtures/test_page.html       | 135 +++++
 .../integration/fixtures/interactive.html     |  32 +-
 .../integration/v2/assets/test_page.html      | 136 +-----
 .../Tests/integration/assets/test_page.html   | 136 +-----
 .../integration/fixtures/form_all_inputs.html | 165 +------
 .../integration/fixtures/interactive.html     |  32 +-
 11 files changed, 581 insertions(+), 1045 deletions(-)
 create mode 100644 libs/cua-driver-fixtures/form_all_inputs.html
 delete mode 100644 libs/cua-driver-fixtures/gesture-playground.html
 create mode 100644 libs/cua-driver-fixtures/gesture_panels.html
 create mode 100644 libs/cua-driver-fixtures/interactive.html
 create mode 100644 libs/cua-driver-fixtures/test_page.html
 mode change 100644 => 120000 libs/cua-driver-rs/tests/integration/fixtures/interactive.html
 mode change 100644 => 120000 libs/cua-driver-rs/tests/integration/v2/assets/test_page.html
 mode change 100644 => 120000 libs/cua-driver/Tests/integration/assets/test_page.html
 mode change 100644 => 120000 libs/cua-driver/Tests/integration/fixtures/form_all_inputs.html
 mode change 100644 => 120000 libs/cua-driver/Tests/integration/fixtures/interactive.html

diff --git a/libs/cua-driver-fixtures/README.md b/libs/cua-driver-fixtures/README.md
index d764a9faa1..7c76e907ca 100644
--- a/libs/cua-driver-fixtures/README.md
+++ b/libs/cua-driver-fixtures/README.md
@@ -1,97 +1,115 @@
 # cua-driver-fixtures
 
-Shared test fixtures for cua-driver (Swift / macOS) and cua-driver-rs (Rust /
-Windows + Linux + macOS). One source of truth; both ports' integration tests
-reference these files.
-
-## What's here
-
-- **`gesture-playground.html`** — single-page interactive harness covering
-  every cua-driver gesture: click (left/right/middle/double), type_text,
-  press_key, hotkey (with modifier-state propagation check), pixel-coord
-  clicks, drag-and-drop, scroll, custom canvas mousedown/move/up, and a form
-  with every HTML input type. State is exposed as a JSON dump that tests can
-  read back via `get_window_state` + DOM reads or via screenshot.
-
-  Each panel has a `data-test=""` attribute for stable test targeting.
-
-  Open in any browser:
-  ```bash
-  open libs/cua-driver-fixtures/gesture-playground.html         # macOS
-  start libs\cua-driver-fixtures\gesture-playground.html        # Windows
-  xdg-open libs/cua-driver-fixtures/gesture-playground.html     # Linux
-  ```
-
-- **`interactive.html`** (planned) — keep as a thin redirect to
-  `gesture-playground.html` for backward compatibility with existing
-  fixtures that hard-code the filename. Both ports' `Tests/integration/`
-  trees previously had their own copy; canonical source is now here.
-
-## Migration plan
-
-1. ✅ Add this dir with the new gesture-playground.html
-2. Update `libs/cua-driver/Tests/integration/conftest.py` to resolve
-   fixtures from this shared dir (with a fallback to the local copy for
-   in-flight branches)
-3. Same for `libs/cua-driver-rs/tests/integration/conftest.py` and the v2
-   harness
-4. After both ports compile + tests pass against the shared dir, remove the
-   duplicated copies under each port
-
-## State exposed by `gesture-playground.html`
-
-The page maintains a single `state` object updated on every interaction.
-A `
` panel renders it as JSON; tests read back via
-either:
-
-- `cua-driver get_window_state` + filter on the JSON text inside `#state-dump`
-- DOM read via the browser's accessibility tree (each panel has a stable
-  `data-test` attribute and inner elements have stable `id`s)
-- `cua-driver screenshot` + OCR (last resort for visual regression)
-
-### State fields
-
-| Panel | State field | Updated on |
-|-------|-------------|------------|
-| `1 · click counter` | `state.counter` (int) | `click` on `#counter-btn` |
-| `2 · click types` | `state.multi` (`{type, at}`) | left/right/middle/dbl `click` |
-| `3 · type_text mirror` | `state.text` (str) | `input` event on `#textbox` |
-| `4 · keyboard` | `state.key` (`{key, code, ctrl, alt, shift, meta}`) | `keydown` on `#keyfocus` |
-| `5 · pixel-coord click` | `state.coord` (`[x, y]`), `state.coord_dist` (px from (60,60)) | `click` on `#coord-area` |
-| `6 · drag` | `state.drag` (`{dropped_from, dropped_at}`) | `drop` on `#drag-target` |
-| `7 · scroll` | `state.scroll` (int, px) | `scroll` on `#scroll-area` |
-| `8 · canvas` | `state.canvas` (`{type, x, y}`) | `mousedown/move/up` on `#canvas` |
-| `9 · form` | `state.form` (form data) | `submit` on `#big-form` |
-
-### Why this matters for cua-driver verification
-
-- **Modifier-state propagation**: panel 4 reports `ctrl=true/false` from the
-  keyboard event. After `hotkey ["ctrl", "s"]`, you can verify `state.key.ctrl
-  === true` — that proves SendInput is correctly updating GetKeyState (the
-  PostMessage-based path doesn't).
-- **Pixel accuracy**: panel 5 reports distance from the ring center (60, 60).
-  After `click(x=60, y=60)`, distance should be ~0. After `click(x=61, y=58)`
-  it should be ~2. Lets you assert pixel-level click precision.
-- **Custom canvas vs DOM events**: panel 8 captures `mousedown` events on a
-  canvas. PostMessage WM_LBUTTONDOWN typically does NOT fire `mousedown` on
-  HTML5 canvases; SendInput does. Same model as Audacity timeline / GIMP /
-  Blender viewport behavior.
-- **Drag**: panel 6 verifies the full HTML5 drag-and-drop sequence
-  (`dragstart` → `dragover` → `drop`). Useful for asserting cua-driver's
-  `drag` tool fires all three.
-- **Scroll**: panel 7's scroll position is a simple int the test can read
-  back to verify `scroll` direction + amount.
+Canonical HTML test fixtures for cua-driver (Swift / macOS) and
+cua-driver-rs (Rust / Windows + Linux + macOS). One source of truth; both
+ports' integration tests reference these files via relative symlinks from
+their existing `Tests/integration/fixtures/` and `assets/` trees.
+
+## Files
+
+| File | Purpose |
+|------|---------|
+| `interactive.html` | 31-line minimal click + type-mirror harness. Stable IDs: `#counter`, `#clicker`, `#textbox`, `#typed`. Used by `test_background_focus.py` (both ports) — opened via `file://`. |
+| `form_all_inputs.html` | Every HTML5 input type (text, password, email, number, tel, textarea, select, checkbox, radio, range, date, color). Submit handler stores result in `window._submitted`; live read via `getFieldValues()`. Used by `test_drag_slider_delivery.py`, `test_hermes_form_fill*.py`. |
+| `test_page.html` | Richer Safari/Chrome/Electron harness — button counter, text input, checkbox, dropdown, textarea, link, canvas. Served via local `html_server` HTTP fixture. Used by `test_chrome.py`, `test_safari.py`, `test_electron.py` (both ports). |
+| `gesture_panels.html` | **New** — extends the same ID-convention style as `test_page.html` with panels for the four gestures not currently covered by the v2 harness: hotkey + modifier-state propagation, pixel-coord pinpoint accuracy, drag-and-drop event sequence, scroll position. See **Why** below. |
+
+All four files are vanilla HTML — no external deps, no build step. Open
+directly in any browser:
+
+```bash
+open libs/cua-driver-fixtures/test_page.html
+```
+
+## Where each port references the canonical copy
+
+Existing port paths are preserved as symlinks. Tests that read these
+paths (`os.path.join(_THIS_DIR, "fixtures", "interactive.html")`,
+`f"{html_server}/test_page.html"`) work unchanged.
+
+```
+libs/cua-driver/Tests/integration/
+├── fixtures/
+│   ├── interactive.html      → ../../../../cua-driver-fixtures/interactive.html
+│   └── form_all_inputs.html  → ../../../../cua-driver-fixtures/form_all_inputs.html
+└── assets/
+    └── test_page.html        → ../../../../cua-driver-fixtures/test_page.html
+
+libs/cua-driver-rs/tests/integration/
+├── fixtures/
+│   └── interactive.html      → ../../../../cua-driver-fixtures/interactive.html
+└── v2/assets/
+    └── test_page.html        → ../../../../../cua-driver-fixtures/test_page.html
+```
+
+Edits to a fixture now propagate to **both ports** in a single commit.
+Drift is impossible by construction.
+
+## Why `gesture_panels.html` was added
+
+`test_page.html` covers Safari/Chrome/Electron interaction surfaces well
+but doesn't probe four gestures that the Windows cua-driver-rs stress
+test (May 2026) showed need explicit harness coverage:
+
+1. **Hotkey + modifier-state propagation** (`#hotkey-focus` + `#hotkey-status`):
+   PostMessage-based hotkey paths inject keystrokes without updating the
+   OS modifier state, so the page-level `KeyboardEvent.ctrlKey` reads
+   `false` even after a routed `ctrl+s`. SendInput updates `GetKeyState`,
+   so the same event reads `ctrl=true`. The status div prints
+   `ctrl=true|false` directly — single assertion per routing path.
+
+2. **Pixel-coord pinpoint** (`#coord-area` + `#coord-status`): a known
+   target at viewport `(60, 60)` inside the box; status shows the
+   distance from the target in pixels. Asserting `dist < 2` is a clean
+   precision test for coordinate-routed clicks.
+
+3. **Drag-and-drop sequence** (`#drag-source` + `#drag-target` +
+   `#drag-status`): the page records each HTML5 drag event in order so
+   tests can assert the full `dragstart → dragover → drop` chain reached
+   the page (not just that the driver moved the cursor).
+
+4. **Scroll position** (`#scroll-area` + `#scroll-status`): a scrollable
+   container whose `scrollTop` is printed live — tests can assert scroll
+   direction + amount after a routed scroll gesture.
+
+Like `test_page.html`, each panel exposes a global accessor:
+`window.getGesturePanelState()` returns a JSON snapshot of all four
+panels' state. Tests can poll it via `browser_eval` (when running with
+Chromium's `--remote-debugging-port`) or read the visible status divs
+via UIA / OCR.
+
+### Known browser-coverage gaps (May 2026)
+
+When driving Chromium-based browsers (Edge / Chrome / Electron) on
+Windows, the integration tests should be aware of:
+
+- **GPU-composited content blanks PrintWindow.** `cua-driver screenshot`
+  captures the browser chrome + a white body. Workaround: launch the
+  target browser with `--remote-debugging-port=` and use
+  `browser_eval` for both inputs and verification.
+- **Chromium auto-disables full a11y** unless an AT-grade client is
+  detected. UIA on the rendered DOM returns empty until the page is
+  flagged accessibility-active. Workaround: enable Chromium's
+  `--force-renderer-accessibility` flag (or rely on `--remote-debugging-port`).
+- **`hotkey` SendInput injection requires the daemon at UIAccess
+  integrity.** Otherwise `SetForegroundWindow` is rejected and the
+  injected events land on the wrong window — `SendInput inserted only 0
+  of 4 events`. Workaround: run `cua-driver` via the
+  `cua-driver-uia.exe` worker (PR #1593) which carries the manifest.
+
+These are environmental requirements for any DOM-level verification of
+gesture behavior in a Chromium target on Windows; they don't affect the
+canonical fixtures, only how they're driven.
 
 ## Adding new fixtures
 
-If you need a fixture for a specific bug or gesture not covered here:
+1. Add the new HTML file directly to this dir
+2. Document it in the table above (purpose + stable IDs + who uses it)
+3. Add a relative symlink under whichever port(s) need it at the path
+   their tests already use
+4. Commit the new file + symlink(s) in the same PR
 
-1. Add it under `libs/cua-driver-fixtures/` (NOT under
-   `libs/cua-driver/Tests/integration/fixtures/` or
-   `libs/cua-driver-rs/tests/integration/fixtures/` — those dirs are
-   deprecated, see Migration plan above)
-2. Document the state shape it exposes in this README
-3. Reference from both Swift + Rust integration test trees
+Do **not** create copies under each port's tree — symlink-only.
 
 ## License
 
diff --git a/libs/cua-driver-fixtures/form_all_inputs.html b/libs/cua-driver-fixtures/form_all_inputs.html
new file mode 100644
index 0000000000..74ae4addfe
--- /dev/null
+++ b/libs/cua-driver-fixtures/form_all_inputs.html
@@ -0,0 +1,164 @@
+
+
+
+
+All Inputs Test Form
+
+
+
+
All Inputs Test Form

+

+
+  

+    
+    
+  

+
+  

+    
+    
+  

+
+  

+    
+    
+  

+
+  

+    
+    
+  

+
+  

+    
+    
+  

+
+  

+    
+    
+  

+
+  

+    
+    
+  

+
+  

+    

+    
+    
+  

+
+  

+    

+    
+      
+    
+      
+    
+    
+  

+
+  

+    
+    
+    50
+  

+
+  

+    
+    
+  

+
+  

+    

+    
+  

+
+  
+

+
+
✓ Form Submitted!

+

+
+
+
+
diff --git a/libs/cua-driver-fixtures/gesture-playground.html b/libs/cua-driver-fixtures/gesture-playground.html
deleted file mode 100644
index d52f0020b7..0000000000
--- a/libs/cua-driver-fixtures/gesture-playground.html
+++ /dev/null
@@ -1,461 +0,0 @@
-
-
-
-
-cua-driver gesture playground
-
-
-
-
-
-

-  
cua-driver gesture playground

-  
A self-contained HTML page exercising every cua-driver Windows + macOS tool. Open this file in any browser; drive it via the cua-driver MCP. Every interaction updates a state display you can read back via get_window_state or screenshot.

-

-
-

-
-  
-  

-    
1 · click counter

-    
-    
clicks: 0

-    
Tool: click(element_index) via UIA — counter increments per click event.

-  

-
-  
-  

-    
2 · click types

-    
-    
last: —

-    
Tools: click, right_click, double_click. Triple-click via click(count=3).

-  

-
-  
-  

-    
3 · type_text mirror

-    
-    

-    
Tool: type_text. Type any Unicode; mirror updates on every char.

-  

-
-  
-  

-    
4 · press_key + hotkey detector

-    
-    
(focus the field above and press a key)

-    
Display shows ctrl=… alt=… shift=… key="…" for the most recent keydown. Use to verify modifier state of hotkey calls (should report all 4 fields, not just the bare key — that proves SendInput modifier-state propagation, not just PostMessage).

-  

-
-  
-  

-    
5 · pixel-coord click target

-    

-      

-      
-    

-    
last click @ 
-      —
-       · distance from target (60, 60):
-      —
-    

-    
Tools: click(x, y). The blue ring is centered at the panel-local (60, 60). Use the distance badge to verify pixel-accuracy.

-  

-
-  
-  

-    
6 · drag-and-drop

-    

-      
drag me

-      
drop here

-    

-    
last drop: —

-    
Tool: drag. Should fire dragstart on source, dragover on target, drop on release.

-  

-
-  
-  

-    
7 · scroll position

-    

-      
-    

-    
scrollTop: 0
-       of …

-    
Tool: scroll. Use to verify scroll-amount math + direction.

-  

-
-  
-  

-    
8 · canvas (custom-drawn) — tests pixel input on a non-DOM widget

-    
-    
last canvas event: —

-    
Captures mousedown/move/up events. Custom-drawn surfaces in apps like Audacity / GIMP / Blender behave similarly — PostMessage WM_LBUTTONDOWN typically does NOT fire mousedown here; SendInput-routed clicks do.

-  

-
-  
-  

-    
9 · form (all input types — backward-compat with existing fixtures)

-    

-      

-        
-        
-        
-        
-      

-      

-        
-        
-        
-        
-      

-      

-        
-        
-      

-      

-        
-        
-        
-        
-        
-      

-      

-        
-        not submitted
-      

-    

-  

-
-  
-  

-    
10 · cumulative state dump (read this back for end-to-end verification)

-    

-      state JSON (auto-updates)
-      

-    

-    
-  

-
-

-
-
-
-
diff --git a/libs/cua-driver-fixtures/gesture_panels.html b/libs/cua-driver-fixtures/gesture_panels.html
new file mode 100644
index 0000000000..8b9d196a83
--- /dev/null
+++ b/libs/cua-driver-fixtures/gesture_panels.html
@@ -0,0 +1,140 @@
+
+
+
+
+CUA Driver Gesture Panels (test_page.html companion)
+
+
+
+
+

+  
Hotkey / modifier-state probe

+  

+    Focus this field, then send hotkey ["ctrl","s"]. The status line must report
+    ctrl=true key=s. PostMessage-only paths report ctrl=false because
+    the OS modifier state is never updated. SendInput-routed hotkeys report ctrl=true.
+  

+  
+  
hotkey: (no key seen)

+

+
+

+  
Pixel-coordinate pinpoint

+  

+    Click on the blue dot at (60, 60) inside the box. Status shows the absolute viewport
+    pixel and the offset from the target center — <2px is a passing precision test.
+  

+  

+    

+  

+  
coord: (no click)

+

+
+

+  
Drag and drop

+  

+    Drag the blue square into the dashed target. Status must show all three events:
+    dragstart → dragover → drop with the source label.
+  

+  
DRAG ME

+  
drop here

+  
drag: (no events)

+

+
+

+  
Scroll

+  

+    Scroll this container with the scroll tool. Status shows the current
+    scrollTop in pixels.
+  

+  

+    
scroll me

+  

+  
scroll: 0

+

+
+
+
+
diff --git a/libs/cua-driver-fixtures/interactive.html b/libs/cua-driver-fixtures/interactive.html
new file mode 100644
index 0000000000..e67b6fee1f
--- /dev/null
+++ b/libs/cua-driver-fixtures/interactive.html
@@ -0,0 +1,31 @@
+
+
+
+
+CUA Driver Test Page
+
+
+
+
Interactive Test Page

+
clicks: 0

+
+
+
typed: 

+
+
+
diff --git a/libs/cua-driver-fixtures/test_page.html b/libs/cua-driver-fixtures/test_page.html
new file mode 100644
index 0000000000..cf07c04b14
--- /dev/null
+++ b/libs/cua-driver-fixtures/test_page.html
@@ -0,0 +1,135 @@
+
+
+
+
+CUA Driver Test Page v2
+
+
+
+
+

+  
Button

+  
clicks: 0

+  
+

+
+

+  
Text Input

+  
+  
typed: 

+

+
+

+  
Checkbox

+  
+  unchecked
+

+
+

+  
Select / Dropdown

+  
+  
selected: Option A

+

+
+

+  
Textarea

+  
+  
content: 

+

+
+

+  
Link

+  Test Link
+  
+

+
+

+  
Canvas

+  
+  
canvas: no click

+

+
+

+
+
+
+
diff --git a/libs/cua-driver-rs/tests/integration/fixtures/interactive.html b/libs/cua-driver-rs/tests/integration/fixtures/interactive.html
deleted file mode 100644
index e67b6fee1f..0000000000
--- a/libs/cua-driver-rs/tests/integration/fixtures/interactive.html
+++ /dev/null
@@ -1,31 +0,0 @@
-
-
-
-
-CUA Driver Test Page
-
-
-
-
Interactive Test Page

-
clicks: 0

-
-
-
typed: 

-
-
-
diff --git a/libs/cua-driver-rs/tests/integration/fixtures/interactive.html b/libs/cua-driver-rs/tests/integration/fixtures/interactive.html
new file mode 120000
index 0000000000..d053b042a9
--- /dev/null
+++ b/libs/cua-driver-rs/tests/integration/fixtures/interactive.html
@@ -0,0 +1 @@
+../../../../cua-driver-fixtures/interactive.html
\ No newline at end of file
diff --git a/libs/cua-driver-rs/tests/integration/v2/assets/test_page.html b/libs/cua-driver-rs/tests/integration/v2/assets/test_page.html
deleted file mode 100644
index cf07c04b14..0000000000
--- a/libs/cua-driver-rs/tests/integration/v2/assets/test_page.html
+++ /dev/null
@@ -1,135 +0,0 @@
-
-
-
-
-CUA Driver Test Page v2
-
-
-
-
-

-  
Button

-  
clicks: 0

-  
-

-
-

-  
Text Input

-  
-  
typed: 

-

-
-

-  
Checkbox

-  
-  unchecked
-

-
-

-  
Select / Dropdown

-  
-  
selected: Option A

-

-
-

-  
Textarea

-  
-  
content: 

-

-
-

-  
Link

-  Test Link
-  
-

-
-

-  
Canvas

-  
-  
canvas: no click

-

-
-

-
-
-
-
diff --git a/libs/cua-driver-rs/tests/integration/v2/assets/test_page.html b/libs/cua-driver-rs/tests/integration/v2/assets/test_page.html
new file mode 120000
index 0000000000..57fd293961
--- /dev/null
+++ b/libs/cua-driver-rs/tests/integration/v2/assets/test_page.html
@@ -0,0 +1 @@
+../../../../../cua-driver-fixtures/test_page.html
\ No newline at end of file
diff --git a/libs/cua-driver/Tests/integration/assets/test_page.html b/libs/cua-driver/Tests/integration/assets/test_page.html
deleted file mode 100644
index cf07c04b14..0000000000
--- a/libs/cua-driver/Tests/integration/assets/test_page.html
+++ /dev/null
@@ -1,135 +0,0 @@
-
-
-
-
-CUA Driver Test Page v2
-
-
-
-
-

-  
Button

-  
clicks: 0

-  
-

-
-

-  
Text Input

-  
-  
typed: 

-

-
-

-  
Checkbox

-  
-  unchecked
-

-
-

-  
Select / Dropdown

-  
-  
selected: Option A

-

-
-

-  
Textarea

-  
-  
content: 

-

-
-

-  
Link

-  Test Link
-  
-

-
-

-  
Canvas

-  
-  
canvas: no click

-

-
-

-
-
-
-
diff --git a/libs/cua-driver/Tests/integration/assets/test_page.html b/libs/cua-driver/Tests/integration/assets/test_page.html
new file mode 120000
index 0000000000..2771775117
--- /dev/null
+++ b/libs/cua-driver/Tests/integration/assets/test_page.html
@@ -0,0 +1 @@
+../../../../cua-driver-fixtures/test_page.html
\ No newline at end of file
diff --git a/libs/cua-driver/Tests/integration/fixtures/form_all_inputs.html b/libs/cua-driver/Tests/integration/fixtures/form_all_inputs.html
deleted file mode 100644
index 74ae4addfe..0000000000
--- a/libs/cua-driver/Tests/integration/fixtures/form_all_inputs.html
+++ /dev/null
@@ -1,164 +0,0 @@
-
-
-
-
-All Inputs Test Form
-
-
-
-
All Inputs Test Form

-

-
-  

-    
-    
-  

-
-  

-    
-    
-  

-
-  

-    
-    
-  

-
-  

-    
-    
-  

-
-  

-    
-    
-  

-
-  

-    
-    
-  

-
-  

-    
-    
-  

-
-  

-    

-    
-    
-  

-
-  

-    

-    
-      
-    
-      
-    
-    
-  

-
-  

-    
-    
-    50
-  

-
-  

-    
-    
-  

-
-  

-    

-    
-  

-
-  
-

-
-
✓ Form Submitted!

-

-
-
-
-
diff --git a/libs/cua-driver/Tests/integration/fixtures/form_all_inputs.html b/libs/cua-driver/Tests/integration/fixtures/form_all_inputs.html
new file mode 120000
index 0000000000..39e40653b3
--- /dev/null
+++ b/libs/cua-driver/Tests/integration/fixtures/form_all_inputs.html
@@ -0,0 +1 @@
+../../../../cua-driver-fixtures/form_all_inputs.html
\ No newline at end of file
diff --git a/libs/cua-driver/Tests/integration/fixtures/interactive.html b/libs/cua-driver/Tests/integration/fixtures/interactive.html
deleted file mode 100644
index e67b6fee1f..0000000000
--- a/libs/cua-driver/Tests/integration/fixtures/interactive.html
+++ /dev/null
@@ -1,31 +0,0 @@
-
-
-
-
-CUA Driver Test Page
-
-
-
-
Interactive Test Page

-
clicks: 0

-
-
-
typed: 

-
-
-
diff --git a/libs/cua-driver/Tests/integration/fixtures/interactive.html b/libs/cua-driver/Tests/integration/fixtures/interactive.html
new file mode 120000
index 0000000000..d053b042a9
--- /dev/null
+++ b/libs/cua-driver/Tests/integration/fixtures/interactive.html
@@ -0,0 +1 @@
+../../../../cua-driver-fixtures/interactive.html
\ No newline at end of file