diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 5acc27fa22..0000000000 --- a/.dockerignore +++ /dev/null @@ -1,34 +0,0 @@ -# Version control -.git -.github -.gitignore - -# Environment and cache -.venv -.env -.env.local -__pycache__ -*.pyc -*.pyo -*.pyd -.Python -.pytest_cache -.pdm-build - -# Distribution / packaging -dist -build -*.egg-info - -# Development -.vscode -.idea -*.swp -*.swo - -# Docs -docs/site - -# Docker -Dockerfile -.dockerignore diff --git a/.github/scripts/tests/test_cua_driver_release_wiring.py b/.github/scripts/tests/test_cua_driver_release_wiring.py index bdeb74c38f..febd0fe4c1 100644 --- a/.github/scripts/tests/test_cua_driver_release_wiring.py +++ b/.github/scripts/tests/test_cua_driver_release_wiring.py @@ -43,6 +43,9 @@ def test_release_reminder_tracks_rust_driver(self) -> None: workflow = self.read(".github/workflows/ci-release-reminder.yml") self.assertIn('["libs/cua-driver/rust/"]="cua-driver-rs"', workflow) + self.assertIn("cua-driver desktop release validation", workflow) + self.assertIn("e2e-rust-linux.yml", workflow) + self.assertIn("e2e-rust-linux-wayland.yml", workflow) def test_unreleased_digest_tracks_rust_driver(self) -> None: workflow = self.read(".github/workflows/release-unreleased-digest.yml") diff --git a/.github/workflows/cd-rust-cua-driver.yml b/.github/workflows/cd-rust-cua-driver.yml index 312dded99c..d4707e89e5 100644 --- a/.github/workflows/cd-rust-cua-driver.yml +++ b/.github/workflows/cd-rust-cua-driver.yml @@ -74,16 +74,14 @@ jobs: - name: Install base tooling (container is bare) run: | apt-get update - # The PipeWire / libei deps the native-Wayland portal path uses - # are intentionally NOT installed here: bullseye ships PipeWire - # 0.3.19 (too old for libspa-sys 0.8, which needs >= 0.3.40) and - # has no libei-dev at all. They sit behind the `portal-libei` - # Cargo feature (disabled by default) and are wired in via the - # Nix build. The wlroots screencopy + virtual-pointer paths and - # the X11 fallback work without them. + # PipeWire ScreenCast capture remains Nix/modern-build only because + # bullseye's 0.3.19 headers are too old for libspa-sys 0.8. Portal + # RemoteDesktop/libei input is pure Rust plus libxkbcommon and ships + # in these portable release binaries. apt-get install -y --no-install-recommends \ git ca-certificates curl build-essential pkg-config \ - libx11-dev libxi-dev libxtst-dev libxext-dev libwayland-dev + libx11-dev libxi-dev libxtst-dev libxext-dev libwayland-dev \ + libxkbcommon-dev - uses: actions/checkout@v4 - name: Determine version id: version @@ -109,7 +107,7 @@ jobs: rust-cua-driver-rs-linux-${{ matrix.arch }}- - name: Build (release) working-directory: libs/cua-driver/rust - run: cargo build -p cua-driver --release --target ${{ matrix.target }} + run: cargo build -p cua-driver --release --features portal-input --target ${{ matrix.target }} - name: Package working-directory: libs/cua-driver/rust run: | diff --git a/.github/workflows/ci-cua-driver-interactive-linux.yml b/.github/workflows/ci-cua-driver-interactive-linux.yml deleted file mode 100644 index d602841dad..0000000000 --- a/.github/workflows/ci-cua-driver-interactive-linux.yml +++ /dev/null @@ -1,128 +0,0 @@ -name: "CI: cua-driver interactive modality suite (Linux)" - -# Runs the cua-driver INTERACTIVE `#[ignore]` modality/harness tests on a real -# (headless) Linux GUI session, so their BEHAVIOR — not just compilation — is -# guarded on every cua-driver change. These tests need a display, the AT-SPI -# accessibility bus, and the GTK3 harness app; the rest of CI only compiles them. -# -# What runs here: -# - modality_capture_mode_test (ax→tree-only / vision→image-only / som→both) -# - modality_desktop_scope_linux_test (window-less screen-absolute click via XTest + gate) -# -# How the GUI session is provided: Xvfb for the display, a per-job D-Bus session -# bus (so AT-SPI's `org.a11y.Bus` can activate), at-spi2-core for the bridge, and -# the GTK3/PyGObject runtime for the harness. The tests skip-with-note when the -# a11y bus is unavailable rather than false-failing, so a vacuous run is visible -# (no green-for-nothing surprise) without breaking the build. -# -# Companion to nix-build.yml / nix-wayland.yml (real desktops, broad app matrix). -# This lane is narrow + fast and runs on free GitHub runners. Trigger: PRs that -# touch the Rust driver or the harness, push to main, and manual dispatch. - -on: - pull_request: - paths: - - "libs/cua-driver/rust/**" - - "libs/cua-driver/tests/fixtures/**" - - ".github/workflows/ci-cua-driver-interactive-linux.yml" - push: - branches: [main] - paths: - - "libs/cua-driver/rust/**" - - "libs/cua-driver/tests/fixtures/**" - - ".github/workflows/ci-cua-driver-interactive-linux.yml" - workflow_dispatch: - -permissions: - contents: read - -jobs: - interactive-modality: - name: Interactive modality suite (Xvfb + AT-SPI) - runs-on: ubuntu-latest - timeout-minutes: 30 - - steps: - - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - - name: Install GUI + accessibility + harness runtime deps - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ - xvfb dbus-x11 at-spi2-core openbox \ - libgtk-3-0 gir1.2-gtk-3.0 python3-gi \ - build-essential libglib2.0-dev libgtk-3-dev \ - libwebkit2gtk-4.1-dev libssl-dev libxdo-dev \ - libayatana-appindicator3-dev librsvg2-dev \ - libxtst6 libxtst-dev libx11-dev libxext-dev \ - ffmpeg - - - name: Rust toolchain - uses: dtolnay/rust-toolchain@stable - - - name: Cache cargo - uses: Swatinem/rust-cache@9d47c6ad4b02e050fd481d890b2ea34778fd09d6 # v2 - with: - workspaces: "libs/cua-driver/rust -> target" - - - name: Build cua-driver (release — the test harness prefers release) - working-directory: libs/cua-driver/rust - run: cargo build --release -p cua-driver - - - name: Build the GTK3 harness app - run: bash libs/cua-driver/tests/fixtures/build/linux.sh - - - name: AT-SPI reachability probe (diagnostic, non-fatal) - run: | - cat > /tmp/atspi_probe.py <<'PY' - import gi - gi.require_version("Atspi", "2.0") - from gi.repository import Atspi - print("[interactive-ci] AT-SPI reachable, desktop count:", Atspi.get_desktop_count()) - PY - - - name: Run the interactive modality suite under Xvfb + a session a11y bus - working-directory: libs/cua-driver/rust - env: - NO_AT_BRIDGE: "0" - GTK_A11Y: "1" - run: | - set -euo pipefail - # Xvfb provides the display; dbus-run-session provides a session bus on - # which AT-SPI's org.a11y.Bus auto-activates; GTK3's atk-bridge then - # registers the harness on the accessibility tree. - xvfb-run -a --server-args="-screen 0 1920x1080x24" \ - dbus-run-session -- bash -c ' - set -e - # openbox manages/places the harness window for AT-SPI extents. - openbox & - sleep 2 - python3 /tmp/atspi_probe.py || echo "[interactive-ci] AT-SPI probe inconclusive — AX assertions will skip if the bus is unavailable" - - # capture_mode matrix (ax/vision/som): real assertions, headless-safe. - cargo test -p cua-driver --test modality_capture_mode_test \ - -- --ignored --nocapture --test-threads=1 - - # desktop-scope: run the GATE here (the desktop_scope_disabled - # contract). The *landing* assertion (counter advances on a - # window-less click) needs a real display + WM to deliver the - # screen-absolute XTest click — Xvfb does not faithfully deliver it, - # so that test runs on the real-desktop lanes (the GNOME/Azure VM run - # + the recorded artifact), not here. See nix-wayland.yml for the - # real-session follow-up that can host the landing assertion too. - cargo test -p cua-driver --test modality_desktop_scope_linux_test \ - window_scope_rejects_windowless_click \ - -- --ignored --nocapture --test-threads=1 - ' - - - name: Upload any recordings / artifacts - if: always() - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 - with: - name: interactive-modality-artifacts - path: | - /tmp/*.mp4 - libs/cua-driver/rust/target/release/*.log - if-no-files-found: ignore - retention-days: 7 diff --git a/.github/workflows/ci-lint-typescript.yml b/.github/workflows/ci-lint-typescript.yml index 6307335b83..2b8a1c58c6 100644 --- a/.github/workflows/ci-lint-typescript.yml +++ b/.github/workflows/ci-lint-typescript.yml @@ -22,14 +22,14 @@ jobs: - name: Set up pnpm uses: pnpm/action-setup@v4 + with: + package_json_file: libs/typescript/package.json - name: Install Node dependencies - run: | - pnpm install --frozen-lockfile - pnpm -C libs/typescript install --frozen-lockfile + run: pnpm -C libs/typescript install --frozen-lockfile - name: TypeScript typecheck - run: node ./scripts/typescript-typecheck.js + run: pnpm -C libs/typescript typecheck - name: Prettier check - run: pnpm prettier --check "libs/typescript/**/*.{ts,tsx,js,jsx,json,md,yaml,yml}" + run: pnpm -C libs/typescript format:check diff --git a/.github/workflows/ci-nix-linux.yml b/.github/workflows/ci-nix-linux.yml index 611a3b0cbc..127577a3fb 100644 --- a/.github/workflows/ci-nix-linux.yml +++ b/.github/workflows/ci-nix-linux.yml @@ -1,14 +1,13 @@ name: "CI: Nix Linux Rust source" -# This is the automatic Nix lane. Desktop-heavy legacy checks live in -# nix-build.yml and nix-wayland.yml and are maintainer-dispatched instead. +# Nix owns reproducible Linux package and Rust-source builds. Interactive +# desktop behavior is owned by the Rust E2E workflows. on: pull_request: paths: - "flake.nix" - "flake.lock" - - "nix/cua-driver/package.nix" - - "nix/cua-driver/tests/rust-unit.nix" + - "nix/cua-driver/**" - "libs/cua-driver/rust/**" - ".github/workflows/ci-nix-linux.yml" push: @@ -16,8 +15,7 @@ on: paths: - "flake.nix" - "flake.lock" - - "nix/cua-driver/package.nix" - - "nix/cua-driver/tests/rust-unit.nix" + - "nix/cua-driver/**" - "libs/cua-driver/rust/**" - ".github/workflows/ci-nix-linux.yml" workflow_dispatch: @@ -41,5 +39,10 @@ jobs: with: extra_nix_config: | experimental-features = nix-command flakes - - name: Run source-built Rust check - run: nix build .#checks.x86_64-linux.cua-driver-linux-rust-unit --print-build-logs --show-trace + - name: Build package and source-owned Rust tests + run: >- + nix build + .#checks.x86_64-linux.cua-compositor-build + .#checks.x86_64-linux.cua-driver-build + .#checks.x86_64-linux.cua-driver-linux-rust-unit + --print-build-logs --show-trace diff --git a/.github/workflows/ci-release-reminder.yml b/.github/workflows/ci-release-reminder.yml index dea14f8774..791aaa45c3 100644 --- a/.github/workflows/ci-release-reminder.yml +++ b/.github/workflows/ci-release-reminder.yml @@ -111,6 +111,14 @@ jobs: BODY+="\nOr add \`no-release\` to skip." fi + if printf '%s\n' "${AFFECTED[@]}" | grep -qx 'cua-driver-rs'; then + BODY+="\n\n### cua-driver desktop release validation\n\n" + BODY+="Before adding the release label, a maintainer records green runs on the exact release SHA:\n" + BODY+="- [ ] [Linux X11 canonical matrix](https://github.com/${REPO}/actions/workflows/e2e-rust-linux.yml)\n" + BODY+="- [ ] [Linux Sway canonical matrix](https://github.com/${REPO}/actions/workflows/e2e-rust-linux-wayland.yml) with environment \`sway\`\n" + BODY+="- [ ] Result links and remaining experimental environment gaps are recorded in the PR\n" + fi + # Delete previous reminder comment if exists, then post new one EXISTING_COMMENT_ID=$(gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \ --jq ".[] | select(.body | contains(\"${COMMENT_MARKER}\")) | .id" 2>/dev/null | head -1) diff --git a/.github/workflows/ci-rust-linux.yml b/.github/workflows/ci-rust-linux.yml index e2d050aa78..40c2cfee03 100644 --- a/.github/workflows/ci-rust-linux.yml +++ b/.github/workflows/ci-rust-linux.yml @@ -48,7 +48,10 @@ jobs: sudo apt-get update sudo apt-get install -y --no-install-recommends \ clang pkg-config libdbus-1-dev libpipewire-0.3-dev libspa-0.2-dev \ - libei-dev libx11-dev libxi-dev libxtst-dev libxext-dev + libei-dev libxkbcommon-dev libx11-dev libxi-dev libxtst-dev libxext-dev - name: Run Linux Rust tests working-directory: libs/cua-driver/rust run: cargo test -p cua-driver -p cua-driver-core -p cua-driver-testkit -p platform-linux --all-targets --locked + - name: Compile portable GNOME/KDE portal input + working-directory: libs/cua-driver/rust + run: cargo check -p cua-driver --features portal-input --locked diff --git a/.github/workflows/ci-rust-windows.yml b/.github/workflows/ci-rust-windows.yml index facc3f2fa2..cbd7af1138 100644 --- a/.github/workflows/ci-rust-windows.yml +++ b/.github/workflows/ci-rust-windows.yml @@ -10,7 +10,6 @@ on: - "libs/cua-driver/rust/crates/cua-driver-testkit/**" - "libs/cua-driver/rust/crates/platform-windows/**" - "libs/cua-driver/rust/crates/cua-driver-uia/**" - - "libs/cua-driver/rust/crates/focus-monitor-win/**" - "libs/cua-driver/tests/fixtures/**" - ".github/workflows/ci-rust-windows.yml" push: @@ -43,4 +42,4 @@ jobs: working-directory: libs/cua-driver/rust # Compile every Rust target without executing desktop-dependent integration # tests; interactive behavior belongs in e2e-rust-windows.yml. - run: cargo test -p cua-driver -p cua-driver-core -p cua-driver-testkit -p platform-windows -p cua-driver-uia -p focus-monitor-win --all-targets --no-run --locked + run: cargo test -p cua-driver -p cua-driver-core -p cua-driver-testkit -p platform-windows -p cua-driver-uia --all-targets --no-run --locked diff --git a/.github/workflows/claude-auto-fix.yml b/.github/workflows/claude-auto-fix.yml index f978d156fb..563b90a206 100644 --- a/.github/workflows/claude-auto-fix.yml +++ b/.github/workflows/claude-auto-fix.yml @@ -252,7 +252,7 @@ jobs: TYPESCRIPT TOOLS: - prettier: code formatter - - TypeScript: type checking via node ./scripts/typescript-typecheck.js + - TypeScript: type checking via pnpm -C libs/typescript typecheck - pnpm: package manager TASK: @@ -269,10 +269,10 @@ jobs: - Formatting: uv run black . - Linting: uv run ruff check . --fix For TypeScript lint failures: - - Formatting: pnpm prettier --write "**/*.{ts,tsx,js,jsx,json,md,yaml,yml}" + - Formatting: pnpm -C libs/typescript format After auto-fixing, verify the fix passes: - Python: uv run isort --check-only . && uv run black --check . && uv run ruff check . - - TypeScript: pnpm prettier --check "**/*.{ts,tsx,js,jsx,json,md,yaml,yml}" + - TypeScript: pnpm -C libs/typescript format:check FIXING TEST FAILURES: For Python test failures: @@ -280,7 +280,7 @@ jobs: - The test environment uses CUA_TELEMETRY_ENABLED=false - Tests use pytest-asyncio with asyncio_mode=auto For TypeScript build/type failures: - - Check types: node ./scripts/typescript-typecheck.js + - Check types: pnpm -C libs/typescript typecheck - Build: cd libs/typescript && pnpm install && pnpm build IMPORTANT: diff --git a/.github/workflows/e2e-rust-linux-wayland.yml b/.github/workflows/e2e-rust-linux-wayland.yml index 1d80fea196..3bd8192bf0 100644 --- a/.github/workflows/e2e-rust-linux-wayland.yml +++ b/.github/workflows/e2e-rust-linux-wayland.yml @@ -1,10 +1,32 @@ -name: "E2E: Rust Linux Wayland interactive" +name: "E2E: Rust Linux Wayland" on: workflow_dispatch: inputs: ref: - description: "Optional commit, branch, or tag override" + description: "Optional full 40-character commit SHA; defaults to dispatch SHA" + required: false + default: "" + environment: + description: "Wayland environment to validate" + required: false + type: choice + default: sway + options: + - sway + - cua-compositor + lane: + description: "Diagnostic lane; canonical dispatches use all" + required: false + type: choice + default: all + options: + - all + - shared + - native + - capture + cell_filter: + description: "Optional shared-cell substring for targeted diagnostics" required: false default: "" @@ -13,28 +35,70 @@ permissions: actions: read jobs: + source: + name: "Resolve exact source" + runs-on: ubuntu-latest + outputs: + sha: ${{ steps.resolve.outputs.sha }} + lanes: ${{ steps.resolve.outputs.lanes }} + steps: + - id: resolve + name: Validate source SHA + shell: bash + env: + REQUESTED_SHA: ${{ inputs.ref }} + DISPATCH_SHA: ${{ github.sha }} + run: | + sha="${REQUESTED_SHA:-$DISPATCH_SHA}" + if [[ ! "$sha" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "ref must be a full 40-character commit SHA" >&2 + exit 2 + fi + echo "sha=${sha,,}" >> "$GITHUB_OUTPUT" + case "${{ inputs.lane }}" in + all) echo 'lanes=["shared","native","capture"]' >> "$GITHUB_OUTPUT" ;; + shared|native|capture) printf 'lanes=["%s"]\n' "${{ inputs.lane }}" >> "$GITHUB_OUTPUT" ;; + *) echo "unsupported lane: ${{ inputs.lane }}" >&2; exit 2 ;; + esac + wayland: - name: "Linux / native Wayland Rust matrix" + name: "Linux / ${{ inputs.environment }} / ${{ matrix.lane }}" + needs: source runs-on: ubuntu-latest timeout-minutes: 90 + strategy: + fail-fast: false + matrix: + lane: ${{ fromJSON(needs.source.outputs.lanes) }} + env: + CUA_E2E_SOURCE_SHA: ${{ needs.source.outputs.sha }} + CUA_E2E_INTERNAL_LANE: ${{ matrix.lane }} + CUA_E2E_CELL_FILTER: ${{ inputs.cell_filter }} + CUA_E2E_WAYLAND_SESSION: ${{ inputs.environment }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: - ref: ${{ inputs.ref || github.ref }} + ref: ${{ needs.source.outputs.sha }} - name: Install Nix uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30 with: extra_nix_config: | experimental-features = nix-command flakes - name: Run complete native Wayland matrix - run: >- - nix develop .#cua-driver-wayland-e2e - -c scripts/ci/linux/run-rust-e2e-wayland.sh + shell: bash + run: | + if [[ "${{ inputs.environment }}" == "cua-compositor" ]]; then + nix develop .#cua-driver-inject-e2e \ + -c scripts/ci/linux/run-rust-e2e-inject.sh + else + nix develop .#cua-driver-wayland-e2e \ + -c scripts/ci/linux/run-rust-e2e-wayland.sh + fi - name: Upload Wayland results if: always() uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 with: - name: rust-linux-wayland + name: rust-linux-wayland-${{ inputs.environment }}-${{ matrix.lane }} path: artifacts/cua-driver/linux if-no-files-found: ignore compression-level: 0 @@ -42,36 +106,78 @@ jobs: summary: if: always() - needs: [wayland] - name: "Linux / native Wayland summary" + needs: [source, wayland] + name: "Linux / pure Wayland summary" runs-on: ubuntu-latest timeout-minutes: 10 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: - ref: ${{ inputs.ref || github.ref }} + ref: ${{ needs.source.outputs.sha }} - name: Download Wayland results continue-on-error: true uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 with: - name: rust-linux-wayland - path: artifacts/rust-linux-wayland + pattern: rust-linux-wayland-${{ inputs.environment }}-* + path: artifacts - name: Publish typed matrix summary shell: bash env: GH_TOKEN: ${{ github.token }} run: | - summary="artifacts/rust-linux-wayland/summary.md" - artifact_id=$(gh api \ - "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts?per_page=100" \ - --jq '.artifacts[] | select(.name == "rust-linux-wayland") | .id' | head -n 1) - if [[ -f "$summary" && -n "$artifact_id" ]]; then - artifact_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts/$artifact_id" - scripts/ci/link-e2e-evidence.sh "$summary" "$artifact_url" >> "$GITHUB_STEP_SUMMARY" - elif [[ -f "$summary" ]]; then - cat "$summary" >> "$GITHUB_STEP_SUMMARY" - else - echo "# CUA Rust Linux Wayland E2E" >> "$GITHUB_STEP_SUMMARY" - echo >> "$GITHUB_STEP_SUMMARY" - echo "No typed matrix summary was produced; inspect the Wayland job log." >> "$GITHUB_STEP_SUMMARY" + output=matrix-summary.md + artifacts_json=$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts?per_page=100") + { + echo "# CUA Rust Linux Wayland E2E: ${{ inputs.environment }}" + echo + echo "Shared, native, and capture lanes run independently on clean runners." + } > "$output" + found=0 + while IFS= read -r summary; do + found=1 + artifact=$(basename "$(dirname "$summary")") + artifact_id=$(jq -r --arg name "$artifact" \ + '.artifacts[] | select(.name == $name) | .id' <<< "$artifacts_json" | head -n 1) + echo >> "$output" + echo "## $artifact" >> "$output" + if [[ -n "$artifact_id" && "$artifact_id" != "null" ]]; then + artifact_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts/$artifact_id" + scripts/ci/link-e2e-evidence.sh "$summary" "$artifact_url" >> "$output" + else + cat "$summary" >> "$output" + fi + done < <(find artifacts -type f -name summary.md -print | sort) + if [[ "$found" == 0 ]]; then + echo >> "$output" + echo "No typed lane summary was produced; inspect the lane logs." >> "$output" fi + { + echo + echo "## Trajectory videos" + echo + echo "| Lane | Videos | Artifact |" + echo "| --- | ---: | --- |" + } >> "$output" + for lane in shared native capture; do + artifact="rust-linux-wayland-${{ inputs.environment }}-$lane" + recording_dir="artifacts/$artifact/recordings" + video_count=0 + [[ -d "$recording_dir" ]] && video_count=$(find "$recording_dir" -type f -name recording.mp4 | wc -l | tr -d ' ') + artifact_id=$(jq -r --arg name "$artifact" \ + '.artifacts[] | select(.name == $name) | .id' <<< "$artifacts_json" | head -n 1) + if [[ -n "$artifact_id" && "$artifact_id" != "null" ]]; then + artifact_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts/$artifact_id" + link="[Open artifact]($artifact_url)" + else + link="Not produced" + fi + echo "| $lane | $video_count | $link |" >> "$output" + done + cat "$output" >> "$GITHUB_STEP_SUMMARY" + - name: Upload combined summary + if: always() + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 + with: + name: rust-linux-wayland-matrix-summary + path: matrix-summary.md + if-no-files-found: error diff --git a/.github/workflows/e2e-rust-linux.yml b/.github/workflows/e2e-rust-linux.yml index 147462dd79..d86e4c5894 100644 --- a/.github/workflows/e2e-rust-linux.yml +++ b/.github/workflows/e2e-rust-linux.yml @@ -4,29 +4,62 @@ on: workflow_dispatch: inputs: ref: - description: "Commit, branch, or tag to test" - required: true - default: "main" - suite: - description: "Rust desktop suite" - required: true + description: "Optional full 40-character commit SHA; defaults to dispatch SHA" + required: false + default: "" + lane: + description: "Diagnostic lane; canonical dispatches use all" + required: false type: choice - options: [shared, modality, all] - default: shared + default: all + options: + - all + - shared + - native + - capture + cell_filter: + description: "Optional shared-cell substring for targeted diagnostics" + required: false + default: "" permissions: contents: read + actions: read jobs: + source: + name: "Resolve exact source" + runs-on: ubuntu-latest + outputs: + sha: ${{ steps.resolve.outputs.sha }} + steps: + - id: resolve + name: Validate source SHA + shell: bash + env: + REQUESTED_SHA: ${{ inputs.ref }} + DISPATCH_SHA: ${{ github.sha }} + run: | + sha="${REQUESTED_SHA:-$DISPATCH_SHA}" + if [[ ! "$sha" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "ref must be a full 40-character commit SHA" >&2 + exit 2 + fi + echo "sha=${sha,,}" >> "$GITHUB_OUTPUT" + shared: - if: inputs.suite == 'shared' || inputs.suite == 'all' - name: "Linux / shared Electron" + if: inputs.lane == 'all' || inputs.lane == 'shared' + name: "Linux / shared Electron + Tauri" + needs: source runs-on: ubuntu-latest timeout-minutes: 60 + env: + CUA_E2E_SOURCE_SHA: ${{ needs.source.outputs.sha }} + CUA_E2E_CELL_FILTER: ${{ inputs.cell_filter }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: - ref: ${{ inputs.ref }} + ref: ${{ needs.source.outputs.sha }} - uses: dtolnay/rust-toolchain@stable - name: Install GUI and Rust build dependencies run: | @@ -38,10 +71,12 @@ jobs: libwebkit2gtk-4.1-dev libssl-dev libxdo-dev \ libayatana-appindicator3-dev librsvg2-dev ffmpeg - name: Run shared Rust behavior matrix + env: + CUA_E2E_INTERNAL_LANE: shared run: | xvfb-run -a --server-args="-screen 0 1920x1080x24" \ dbus-run-session -- bash -lc \ - "scripts/ci/linux/run-rust-e2e.sh --suite shared" + "openbox >/tmp/cua-openbox.log 2>&1 & sleep 2; scripts/ci/linux/run-rust-e2e.sh" - name: Upload shared results if: always() uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 @@ -52,15 +87,18 @@ jobs: compression-level: 0 retention-days: 14 - modality: - if: inputs.suite == 'modality' || inputs.suite == 'all' - name: "Linux / modality and desktop scope" + native: + if: inputs.lane == 'all' || inputs.lane == 'native' + name: "Linux / GTK3 native harness" + needs: source runs-on: ubuntu-latest timeout-minutes: 60 + env: + CUA_E2E_SOURCE_SHA: ${{ needs.source.outputs.sha }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: - ref: ${{ inputs.ref }} + ref: ${{ needs.source.outputs.sha }} - uses: dtolnay/rust-toolchain@stable - name: Install GUI and Rust build dependencies run: | @@ -71,16 +109,57 @@ jobs: libspa-0.2-dev libei-dev libx11-dev libxi-dev libxtst-dev libxext-dev \ libwebkit2gtk-4.1-dev libssl-dev libxdo-dev \ libayatana-appindicator3-dev librsvg2-dev ffmpeg - - name: Run modality Rust matrix + - name: Run GTK3 native Rust harness + env: + CUA_E2E_INTERNAL_LANE: native run: | xvfb-run -a --server-args="-screen 0 1920x1080x24" \ dbus-run-session -- bash -lc \ - "scripts/ci/linux/run-rust-e2e.sh --suite modality" - - name: Upload modality results + "openbox >/tmp/cua-openbox.log 2>&1 & sleep 2; scripts/ci/linux/run-rust-e2e.sh" + - name: Upload native results if: always() uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 with: - name: rust-linux-modality + name: rust-linux-native + path: artifacts/cua-driver/linux + if-no-files-found: ignore + compression-level: 0 + retention-days: 14 + + capture: + if: inputs.lane == 'all' || inputs.lane == 'capture' + name: "Linux / capture and desktop scope" + needs: source + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + CUA_E2E_SOURCE_SHA: ${{ needs.source.outputs.sha }} + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: ${{ needs.source.outputs.sha }} + - uses: dtolnay/rust-toolchain@stable + - name: Install GUI and Rust build dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + xvfb dbus-x11 at-spi2-core openbox python3-gi gir1.2-gtk-3.0 \ + libgtk-3-dev clang pkg-config libdbus-1-dev libpipewire-0.3-dev \ + libspa-0.2-dev libei-dev libx11-dev libxi-dev libxtst-dev libxext-dev \ + libwebkit2gtk-4.1-dev libssl-dev libxdo-dev \ + libayatana-appindicator3-dev librsvg2-dev ffmpeg + - name: Run capture and desktop-scope contracts + env: + CUA_E2E_INTERNAL_LANE: capture + run: | + xvfb-run -a --server-args="-screen 0 1920x1080x24" \ + dbus-run-session -- bash -lc \ + "openbox >/tmp/cua-openbox.log 2>&1 & sleep 2; scripts/ci/linux/run-rust-e2e.sh" + - name: Upload capture results + if: always() + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 + with: + name: rust-linux-capture path: artifacts/cua-driver/linux if-no-files-found: ignore compression-level: 0 @@ -88,31 +167,89 @@ jobs: summary: if: always() - needs: [shared, modality] + needs: [source, shared, native, capture] name: "Linux / matrix summary" runs-on: ubuntu-latest timeout-minutes: 10 steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: ${{ needs.source.outputs.sha }} - name: Download lane results uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 with: path: artifacts - name: Publish matrix summary shell: bash + env: + GH_TOKEN: ${{ github.token }} run: | + summary_path=matrix-summary.md { echo "# CUA Rust Linux E2E matrix" echo echo "The lane jobs above are independent; a failure in one lane does not hide the others." echo - } >> "$GITHUB_STEP_SUMMARY" + } > "$summary_path" + artifacts_json=$(gh api \ + "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts?per_page=100") found=0 while IFS= read -r summary; do found=1 - echo "## $(basename "$(dirname "$summary")")" >> "$GITHUB_STEP_SUMMARY" - cat "$summary" >> "$GITHUB_STEP_SUMMARY" - echo >> "$GITHUB_STEP_SUMMARY" + artifact=$(basename "$(dirname "$summary")") + echo "## $artifact" >> "$summary_path" + artifact_id=$(jq -r --arg name "$artifact" \ + '.artifacts[] | select(.name == $name) | .id' <<< "$artifacts_json" | head -n 1) + if [[ -n "$artifact_id" && "$artifact_id" != "null" ]]; then + artifact_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts/$artifact_id" + scripts/ci/link-e2e-evidence.sh "$summary" "$artifact_url" >> "$summary_path" + else + cat "$summary" >> "$summary_path" + fi + echo >> "$summary_path" done < <(find artifacts -type f -name summary.md -print | sort) if [[ "$found" == 0 ]]; then - echo "No lane summary artifact was produced." >> "$GITHUB_STEP_SUMMARY" + echo "No lane summary artifact was produced." >> "$summary_path" fi + { + echo + echo "## Trajectory videos" + echo + echo "Full-desktop MP4s are retained for 14 days inside each lane artifact." + echo + echo "| Lane | Videos | Artifact |" + echo "| --- | ---: | --- |" + } >> "$summary_path" + while IFS='|' read -r lane artifact; do + recording_dir="artifacts/$artifact/recordings" + if [[ -d "$recording_dir" ]]; then + video_count=$(find "$recording_dir" -type f -name recording.mp4 | wc -l | tr -d ' ') + else + video_count=0 + fi + artifact_id=$(jq -r --arg name "$artifact" \ + '.artifacts[] | select(.name == $name) | .id' <<< "$artifacts_json" | head -n 1) + if [[ -n "$artifact_id" && "$artifact_id" != "null" ]]; then + artifact_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts/$artifact_id" + artifact_link="[Download $artifact]($artifact_url)" + else + artifact_link="Not produced in this dispatch" + fi + echo "| $lane | $video_count | $artifact_link |" >> "$summary_path" + done <<'EOF' + Shared Electron + Tauri|rust-linux-shared + GTK3 native|rust-linux-native + Capture + desktop scope|rust-linux-capture + EOF + { + echo + echo "Each evidence link opens its owning lane artifact; the row text is the exact \`recordings/-pid-/recording.mp4\` path, with an adjacent \`trajectory.json\`." + } >> "$summary_path" + cat "$summary_path" >> "$GITHUB_STEP_SUMMARY" + - name: Upload matrix summary + if: always() + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 + with: + name: rust-linux-matrix-summary + path: matrix-summary.md + if-no-files-found: error diff --git a/.github/workflows/e2e-rust-windows.yml b/.github/workflows/e2e-rust-windows.yml index f567bde6de..0047999813 100644 --- a/.github/workflows/e2e-rust-windows.yml +++ b/.github/workflows/e2e-rust-windows.yml @@ -4,15 +4,9 @@ on: workflow_dispatch: inputs: ref: - description: "Commit, branch, or tag to test" - required: true - default: "main" - suite: - description: "Rust desktop suite" - required: true - type: choice - options: [default, guard, shared, native, modality, all] - default: shared + description: "Optional full 40-character commit SHA; defaults to dispatch SHA" + required: false + default: "" runner: description: "Runner label; use the Azure RDP runner label for VM e2e" required: true @@ -23,83 +17,37 @@ permissions: actions: read jobs: - default: - if: inputs.suite == 'default' || inputs.suite == 'all' - name: "Windows / default Rust tests" - runs-on: ${{ inputs.runner }} - timeout-minutes: 90 - steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - ref: ${{ inputs.ref }} - - name: Ensure FFmpeg for trajectory video - shell: pwsh - run: | - if (-not (Get-Command ffmpeg.exe -ErrorAction SilentlyContinue)) { - choco install ffmpeg -y --no-progress - } - ffmpeg -version - ffprobe -version - - name: Run default Rust tests - shell: pwsh - run: .\scripts\ci\windows\run-rust-e2e.ps1 -Suite default -RequireGui - - name: Collect logs - if: always() - shell: pwsh - run: .\scripts\ci\windows\collect-artifacts.ps1 - - name: Upload default results - if: always() - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 - with: - name: rust-windows-default - path: artifacts/cua-driver/windows - if-no-files-found: ignore - compression-level: 0 - retention-days: 14 - - guard: - if: inputs.suite == 'guard' || inputs.suite == 'all' - name: "Windows / UX guards" - runs-on: ${{ inputs.runner }} - timeout-minutes: 90 + source: + name: "Resolve exact source" + runs-on: ubuntu-latest + outputs: + sha: ${{ steps.resolve.outputs.sha }} steps: - - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - ref: ${{ inputs.ref }} - - name: Ensure FFmpeg for trajectory video - shell: pwsh + - id: resolve + name: Validate source SHA + shell: bash + env: + REQUESTED_SHA: ${{ inputs.ref }} + DISPATCH_SHA: ${{ github.sha }} run: | - if (-not (Get-Command ffmpeg.exe -ErrorAction SilentlyContinue)) { - choco install ffmpeg -y --no-progress - } - ffmpeg -version - ffprobe -version - - name: Run UX guard tests - shell: pwsh - run: .\scripts\ci\windows\run-rust-e2e.ps1 -Suite guard -RequireGui - - name: Collect logs - if: always() - shell: pwsh - run: .\scripts\ci\windows\collect-artifacts.ps1 - - name: Upload guard results - if: always() - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 - with: - name: rust-windows-guard - path: artifacts/cua-driver/windows - if-no-files-found: ignore - compression-level: 0 - retention-days: 14 + sha="${REQUESTED_SHA:-$DISPATCH_SHA}" + if [[ ! "$sha" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "ref must be a full 40-character commit SHA" >&2 + exit 2 + fi + echo "sha=${sha,,}" >> "$GITHUB_OUTPUT" shared: - if: inputs.suite == 'shared' || inputs.suite == 'all' name: "Windows / shared Electron + Tauri" + needs: source runs-on: ${{ inputs.runner }} timeout-minutes: 90 + env: + CUA_E2E_SOURCE_SHA: ${{ needs.source.outputs.sha }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: - ref: ${{ inputs.ref }} + ref: ${{ needs.source.outputs.sha }} - name: Ensure FFmpeg for trajectory video shell: pwsh run: | @@ -110,7 +58,9 @@ jobs: ffprobe -version - name: Run shared Rust behavior matrix shell: pwsh - run: .\scripts\ci\windows\run-rust-e2e.ps1 -Suite shared -RequireGui + env: + CUA_E2E_INTERNAL_LANE: shared + run: .\scripts\ci\windows\run-rust-e2e.ps1 -RequireGui - name: Collect logs if: always() shell: pwsh @@ -126,14 +76,16 @@ jobs: retention-days: 14 native: - if: inputs.suite == 'native' || inputs.suite == 'all' - name: "Windows / native WPF + WebView2" + name: "Windows / native WPF + WinUI3 + WebView2" + needs: source runs-on: ${{ inputs.runner }} timeout-minutes: 90 + env: + CUA_E2E_SOURCE_SHA: ${{ needs.source.outputs.sha }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: - ref: ${{ inputs.ref }} + ref: ${{ needs.source.outputs.sha }} - name: Ensure FFmpeg for trajectory video shell: pwsh run: | @@ -144,7 +96,9 @@ jobs: ffprobe -version - name: Run native Rust harnesses shell: pwsh - run: .\scripts\ci\windows\run-rust-e2e.ps1 -Suite native -RequireGui + env: + CUA_E2E_INTERNAL_LANE: native + run: .\scripts\ci\windows\run-rust-e2e.ps1 -RequireGui - name: Collect logs if: always() shell: pwsh @@ -159,15 +113,17 @@ jobs: compression-level: 0 retention-days: 14 - modality: - if: inputs.suite == 'modality' || inputs.suite == 'all' - name: "Windows / modality input E2E" + capture: + name: "Windows / capture and desktop scope" + needs: source runs-on: ${{ inputs.runner }} timeout-minutes: 90 + env: + CUA_E2E_SOURCE_SHA: ${{ needs.source.outputs.sha }} steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: - ref: ${{ inputs.ref }} + ref: ${{ needs.source.outputs.sha }} - name: Ensure FFmpeg for trajectory video shell: pwsh run: | @@ -176,18 +132,20 @@ jobs: } ffmpeg -version ffprobe -version - - name: Run modality input E2E + - name: Run capture and desktop-scope contracts shell: pwsh - run: .\scripts\ci\windows\run-rust-e2e.ps1 -Suite modality -RequireGui + env: + CUA_E2E_INTERNAL_LANE: capture + run: .\scripts\ci\windows\run-rust-e2e.ps1 -RequireGui - name: Collect logs if: always() shell: pwsh run: .\scripts\ci\windows\collect-artifacts.ps1 - - name: Upload modality results + - name: Upload capture results if: always() uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4 with: - name: rust-windows-modality + name: rust-windows-capture path: artifacts/cua-driver/windows if-no-files-found: ignore compression-level: 0 @@ -195,11 +153,14 @@ jobs: summary: if: always() - needs: [default, guard, shared, native, modality] + needs: [source, shared, native, capture] name: "Windows / matrix summary" runs-on: ubuntu-latest timeout-minutes: 10 steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + ref: ${{ needs.source.outputs.sha }} - name: Download lane results uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4 with: @@ -216,19 +177,27 @@ jobs: echo "The lane jobs above are independent; a failure in one lane does not hide the others." echo } > "$summary_path" + artifacts_json=$(gh api \ + "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts?per_page=100") found=0 while IFS= read -r summary; do found=1 - echo "## $(basename "$(dirname "$summary")")" >> "$summary_path" - cat "$summary" >> "$summary_path" + artifact=$(basename "$(dirname "$summary")") + echo "## $artifact" >> "$summary_path" + artifact_id=$(jq -r --arg name "$artifact" \ + '.artifacts[] | select(.name == $name) | .id' <<< "$artifacts_json" | head -n 1) + if [[ -n "$artifact_id" && "$artifact_id" != "null" ]]; then + artifact_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts/$artifact_id" + scripts/ci/link-e2e-evidence.sh "$summary" "$artifact_url" >> "$summary_path" + else + cat "$summary" >> "$summary_path" + fi echo >> "$summary_path" done < <(find artifacts -type f -name summary.md -print | sort) if [[ "$found" == 0 ]]; then echo "No lane summary artifact was produced." >> "$summary_path" fi - artifacts_json=$(gh api \ - "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts?per_page=100") { echo echo "## Trajectory videos" @@ -255,15 +224,13 @@ jobs: fi echo "| $lane | $video_count | $artifact_link |" >> "$summary_path" done <<'EOF' - Default Rust|rust-windows-default - UX guards|rust-windows-guard Electron + Tauri|rust-windows-shared WPF + WinUI3 + WebView2|rust-windows-native - Modality input|rust-windows-modality + Capture + desktop scope|rust-windows-capture EOF { echo - echo "Each artifact stores videos under \`recordings//recording.mp4\` with a matching \`trajectory.json\`." + echo "Each evidence link opens its owning lane artifact; the row text is the exact \`recordings/-pid-/recording.mp4\` path, with an adjacent \`trajectory.json\`." } >> "$summary_path" cat "$summary_path" >> "$GITHUB_STEP_SUMMARY" - name: Upload matrix summary diff --git a/.github/workflows/nix-build.yml b/.github/workflows/nix-build.yml deleted file mode 100644 index 8cf029dbaa..0000000000 --- a/.github/workflows/nix-build.yml +++ /dev/null @@ -1,437 +0,0 @@ -name: Nix Build & Integration Tests - -on: - # The legacy GUI/toolkit matrix is expensive and includes GIF diagnostics. - # It remains available for maintainers, while ci-nix-linux.yml owns the - # automatic source-built Rust check. - workflow_dispatch: - -permissions: - id-token: write - contents: read - pull-requests: write - -env: - AWS_REGION: us-west-2 - NIX_CACHE_BUCKET: trycua-nix-cache - NIX_CACHE_SECRET: nix-cache/trycua-nix-cache/signing-key - -jobs: - nix-checks: - name: ${{ matrix.name }} - runs-on: ubuntu-latest - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - - name: NixOS integration test - check_attr: cua-driver-integration - timeout_minutes: 8 - visual: false - result_link: result-cua-driver-integration - artifact_name: "" - - name: NixOS set_config persistence test - check_attr: cua-driver-set-config - timeout_minutes: 8 - visual: false - result_link: result-cua-driver-set-config - artifact_name: "" - - name: Linux cursor click GIF test - check_attr: cua-driver-linux-cursor-click-gif - timeout_minutes: 12 - visual: true - result_link: result-linux-cursor-click-gif - artifact_name: cua-driver-linux-cursor-click-gif - - name: Linux background terminal GIF test - check_attr: cua-driver-linux-background-terminal-gif - timeout_minutes: 12 - visual: true - result_link: result-linux-background-terminal-gif - artifact_name: cua-driver-linux-background-terminal-gif - - name: Linux parallel multi-cursor drag test - check_attr: cua-driver-linux-parallel-drag-xserver - timeout_minutes: 20 - visual: true - result_link: result-linux-parallel-drag-xserver - artifact_name: cua-driver-linux-parallel-drag-xserver - # NOTE: the older "parallel multi-cursor drag GIF" scenario - # (linux-parallel-drag-gif.nix) is intentionally not in this matrix — - # it hand-launches Xorg, which can't get a VT/seat in the emulated - # nixos-test VM. The services.xserver entry above supersedes it; the - # old scenario file is kept for local/real-X runs. - # Full entries (CDP / Tk focus-free-write overrides) — kept as-is. - - name: Linux background GUI test (chromium) - check_attr: cua-driver-linux-background-gui-chromium - timeout_minutes: 25 - visual: true - result_link: result-linux-background-gui-chromium - artifact_name: cua-driver-linux-background-gui-chromium - - name: Linux background GUI test (tk) - check_attr: cua-driver-linux-background-gui-tk - timeout_minutes: 15 - visual: true - result_link: result-linux-background-gui-tk - artifact_name: cua-driver-linux-background-gui-tk - # Real-app READ-ONLY skeleton matrix (5 per toolkit category). - # GTK3 - - name: Linux background GUI test (gtk3-gedit) - check_attr: cua-driver-linux-background-gui-gtk3-gedit - timeout_minutes: 15 - visual: true - result_link: result-linux-background-gui-gtk3-gedit - artifact_name: cua-driver-linux-background-gui-gtk3-gedit - - name: Linux background GUI test (gtk3-mousepad) - check_attr: cua-driver-linux-background-gui-gtk3-mousepad - timeout_minutes: 15 - visual: true - result_link: result-linux-background-gui-gtk3-mousepad - artifact_name: cua-driver-linux-background-gui-gtk3-mousepad - # gtk3-geany temporarily disabled (huge AT-SPI tree grinds the emulated VM; job times out) - # - name: Linux background GUI test (gtk3-geany) - # check_attr: cua-driver-linux-background-gui-gtk3-geany - # timeout_minutes: 15 - # visual: true - # result_link: result-linux-background-gui-gtk3-geany - # artifact_name: cua-driver-linux-background-gui-gtk3-geany - - name: Linux background GUI test (gtk3-scite) - check_attr: cua-driver-linux-background-gui-gtk3-scite - timeout_minutes: 15 - visual: true - result_link: result-linux-background-gui-gtk3-scite - artifact_name: cua-driver-linux-background-gui-gtk3-scite - # gtk3-abiword temporarily disabled (huge AT-SPI tree grinds the emulated VM; job times out) - # - name: Linux background GUI test (gtk3-abiword) - # check_attr: cua-driver-linux-background-gui-gtk3-abiword - # timeout_minutes: 15 - # visual: true - # result_link: result-linux-background-gui-gtk3-abiword - # artifact_name: cua-driver-linux-background-gui-gtk3-abiword - # GTK4 - - name: Linux background GUI test (gtk4-characters) - check_attr: cua-driver-linux-background-gui-gtk4-characters - timeout_minutes: 15 - visual: true - result_link: result-linux-background-gui-gtk4-characters - artifact_name: cua-driver-linux-background-gui-gtk4-characters - # Qt5 - - name: Linux background GUI test (qt5-manuskript) - check_attr: cua-driver-linux-background-gui-qt5-manuskript - timeout_minutes: 15 - visual: true - result_link: result-linux-background-gui-qt5-manuskript - artifact_name: cua-driver-linux-background-gui-qt5-manuskript - - name: Linux background GUI test (qt5-klog) - check_attr: cua-driver-linux-background-gui-qt5-klog - timeout_minutes: 15 - visual: true - result_link: result-linux-background-gui-qt5-klog - artifact_name: cua-driver-linux-background-gui-qt5-klog - - name: Linux background GUI test (qt5-openambit) - check_attr: cua-driver-linux-background-gui-qt5-openambit - timeout_minutes: 15 - visual: true - result_link: result-linux-background-gui-qt5-openambit - artifact_name: cua-driver-linux-background-gui-qt5-openambit - # Qt6 - - name: Linux background GUI test (qt6-kate) - check_attr: cua-driver-linux-background-gui-qt6-kate - timeout_minutes: 15 - visual: true - result_link: result-linux-background-gui-qt6-kate - artifact_name: cua-driver-linux-background-gui-qt6-kate - - name: Linux background GUI test (qt6-kcalc) - check_attr: cua-driver-linux-background-gui-qt6-kcalc - timeout_minutes: 15 - visual: true - result_link: result-linux-background-gui-qt6-kcalc - artifact_name: cua-driver-linux-background-gui-qt6-kcalc - - name: Linux background GUI test (qt6-okular) - check_attr: cua-driver-linux-background-gui-qt6-okular - timeout_minutes: 15 - visual: true - result_link: result-linux-background-gui-qt6-okular - artifact_name: cua-driver-linux-background-gui-qt6-okular - - name: Linux background GUI test (qt6-qownnotes) - check_attr: cua-driver-linux-background-gui-qt6-qownnotes - timeout_minutes: 15 - visual: true - result_link: result-linux-background-gui-qt6-qownnotes - artifact_name: cua-driver-linux-background-gui-qt6-qownnotes - # Electron (heavy: more memory + 25-min timeout) - - name: Linux background GUI test (electron-zettlr) - check_attr: cua-driver-linux-background-gui-electron-zettlr - timeout_minutes: 25 - visual: true - result_link: result-linux-background-gui-electron-zettlr - artifact_name: cua-driver-linux-background-gui-electron-zettlr - - name: Linux background GUI test (electron-joplin) - check_attr: cua-driver-linux-background-gui-electron-joplin - timeout_minutes: 25 - visual: true - result_link: result-linux-background-gui-electron-joplin - artifact_name: cua-driver-linux-background-gui-electron-joplin - - name: Linux background GUI test (electron-logseq) - check_attr: cua-driver-linux-background-gui-electron-logseq - timeout_minutes: 25 - visual: true - result_link: result-linux-background-gui-electron-logseq - artifact_name: cua-driver-linux-background-gui-electron-logseq - # Firefox is temporarily disabled: under the emulated CI VM (no KVM) it - # does not surface its window within the launch timeout, so the job - # times out before any AT-SPI subtest runs. The browser/AT-SPI read - # path is already covered by the chromium job. Re-enable once launch is - # made reliable (longer timeout + pre-seeded first-run-free profile). - # - name: Linux background GUI test (firefox) - # check_attr: cua-driver-linux-background-gui-firefox - # timeout_minutes: 25 - # visual: false - # result_link: result-linux-background-gui-firefox - # artifact_name: "" - - steps: - - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - - name: Configure AWS Credentials via OIDC - uses: aws-actions/configure-aws-credentials@ff717079ee2060e4bcee96c4779b553acc87447c # v4 - with: - role-to-assume: arn:aws:iam::296062593712:role/github-actions-nix-cache - aws-region: ${{ env.AWS_REGION }} - - - name: Get Nix signing key - id: nix-key - run: | - SECRET=$(aws secretsmanager get-secret-value \ - --secret-id "${{ env.NIX_CACHE_SECRET }}" \ - --query 'SecretString' --output text) - - SECRET_KEY=$(echo "$SECRET" | jq -r '.secret_key') - echo "::add-mask::$SECRET_KEY" - echo "$SECRET_KEY" > "${{ runner.temp }}/signing-key.sec" - chmod 600 "${{ runner.temp }}/signing-key.sec" - - PUBLIC_KEY=$(echo "$SECRET" | jq -r '.public_key') - echo "public_key=$PUBLIC_KEY" >> "$GITHUB_OUTPUT" - - - name: Setup AWS credentials file for Nix - run: | - mkdir -p ~/.aws - printf '[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\nregion = %s\n' \ - "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$AWS_SESSION_TOKEN" "$AWS_REGION" > ~/.aws/credentials - chmod 600 ~/.aws/credentials - - sudo mkdir -p /root/.aws - sudo bash -c "printf '[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\nregion = %s\n' \ - '$AWS_ACCESS_KEY_ID' '$AWS_SECRET_ACCESS_KEY' '$AWS_SESSION_TOKEN' '$AWS_REGION' > /root/.aws/credentials" - sudo chmod 600 /root/.aws/credentials - - - name: Install Nix - uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30 - with: - extra_nix_config: | - experimental-features = nix-command flakes - # Container-based NixOS integration tests (testers.nixosTest with - # `containers`) run under systemd-nspawn instead of QEMU: no KVM - # needed, much faster/cheaper. The nspawn builds require an allocated - # UID range and cgroup delegation, advertised here on the builder. - extra-experimental-features = auto-allocate-uids cgroups - auto-allocate-uids = true - extra-system-features = uid-range - substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }} https://cache.nixos.org - trusted-substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }} - trusted-public-keys = ${{ steps.nix-key.outputs.public_key }} cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - - - name: Run ${{ matrix.name }} - timeout-minutes: ${{ matrix.timeout_minutes }} - run: | - nix build .#checks.x86_64-linux.${{ matrix.check_attr }} \ - --print-build-logs --show-trace \ - -o ${{ matrix.result_link }} - - - name: Collect GIF artifacts - if: always() && matrix.visual - run: | - mkdir -p artifacts - find -L "${{ matrix.result_link }}/" \( -name '*.gif' -o -name '*.png' -o -name '*.json' \) -type f -exec cp {} artifacts/ \; 2>/dev/null || true - ls -la artifacts/ 2>/dev/null || echo "No visual artifacts found" - - - name: Upload GIF artifacts - if: always() && matrix.visual - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: ${{ matrix.artifact_name }} - path: artifacts/* - if-no-files-found: warn - - - name: Sign and upload to Nix cache - if: always() - run: | - echo "Signing and uploading build artifacts to Nix cache..." - nix store sign --key-file "${{ runner.temp }}/signing-key.sec" --all - nix copy --to "s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }}&want-mass-query=true" --all -L - - - name: Cleanup signing key - if: always() - run: rm -f "${{ runner.temp }}/signing-key.sec" - - comment-linux-visual-artifacts: - name: Comment Linux visual artifacts - if: always() && github.event_name == 'pull_request' - needs: [nix-checks] - runs-on: ubuntu-latest - steps: - - name: Comment Linux visual artifacts on PR - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 - with: - script: | - const marker = ''; - const runUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - const artifactNames = [ - 'cua-driver-linux-cursor-click-gif', - 'cua-driver-linux-background-terminal-gif', - 'cua-driver-linux-parallel-drag-xserver', - 'cua-driver-linux-background-gui-chromium', - 'cua-driver-linux-background-gui-tk', - 'cua-driver-linux-background-gui-gtk3-gedit', - 'cua-driver-linux-background-gui-gtk3-mousepad', - 'cua-driver-linux-background-gui-gtk3-scite', - 'cua-driver-linux-background-gui-gtk4-characters', - 'cua-driver-linux-background-gui-qt5-manuskript', - 'cua-driver-linux-background-gui-qt5-klog', - 'cua-driver-linux-background-gui-qt5-openambit', - 'cua-driver-linux-background-gui-qt6-kate', - 'cua-driver-linux-background-gui-qt6-kcalc', - 'cua-driver-linux-background-gui-qt6-okular', - 'cua-driver-linux-background-gui-qt6-qownnotes', - 'cua-driver-linux-background-gui-electron-zettlr', - 'cua-driver-linux-background-gui-electron-joplin', - 'cua-driver-linux-background-gui-electron-logseq', - 'cua-driver-linux-som-overlays', - ]; - - let body = `${marker}\n## Linux visual regression artifacts\n\n`; - body += 'Matrix jobs now run independently. Download visual artifacts from this workflow run.\n'; - body += 'Each background-GUI job uploads a `.gif` of the interaction plus two annotated PNGs '; - body += '(`.png` raw, `-atspi.png` with AT-SPI element boxes); '; - body += 'the `cua-driver-linux-som-overlays` artifact adds `-som.png` cua Set-of-Marks overlays:\n'; - for (const artifactName of artifactNames) { - body += `- \`${artifactName}\`\n`; - } - body += `\n[Open workflow run and download artifacts](${runUrl})\n`; - - const comments = await github.paginate(github.rest.issues.listComments, { - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - per_page: 100, - }); - const existing = comments.find(comment => - comment.user?.type === 'Bot' && comment.body?.includes(marker) - ); - - if (existing) { - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: existing.id, - body, - }); - } else { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body, - }); - } - - # ── Set-of-Marks overlays ─────────────────────────────────────────────────── - # Downstream aggregate job: consumes the raw `.png` screenshots uploaded - # by the background-GUI matrix and emits `-som.png` cua Set-of-Marks - # overlays. Deliberately OFF the hot path — `needs: [nix-checks]` + `if: - # always()` so it runs after the matrix regardless of pass/fail and never - # blocks it. NOTE: `pip install -e libs/python/som` pulls torch + ultralytics + - # easyocr (~GBs) and downloads model weights at first parse, so this job is - # the slow/expensive one; it is intentionally isolated from the build matrix. - som-annotate: - name: Annotate screenshots with cua Set-of-Marks - if: always() - needs: [nix-checks] - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - - name: Download all run artifacts - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 - with: - path: downloaded-artifacts - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 - with: - python-version: "3.12" - - - name: Install cua-som - run: | - python -m pip install --upgrade pip - pip install -e libs/python/som - - - name: Generate Set-of-Marks overlays - run: | - mkdir -p som-overlays - python - <<'PY' - import base64, glob, os, sys, traceback - - # Collect raw background-GUI screenshots: .png produced by the - # skeleton matrix. Exclude the AT-SPI overlays (-atspi) and any prior - # SoM outputs (-som). Artifacts land under downloaded-artifacts//. - candidates = [] - for p in glob.glob("downloaded-artifacts/**/*-background-gui-*.png", recursive=True): - stem = os.path.basename(p) - if stem.endswith("-atspi.png") or stem.endswith("-som.png"): - continue - candidates.append(p) - candidates = sorted(set(candidates)) - print(f"Found {len(candidates)} raw screenshot(s) to annotate:", flush=True) - for p in candidates: - print(f" {p}", flush=True) - - if not candidates: - print("No raw screenshots found; nothing to annotate.", flush=True) - sys.exit(0) - - from som import OmniParser - parser = OmniParser() - - ok = 0 - for p in candidates: - try: - with open(p, "rb") as f: - data = f.read() - result = parser.parse(data) - b64 = result.annotated_image_base64 - # Strip a possible data URL prefix before decoding. - if "," in b64 and b64.strip().startswith("data:"): - b64 = b64.split(",", 1)[1] - out_name = os.path.splitext(os.path.basename(p))[0] + "-som.png" - out_path = os.path.join("som-overlays", out_name) - with open(out_path, "wb") as f: - f.write(base64.b64decode(b64)) - print(f"OK {p} -> {out_path}", flush=True) - ok += 1 - except Exception: - print(f"FAIL {p}", flush=True) - traceback.print_exc() - print(f"Annotated {ok}/{len(candidates)} screenshot(s).", flush=True) - PY - ls -la som-overlays/ 2>/dev/null || echo "No SoM overlays produced" - - - name: Upload Set-of-Marks overlays - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: cua-driver-linux-som-overlays - path: som-overlays/*-som.png - if-no-files-found: warn diff --git a/.github/workflows/nix-screenshot.yml b/.github/workflows/nix-screenshot.yml deleted file mode 100644 index 7d9f8b9a59..0000000000 --- a/.github/workflows/nix-screenshot.yml +++ /dev/null @@ -1,149 +0,0 @@ -name: CUA Driver Screenshot Test - -on: - pull_request: - types: [labeled] - -permissions: - id-token: write - contents: read - pull-requests: write - -env: - AWS_REGION: us-west-2 - NIX_CACHE_BUCKET: trycua-nix-cache - NIX_CACHE_SECRET: nix-cache/trycua-nix-cache/signing-key - -jobs: - screenshot-test: - name: Run cua-driver screenshot test - if: github.event.label.name == 'cua-driver-screenshot' - runs-on: ubuntu-latest - timeout-minutes: 15 - - steps: - - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - - name: Configure AWS Credentials via OIDC - uses: aws-actions/configure-aws-credentials@ff717079ee2060e4bcee96c4779b553acc87447c # v4 - with: - role-to-assume: arn:aws:iam::296062593712:role/github-actions-nix-cache - aws-region: ${{ env.AWS_REGION }} - - - name: Get Nix signing key - id: nix-key - run: | - SECRET=$(aws secretsmanager get-secret-value \ - --secret-id "${{ env.NIX_CACHE_SECRET }}" \ - --query 'SecretString' --output text) - - SECRET_KEY=$(echo "$SECRET" | jq -r '.secret_key') - echo "::add-mask::$SECRET_KEY" - echo "$SECRET_KEY" > "${{ runner.temp }}/signing-key.sec" - chmod 600 "${{ runner.temp }}/signing-key.sec" - - PUBLIC_KEY=$(echo "$SECRET" | jq -r '.public_key') - echo "public_key=$PUBLIC_KEY" >> "$GITHUB_OUTPUT" - - - name: Setup AWS credentials file for Nix - run: | - mkdir -p ~/.aws - printf '[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\nregion = %s\n' \ - "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$AWS_SESSION_TOKEN" "$AWS_REGION" > ~/.aws/credentials - chmod 600 ~/.aws/credentials - - sudo mkdir -p /root/.aws - sudo bash -c "printf '[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\nregion = %s\n' \ - '$AWS_ACCESS_KEY_ID' '$AWS_SECRET_ACCESS_KEY' '$AWS_SESSION_TOKEN' '$AWS_REGION' > /root/.aws/credentials" - sudo chmod 600 /root/.aws/credentials - - - name: Install Nix - uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30 - with: - extra_nix_config: | - experimental-features = nix-command flakes - # Container-based NixOS integration tests (testers.nixosTest with - # `containers`) run under systemd-nspawn instead of QEMU: no KVM - # needed, much faster/cheaper. The nspawn builds require an allocated - # UID range and cgroup delegation, advertised here on the builder. - extra-experimental-features = auto-allocate-uids cgroups - auto-allocate-uids = true - extra-system-features = uid-range - substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }} https://cache.nixos.org - trusted-substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }} - trusted-public-keys = ${{ steps.nix-key.outputs.public_key }} cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - - - name: Run screenshot test - timeout-minutes: 12 - run: nix build .#checks.x86_64-linux.cua-driver-screenshot --print-build-logs --show-trace - - - name: Extract screenshot - if: always() - run: | - if [ -L result ]; then - find -L result/ -name '*.png' -type f -exec cp {} . \; 2>/dev/null || true - fi - ls -la *.png 2>/dev/null || echo "No screenshots found" - - - name: Upload screenshot artifact - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: cua-driver-screenshot - path: "*.png" - if-no-files-found: warn - - - name: Comment screenshot on PR - if: always() - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 - with: - script: | - const fs = require('fs'); - const pngs = fs.readdirSync('.').filter(f => f.endsWith('.png')); - - let body = '## CUA Driver Screenshot Test\n\n'; - - if (pngs.length > 0) { - const artifactUrl = `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; - body += `Screenshot captured by cua-driver's \`get_window_state\` tool from inside a NixOS container.\n\n`; - body += `📸 [Download screenshot artifact](${artifactUrl})\n\n`; - body += '✅ Test passed\n'; - } else { - body += '⚠️ No screenshots were captured. Check the [workflow run](' + - `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + - ') for details.\n'; - } - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: body, - }); - - - name: Remove label - if: always() - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 - with: - script: | - try { - await github.rest.issues.removeLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - name: 'cua-driver-screenshot', - }); - } catch (e) { - console.log('Label already removed or not found:', e.message); - } - - - name: Sign and upload to Nix cache - if: always() - run: | - nix store sign --key-file "${{ runner.temp }}/signing-key.sec" --all - nix copy --to "s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }}&want-mass-query=true" --all -L - - - name: Cleanup signing key - if: always() - run: rm -f "${{ runner.temp }}/signing-key.sec" diff --git a/.github/workflows/nix-wayland.yml b/.github/workflows/nix-wayland.yml deleted file mode 100644 index e651b513b3..0000000000 --- a/.github/workflows/nix-wayland.yml +++ /dev/null @@ -1,199 +0,0 @@ -name: CUA Driver Native-Wayland TDD Suite - -# Native-Wayland reproduction of the cua-driver NixOS tests across five desktop -# sessions (XFCE on labwc/sway, plus KDE and GNOME). The cursor-click-gif and -# background-terminal-gif tests use the EIS-backed cua-compositor injection path -# for deterministic keyboard delivery without relying on compositor focus policy. -# These jobs are maintainer-dispatched desktop diagnostics. They are BLOCKING -# within a dispatched run, but do not charge every PR with the full compositor -# matrix. -# Artifacts (screenshots, GIFs, logs) are uploaded so you can see exactly what -# each compositor does. -on: - # Native-compositor checks are maintainer-dispatched because they are - # desktop e2e coverage, not cheap pull-request unit checks. - workflow_dispatch: - -permissions: - id-token: write - contents: read -env: - AWS_REGION: us-west-2 - NIX_CACHE_BUCKET: trycua-nix-cache - NIX_CACHE_SECRET: nix-cache/trycua-nix-cache/signing-key - -jobs: - wayland-scenarios: - name: ${{ matrix.desktop }} / ${{ matrix.scenario }} - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - desktop: [xfce-labwc, xfce-sway, kde, gnome] - scenario: - - integration - - screenshot - - cursor-click-gif - - background-terminal-gif - - parallel-drag - steps: - - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - - name: Configure AWS Credentials via OIDC - uses: aws-actions/configure-aws-credentials@ff717079ee2060e4bcee96c4779b553acc87447c # v4 - with: - role-to-assume: arn:aws:iam::296062593712:role/github-actions-nix-cache - aws-region: ${{ env.AWS_REGION }} - - - name: Get Nix signing key - id: nix-key - run: | - SECRET=$(aws secretsmanager get-secret-value \ - --secret-id "${{ env.NIX_CACHE_SECRET }}" \ - --query 'SecretString' --output text) - SECRET_KEY=$(echo "$SECRET" | jq -r '.secret_key') - echo "::add-mask::$SECRET_KEY" - echo "$SECRET_KEY" > "${{ runner.temp }}/signing-key.sec" - chmod 600 "${{ runner.temp }}/signing-key.sec" - PUBLIC_KEY=$(echo "$SECRET" | jq -r '.public_key') - echo "public_key=$PUBLIC_KEY" >> "$GITHUB_OUTPUT" - - - name: Setup AWS credentials file for Nix - run: | - mkdir -p ~/.aws - printf '[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\nregion = %s\n' \ - "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$AWS_SESSION_TOKEN" "$AWS_REGION" > ~/.aws/credentials - chmod 600 ~/.aws/credentials - sudo mkdir -p /root/.aws - sudo bash -c "printf '[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\nregion = %s\n' \ - '$AWS_ACCESS_KEY_ID' '$AWS_SECRET_ACCESS_KEY' '$AWS_SESSION_TOKEN' '$AWS_REGION' > /root/.aws/credentials" - sudo chmod 600 /root/.aws/credentials - - - name: Install Nix - uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30 - with: - extra_nix_config: | - experimental-features = nix-command flakes - substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }} https://cache.nixos.org - trusted-substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }} - trusted-public-keys = ${{ steps.nix-key.outputs.public_key }} cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - - - name: Run cua-driver-wayland-${{ matrix.desktop }}-${{ matrix.scenario }} - run: | - nix build ".#checks.x86_64-linux.cua-driver-wayland-${{ matrix.desktop }}-${{ matrix.scenario }}" \ - --print-build-logs --show-trace \ - -o "result-${{ matrix.desktop }}-${{ matrix.scenario }}" - - - name: Collect artifacts - if: always() - run: | - mkdir -p artifacts - find -L "result-${{ matrix.desktop }}-${{ matrix.scenario }}/" \ - \( -name '*.gif' -o -name '*.png' -o -name '*.json' \) -type f \ - -exec cp {} artifacts/ \; 2>/dev/null || true - ls -la artifacts/ 2>/dev/null || echo "No artifacts found" - - - name: Upload artifacts - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: cua-driver-wayland-${{ matrix.desktop }}-${{ matrix.scenario }} - path: artifacts/* - if-no-files-found: warn - - - name: Sign and upload to Nix cache - if: always() - run: | - nix store sign --key-file "${{ runner.temp }}/signing-key.sec" --all || true - nix copy --to "s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }}&want-mass-query=true" --all -L || true - - - name: Cleanup signing key - if: always() - run: rm -f "${{ runner.temp }}/signing-key.sec" - - wayland-background-gui: - name: ${{ matrix.desktop }} / background-gui ${{ matrix.app }} - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - desktop: [xfce-labwc, xfce-sway, kde, gnome] - app: [foot, gtk3-gedit, qt6-kcalc] - steps: - - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - - name: Configure AWS Credentials via OIDC - uses: aws-actions/configure-aws-credentials@ff717079ee2060e4bcee96c4779b553acc87447c # v4 - with: - role-to-assume: arn:aws:iam::296062593712:role/github-actions-nix-cache - aws-region: ${{ env.AWS_REGION }} - - - name: Get Nix signing key - id: nix-key - run: | - SECRET=$(aws secretsmanager get-secret-value \ - --secret-id "${{ env.NIX_CACHE_SECRET }}" \ - --query 'SecretString' --output text) - SECRET_KEY=$(echo "$SECRET" | jq -r '.secret_key') - echo "::add-mask::$SECRET_KEY" - echo "$SECRET_KEY" > "${{ runner.temp }}/signing-key.sec" - chmod 600 "${{ runner.temp }}/signing-key.sec" - PUBLIC_KEY=$(echo "$SECRET" | jq -r '.public_key') - echo "public_key=$PUBLIC_KEY" >> "$GITHUB_OUTPUT" - - - name: Setup AWS credentials file for Nix - run: | - mkdir -p ~/.aws - printf '[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\nregion = %s\n' \ - "$AWS_ACCESS_KEY_ID" "$AWS_SECRET_ACCESS_KEY" "$AWS_SESSION_TOKEN" "$AWS_REGION" > ~/.aws/credentials - chmod 600 ~/.aws/credentials - sudo mkdir -p /root/.aws - sudo bash -c "printf '[default]\naws_access_key_id = %s\naws_secret_access_key = %s\naws_session_token = %s\nregion = %s\n' \ - '$AWS_ACCESS_KEY_ID' '$AWS_SECRET_ACCESS_KEY' '$AWS_SESSION_TOKEN' '$AWS_REGION' > /root/.aws/credentials" - sudo chmod 600 /root/.aws/credentials - - - name: Install Nix - uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30 - with: - extra_nix_config: | - experimental-features = nix-command flakes - substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }} https://cache.nixos.org - trusted-substituters = s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }} - trusted-public-keys = ${{ steps.nix-key.outputs.public_key }} cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= - - - name: Run cua-driver-wayland-${{ matrix.desktop }}-background-gui-${{ matrix.app }} - run: | - nix build ".#checks.x86_64-linux.cua-driver-wayland-${{ matrix.desktop }}-background-gui-${{ matrix.app }}" \ - --print-build-logs --show-trace \ - -o "result-${{ matrix.desktop }}-bg-gui-${{ matrix.app }}" - - - name: Collect artifacts - if: always() - run: | - mkdir -p artifacts - find -L "result-${{ matrix.desktop }}-bg-gui-${{ matrix.app }}/" \ - \( -name '*.gif' -o -name '*.png' -o -name '*.json' \) -type f \ - -exec cp {} artifacts/ \; 2>/dev/null || true - ls -la artifacts/ 2>/dev/null || echo "No artifacts found" - - - name: Upload artifacts - if: always() - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 - with: - name: cua-driver-wayland-${{ matrix.desktop }}-background-gui-${{ matrix.app }} - path: artifacts/* - if-no-files-found: warn - - - name: Sign and upload to Nix cache - if: always() - run: | - nix store sign --key-file "${{ runner.temp }}/signing-key.sec" --all || true - nix copy --to "s3://${{ env.NIX_CACHE_BUCKET }}?region=${{ env.AWS_REGION }}&want-mass-query=true" --all -L || true - - - name: Cleanup signing key - if: always() - run: rm -f "${{ runner.temp }}/signing-key.sec" diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a2e354939e..7fc73c95df 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,11 +11,11 @@ repos: - repo: local hooks: - - id: tsc + - id: typescript-typecheck name: TypeScript type check - entry: node ./scripts/typescript-typecheck.js - language: node - files: \.(ts|tsx)$ + entry: pnpm -C libs/typescript typecheck + language: system + files: ^libs/typescript/.*\.(ts|tsx)$ pass_filenames: false - repo: https://github.com/PyCQA/isort diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 75c349948b..29d7bdc0ce 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,59 +1,67 @@ # Contributing to Cua -We deeply appreciate your interest in contributing to Cua! Whether you're reporting bugs, suggesting enhancements, improving docs, or submitting pull requests, your contributions help improve the project for everyone. - -## Reporting Bugs - -If you've encountered a bug in the project, we encourage you to report it. Please follow these steps: - -1. **Check the Issue Tracker**: Before submitting a new bug report, please check our issue tracker to see if the bug has already been reported. -2. **Create a New Issue**: If the bug hasn't been reported, create a new issue with: - - A clear title and detailed description - - Steps to reproduce the issue - - Expected vs actual behavior - - Your environment (macOS version, cua version) - - Any relevant logs or error messages -3. **Label Your Issue**: Label your issue as a `bug` to help maintainers identify it quickly. - -## Suggesting Enhancements - -We're always looking for suggestions to make Cua better. If you have an idea: - -1. **Check Existing Issues**: See if someone else has already suggested something similar. -2. **Create a New Issue**: If your enhancement is new, create an issue describing: - - The problem your enhancement solves - - How your enhancement would work - - Any potential implementation details - - Why this enhancement would benefit Cua users - -## Code Formatting - -We follow strict code formatting guidelines to ensure consistency across the codebase. Before submitting any code: - -1. **Review Our Format Guide**: Please review our [Code Formatting Standards](Development.md#code-formatting-standards) section in the Getting Started guide. -2. **Configure Your IDE**: We recommend using the workspace settings provided in `.vscode/` for automatic formatting. -3. **Run Formatting Tools**: Always run the formatting tools before submitting a PR: - ```bash - # For Python code - uv run black . - uv run isort . - uv run ruff check --fix . - ``` -4. **Validate Your Code**: Ensure your code passes all checks: - ```bash - uv run mypy . - ``` -5. Every time you try to commit code, a pre-commit hook will automatically run the formatting and validation tools. If any issues are found, the commit will be blocked until they are resolved. Please make sure to address any issues reported by the pre-commit hook before attempting to commit again. Once all issues are resolved, you can proceed with your commit. +Thanks for contributing to Cua. The repository includes Python and TypeScript +SDKs, a Rust desktop driver, Swift virtualization tools, container images, and +public documentation. Start with the component that owns the behavior you want +to change. + +## Report a Bug + +Before opening an issue, search the existing issue tracker. Include: + +- a concise description and reproducible steps; +- expected and actual behavior; +- Cua package or driver version; +- operating system, window system, and application when relevant; +- logs, structured errors, screenshots, or recordings that help reproduce it. + +Do not include credentials or private application data. + +## Propose a Change + +For feature requests, describe the user problem and the expected behavior +before prescribing an implementation. Mention affected platforms and existing +workarounds when known. + +## Submit Code + +1. Read [`Development.md`](Development.md) and the guide next to the component. +2. Keep changes scoped to the component that owns the behavior. +3. Add or update tests that observe the public effect of the change. +4. Run the applicable commands in [`TESTING.md`](TESTING.md). +5. Run the formatters and linters owned by the changed component. +6. Open a focused pull request that explains behavior, validation, and known gaps. + +Root pre-commit hooks are optional local helpers. Install them with: + +```bash +uv sync --group dev +uv run pre-commit install +``` + +Mypy is configured but is not currently a pre-commit gate. Rust, TypeScript, +Swift, and documentation checks remain component-owned. + +## Desktop Behavior Changes + +cua-driver behavior must be verified through the canonical Rust harnesses. A +successful tool response alone is not evidence that an action reached the +application. Delivery tests should observe fixture state and attach focus, +z-order, cursor, leaked-input, capture, or refusal oracles as required. + +Do not weaken a test to match the current driver. Add a capability, return an +exact structured refusal, or record the behavior as an explicit gap. ## Documentation -Documentation improvements are always welcome. You can: +Public documentation lives under `docs/content/docs` and follows Diataxis. See +[`docs/README.md`](docs/README.md) before adding a page. Contributor-only plans, +journals, and implementation notes belong next to their component. -- Fix typos or unclear explanations -- Add examples and use cases -- Improve API documentation -- Add tutorials or guides +Documentation changes should pass generator drift, hygiene, internal links, +and the production Fumadocs build. -For detailed instructions on setting up your development environment and submitting code contributions, please see our [Developer-Guide](Development.md). +## Community -Feel free to join our [Discord community](https://discord.com/invite/mVnXXpdE85) to discuss ideas or get help with your contributions. +For design discussion and contributor help, join the +[Cua Discord community](https://discord.com/invite/mVnXXpdE85). diff --git a/Development.md b/Development.md index f652882895..836a27804b 100644 --- a/Development.md +++ b/Development.md @@ -1,346 +1,145 @@ -# Development Guide +# Development -This guide covers setting up and developing the Cua monorepo. +This file is the contributor map for the Cua monorepo. Each component owns its +detailed setup, build, and test instructions. Start here, then follow the guide +next to the code you plan to change. -## Project Structure +## Choose a Component -The project is organized as a monorepo with these main packages: +| Area | Main paths | Toolchain | Start here | +| ---------------------------- | ----------------------------------------------------------- | ----------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | +| cua-driver | `libs/cua-driver/rust`, `libs/cua-driver/python` | Rust, platform SDKs, Nix on Linux | [`libs/cua-driver/README.md`](libs/cua-driver/README.md), [`libs/cua-driver/rust/README.md`](libs/cua-driver/rust/README.md) | +| Python SDKs and services | `libs/python`, `libs/cua-bench` | Python 3.12, uv | Package `pyproject.toml` and README | +| TypeScript SDKs | `libs/typescript` | Node.js, pnpm | [`libs/typescript/README.md`](libs/typescript/README.md) and its `package.json` scripts | +| CuaBot | `libs/cuabot` | Node.js, pnpm | [`libs/cuabot/README.md`](libs/cuabot/README.md) | +| Lume | `libs/lume` | Swift, Xcode command-line tools | [`libs/lume/Development.md`](libs/lume/Development.md) | +| Sandbox and container images | `libs/kasm`, `libs/lumier`, `libs/qemu-docker`, `libs/xfce` | Docker plus the component toolchain | README or Development file in the component | +| Public documentation | `docs/content/docs` | Node.js, pnpm, Fumadocs | [`docs/README.md`](docs/README.md) | +| Samples | `samples` | Depends on the sample | README next to the sample | -**Python Packages** (located in `libs/python/`): +The repository changes quickly. Directory listings in this root guide are an +orientation aid, not a package registry. Release targets come from +[`.github/workflows/release-bump-version.yml`](.github/workflows/release-bump-version.yml), +and test ownership comes from CI plus the component guides. -- `core/` - Base package with telemetry support -- `computer/` - Computer-use interface (CUI) library -- `agent/` - AI agent library with multi-provider support -- `som/` - Set-of-Mark parser -- `computer-server/` - Server component for VM -- `mcp-server/` - MCP server implementation -- `bench-ui/` - Benchmark UI utilities +## Common Setup -**Other Packages**: - -- `libs/lume/` - Lume CLI (Swift) -- `libs/typescript/` - TypeScript packages including `cua-cli` -- `libs/cuabot/` - CuaBot multi-agent computer-use sandbox CLI - -All Python packages are part of a [uv workspace](https://docs.astral.sh/uv/concepts/projects/workspaces/) which manages a shared virtual environment and dependencies. - -## Quick Start - -1. **Install Lume CLI** (required for local VM management): - - ```bash - /bin/bash -c "$(curl -fsSL https://cua.ai/lume/install.sh)" - ``` - -2. **Clone the repository**: - - ```bash - git clone https://github.com/trycua/cua.git - cd cua - ``` - -3. **Create `.env.local`** in the root directory with your API keys: - - ```bash - ANTHROPIC_API_KEY=your_anthropic_key_here - OPENAI_API_KEY=your_openai_key_here - ``` - -4. **Install Node.js dependencies**: - - ```bash - npm install -g pnpm # if not already installed - pnpm install - ``` - -5. **Install Python dependencies**: - - ```bash - pip install uv # if not already installed - uv sync - ``` - -6. **Open workspace in VS Code/Cursor**: - - ```bash - # For Python development - code .vscode/py.code-workspace - - # For Lume (Swift) development - code .vscode/lume.code-workspace - ``` - -7. **Install pre-commit hooks**: - ```bash - uv run pre-commit install - ``` - -
-Why use workspace files? - -Using the workspace file is strongly recommended as it: - -- Sets up correct Python environments for each package -- Configures proper import paths -- Enables debugging configurations -- Maintains consistent settings across packages - -
- -## Python Development - -### Requirements - -- **Python 3.12+** (see [`pyproject.toml`](./pyproject.toml) for exact requirements) -- **uv** - Python package manager -- **pnpm** - Node.js package manager - -### Setup - -Install all workspace dependencies with a single command: +Clone the repository and enter it: ```bash -uv sync +git clone https://github.com/trycua/cua.git +cd cua ``` -This installs all dependencies in the virtual environment `.venv`. Each Cua package is installed in editable mode, so changes to source code are immediately reflected. - -The `.venv` environment is configured as the default VS Code Python interpreter in [`.vscode/settings.json`](.vscode/settings.json). - -## Code Formatting - -The Cua project follows strict code formatting standards to ensure consistency across all packages. - -### Python Formatting - -#### Tools - -- **[Black](https://black.readthedocs.io/)** - Code formatter -- **[isort](https://pycqa.github.io/isort/)** - Import sorter -- **[Ruff](https://beta.ruff.rs/docs/)** - Fast linter and formatter -- **[MyPy](https://mypy.readthedocs.io/)** - Static type checker (configured but not enforced in pre-commit) - -All tools are automatically installed when you run `uv sync`. - -#### Configuration - -Formatting configuration is defined in [`pyproject.toml`](./pyproject.toml). See the `[tool.black]`, `[tool.ruff]`, `[tool.mypy]`, and `[tool.isort]` sections for all settings. - -#### Key Rules - -- **Line Length**: Maximum of 100 characters -- **Python Version**: Code must be compatible with Python 3.12+ -- **Imports**: Automatically sorted (using Ruff's "I" rule) -- **Type Hints**: Required for all function definitions (strict mypy mode) - -
-IDE configuration details - -##### Python-specific settings - -Python-specific IDE settings are configured in [`.vscode/settings.json`](.vscode/settings.json), including: - -- Python interpreter path -- Format on save -- Code actions on save -- Black formatter configuration -- Ruff and MyPy integration - -##### JS/TS-specific settings +Install only the toolchains required by your component: -JavaScript/TypeScript formatting settings are also in [`.vscode/settings.json`](.vscode/settings.json), ensuring Prettier is used for all JS/TS files. +| Work | Required tools | +| --------------------------- | --------------------------------------------------------------------------- | +| Python packages | Python 3.12 and [uv](https://docs.astral.sh/uv/) | +| TypeScript, CuaBot, or docs | The Node.js version required by the component and its declared pnpm version | +| cua-driver | Rust plus the target OS SDK; Nix for the reproducible Linux lanes | +| Lume | macOS, Swift, and Xcode command-line tools | +| Images | Docker or the image-specific builder documented by the component | -##### Recommended VS Code Extensions +The root uv workspace contains only the members declared in +[`pyproject.toml`](pyproject.toml). Other directories under `libs/python` are +independent packages; use their own `pyproject.toml` and CI workflow. -- **Black Formatter** – `ms-python.black-formatter` -- **Ruff** – `charliermarsh.ruff` -- **Pylance** – `ms-python.vscode-pylance` -- **isort** – `ms-python.isort` -- **Prettier** – `esbenp.prettier-vscode` -- **Mypy Type Checker** – `ms-python.mypy-type-checker` +The root Node package installs repository-wide Prettier only. Run `pnpm +install` inside `libs/typescript`, `libs/cuabot`, or `docs` for those +components' dependencies. -> VS Code will automatically suggest installing the recommended extensions when you open the workspace. +API keys are not required for ordinary builds and deterministic tests. Add +credentials only for a test or example that explicitly calls an external +provider, and never commit them. -
+## Root Formatting Hooks -
-Manual formatting commands - -To manually format code: +Install the root Python development tools and optional Git hooks with: ```bash -# Format all Python files using Black -uv run black . - -# Sort imports using isort -uv run isort . - -# Run Ruff linter with auto-fix -uv run ruff check . - -# Run type checking with MyPy -uv run mypy . +uv sync --group dev +uv run pre-commit install ``` -
- -#### Pre-commit Validation - -Before submitting a pull request, ensure your code passes all formatting checks. - -**Recommended: Run all hooks via pre-commit** +Run all configured hooks against the repository with: ```bash -uv run pre-commit run +uv run pre-commit run --all-files ``` -This automatically runs Black, Ruff, isort, Prettier, TypeScript type checking, and other configured hooks. See [`.pre-commit-config.yaml`](.pre-commit-config.yaml) for the complete list. - -> **Note:** MyPy is currently disabled in pre-commit hooks due to untyped codebase, but it's still configured and can be run manually. +The hooks currently run Prettier, the TypeScript workspace typecheck, Black, +isort, and Ruff. Install the TypeScript workspace dependencies before running +the typecheck hook. Mypy is configured in `pyproject.toml` but is not a +pre-commit gate. -
-Run individual tools manually +For a read-only repository-wide formatting check: ```bash -# Python checks -uv run black --check . -uv run isort --check . -uv run ruff check . -uv run mypy . - -# JavaScript/TypeScript checks +pnpm install --frozen-lockfile pnpm prettier:check ``` -
- -### JavaScript / TypeScript Formatting - -The project uses **Prettier** to ensure consistent formatting across all JS/TS/JSON/Markdown/YAML files. - -#### Installation - -All Node.js dependencies are managed via `pnpm`: - -```bash -npm install -g pnpm # if not already installed -pnpm install -``` - -#### Usage - -- **Check formatting** (without making changes): - - ```bash - pnpm prettier:check - ``` - -- **Automatically format files**: - - ```bash - pnpm prettier:format - ``` +Component-specific Rust, TypeScript, Swift, and documentation checks remain in +their component guides. See [`TESTING.md`](TESTING.md) for the test map. -#### VS Code Integration +## cua-driver Development -The workspace config ensures Prettier is used automatically for JS/TS/JSON/Markdown/YAML files. Ensure `editor.formatOnSave` is enabled in VS Code. +cua-driver has three distinct validation layers: -### Swift Code (Lume) +1. Rust unit and protocol tests that do not require a target GUI application. +2. Source-built harness E2E tests that drive Electron, Tauri, and native toolkit + fixtures in a real user desktop session. +3. Optional real-application checks for software that is not part of the + canonical harness catalog. -For Swift code in the `libs/lume` directory: +The Rust harnesses are the source of truth for desktop behavior. Python tests +do not duplicate that matrix. Start with: -- Follow the [Swift API Design Guidelines](https://www.swift.org/documentation/api-design-guidelines/) -- Use SwiftFormat for consistent formatting -- Code will be automatically formatted on save when using the lume workspace +- [`libs/cua-driver/rust/README.md`](libs/cua-driver/rust/README.md) for the Cargo workspace. +- [`libs/cua-driver/rust/crates/cua-driver/tests/README.md`](libs/cua-driver/rust/crates/cua-driver/tests/README.md) for test ownership. +- [`scripts/ci/README.md`](scripts/ci/README.md) for canonical OS runners. +- [Platform support](https://cua.ai/docs/reference/cua-driver/platform-support) for current capability boundaries. +- [How Cua Driver is validated](https://cua.ai/docs/concepts/how-cua-driver-is-validated) for the public evidence model. +- [Platform roadmap](https://cua.ai/docs/reference/cua-driver/platform-roadmap) for remaining work and platform boundaries. -Refer to [`libs/lume/Development.md`](./libs/lume/Development.md) for detailed Lume development instructions. +Windows and macOS desktop tests need a real user session. Windows requires an +active console or RDP session. macOS requires a logged-in session with +Accessibility and Screen Recording permissions. The hosted Linux Sway and +nested-compositor runners create controlled sessions; GNOME, KDE, and real +Xorg validation use an existing graphical login. -## Releasing Packages +## Documentation -Cua uses an automated GitHub Actions workflow to bump package versions and publish to PyPI/NPM. - -> **Note:** The main branch is currently not protected. If branch protection is enabled in the future, the github-actions bot must be added to the bypass list for these workflows to commit directly. - -### Version Bump & Publish Workflow - -All packages are managed through a single consolidated workflow: [Bump Version & Publish](https://github.com/trycua/cua/actions/workflows/release-bump-version.yml) - -**Supported packages:** - -**Python (PyPI):** - -- `pypi/agent` - AI agent library -- `pypi/auto` - Cross-platform automation library (mouse, keyboard, screen, window, clipboard, shell) -- `pypi/bench` - Benchmark toolkit for computer-use RL environments -- `pypi/computer` - Computer-use interface library -- `pypi/computer-server` - Server component for VM -- `pypi/core` - Base package with telemetry -- `pypi/mcp-server` - MCP server implementation -- `pypi/som` - Set-of-Mark parser - -**JavaScript/TypeScript (NPM):** - -- `npm/cli` - Cua command-line interface -- `npm/computer` - Computer client for TypeScript -- `npm/core` - Core TypeScript utilities -- `npm/cuabot` - Multi-agent computer-use sandbox - -**Docker:** - -- `docker/cuabot` - CuaBot container image - -**How to use:** - -1. Navigate to the [Bump Version & Publish workflow](https://github.com/trycua/cua/actions/workflows/release-bump-version.yml) -2. Click the "Run workflow" button in the GitHub UI -3. Select the **service/package** from the dropdown (e.g., `pypi/computer` or `npm/cli`) -4. Select the **bump type** (patch/minor/major) from the second dropdown -5. Click "Run workflow" to start the process - -**What happens automatically:** - -1. Version is bumped in the package configuration file -2. Changes are committed and pushed to main -3. Package is automatically published to PyPI or NPM -4. For `npm/cli`: Binaries are built and a GitHub release is created - -> **Note:** For `pypi/computer`, the workflow also automatically bumps `pypi/agent` to maintain version compatibility. - -
-Local Testing (Advanced) - -The Makefile provides utility targets for local testing only: - -```bash -# Test version bump locally (dry run) -make dry-run-patch-core - -# View current versions -make show-versions -``` +Public docs use Fumadocs and follow Diataxis: -**Note:** For production releases, always use the GitHub Actions workflows above instead of running Makefile commands directly. +- tutorials teach a first success; +- how-to guides solve a specific task; +- concepts explain constraints and design; +- reference pages state commands, contracts, support, and limits. -
+Run documentation commands from `docs`; see [`docs/README.md`](docs/README.md). +Contributor-only implementation notes should remain next to their component +instead of entering the public docs navigation. ---- +## Releases -
-Per-package tag patterns +Maintainers release packages through the +[CD: Bump Version](https://github.com/trycua/cua/actions/workflows/release-bump-version.yml) +workflow. Its `service` input is the current release-target registry. Each +package's `.bumpversion.cfg`, Cargo manifest, or package manifest owns its +version and tag format. -Each package uses its own tag format defined in `.bumpversion.cfg`: +Do not duplicate the complete release-target list or example versions in this +guide. They become stale as components are added. The workflow bumps one target +at a time and tag-triggered CD workflows perform publication. -- **cua-agent**: `agent-v{version}` (e.g., `agent-v0.4.35`) -- **cua-auto**: `auto-v{version}` (e.g., `auto-v0.1.0`) -- **cua-bench**: `bench-v{version}` (e.g., `bench-v0.1.0`) -- **cua-computer**: `computer-v{version}` (e.g., `computer-v0.4.7`) -- **cua-computer-server**: `computer-server-v{version}` (e.g., `computer-server-v0.1.27`) -- **cua-core**: `core-v{version}` (e.g., `core-v0.1.9`) -- **cua-mcp-server**: `mcp-server-v{version}` (e.g., `mcp-server-v0.1.14`) -- **cua-som**: `som-v{version}` (e.g., `som-v0.1.3`) -- **cuabot**: `cuabot-v{version}` (e.g., `cuabot-v1.0.0`) -- **cuabot (docker)**: `docker-cuabot-v{version}` (e.g., `docker-cuabot-v1.0.0`) +The root `Makefile` provides local version inspection and dry-run helpers. It +does not publish production releases. -
+## Generated and Local Files - +Do not commit build products, staged harness binaries, local VM artifacts, +credentials, permission databases, or editor-specific state. Promote an +artifact into source control only when it becomes a stable fixture, sample, or +maintained document. diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index a07d20cb72..0000000000 --- a/Dockerfile +++ /dev/null @@ -1,56 +0,0 @@ -FROM python:3.12-slim - -# Set environment variables -ENV PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 \ - PIP_NO_CACHE_DIR=1 \ - PIP_DISABLE_PIP_VERSION_CHECK=1 \ - PYTHONPATH="/app/libs/python/core:/app/libs/python/computer:/app/libs/python/agent:/app/libs/python/som:/app/libs/python/computer-server:/app/libs/python/mcp-server" - -# Install system dependencies for ARM architecture -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - build-essential \ - libgl1 \ - libglib2.0-0 \ - libxcb-xinerama0 \ - libxkbcommon-x11-0 \ - cmake \ - pkg-config \ - curl \ - iputils-ping \ - net-tools \ - sed \ - xxd \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Set working directory -WORKDIR /app - -# Copy the entire project temporarily -# We'll mount the real source code over this at runtime -COPY . /app/ - -# Create a simple .env.local file for build.sh -RUN echo "PYTHON_BIN=python" > /app/.env.local - -# Modify build.sh to skip virtual environment creation -RUN sed -i 's/python -m venv .venv/echo "Skipping venv creation in Docker"/' /app/scripts/build.sh && \ - sed -i 's/source .venv\/bin\/activate/echo "Skipping venv activation in Docker"/' /app/scripts/build.sh && \ - sed -i 's/find . -type d -name ".venv" -exec rm -rf {} +/echo "Skipping .venv removal in Docker"/' /app/scripts/build.sh && \ - chmod +x /app/scripts/build.sh - -# Run the build script to install dependencies -RUN cd /app && ./scripts/build.sh - -# Clean up the source files now that dependencies are installed -# When we run the container, we'll mount the actual source code -RUN rm -rf /app/* /app/.??* - -# Note: This Docker image doesn't contain the lume executable (macOS-specific) -# Instead, it relies on connecting to a lume server running on the host machine -# via host.docker.internal:7777 - -# Default command -CMD ["bash"] \ No newline at end of file diff --git a/README.md b/README.md index 04a52135fd..fd7d0edca5 100644 --- a/README.md +++ b/README.md @@ -59,9 +59,9 @@ --- -## Cua Drivers - Background computer-use on macOS and Windows, with Linux pre-release +## Cua Drivers - Background computer-use on macOS, Windows, and Linux -Drive native desktop apps **in the background**. Agents click, type, and verify without stealing the cursor or focus. Use the same CLI and MCP server on macOS and Windows from Claude Code, Cursor, Codex, OpenClaw, and custom clients. Linux support is available as a pre-release backend while platform testing is still in progress. +Drive native desktop apps **in the background**. Agents click, type, and verify without stealing the cursor or focus. Use the same CLI and MCP server on macOS, Windows, and Linux from Claude Code, Cursor, Codex, OpenClaw, and custom clients. Linux supports X11 and compositor-specific Wayland routes with explicit limits for raw background input. **macOS / Linux** @@ -93,7 +93,6 @@ Build agents that see screens, click buttons, and complete tasks autonomously. O pip install cua ``` - ```python # Requires Python 3.11 or later from cua import Sandbox, Image @@ -120,7 +119,6 @@ async with Sandbox.ephemeral(Image.linux()) as sb: # or .macos() .windows() .a Evaluate computer-use agents on OSWorld, ScreenSpot, Windows Arena, and custom tasks. Export trajectories for training. - ```bash # Clone, install, and create base image git clone https://github.com/trycua/cua && cd cua/cua-bench @@ -138,7 +136,6 @@ cb run dataset datasets/cua-bench-basic --agent cua-agent --max-parallel 4 Create and manage macOS/Linux VMs with near-native performance on Apple Silicon using Apple's Virtualization.Framework. - ```bash # Install Lume /bin/bash -c "$(curl -fsSL https://cua.ai/lume/install.sh)" @@ -163,15 +160,15 @@ of Setup Assistant on its first display boot; see [issue #2155](https://github.c ## Packages -| Package | Description | -| --------------------------------------------------------------------------- | ---------------------------------------------------------- | -| [cua-driver](libs/cua-driver/README.md) | Background computer-use agent for macOS, Windows, and Linux | -| [cua-agent](https://cua.ai/docs/cua/reference/agent-sdk) | AI agent framework for computer-use tasks | -| [cua-sandbox](https://cua.ai/docs/cua/reference/sandbox-sdk) | SDK for creating and controlling sandboxes | -| [cua-computer-server](https://cua.ai/docs/cua/reference/sandbox-sdk) | Driver for UI interactions and code execution in sandboxes | -| [cua-bench](https://cua.ai/docs/cuabench) | Benchmarks and RL environments for computer-use | -| [lume](https://cua.ai/docs/lume) | macOS/Linux VM management on Apple Silicon | -| [lumier](https://cua.ai/docs/lume/guide/advanced/lumier) | Docker-compatible interface for Lume VMs | +| Package | Description | +| -------------------------------------------------------------------- | ----------------------------------------------------------- | +| [cua-driver](libs/cua-driver/README.md) | Background computer-use agent for macOS, Windows, and Linux | +| [cua-agent](https://cua.ai/docs/cua/reference/agent-sdk) | AI agent framework for computer-use tasks | +| [cua-sandbox](https://cua.ai/docs/cua/reference/sandbox-sdk) | SDK for creating and controlling sandboxes | +| [cua-computer-server](https://cua.ai/docs/cua/reference/sandbox-sdk) | Driver for UI interactions and code execution in sandboxes | +| [cua-bench](https://cua.ai/docs/cuabench) | Benchmarks and RL environments for computer-use | +| [lume](https://cua.ai/docs/lume) | macOS/Linux VM management on Apple Silicon | +| [lumier](https://cua.ai/docs/lume/guide/advanced/lumier) | Docker-compatible interface for Lume VMs | ## Resources diff --git a/TESTING.md b/TESTING.md index 38600a95ad..374f8fe03c 100644 --- a/TESTING.md +++ b/TESTING.md @@ -1,106 +1,135 @@ -# Testing Guide for Cua +# Testing -Quick guide to running tests and understanding the test architecture. +Cua is a multi-language monorepo. There is no single root command that proves +every package, desktop, VM, and image. Run the tests owned by the components you +changed and use the corresponding CI workflow as the executable source of +truth. -## 🚀 Quick Start +## Test Map -```bash -# Install dependencies -pip install pytest pytest-asyncio pytest-mock pytest-cov +| Area | Deterministic tests | Integration or E2E owner | +| -------------------- | ----------------------------------------------------- | -------------------------------------------------------------------------------- | +| Python SDKs | Package `tests/` directories with pytest | Package-specific integration tests and `tests/integration` | +| TypeScript SDKs | Package Vitest/typecheck scripts | Package-owned integration tests | +| cua-driver | Rust unit, schema, protocol, and compile tests | Canonical Rust desktop harnesses on Windows, macOS, Linux X11, and Linux Wayland | +| Lume | Swift package tests | VM and unattended-setup checks documented by Lume | +| Public docs | Generator drift, hygiene, links, and production build | Rendered Fumadocs site | +| Images and sandboxes | Component build and schema tests | Image-specific smoke or VM tests | -# Install package -cd libs/python/core -pip install -e . +Path-filtered CI avoids running unrelated operating systems, so a green job for +one component does not validate another component. -# Run tests -export CUA_TELEMETRY_ENABLED=false # or $env:CUA_TELEMETRY_ENABLED="false" on Windows -pytest tests/ -v -``` +## Python -## 🧪 Running Tests +For a member of the root uv workspace: ```bash -# All packages -pytest libs/python/*/tests/ -v +uv sync --group test +CUA_TELEMETRY_ENABLED=false uv run pytest libs/python//tests -v +``` + +Packages outside the root uv workspace should be installed from their own +`pyproject.toml`. The current package matrix and installation sequence live in +[`.github/workflows/ci-test-python.yml`](.github/workflows/ci-test-python.yml). -# Specific package -cd libs/python/core && pytest tests/ -v +## TypeScript -# With coverage -pytest tests/ --cov --cov-report=html +Run from `libs/typescript`: -# Specific test -pytest tests/test_telemetry.py::TestTelemetryEnabled::test_telemetry_enabled_by_default -v +```bash +pnpm install --frozen-lockfile +pnpm test +pnpm typecheck +pnpm format:check ``` -## 🏗️ Test Architecture +Use a package's own `package.json` scripts when working outside that workspace, +including CuaBot and the documentation site. -**Principles**: SRP (Single Responsibility) + Vertical Slices + Testability +## cua-driver Unit and Protocol Tests -``` -libs/python/ -├── core/tests/ # Tests ONLY core -├── agent/tests/ # Tests ONLY agent -└── computer/tests/ # Tests ONLY computer +Run from `libs/cua-driver/rust`. Focused examples: + +```bash +cargo test -p cua-driver-core --locked +cargo test -p cua-driver --test protocol_mcp_test --locked ``` -Each test file = ONE feature. Each test class = ONE concern. +Linux source and package checks run through Nix. Windows and Linux compile gates +are split into OS-specific workflows. See +[`libs/cua-driver/rust/README.md`](libs/cua-driver/rust/README.md) for workspace +commands. -## ➕ Adding New Tests +Unit and protocol tests do not prove that desktop input reached a real +application. -1. Create `test_*.py` in the appropriate package's `tests/` directory -2. Follow the pattern: +## cua-driver Harness E2E -```python -"""Unit tests for my_feature.""" -import pytest -from unittest.mock import patch +The canonical desktop suites build repository-owned applications, drive them +through the Rust driver, and verify application or desktop state independently +from the tool response. Foreground/background delivery and AX/PX addressing are +dimensions of each action row. -class TestMyFeature: - """Test MyFeature class.""" +Canonical entry points: - def test_initialization(self): - """Test that feature initializes.""" - from my_package import MyFeature - feature = MyFeature() - assert feature is not None +```text +Linux X11/session: scripts/ci/linux/run-rust-e2e.sh +Linux Sway: scripts/ci/linux/run-rust-e2e-wayland.sh +Linux nested: scripts/ci/linux/run-rust-e2e-inject.sh +Linux GNOME/KDE: scripts/ci/linux/run-rust-e2e-desktop.sh +Linux real Xorg: scripts/ci/linux/run-rust-e2e-desktop.sh xorg +Windows: .\scripts\ci\windows\run-rust-e2e.ps1 -RequireGui +macOS: scripts/ci/macos/run-rust-e2e.sh ``` -3. Mock external dependencies: +The hosted Sway and nested-compositor runners create controlled sessions. +GNOME, KDE, real Xorg, Windows, and macOS use an existing graphical login. The +suites are often maintainer-triggered and retain typed case/results, +screenshots, accessibility state, trajectories, logs, and video where the lane +supports it. The reporter rejects missing rows, false-success responses, +undeclared outcomes, and incomplete required evidence. -```python -@pytest.fixture -def mock_api(): - with patch("my_package.api_client") as mock: - yield mock -``` +See: -## 🔄 CI/CD +- [`libs/cua-driver/docs/test-harnesses-guide.md`](libs/cua-driver/docs/test-harnesses-guide.md) +- [`libs/cua-driver/docs/test-matrix.md`](libs/cua-driver/docs/test-matrix.md) +- [Platform support and validation](https://cua.ai/docs/reference/cua-driver/platform-support) -Tests run automatically on every PR via GitHub Actions (`.github/workflows/ci-python-tests.yml`): +## Lume -- Matrix strategy: each package tested separately -- Python 3.12 -- ~2 minute runtime +Run from `libs/lume`: -## 🐛 Troubleshooting +```bash +swift test +``` -**ModuleNotFoundError**: Run `pip install -e .` in package directory +VM-dependent and unattended-setup checks have additional prerequisites in +[`libs/lume/Development.md`](libs/lume/Development.md). -**Tests fail in CI but pass locally**: Set `CUA_TELEMETRY_ENABLED=false` +## Public Documentation -**Async tests error**: Install `pytest-asyncio` and use `@pytest.mark.asyncio` +Run from `docs`: -**Mock not working**: Patch at usage location, not definition: +```bash +pnpm install --frozen-lockfile +pnpm docs:check +pnpm docs:check-hygiene +pnpm docs:check-links +pnpm build +``` -```python -# ✅ Right -@patch("my_package.module.external_function") +The production build validates MDX compilation and static route generation. +The generator check prevents generated CLI and API references from drifting +from source. -# ❌ Wrong -@patch("external_library.function") -``` +## Before Opening a Pull Request ---- +1. Run focused tests while developing. +2. Run the complete deterministic test owner for every component changed. +3. Run the affected interactive E2E lane when desktop behavior or its contract changes. +4. Run formatting and documentation checks for modified files. +5. Record any test that could not run and why. -**Questions?** Check existing tests for examples or open an issue. +Do not turn missing dependencies, desktop sessions, fixtures, or permissions +into a reduced green run. Environment failures and unsupported capabilities +must remain visible. diff --git a/docs/content/docs/concepts/how-cua-driver-is-validated.mdx b/docs/content/docs/concepts/how-cua-driver-is-validated.mdx new file mode 100644 index 0000000000..2272a0d262 --- /dev/null +++ b/docs/content/docs/concepts/how-cua-driver-is-validated.mdx @@ -0,0 +1,119 @@ +--- +title: "How Cua Driver Is Validated" +description: "Why Cua Driver uses source-built desktop harnesses, independent application oracles, and retained evidence to validate behavior between releases." +--- + +# How Cua Driver Is Validated + +Desktop automation crosses boundaries that ordinary unit tests cannot observe. +A request can be valid, reach an operating-system API, and return success while +the target application receives nothing. Cua Driver therefore treats protocol +correctness and observed desktop behavior as two different kinds of evidence. + +## Two layers answer different questions + +Unit and protocol tests answer whether the driver made a deterministic decision +correctly. They cover schemas, transport, sessions, element identity, route +selection, capture helpers, coordinate conversion, and structured errors. They +run without a target GUI application and catch inexpensive regressions early. + +Harness end-to-end tests answer whether an action actually reached a real +desktop surface. They build a small application from source, launch it in a +real graphical user session, drive it through the Rust driver, and inspect +state owned by the application or desktop rather than trusting the response. + +Neither layer replaces the other. An E2E result does not exhaustively test +protocol edge cases, and a unit test cannot prove that a click changed an +application. + +## The harnesses model representative surfaces + +The shared harness presents the same deterministic web behavior through +Electron and Tauri on every operating system. macOS also hosts it in WKWebView. +Native harnesses exercise the accessibility and windowing APIs that a shared +web renderer cannot represent. + +| Platform | Shared surfaces | Native surfaces | +| -------- | ------------------------------ | -------------------------- | +| Windows | Electron and Tauri | WPF, WinUI 3, and WebView2 | +| macOS | Electron, Tauri, and WKWebView | AppKit and SwiftUI | +| Linux | Electron and Tauri | GTK 3 on X11 and Wayland | + +These applications are fixtures, not mocks. Each is compiled and launched as a +real process. Their purpose is to expose deterministic state for clicks, text, +keys, scrolling, dragging, child windows, controls, and editor behavior. This +makes a failure reproducible without depending on the changing state of an +installed third-party application. + +## One catalog describes each behavior cell + +The Rust catalog records the dimensions that affect delivery: + +| Dimension | Examples | +| --------------- | ------------------------------------------------------------------------------------------------------ | +| Action | left click, right click, double click, type text, key, hotkey, scroll, drag, child window, editor save | +| Addressing | AX element or PX coordinate | +| Delivery | foreground or background | +| Scope | target window or full desktop | +| Surface | shared renderer, native toolkit, embedded web view, or compositor | +| Expected result | delivered or one exact structured refusal | + +Foreground and background are dimensions of an action, not separate test +families. When a surface supports both delivery modes, both belong in the same +catalog. A platform-specific runner establishes the desktop session and +collects evidence, but it does not redefine the expected behavior. + +## Independent oracles define success + +A successful driver response is not a passing E2E result. A delivered action +must change state that the fixture or desktop independently owns. + +| Oracle | What it establishes | +| -------------------- | --------------------------------------------------------------------------------------- | +| Fixture state | The application recorded the click, text, key, selection, scroll, drag, or saved state. | +| Accessibility state | UIA, AX, or AT-SPI reports the expected control value or structure. | +| Pixel state | Before and after images contain the required visible change. | +| Focus and z-order | A background action did not activate or raise the target. | +| Cursor state | The physical user cursor did not move when background delivery promised that property. | +| Leaked-input journal | The foreground sentinel did not receive input intended for the background target. | +| Protocol state | An unsupported route returned the exact declared refusal. | + +Background checks combine target-state and side-effect oracles. This matters +because input delivered to the wrong foreground application is worse than an +honest refusal. A refusal passes only when its exact code is expected and the +desktop remains unchanged. + +## Evidence makes a result auditable + +Canonical hosted GUI runs retain the typed result for every declared cell, the +source commit, per-cell desktop video, before and after state, the tool +trajectory, fixture journals, driver logs, and environment preflight. The +GitHub Actions summary links each matrix row to its artifacts. + +The reporter rejects missing or duplicate rows, undeclared outcomes, +contradictory results, and incomplete required evidence. A failed environment +cannot silently turn into a smaller green matrix, and an `ok` response without +an observed effect remains a failure. + +## Release validation balances cost and fidelity + +Deterministic unit, protocol, compile, and packaging checks run automatically +where configured. Interactive E2E suites require a real graphical session and +are maintainer-triggered because they are slower and more environment-sensitive. +Windows and Linux have dispatchable GitHub Actions workflows; macOS runs the +same Rust catalog in a logged-in session with Accessibility and Screen +Recording permissions. + +Each accepted run builds the driver and fixtures from one exact source commit. +Support claims change only after an unchanged behavior cell produces new +application-owned evidence or an explicit structured refusal. Skips, weaker +oracles, and command-return-only checks do not establish support. + +## Related reference + +- [Platform support](/reference/cua-driver/platform-support) records currently + proven environments and limitations. +- [Platform roadmap](/reference/cua-driver/platform-roadmap) records remaining + engineering work and hard platform boundaries. +- [Interface contracts](/reference/cua-driver/contracts) defines public driver + behavior and refusal semantics. diff --git a/docs/content/docs/concepts/index.mdx b/docs/content/docs/concepts/index.mdx index 4f317ee608..37cd8a7fe9 100644 --- a/docs/content/docs/concepts/index.mdx +++ b/docs/content/docs/concepts/index.mdx @@ -5,4 +5,4 @@ description: "Understand the design ideas behind Cua and how its main pieces fit Use these pages when you want the model behind Cua rather than a step-by-step guide or API table. They explain what computer use means in Cua, how Cua Driver keeps the desktop usable while it acts, and how Cua Sandbox gives an agent a disposable computer. -Start with [What is computer use?](/concepts/what-is-computer-use) for the basic model. Read [Best-effort background](/concepts/the-no-foreground-contract) to understand Cua Driver's default behavior on a shared machine, then [Capture and delivery modalities](/concepts/capture-and-delivery-modalities) for the action axes. Read [How sandboxes work](/concepts/how-sandboxes-work) when you need the model for disposable computers. Read [How Lume prepares vanilla macOS VMs](/concepts/how-lume-unattended-setup-works) for the local VM setup model, and [How SIP works in Lume VMs](/concepts/how-sip-works-in-lume-vms) for signed boot-policy changes. +Start with [What is computer use?](/concepts/what-is-computer-use) for the basic model. Read [Best-effort background](/concepts/the-no-foreground-contract) to understand Cua Driver's default behavior on a shared machine, then [Capture and delivery modalities](/concepts/capture-and-delivery-modalities) for the action axes. [How Cua Driver is validated](/concepts/how-cua-driver-is-validated) explains why desktop support requires application-owned E2E evidence in addition to unit tests. Read [How sandboxes work](/concepts/how-sandboxes-work) when you need the model for disposable computers. Read [How Lume prepares vanilla macOS VMs](/concepts/how-lume-unattended-setup-works) for the local VM setup model, and [How SIP works in Lume VMs](/concepts/how-sip-works-in-lume-vms) for signed boot-policy changes. diff --git a/docs/content/docs/concepts/meta.json b/docs/content/docs/concepts/meta.json index 55e4f5d72f..961c521fd3 100644 --- a/docs/content/docs/concepts/meta.json +++ b/docs/content/docs/concepts/meta.json @@ -1 +1 @@ -{ "title": "Concepts", "icon": "Lightbulb", "pages": ["index", "what-is-computer-use", "the-no-foreground-contract", "capture-and-delivery-modalities", "how-sandboxes-work", "how-lume-unattended-setup-works", "how-sip-works-in-lume-vms"] } +{ "title": "Concepts", "icon": "Lightbulb", "pages": ["index", "what-is-computer-use", "the-no-foreground-contract", "capture-and-delivery-modalities", "how-cua-driver-is-validated", "how-sandboxes-work", "how-lume-unattended-setup-works", "how-sip-works-in-lume-vms"] } diff --git a/docs/content/docs/reference/cua-driver/development.mdx b/docs/content/docs/reference/cua-driver/development.mdx index b2b17944d6..6ffe494c12 100644 --- a/docs/content/docs/reference/cua-driver/development.mdx +++ b/docs/content/docs/reference/cua-driver/development.mdx @@ -23,8 +23,32 @@ they stay close to implementation changes. - `libs/cua-driver/rust/README.md` maps the Cargo workspace and test classes. - `libs/cua-driver/rust/crates/cua-driver/tests/README.md` explains Rust test naming and ignored GUI lanes. - `libs/cua-driver/tests/fixtures/README.md` explains source-built harness apps and staged outputs. -- `libs/cua-driver/rust/crates/cua-driver/tests/README.md` explains Rust test naming and ignored GUI lanes. - `libs/cua-driver/scripts/README.md` explains install and VM sync helpers. +## Desktop Validation Entry Points + +The canonical GUI runners execute the complete Rust harness catalog. Their +internal CI lanes may split shared, native, and capture owners for reporting, +but contributors do not select those partitions directly. + +```text +Linux X11/session: scripts/ci/linux/run-rust-e2e.sh +Linux Sway: scripts/ci/linux/run-rust-e2e-wayland.sh +Linux nested: scripts/ci/linux/run-rust-e2e-inject.sh +Linux GNOME/KDE: scripts/ci/linux/run-rust-e2e-desktop.sh +Linux real Xorg: scripts/ci/linux/run-rust-e2e-desktop.sh xorg +Windows: .\scripts\ci\windows\run-rust-e2e.ps1 -RequireGui +macOS: scripts/ci/macos/run-rust-e2e.sh +``` + +The Sway and nested-compositor runners create controlled sessions. GNOME, KDE, +real Xorg, Windows, and macOS use an existing graphical login. Windows needs an +interactive console or RDP session, and macOS needs a logged-in session with +the required Accessibility and Screen Recording permissions. See +[Platform support](/reference/cua-driver/platform-support) for current capability +boundaries, [How Cua Driver is validated](/concepts/how-cua-driver-is-validated) +for the evidence model, and [Platform roadmap](/reference/cua-driver/platform-roadmap) +for remaining work. + Build artifacts, VM logs, and local verification journals should stay out of git unless they have been promoted into a stable fixture or contributor doc. diff --git a/docs/content/docs/reference/cua-driver/limits.mdx b/docs/content/docs/reference/cua-driver/limits.mdx index 30654104a2..d2ce92a885 100644 --- a/docs/content/docs/reference/cua-driver/limits.mdx +++ b/docs/content/docs/reference/cua-driver/limits.mdx @@ -1,11 +1,16 @@ --- title: Known Limits -description: Documented behavioral limits of Cua Driver and available workarounds +description: Documented behavioral limits of Cua Driver and available alternatives --- import { Callout } from 'fumadocs-ui/components/callout'; -Cua Driver uses best-effort background delivery for every app it can reach via accessibility or routed pixel input. A handful of targets and platform quirks fall outside that envelope. The macOS cases come first, followed by the Linux session-stack limits. +Cua Driver attempts background delivery only when the operating system and +target expose a route that can be addressed safely. Unsupported shapes return a +structured refusal instead of reporting a silent success. This page lists +target-specific constraints and available alternatives; see +[Platform Support](/reference/cua-driver/platform-support) for the broader +operating-system matrix. --- @@ -77,24 +82,42 @@ Cua Driver uses best-effort background delivery for every app it can reach via a --- -## Native Wayland apps can't receive synthetic keystrokes +## Native Wayland background keyboard input is focus-bound -**Affected:** GTK/Qt apps running as native Wayland clients on GNOME Mutter or KDE KWin (no X11 surface). +**Affected:** Unfocused GTK/Qt apps running as native clients on standard +Wayland compositors, including Sway/wlroots, GNOME/Mutter, and KDE/KWin (no X11 +surface). -**Symptom:** `press_key` / `hotkey`, and `type_text` into a field that is not AT-SPI-editable, return success but the keystroke never reaches the app. Clicks and element actions on the same app work fine. +**Symptom:** A background `press_key`, `hotkey`, or `type_text` request for a +field that is not AT-SPI-editable returns structured +`background_unavailable`. Foreground delivery may also refuse when the desktop +has no target-addressable activation or raw-input backend. -**Cause:** Wayland blocks one client from synthesizing input into another by design. The compositor-cooperative paths that would re-enable it are not free here: `libei` requires a one-time RemoteDesktop-portal grant, and `wtype` (virtual-keyboard) is wlroots-only, so it fails on Mutter/KWin. +**Cause:** Wayland blocks one ordinary client from targeting another client's +surface with synthetic input. Portal/libei input on GNOME and KDE follows the +compositor's active seat and requires a RemoteDesktop grant. The virtual +keyboard used on wlroots compositors is also focus-bound, even though it does +not require the same portal route. **Workarounds:** 1. Type into accessible text fields with `type_text`. AT-SPI `insertText` writes the field directly, with no synthetic key event involved. 2. Drive controls by `element_index` (`click`, `set_value`) instead of keyboard shortcuts where an equivalent control exists. -3. Run the app under XWayland (`GDK_BACKEND=x11` / `QT_QPA_PLATFORM=xcb`). It then exposes an X11 surface and the X11 `XTEST` keyboard path applies. +3. Retry with `delivery_mode:"foreground"` when the desktop exposes a verified + activation and input adapter. +4. Run the app under XWayland (`GDK_BACKEND=x11` / `QT_QPA_PLATFORM=xcb`). It + then exposes an X11 surface and the X11 keyboard paths apply. - This is keyboard-only. Background **clicks and element actions** land on native Wayland via AT-SPI; coordinate (vision) clicks land too, resolved to the element under the pixel. + Background element actions can land through AT-SPI. A coordinate left click + can also land when its point resolves to an actionable AT-SPI element. This + does not establish arbitrary raw background PX delivery. +The opt-in nested `cua-compositor` is a separate, compositor-owned environment. +Its private per-surface injection protocol is not constrained like an ordinary +client on Sway, GNOME, or KDE, and its capabilities are documented separately. + --- ## GTK4 reports (0,0) screen coordinates over AT-SPI (handled) diff --git a/docs/content/docs/reference/cua-driver/mcp-tools.mdx b/docs/content/docs/reference/cua-driver/mcp-tools.mdx index 1941cee49d..bbbac596c2 100644 --- a/docs/content/docs/reference/cua-driver/mcp-tools.mdx +++ b/docs/content/docs/reference/cua-driver/mcp-tools.mdx @@ -211,7 +211,7 @@ from_zoom: set true after a zoom call to auto-translate zoom-image pixel coordin - `button` (string, optional): Mouse button. Default: "left" — omit for legacy left-click behaviour. Pixel path uses the matching CGEvent primitive; AX path maps "right" to AXShowMenu and falls back to a pixel middle-click at the element's center for "middle". - `count` (integer, optional): Click count (pixel path only). Default 1. - `debug_image_out` (string, optional): Optional file path. When set on a pixel-addressed click, captures a fresh screenshot, draws a red crosshair at (x, y), and writes the PNG. Use to verify coordinate spaces. Requires window_id; incompatible with from_zoom. -- `delivery_mode` (string, optional): Best-effort-background ladder rung for a PIXEL click (default "background"). "background": post the CGEvent to the pid without fronting. "foreground": briefly front the window, click, restore the prior frontmost — the explicit last resort for surfaces that drop background synthetic clicks. Requires window_id. A click is never driver-verifiable (no read-back), so both report verified:false — confirm the effect via screenshot. Use the agent loop: background AX (element_index) → screenshot → background pixel (x/y) → screenshot → delivery_mode:"foreground". +- `delivery_mode` (string, optional): Best-effort-background ladder rung (default "background"). "background": perform the AX action or post the CGEvent without fronting. "foreground": briefly front the window, act, let transient UI settle, then restore the prior frontmost app. Requires window_id. A click is never driver-verifiable (no read-back), so both report verified:false — confirm the effect via screenshot. Use the agent loop: background AX (element_index) → screenshot → background pixel (x/y) → screenshot → delivery_mode:"foreground". - `element_index` (integer, optional): Element index from last get_window_state. REQUIRES `pid` and `window_id` to be passed alongside it — element_index alone (no pid) fails fast with "Missing required integer field: pid"; it is not a silent no-op. - `element_token` (string, optional): Opaque per-snapshot element handle from `structuredContent.elements[].element_token` of the last get_window_state. Takes precedence over element_index when both supplied. Returns an explicit "stale" error if the snapshot has been superseded — re-snapshot in that case. - `from_zoom` (boolean, optional): When true, x and y are in the last zoom image for this pid; driver translates back to full-window coordinates. @@ -483,12 +483,12 @@ Actions: - `action` (string, required): Action to perform. - `attributes` (array of string, optional): Element attributes to include in query_dom results. - `bundle_id` (string, optional): Bundle ID of the browser. Required for enable_javascript_apple_events (macOS only). -- `cdp_port` (integer, optional): Optional, for insert_text/type_keystrokes only: use this exact CDP port instead of auto-discovering one from pid. Needed when the port was opened via the browser's own remote-debugging toggle rather than a launch-time flag, since that path may not answer the auto-discovery probe. +- `cdp_port` (integer, optional): Optional, for execute_javascript/insert_text/type_keystrokes: use this exact CDP port instead of auto-discovering one from pid. Needed when the port was opened via the browser's own remote-debugging toggle rather than a launch-time flag, since that path may not answer the auto-discovery probe. - `css_selector` (string, optional): CSS selector for query_dom (e.g. 'a', 'button', 'input', 'h1'-'h6', 'p', 'img', 'select', '*'). - `javascript` (string, optional): JavaScript to execute. Required for execute_javascript. - `pid` (integer, optional): Target process ID. - `selector` (string, optional): CSS selector for click_element (e.g. 'button.submit', '#login a'). -- `target_url_contains` (string, optional): Optional, for insert_text/type_keystrokes only: pick the browser tab whose URL contains this substring instead of whichever tab is found first. Use this on a multi-tab browser — there's no built-in link between window_id and which tab a CDP call reaches. +- `target_url_contains` (string, optional): Optional, for execute_javascript/insert_text/type_keystrokes: require exactly one browser tab whose URL contains this substring. Use this on a multi-tab browser — there's no built-in link between window_id and which tab a CDP call reaches. - `text` (string, optional): Text to insert or type. Required for insert_text and type_keystrokes. The target field must already have DOM focus (click/focus it first). - `user_has_confirmed_enabling` (boolean, optional): Must be true to proceed with enable_javascript_apple_events. This will quit and relaunch the browser. - `window_id` (integer, optional): Target window ID from list_windows. @@ -503,10 +503,13 @@ Actions: Start trajectory recording. Every subsequent action-tool invocation (click, right_click, scroll, type_text, press_key, hotkey, set_value) writes a turn folder under `output_dir`: +- `before_state.json` / `after_state.json` — application AX/UIA/AT-SPI state immediately before and after the action. +- `before.png` / `after.png` — target-window screenshots immediately before and after the action. +- `evidence.json` — capture status and a stable classification when an expected artifact could not be captured. - `app_state.json` — post-action AX/UIA snapshot for the target pid. -- `screenshot.png` — post-action per-window screenshot of the target's frontmost on-screen window. +- `screenshot.png` — compatibility alias of `after.png`. - `action.json` — tool name, full input arguments, result summary, pid, click point (when applicable), ISO-8601 timestamp. -- `click.png` — for click-family actions only, `screenshot.png` with a red dot drawn at the click point. +- `click.png` — for click-family actions only, `before.png` with a red marker at the click point. Turn folders are named `turn-00001/`, `turn-00002/`, etc. Turn numbering restarts at 1 each time recording is (re-)started. diff --git a/docs/content/docs/reference/cua-driver/meta.json b/docs/content/docs/reference/cua-driver/meta.json index 2773873c78..f90c04713f 100644 --- a/docs/content/docs/reference/cua-driver/meta.json +++ b/docs/content/docs/reference/cua-driver/meta.json @@ -1 +1,17 @@ -{ "title": "Cua Driver", "pages": ["cli-reference", "macos-permissions", "embedding", "mcp-tools", "mcp-tool-notes", "action-selection-policy", "contracts", "process-model", "limits", "development"] } +{ + "title": "Cua Driver", + "pages": [ + "cli-reference", + "macos-permissions", + "embedding", + "mcp-tools", + "mcp-tool-notes", + "action-selection-policy", + "contracts", + "platform-support", + "platform-roadmap", + "process-model", + "limits", + "development" + ] +} diff --git a/docs/content/docs/reference/cua-driver/platform-roadmap.mdx b/docs/content/docs/reference/cua-driver/platform-roadmap.mdx new file mode 100644 index 0000000000..fdf5167777 --- /dev/null +++ b/docs/content/docs/reference/cua-driver/platform-roadmap.mdx @@ -0,0 +1,110 @@ +--- +title: Platform Roadmap +description: Remaining Cua Driver platform work, evidence gaps, and operating-system boundaries without implied delivery dates. +--- + +This roadmap records capability and validation work for Cua Driver. It does not +promise dates. A capability moves to [Platform Support](/reference/cua-driver/platform-support) +only after a canonical Rust harness observes the required application and +desktop state. + +## Status vocabulary + +| Status | Meaning | +| ----------------- | ------------------------------------------------------------------------------------------------- | +| Proven | Canonical harness evidence supports the current public claim. | +| Evidence gap | Code or platform APIs may support the behavior, but representative harness evidence is missing. | +| Engineering work | A concrete driver, fixture, observer, or runner change is required. | +| Experimental | An opt-in backend or environment has evidence but is not part of the general support contract. | +| Platform boundary | The ordinary OS security or windowing model does not expose a safe general route for the behavior. | + +An evidence gap does not mean a behavior is impossible. A platform boundary +does not prevent narrower semantic routes, foreground delivery, or operation +inside an environment that owns the compositor. + +## Cross-platform priorities + +| Area | Current state | Next acceptance condition | +| ---------------------------- | --------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------- | +| Shared application catalog | Electron and Tauri use one typed Rust behavior catalog across operating systems. | Keep the same action, AX/PX, foreground/background, scope, and oracle dimensions on every representative host. | +| Native application catalogs | Windows, macOS, and Linux have toolkit-specific source-built harnesses. | Expand native rows toward the shared action cross-product where the toolkit exposes an equivalent behavior. | +| Background safety | Background cells attach focus, z-order, cursor, and leaked-input oracles. | Add the same side-effect owners to every new background delivery or refusal row. | +| Release evidence | Hosted GUI lanes retain typed results, trajectories, screenshots, logs, and per-cell video. | Require accepted exact-source-SHA runs for affected platforms before changing release support claims. | +| Representative environments | Hosted lanes cover Windows, Linux X11, Sway, and the nested compositor; macOS uses a logged-in host. | Close named GNOME, KDE, real-Xorg, and renderer evidence gaps without weakening fixtures or oracles. | + +## Windows + +| Work item | Status | Acceptance condition | +| ------------------------------------------ | ---------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| Broader WPF PX gestures and keyboard paths | Evidence gap | Typed foreground/background rows observe native control state and all required desktop side effects. | +| WinUI 3 pointer and background coverage | Evidence gap | Right click, double click, drag, scroll, and keyboard contracts are declared and empirically observed or refused. | +| WebView2 native-input coverage | Evidence gap | Native pointer and keyboard cells complement the existing page/CDP and background-left-click evidence. | +| Elevated-integrity boundary | Engineering work | A controlled fixture proves the `background_uipi_blocked` contract across process integrity levels. | + +Windows integrity isolation is a platform boundary: a lower-integrity process +cannot generally inject input into a higher-integrity target. The roadmap item +is to detect and prove that refusal, not bypass the operating-system boundary. + +## macOS + +| Work item | Status | Acceptance condition | +| ---------------------------------------- | ---------------- | ---------------------------------------------------------------------------------------------------------- | +| Native AppKit action cross-product | Evidence gap | Press key, hotkey, AX-addressed pointer gestures, and additional controls have application-owned evidence. | +| SwiftUI transient-window discovery | Engineering work | The opened popover or panel is independently visible through targeted window and AX enumeration. | +| Background drag and renderer scroll gaps | Engineering work | Each shape either delivers without focus or cursor side effects, or returns its exact documented refusal. | +| Optional installed-app confidence checks | Proven, optional | TextEdit and Calculator checks remain supporting evidence rather than replacements for repo-local fixtures. | + +Accessibility and Screen Recording consent remain platform prerequisites. +Off-Space SwiftUI tree stripping, minimized keyboard commits, and applications +that accept only HID-tap input are platform or target boundaries; their safe +alternatives remain documented in [Known Limits](/reference/cua-driver/limits). + +## Linux X11 + +| Work item | Status | Acceptance condition | +| --------------------------------- | ------------ | ----------------------------------------------------------------------------------------------------- | +| Hosted Openbox/Xvfb catalog | Proven | Preserve the complete shared and GTK catalog with exact delivery or refusal outcomes. | +| Real-Xorg MPX and uinput behavior | Evidence gap | A maintainer lane proves routes that Xvfb cannot represent, including multi-pointer desktop behavior. | +| Additional toolkit confidence | Evidence gap | Add a surface only when it represents a materially different input or accessibility contract. | + +X11 permits more synthetic-input routes than Wayland, but an X server accepting +an event does not prove the target toolkit handled it. Application-owned state +remains required. + +## Linux Wayland + +| Environment | Work item | Status | Acceptance condition | +| ----------------------- | --------------------------------------------- | ---------------- | ------------------------------------------------------------------------------------------------------------------------- | +| Sway/wlroots | Complete accepted Electron shared catalog | Engineering work | Every declared cell delivers or returns its exact refusal through unchanged fixture and side-effect oracles. | +| Sway/wlroots | Representative Tauri/WebKitGTK | Evidence gap | Run the shared catalog with a usable renderer accessibility tree on real DRM/EGL hardware. | +| GNOME/Mutter | Shared renderer catalog and portal video | Evidence gap | Electron and representative WebKitGTK rows pass with reporter-owned per-cell recordings. | +| KDE/KWin | Target-addressable activation and full matrix | Engineering work | A supported KWin adapter activates the selected target and an independent observer verifies it before portal input. | +| Nested `cua-compositor` | Complete shared catalog and protocol coverage | Experimental | One accepted full catalog proves renderer behavior, Unicode text, and canonical parallel drag. | +| Other wlroots | Representative compatibility | Evidence gap | Add a lane only after a user report demonstrates behavior that the Sway lane does not represent. | + +### Standard Wayland boundary + +An ordinary client on a standard Wayland compositor cannot generally send raw +pointer or keyboard input to an arbitrary occluded, unfocused surface. +Reconstructed coordinates solve target geometry, but they do not change which +surface receives compositor-seat input. Semantic AT-SPI actions and PX-addressed +hit-testing can still operate in the background when a coordinate resolves to +an actionable accessible element. + +The optional nested `cua-compositor` has a different capability because it owns +the compositor and can route input directly to a selected client surface. Work +in that lane can improve controlled nested sessions, but cannot establish a +general Sway, GNOME, KDE, or stock-Wayland claim. + +## Promotion rules + +A roadmap item becomes supported only when all of the following are true: + +- the behavior is a typed Rust case rather than a second OS-specific matrix; +- delivery changes fixture-owned state, or refusal returns the exact declared code; +- every required focus, z-order, cursor, leaked-input, capture, and scope oracle passes; +- the result identifies the operating system, window system, surface, delivery route, and source commit; +- representative evidence is retained and the public support reference is updated. + +For the reasoning behind these requirements, see +[How Cua Driver is validated](/concepts/how-cua-driver-is-validated). diff --git a/docs/content/docs/reference/cua-driver/platform-support.mdx b/docs/content/docs/reference/cua-driver/platform-support.mdx new file mode 100644 index 0000000000..3db560e6b8 --- /dev/null +++ b/docs/content/docs/reference/cua-driver/platform-support.mdx @@ -0,0 +1,91 @@ +--- +title: Platform Support +description: Current operating-system and window-system support, proven surfaces, and known capability boundaries. +--- + +import { Callout } from 'fumadocs-ui/components/callout'; + +cua-driver supports Windows, macOS, and Linux. Support is defined by observed +behavior in a real application, not by whether a tool call returned success. +The exact delivery route depends on the operating system, window system, +application toolkit, action, and whether the target may be brought forward. + +For definitions of AX, PX, foreground, background, window scope, and desktop +scope, see [Capture and delivery modalities](/concepts/capture-and-delivery-modalities). + +## Support levels + +| Level | Meaning | +| --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | +| Supported | A canonical Rust harness test proves the result against application-owned or desktop-owned state. | +| Supported with limits | Common paths are proven, but the platform or window system cannot safely provide every delivery shape. Unsupported paths return a structured refusal. | +| Experimental | The backend exists, but representative coverage is incomplete. Do not assume unlisted actions work. | + +## Platform overview + +| Platform | Window system and automation APIs | Current state | +| ------------- | -------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Windows | Win32, UI Automation (UIA), native input, and targeted window messages | Supported. Canonical coverage includes Electron, Tauri, WPF, WinUI 3, and WebView2. Some background Chromium gestures and elevated-integrity boundaries remain unavailable or unproven. | +| macOS | AppKit, Accessibility (AX), Quartz/HID, and ScreenCaptureKit | Supported. Canonical coverage includes Electron, Tauri, AppKit, SwiftUI, and WKWebView. Accessibility and screen-recording permissions are required. Some background scroll and drag shapes return structured refusals. | +| Linux X11 | X11/EWMH, XTest, AT-SPI, and toolkit accessibility bridges | Supported with toolkit-specific limits. Foreground input and semantic background actions are broadly covered. Toolkits that reject synthetic background events receive an explicit refusal instead of a silent success. | +| Linux Wayland | AT-SPI plus compositor-specific discovery, capture, activation, and portal input | Supported with compositor-specific limits. Semantic background actions work where the application exposes them. Raw input cannot generally be sent to an arbitrary occluded surface. | + +## Linux window systems + +Linux support is recorded per window system and Wayland compositor. "Wayland" +is not one uniform automation API: compositors expose different discovery, +capture, activation, and input protocols. + +| Environment | State | What is proven | Main limits | +| ------------------------------- | --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| X11/Xorg | Supported | Window discovery and capture, AT-SPI trees and actions, foreground pointer and keyboard input, semantic background delivery, desktop scope, and structured refusals | Raw background delivery depends on the target toolkit. X11 accepting an event does not prove the application handled it. | +| Sway (wlroots reference lane) | Supported with limits | Native discovery, screencopy, GTK controls, partial Electron shared coverage, and the complete Electron PX-background group | Focus-bound background input returns structured refusals. A complete accepted shared renderer matrix remains open. Other wlroots compositors are expected to share protocol support but are not yet proven. | +| GNOME/Mutter | Supported with limits | AT-SPI actions, GTK controls, compositor-backed window geometry and capture, verified foreground activation, and portal/libei foreground input | The bundled WinRects Shell helper and one Shell-session restart are prerequisites for authoritative geometry and activation. Portal video recording remains incomplete. | +| KDE/KWin | Experimental | Plasma 6 session startup, GTK AT-SPI discovery, generic discovery where exposed, and portal interface availability | cua-driver does not yet have a target-addressable KWin activation adapter, and no complete behavioral matrix is accepted. Focus-bound input refuses rather than risking delivery to the wrong application. | +| `cua-compositor` nested session | Experimental | Native GTK behavior, capture and scope, private route metadata, independent observation, and per-cell video | The complete shared renderer matrix is not accepted. Unicode text and a canonical parallel-drag row remain unproven; this route does not establish a stock-Wayland capability. | +| XWayland | Supported with limits | X11 routes are used when the application exposes a real X11 window; native Wayland and AT-SPI fallbacks cover mixed sessions | Capabilities depend on whether the application is actually using X11 or native Wayland. | + +### Wayland background AX and PX + +**Background AX works when the target exposes a semantic AT-SPI action.** For +example, the driver can invoke an accessible button without raising its window. +The passing test must also prove that focus and z-order did not change and that +input did not leak into the foreground application. + +**Background PX left click is proven for GTK controls on Sway and GNOME when +the coordinate resolves to an actionable AT-SPI element.** A caller still +addresses the target by pixel, but the safe delivery route hit-tests that point +and invokes its semantic action. This result does not establish raw background +PX delivery, other pointer actions, or the same route on Electron and Tauri. + +**Arbitrary raw background PX is not available to an ordinary client on a +standard Wayland compositor.** Reconstructing a window's coordinate system +tells cua-driver where the target is, but portal/libei and virtual input still +deliver through the compositor's active seat. An occluding surface therefore +receives a raw event sent at that screen coordinate. + +cua-driver also contains an opt-in exception: the nested `cua-compositor` +backend owns the compositor and can route `wl_pointer` and `wl_keyboard` events +directly to a selected client surface through `CUA_INJECT_SOCKET`, without +changing seat focus. This implements focus-free raw click, text, named-key, and +multi-pointer drag paths. The backend is covered by the converged typed matrix, +but remains experimental until the shared renderer matrix also passes. Unicode +text, multi-pointer behavior, canvas/game input, and any row without external +fixture evidence remain unproven. + + + A structured `background_unavailable` or `background_occluded` result is part of the contract. It + means cua-driver refused an unsafe or unsupported route before it could disturb the user's active + desktop. It is not a silent success. + + +## Related documentation + +- [How Cua Driver is validated](/concepts/how-cua-driver-is-validated) explains + why unit tests and application-owned E2E evidence have different roles. +- [Platform roadmap](/reference/cua-driver/platform-roadmap) records the + remaining engineering work, evidence gaps, and platform boundaries. +- [Known limits](/reference/cua-driver/limits) lists target-specific constraints + and available alternatives. +- [Development](/reference/cua-driver/development) provides contributor entry + points and canonical validation commands. diff --git a/docs/content/docs/reference/lume/cli-reference.mdx b/docs/content/docs/reference/lume/cli-reference.mdx index 768b3cb96c..ac93171900 100644 --- a/docs/content/docs/reference/lume/cli-reference.mdx +++ b/docs/content/docs/reference/lume/cli-reference.mdx @@ -7,12 +7,12 @@ description: Command Line Interface reference for Lume AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY Generated by: npx tsx scripts/docs-generators/lume.ts Source: lume dump-docs --type cli - Version: 0.3.13 + Version: 0.3.14 */} A lightweight CLI and local API server to build, run and manage macOS VMs. -Documented against Lume **0.3.13**. Run `lume --version` for your installed version. +Documented against Lume **0.3.14**. Run `lume --version` for your installed version. For installation steps, see [Install Lume](/how-to-guides/lume/install-lume). diff --git a/docs/content/docs/reference/lume/http-api.mdx b/docs/content/docs/reference/lume/http-api.mdx index 268fd8d288..109915c950 100644 --- a/docs/content/docs/reference/lume/http-api.mdx +++ b/docs/content/docs/reference/lume/http-api.mdx @@ -7,14 +7,14 @@ description: HTTP API reference for Lume server AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY Generated by: npx tsx scripts/docs-generators/lume.ts Source: lume dump-docs --type api - Version: 0.3.13 + Version: 0.3.14 */} import { Tabs, Tab } from 'fumadocs-ui/components/tabs'; HTTP API for managing macOS and Linux virtual machines -Documented against Lume **0.3.13**. Run `lume --version` for your installed version. +Documented against Lume **0.3.14**. Run `lume --version` for your installed version. ## Default URL diff --git a/flake.nix b/flake.nix index d18f04e733..8211d87744 100644 --- a/flake.nix +++ b/flake.nix @@ -21,17 +21,7 @@ ( system: let - pkgs = import nixpkgs { - inherit system; - # Several Electron-based apps in the background-GUI test matrix - # (logseq, joplin, zettlr) pin Electron releases that nixos-26.05 - # flags as EOL/insecure. These are read-only smoke tests running in - # throwaway CI containers, so permit insecure Electron specifically - # (version-agnostic, so a nixpkgs bump to a newer EOL Electron keeps - # working without editing an exact version string here). - config.allowInsecurePredicate = - pkg: nixpkgs.lib.hasPrefix "electron" (nixpkgs.lib.getName pkg); - }; + pkgs = import nixpkgs { inherit system; }; rustSrc = ./libs/cua-driver/rust; @@ -39,223 +29,108 @@ inherit pkgs; src = rustSrc; }; + + cuaCompositorPackage = pkgs.callPackage ./nix/cua-driver/compositor { }; + + # nixpkgs builds the AT-SPI launcher for NixOS's system profile. + # The E2E shell also runs on non-NixOS hosts such as GitHub's Ubuntu + # image, so point its private accessibility bus at store binaries. + hostAtSpi = pkgs.at-spi2-core.overrideAttrs (old: { + mesonFlags = map ( + flag: + if pkgs.lib.hasPrefix "-Ddbus_daemon=" flag then + "-Ddbus_daemon=${pkgs.dbus}/bin/dbus-daemon" + else if pkgs.lib.hasPrefix "-Ddbus_broker=" flag then + "-Ddbus_broker=${pkgs.dbus-broker}/bin/dbus-broker-launch" + else + flag + ) old.mesonFlags; + }); + + waylandE2eLibraries = with pkgs; [ + alsa-lib + cairo + cups + dbus + expat + glib + gtk3 + libayatana-appindicator + libdrm + libei + libgbm + librsvg + libsoup_3 + libx11 + libxcb + libxcomposite + libxdamage + libxext + libxfixes + libxi + libxkbcommon + libxrandr + libxtst + mesa + nspr + nss + openssl + pango + pipewire + webkitgtk_4_1 + ]; + + waylandE2eShell = extraPackages: pkgs.mkShell { + # hostAtSpi is referenced by absolute launcher path below, but is + # deliberately not a shell package: adding its rebuilt library and + # typelib hooks alongside GTK's stock AT-SPI closure loads two ATK + # copies and crashes PyGObject during Gtk import. + packages = (with pkgs; [ + cargo + clang + dbus + ffmpeg + gobject-introspection + grim + jq + nodejs + pkg-config + procps + rustc + rustfmt + sway + wf-recorder + wtype + # Keep the GTK3 fixture on the mature Python/PyGObject combination. + (python312.withPackages (pythonPackages: [ pythonPackages.pygobject3 ])) + ]) ++ extraPackages; + buildInputs = waylandE2eLibraries; + LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath waylandE2eLibraries; + shellHook = '' + export NO_AT_BRIDGE=0 + export CUA_AT_SPI_BUS_LAUNCHER="${hostAtSpi}/libexec/at-spi-bus-launcher" + export XDG_DATA_DIRS="${hostAtSpi}/share''${XDG_DATA_DIRS:+:$XDG_DATA_DIRS}" + ''; + }; in { packages = { + cua-compositor = cuaCompositorPackage; cua-driver = cuaDriverPackage; default = cuaDriverPackage; }; - checks = - { - cua-driver-build = cuaDriverPackage; - # Source-built, headless Rust checks. The desktop behavioral - # matrix remains an explicit maintainer-dispatched e2e lane. - cua-driver-linux-rust-unit = import ./nix/cua-driver/tests/rust-unit.nix { - inherit pkgs; - src = rustSrc; - }; - } - // pkgs.lib.optionalAttrs (system == "x86_64-linux") { - # NixOS container integration test (x86_64-linux only) - cua-driver-integration = import ./nix/cua-driver/tests/integration.nix { - inherit pkgs; - inherit (pkgs) lib; - cuaDriverModule = { - imports = [ ./nix/cua-driver/module.nix ]; - services.cua-driver.package = cuaDriverPackage; - }; - }; - - # set_config persistence test — regression for #1923 (fixed in - # #1928): the {key, value} write shape must persist and read back - # via get_config (it was silently dropped on Linux before). - cua-driver-set-config = import ./nix/cua-driver/tests/set-config.nix { - inherit pkgs; - inherit (pkgs) lib; - cuaDriverModule = { - imports = [ ./nix/cua-driver/module.nix ]; - services.cua-driver.package = cuaDriverPackage; - }; - }; - - # Screenshot test — uses cua-driver's own get_window_state tool - # to capture a screenshot via MCP, proving the driver can see the display - cua-driver-screenshot = import ./nix/cua-driver/tests/screenshot.nix { - inherit pkgs; - inherit (pkgs) lib; - cuaDriverModule = { - imports = [ ./nix/cua-driver/module.nix ]; - services.cua-driver.package = cuaDriverPackage; - }; - }; - - cua-driver-linux-cursor-click-gif = import ./nix/cua-driver/tests/linux-cursor-click-gif.nix { - inherit pkgs; - inherit (pkgs) lib; - cuaDriverModule = { - imports = [ ./nix/cua-driver/module.nix ]; - services.cua-driver.package = cuaDriverPackage; - }; - }; - - cua-driver-linux-background-terminal-gif = import ./nix/cua-driver/tests/linux-background-terminal-gif.nix { - inherit pkgs; - inherit (pkgs) lib; - cuaDriverModule = { - imports = [ ./nix/cua-driver/module.nix ]; - services.cua-driver.package = cuaDriverPackage; - }; - }; - - # Multi-cursor (MPX) parallel-drag test on a REAL Xorg brought up - # by NixOS services.xserver (dummy video + libinput, on a seat via - # a display manager). This is the CI-viable replacement for the - # hand-launched-Xorg linux-parallel-drag-gif.nix (which timed out - # because a self-launched Xorg couldn't get a VT/seat in the - # emulated nixos-test VM). Proves uinput slaves enumerate as X - # devices, two cursors draw concurrent window-targeted events, and - # the shield grab keeps focus off the drag. - cua-driver-linux-parallel-drag-xserver = import ./nix/cua-driver/tests/linux-parallel-drag-xserver.nix { - inherit pkgs; - inherit (pkgs) lib; - cuaDriverModule = { - imports = [ ./nix/cua-driver/module.nix ]; - services.cua-driver.package = cuaDriverPackage; - }; - }; + checks = { + cua-compositor-build = cuaCompositorPackage; + cua-driver-build = cuaDriverPackage; + cua-driver-linux-rust-unit = import ./nix/cua-driver/tests/rust-unit.nix { + inherit pkgs; + src = rustSrc; + }; + }; - # NOTE: cua-driver-linux-parallel-drag-gif (nix/cua-driver/tests/ - # linux-parallel-drag-gif.nix) is intentionally NOT a flake check — - # it hand-launches Xorg, which can't get a VT/seat in the emulated - # GHA nixos-test VM. It is superseded by the services.xserver test - # above and kept only for local/real-X manual runs. - } - // pkgs.lib.optionalAttrs (system == "x86_64-linux") ( - # Background GUI input coverage — one independent matrix job per - # app, proving focus-free typing into real toolkit/browser windows. - pkgs.lib.listToAttrs ( - map ( - app: - pkgs.lib.nameValuePair "cua-driver-linux-background-gui-${app}" ( - import ./nix/cua-driver/tests/linux-background-gui.nix { - inherit pkgs app; - inherit (pkgs) lib; - cuaDriverModule = { - imports = [ ./nix/cua-driver/module.nix ]; - services.cua-driver.package = cuaDriverPackage; - }; - } - ) - # Real-app matrix: 5 apps per toolkit category run as a LENIENT, - # READ-ONLY skeleton (find window + driver page/get_text + GIF; - # focus-free WRITE / typed-text assertions are added later via - # trajectories). chromium keeps the full CDP focus-free-write - # override; tk is the negative-control full entry (Tk `send`). - # "firefox" remains disabled: historically it did not surface its - # window within the launch timeout in CI. Container tests run at - # native speed, so this may now pass — left disabled pending - # verification. - ) [ - "chromium" - "tk" - # GTK3 - "gtk3-gedit" - "gtk3-mousepad" - # gtk3-geany / gtk3-abiword temporarily disabled: their huge - # AT-SPI trees made the bounds walk + recorder grind in the - # previous emulated VM and the jobs timed out. Container tests run - # at native speed, so this may now pass — re-enable once verified - # fast enough for 700+-node trees. - # "gtk3-geany" - "gtk3-scite" - # "gtk3-abiword" - # GTK4 - "gtk4-characters" - # Qt5 - "qt5-manuskript" - "qt5-klog" - "qt5-openambit" - # Qt6 - "qt6-kate" - "qt6-kcalc" - "qt6-okular" - "qt6-qownnotes" - # Electron - "electron-zettlr" - "electron-joplin" - "electron-logseq" - ] - ) - ) - # Native-Wayland TDD matrix — reproduce the cua-driver scenarios on - # real, NATIVE Wayland sessions (XFCE on labwc/wayfire/sway, plus KDE - # and GNOME). Apps run as Wayland clients and the tests never set - # DISPLAY, so the X11-only driver cannot see them: this is a RED suite - # specifying native Wayland support. One check per (desktop × scenario) - # and per (desktop × background-GUI app). See - # nix/cua-driver/tests/wayland/README.md. - // pkgs.lib.optionalAttrs (system == "x86_64-linux") ( - let - # NOTE: xfce-wayfire dropped — the wayfire package fails to - # build in the current nixpkgs pin (wf-config can't link - # -ldoctest), an upstream packaging bug unrelated to cua-driver. - # labwc + sway still cover XFCE-on-wlroots. - waylandDesktops = [ - "xfce-labwc" - "xfce-sway" - "kde" - "gnome" - ]; - waylandScenarios = { - integration = ./nix/cua-driver/tests/wayland/integration.nix; - screenshot = ./nix/cua-driver/tests/wayland/screenshot.nix; - cursor-click-gif = ./nix/cua-driver/tests/wayland/cursor-click-gif.nix; - background-terminal-gif = ./nix/cua-driver/tests/wayland/background-terminal-gif.nix; - parallel-drag = ./nix/cua-driver/tests/wayland/parallel-drag.nix; - }; - waylandBgApps = [ - "foot" - "gtk3-gedit" - "qt6-kcalc" - ]; - waylandModule = { - imports = [ ./nix/cua-driver/module.nix ]; - services.cua-driver.package = cuaDriverPackage; - }; - scenarioChecks = pkgs.lib.listToAttrs ( - pkgs.lib.concatMap ( - desktop: - map ( - scenario: - pkgs.lib.nameValuePair "cua-driver-wayland-${desktop}-${scenario}" ( - import waylandScenarios.${scenario} { - inherit pkgs desktop; - inherit (pkgs) lib; - cuaDriverModule = waylandModule; - } - ) - ) (builtins.attrNames waylandScenarios) - ) waylandDesktops - ); - bgGuiChecks = pkgs.lib.listToAttrs ( - pkgs.lib.concatMap ( - desktop: - map ( - app: - pkgs.lib.nameValuePair "cua-driver-wayland-${desktop}-background-gui-${app}" ( - import ./nix/cua-driver/tests/wayland/background-gui.nix { - inherit pkgs desktop app; - inherit (pkgs) lib; - cuaDriverModule = waylandModule; - } - ) - ) waylandBgApps - ) waylandDesktops - ); - in - scenarioChecks // bgGuiChecks - ); + devShells.cua-driver-wayland-e2e = waylandE2eShell [ ]; + devShells.cua-driver-inject-e2e = waylandE2eShell [ cuaCompositorPackage ]; } ) // { diff --git a/libs/cua-driver/README.md b/libs/cua-driver/README.md index 0c8aef4558..3fe6be4dfe 100644 --- a/libs/cua-driver/README.md +++ b/libs/cua-driver/README.md @@ -18,6 +18,14 @@ Background computer-use driver for any agents. Speaks MCP over stdio; drives nat Start with `rust/README.md`, `rust/crates/cua-driver/tests/README.md`, and `tests/fixtures/README.md` when changing driver behavior or tests. +Contributor documentation: + +- `docs/test-matrix.md` maps unit and canonical harness E2E suites. +- `docs/action-support.md` is the empirical platform behavior ledger. +- `docs/test-harnesses-guide.md` explains fixture and runner ownership. +- `docs/linux-desktop-validation.md` covers representative Linux sessions. +- `docs/linux-support-completion-plan.md` preserves the historical Linux plan. + ## Claude Code computer-use compatibility Standard Claude Code MCP registration: diff --git a/libs/cua-driver/docs/action-support.md b/libs/cua-driver/docs/action-support.md new file mode 100644 index 0000000000..16fe220a5c --- /dev/null +++ b/libs/cua-driver/docs/action-support.md @@ -0,0 +1,80 @@ +# Desktop action support + +This ledger records the canonical Rust harness contracts for Windows, macOS, +and Linux. It is derived from typed `CaseSpec` rows and accepted E2E evidence, +not from a successful driver response alone. + +- **Delivered** means a fixture-owned state change was observed. +- **Refused** means the exact structured refusal code and all required desktop + side-effect oracles passed. +- **Gap** means unsupported or not yet proven. A missing row is never evidence + that an action is impossible. + +AX and PX describe how the target is selected. They do not require the same +delivery backend: a PX target may be hit-tested and delivered through AX/UIA +when that is the background-safe route. + +## Shared web harnesses + +Foreground rows listed here are delivered for both AX and PX. + +| OS | Harness | Background delivered | Background refused | +| --- | --- | --- | --- | +| Windows | Electron | left click and child window AX/PX | right click and double click AX/PX, plus drag PX: `background_occluded`; type text, press key, hotkey, and scroll AX/PX plus editor save AX: `background_unavailable`; `start_minimized` launch: `background_unavailable` when Windows denies the foreground lock, with no process spawned | +| Windows | Tauri | left/right/double click AX/PX, type text AX/PX, press key AX/PX, child window AX/PX, scroll AX, editor save AX | hotkey AX/PX and scroll PX: `background_unavailable`; drag PX: `background_occluded` | +| macOS | Electron | left/right/double click AX/PX, type text AX/PX, press key AX/PX, hotkey AX/PX, child window AX/PX, editor save AX | scroll AX/PX and drag PX: `background_unavailable` | +| macOS | Tauri | left/right/double click AX/PX, type text AX/PX, press key AX/PX, hotkey AX/PX, scroll AX/PX, child window AX/PX, editor save AX | drag PX: `background_unavailable` | +| macOS | WKWebView | left/right/double click AX/PX, type text AX/PX, press key AX/PX, hotkey AX/PX, scroll AX/PX, child window AX/PX, editor save AX | drag PX: `background_unavailable` | +| Linux X11 | Electron | Background AX left click and child-window actions deliver with strict focus, z-order, cursor, and input-leak oracles | Background PX left click, AX/PX right and double click, PX child window and drag, AX/PX keyboard and scroll, and AX editor save return exact `background_unavailable` | +| Linux X11 | Tauri | Background AX/PX left click and child-window actions deliver with strict focus, z-order, cursor, and input-leak oracles | Other background pointer, keyboard, scroll, drag, and editor-save shapes return exact `background_unavailable` | +| Linux Sway | Electron | Run `29199656600` passed 29/36 shared rows. Focused run `29200827296` then passed all 9 PX-background rows at the replacement source, including six delivered/refused rows that were already green and the three refusal rows repaired by preserving real compositor origins. | Foreground AX right/double click and foreground AX/PX hotkey are the four remaining delivery gaps. | +| Linux Sway | Tauri | Native toplevel discovery and fixture launch are exercised | Hosted software-rendered WebKitGTK does not expose a usable renderer AT-SPI tree without DRM/EGL, so shared Tauri results on the headless Sway runner are environment-limited rather than accepted product evidence | + +All shared background rows require fixture state, focus, z-order, cursor, and +no-leaked-input evidence. Foreground rows require fixture state and retain a +video plus before/after turn evidence. + +## Native Windows + +| Harness | Proven contracts | Refusals and gaps | +| --- | --- | --- | +| WPF | Native AX controls; background combo selection, left click, and value changes; PX background left click through UIA hit-testing; foreground pointer gestures | Background F5 and PX drag are `background_unavailable`. Additional PX gesture rows remain unproven. | +| WinUI3 | Current UIA control, value, selection, popup, and slider rows | Background right/double-click refusal behavior and broader PX coverage remain unproven. | +| WebView2 | CDP page operations; native PX background left click through UIA hit-testing | Native keyboard and broader pointer cells remain unproven outside the shared Tauri/Electron hosts. | + +`background_uipi_blocked` is a production refusal code with no canonical +elevated fixture today. It must not be counted as covered until a controlled +integrity-level harness can exercise it. + +## Native macOS + +| Harness | Proven contracts | Refusals and gaps | +| --- | --- | --- | +| AppKit | AX tree/capture; AX background left click, set value, and type text; PX background left/right/double click; PX foreground right/double click and slider drag; AX foreground/background scroll; desktop PX foreground left click | PX background slider drag returns exact `background_unavailable`. Native press key, hotkey, AX-addressed right/double click, and broader control combinations remain unproven. | +| SwiftUI | AX tree/capture; AX background left click and set value; foreground popover-trigger activation | The fixture proves `popover_open=true`, but the transient panel remains absent from targeted AX enumeration. Other native pointer and keyboard combinations remain unproven. | +| WKWebView | Full 36-cell shared-web catalog through the dedicated repo-local native host | PX background drag returns exact `background_unavailable`; the other 35 shared cells deliver. Native host-specific controls are outside this fixture. | + +## Native Linux + +Linux is recorded per display server and compositor because Wayland protocols +are capabilities, not one uniform API. + +| Environment | Proven contracts | Refusals and gaps | +| --- | --- | --- | +| X11/Openbox | Exact run `29194456173` passed all 108 declared outcomes: 71 deliveries and 37 exact refusals. GTK3 AT-SPI actions and values, foreground XTest pointer/keyboard routes, capture, desktop scope, and the complete Electron/Tauri catalog all reached fixture-owned oracles. | Xvfb does not prove the real-Xorg MPX/uinput background pointer route. Toolkits that reject XSendEvent retain exact refusals. | +| Sway/wlroots | Run `29195551765` passed the expanded GTK3 matrix 31/31 and capture/scope 5/5. Shared run `29199656600` passed 29/36 Electron rows, up from 23/36. Focused run `29200827296` passed all 9 Electron PX-background rows after the origin repair. | Stock Wayland cannot target raw focus-bound input at an occluded surface. Foreground Electron AX right/double click and AX/PX hotkey remain open. All 36 Tauri rows remain environment-limited because hosted software-rendered WebKitGTK exposes no usable renderer tree without DRM/EGL. | +| GNOME/Mutter | A real GNOME 46 Wayland run passed the full GTK3 matrix, 31/31. The WinRects helper supplies stable window ids, frame and buffer geometry, stacking, verified activation, stage capture, and the compositor cursor. AT-SPI handles semantic actions; persistent portal/libei sessions deliver foreground PX click, right/double click, drag, scroll, type, key, and hotkey. Background rows either deliver through AT-SPI with focus and leak guards or return the declared exact refusal. | The helper requires installation plus one Shell-session restart. Without it, target-bound foreground input refuses. Portal video and a shared Electron/Tauri GNOME run remain open. | +| KDE/KWin | A Plasma 6 session reached GTK AT-SPI discovery, generic toplevel discovery, and portal-interface preflight. Portal input is compiled into release binaries. | No behavioral matrix is accepted. A target-addressable KWin activation adapter is not implemented, so foreground portal/libei input refuses instead of injecting into the wrong focused app. | +| Nested `cua-compositor` | The optional backend owns its nested session and is covered by typed shared, native, and capture catalogs. Run `29197643541` passed native GTK3 31/31; run `29197887387` passed capture/scope 5/5. Full shared run `29199596935` passed 26/36 Electron rows before the occluded-subtree and wheel-frame repairs. | The lane remains experimental. The accepted full shared run still has 10 failures; focused replacement runs reduce that set but do not promote the environment. Unicode text and a canonical parallel-drag row remain unproven. This private route is not evidence of a stock-Wayland capability. | + +PX background left-click rows may resolve the screen point to an actionable +AT-SPI node. Such a pass proves the public PX-addressed behavior and its desktop +side effects, but it does not prove raw pixel delivery to canvases or games. + +## Maintenance rule + +Update this document only when a typed row is added, removed, or changes +contract and an empirical run supports the change. Keep unsupported delivery +as an exact refusal where the driver can determine it before dispatch. When an +OS API reports success but offers no effect read-back, retain a visible gap +rather than inventing a fixture-specific refusal in production code. diff --git a/libs/cua-driver/docs/cross-platform-ci-test-plan.md b/libs/cua-driver/docs/cross-platform-ci-test-plan.md deleted file mode 100644 index 2dbc1420f8..0000000000 --- a/libs/cua-driver/docs/cross-platform-ci-test-plan.md +++ /dev/null @@ -1,286 +0,0 @@ -# Cross-Platform CI and E2E Test Plan - -Status: implementation plan, with the source-built unit and manual runner -foundation now landed on the working branch - -## Goal - -Make the Rust harnesses the source of truth for user behavior on Linux and -Windows. Keep unit tests cheap and OS-scoped. Run desktop end-to-end tests as a -maintainer-controlled gate with the real session each platform requires. - -The same Rust scenario should run on both platforms whenever the behavior is -supported. The runner may use different platform APIs, but the scenario ID, -application oracle, and outcome rules should stay the same. - -## Test Tiers - -### Tier 1: Rust unit and compile checks - -Run automatically on pull requests when an affected platform or shared Rust -path changes. - -- No desktop session, TCC, RDP, AT-SPI, or Azure credentials. -- Run on `ubuntu-latest` and `windows-latest` in separate jobs. -- Use `cargo test --workspace --all-targets --locked`. -- Run Clippy in the same OS-scoped workflow where its cost remains acceptable. - -Shared code changes run both OS jobs. A platform-only change runs one job. - -### Tier 2: Fast Linux integration checks - -Keep the small Xvfb and AT-SPI checks that do not require a real user desktop. -These checks can remain automatic if their runtime stays short and stable. - -They cover protocol setup, capture modes, accessibility reachability, and -desktop-scope contract errors. They do not replace the behavioral harness. - -### Tier 3: Maintainer e2e gate - -Run the same Rust behavioral matrix against real harness applications. - -- Linux: the manual Rust runner for the canonical matrix; Nix X11 and Nix - Wayland remain supporting compositor checks until the source-built app pilot - is complete. -- Windows: Azure Windows with an active RDP user session. -- Trigger: `workflow_dispatch` with a commit ref, PR number, and suite. -- Optional nightly and post-merge runs can catch drift without slowing every PR. -- The VM or runner never pushes code. - -### Tier 4: Supporting compatibility checks - -Keep toolkit and distro checks that cover behavior outside the canonical harness -matrix. These checks should be labeled as supporting coverage and should not -claim that a user workflow passed when they only prove window discovery or a -non-error response. - -## Canonical Behavioral Matrix - -The Rust test suite owns the scenario list. CI workflows should select a suite, -not duplicate the scenario definitions in YAML or Python. - -### Applications - -- Electron -- Tauri - -### Surfaces - -- Accessibility action path -- Pixel-coordinate path - -### Delivery modes - -- Background -- Foreground - -### Scenarios - -- Calculator click sequence and result oracle -- Editor type, save, and saved-state oracle -- Enter key and hotkey state oracles -- Nested scroll and scroll-offset oracle -- Child-window creation oracle -- Drag and drop-state oracle -- Focus sentinel and window posture checks - -Each result has one of these outcomes: - -```text -pass -expected_refusal -known_gap -not_applicable -unexpected_failure -``` - -An expected refusal must include the structured driver error and capability -reason. A driver response without an external application-state change is not a -pass. - -The runner should emit a common result record: - -```json -{ - "scenario": "editor_type_and_save", - "os": "windows", - "app": "electron", - "surface": "ax", - "delivery": "background", - "outcome": "pass", - "oracle": "editor_status=saved:exact" -} -``` - -Platform-specific applications such as WPF and WinUI 3 remain focused Windows -coverage. GTK and Qt remain focused Linux coverage. They supplement the shared -matrix and do not define cross-platform acceptance. - -## Linux Structure - -Nix remains the Linux desktop test environment. It should build the driver and -repo-local harness applications from source and run the Rust tests against those -outputs. The test should not download a prebuilt application and Linux e2e runs -do not need GIF recording. - -Proposed layout: - -```text -nix/cua-driver/ - package.nix - module.nix - README.md - lib/ - app-catalog.nix - artifacts.nix - atspi-session.nix - mcp-client.nix - wayland-session.nix - x11-session.nix - checks/ - x11/ - harness-electron.nix - harness-tauri.nix - input-cursor-click.nix - input-parallel-drag.nix - service-config-persistence.nix - service-integration.nix - wayland/ - app-background-read.nix - input-background-terminal.nix - input-cursor-click.nix - input-parallel-drag.nix - service-integration.nix -``` - -The current generic `linux-background-gui.nix` should be split. Its read-only -app entries should become supporting checks or be removed after equivalent -canonical harness coverage exists. Its Chromium and Tk write paths should be -retained until the Rust harness covers the same contract. - -Use explicit flake check names such as: - -```text -cua-driver-linux-x11-harness-electron -cua-driver-linux-x11-harness-tauri -cua-driver-linux-x11-service-config-persistence -cua-driver-linux-wayland-gnome-input-background-terminal -``` - -Do not put `gif` in a test name. Screenshots, AX trees, structured results, and -driver logs are enough for routine artifacts. Capture a PNG on failure or when -diagnosing a pixel scenario. - -The testkit needs test-only path overrides so Nix can use immutable build -outputs: - -```text -CUA_TEST_DRIVER_BIN -CUA_TEST_APPS_ROOT -CUA_TEST_WORKSPACE_ROOT -``` - -Local runs keep their current defaults. - -## Windows Structure - -Windows uses the same Rust scenarios and repo-local harness source. The Azure VM -builds the applications on demand inside the active RDP session. - -Workflow and helper names: - -```text -.github/workflows/ci-rust-windows.yml -.github/workflows/e2e-rust-windows.yml -scripts/ci/windows/ - verify-user-session.ps1 - build-harnesses.ps1 - run-rust-e2e.ps1 - collect-artifacts.ps1 -``` - -The Windows e2e workflow now: - -1. Check out or sync the requested commit. -2. Confirm that the target process runs in the active RDP user session, not - Session 0. -3. Build Electron, Tauri, WPF, and WinUI 3 harnesses from source. -4. Build the Rust driver and integration tests. -5. Run the shared behavioral matrix, then Windows-native tests. -6. Collect structured results, UIA trees, screenshots, and driver logs. -7. Leave the VM unable to push or modify the source branch. - -The Windows unit workflow runs on `windows-latest` and does not attempt these -desktop steps. - -## Workflow Names - -Use the distinction between `ci` and `e2e` in the filename and displayed job -name: - -```text -CI: Rust Linux unit -CI: Rust Windows unit -CI: Linux fast integration -E2E: Linux Nix X11 -E2E: Linux Nix Wayland -E2E: Linux Azure desktop -E2E: Windows Azure RDP -``` - -The Nix X11 and Wayland workflows should stop running automatically for every -PR update once the maintainer gate is available. Keep manual dispatch, nightly -runs, and post-merge regression runs. - -## Legacy Test Migration - -Keep an inventory before deleting tests. For each test record its scenario, -oracle, OS, window system, app, and current CI role. - -Classify each test as: - -- `canonical`: a trusted user-behavior acceptance test. -- `supporting`: useful backend or toolkit coverage. -- `diagnostic`: useful during investigation but not a product gate. -- `retire`: redundant or weaker than a canonical replacement. - -Migration order: - -1. Add the testkit path overrides and strict fixture preflight. -2. Land the OS-scoped unit checks and the manual desktop runners. -3. Pilot source-built harness packaging inside Nix before moving app builds - into the Nix sandbox. -4. Add missing scenario coverage before removing an old test. -5. Move weaker tests out of required CI. -6. Delete tests only after the replacement has passed repeatedly on the target - platform. -7. Update the Nix and workflow READMEs to describe the new ownership. - -Do not delete specialized Wayland, MPX, WPF, or WinUI coverage merely because -the shared Electron/Tauri matrix exists. Retain those tests until the shared -matrix covers the behavior they uniquely exercise. - -## Acceptance Criteria - -- Shared scenario IDs and external oracles are identical on Linux and Windows. -- Shared Rust changes run both unit-test jobs. -- Platform-only changes run only the affected OS unit job. -- Linux e2e builds harness applications from source in the Linux desktop runner; - a fully sandboxed Nix app-build pilot remains a separate follow-up. -- Windows e2e builds harness applications from source in the Azure VM. -- No Linux e2e test requires GIF recording. -- No canonical test passes on driver success alone. -- Unsupported background behavior produces a structured refusal or remains a - visible known gap. -- Maintainer e2e runs publish a result table and failure artifacts linked to the - PR. -- Legacy tests are either classified, upgraded, moved out of required CI, or - removed with an equivalent canonical replacement. - -## Non-Goals - -- Do not make Windows use Nix. -- Do not create a second Python behavioral framework. -- Do not download opaque prebuilt desktop fixtures. -- Do not weaken external-state assertions to make a matrix green. -- Do not make Azure or RDP credentials available to untrusted PR code. diff --git a/libs/cua-driver/docs/e2e-ci-reporting.md b/libs/cua-driver/docs/e2e-ci-reporting.md index f460e4affd..bd9ec1a306 100644 --- a/libs/cua-driver/docs/e2e-ci-reporting.md +++ b/libs/cua-driver/docs/e2e-ci-reporting.md @@ -1,67 +1,189 @@ # Rust E2E CI Reporting -The Rust desktop harness is the behavioral source of truth. GitHub Actions only -selects the lane and publishes its results. +Rust owns behavioral case declarations, observations, result classification, +and report validation. OS runners build the environment, execute Rust targets, +and upload the validated artifacts. -## Workflow UI +## Canonical Invocation -The Linux and Windows manual workflows split the expensive suites into -independent jobs. On Windows, `suite=all` covers the default Rust tests, UX -guards, shared Electron/Tauri behavior, WPF/WinUI3/WebView2 harnesses, and -modality-input E2E. The workflow summary job collects their artifacts and -writes one Markdown table to the GitHub Actions run summary. +The contributor-facing invocation runs the complete matrix on every OS and +takes no suite selector: -This means a failure in the shared Electron/Tauri lane does not prevent the -native or modality lane from running. The workflow still fails when a required -lane fails. +```text +Windows: scripts/ci/windows/run-rust-e2e.ps1 -RequireGui +Linux: scripts/ci/linux/run-rust-e2e.sh +macOS: scripts/ci/macos/run-rust-e2e.sh +``` -## Result files +Workflows set a private lane value to fan that matrix into shared, native, and +capture jobs so one failure does not hide another lane. These are execution +partitions, not public suite choices or separate behavioral sources of truth. + +## Execution Order + +Each runner follows the same sequence: + +1. Resolve one immutable source SHA and check out that SHA in every lane. +2. Build the Rust driver and required repo-local fixtures. +3. Run `e2e_environment_preflight_test` once; it verifies `git rev-parse HEAD` + against the workflow's resolved SHA. +4. Abort before behavioral cells if the desktop, permissions, fixture, capture, + or video lifecycle is unavailable. +5. Run the selected Rust integration targets. +6. Validate declarations, results, and evidence with `cua-e2e-report`. +7. Upload the report, logs, and recordings. + +The preflight prevents a single TCC or desktop-session problem from appearing +as the same failure on every behavioral cell. + +## Artifact Files + +Each OS lane writes: + +```text +artifacts/cua-driver// +|-- environment.jsonl +|-- cases.jsonl +|-- results.jsonl +|-- summary.md +|-- environment-preflight.log +|-- .log +`-- recordings/-pid-/ + |-- recording.mp4 + |-- trajectory.json + |-- session.json + |-- cursor.jsonl + `-- turn-*/ + |-- action.json + |-- evidence.json + |-- before_state.json + |-- before.png + |-- after_state.json + |-- after.png + |-- app_state.json + |-- screenshot.png + `-- click.png (click-family turns) +``` -When `CUA_E2E_RESULTS_FILE` is set, Rust shared behavior tests append one JSON -object per host and scenario: +`cases.jsonl` is the executed catalog. `results.jsonl` contains one result for +every declared cell. In canonical mode, the reporter rejects duplicates, +missing results, undeclared results, contradictory statuses, missing required +videos, and missing or invalid expected turn evidence. An `evidence.json` +classification makes capture failure auditable but does not turn it into a +passing artifact. + +The testkit prepares each cell's artifact directory before fixture setup but +does not start `recording.mp4` yet. On Windows GitHub-hosted runners it obtains +the test process's inherited top-level console with `GetConsoleWindow`, verifies +the HostedComputeAgent/runner title and console class, then minimizes it with +`ShowWindow(SW_MINIMIZE)`. Fixture readiness +and the required foreground or sentinel-occluded posture are established after +that cleanup. The cell then calls `start_behavior_recording`, waits 300 ms for +a visible baseline frame, dispatches the action, and keeps recording through +oracle collection. + +`trajectory.json` records the behavioral video phase as `pending`, `started`, +or `finalized`, its start/baseline/finalization timestamps, and the hosted-runner +console cleanup status. Strict reporting +requires `finalized`. A setup or posture failure before the boundary remains in +the test result and logs, and the testkit writes `recording-error.txt`; it does +not create a misleading behavioral clip. + +## Environment Record + +The preflight emits exactly one record: ```json { - "schema": "cua-e2e-result/v1", + "schema": "cua-e2e-environment/v2", + "platform": "macos", + "display_server": "quartz", + "source_sha": "0123456789abcdef0123456789abcdef01234567", + "status": "ready", + "duration_ms": 912, + "message": "" +} +``` + +An error record produces an environment section in `summary.md` and aborts the +lane before case declarations are executed. The same source SHA appears in the +successful typed summary. + +## Case And Result Contract + +A case declaration uses `cua-e2e-case/v2`. A result flattens the same contract +fields into `cua-e2e-result/v2` and adds the observation: + +```json +{ + "schema": "cua-e2e-result/v2", + "cell_id": "windows-tauri-left-click-ax-background", "platform": "windows", - "host": "tauri", - "scenario": "keyboard", - "status": "FAIL", - "message": "application state did not reach key_state=enter", - "duration_ms": 24360 + "display_server": "win32", + "harness": "tauri", + "toolkit": "platform-webview", + "action": "left_click", + "targeting": "ax", + "delivery": "background", + "scope": "window", + "driver_route": "uia_invoke", + "expected_behavior": { "kind": "deliver" }, + "oracles": ["fixture_state", "focus", "z_order", "no_leaked_input", "cursor"], + "test_status": "pass", + "observed_behavior": "delivered", + "passed_oracles": ["fixture_state", "focus", "z_order", "no_leaked_input", "cursor"], + "duration_ms": 1482, + "message": "", + "evidence": { + "video": "recordings/windows-tauri-left-click-ax-background-pid123-001/recording.mp4", + "trajectory": "recordings/windows-tauri-left-click-ax-background-pid123-001/trajectory.json" + } } ``` -When `CUA_E2E_SUMMARY_FILE` is set, the same test writes a Markdown row for the -GitHub summary. The CI scripts also parse each Cargo `test ... ok/FAILED` line, -so native, guard, default, and modality suites appear as individual test rows -as well as lane-level rows. Full logs are retained alongside them. +`targeting` describes how the action identifies its target: `ax`, `px`, +`page`, or `not_applicable`. It is separate from screenshot/tree capture. -Statuses are deliberately explicit: +## Classification Rules -- `PASS`: the external application-state oracle passed. -- `FAIL`: the driver response or external oracle failed. -- `SKIP`: the fixture was unavailable and was not required by the invocation. +`test_status` answers whether the cell met its declared contract. +`observed_behavior` records what the driver did. -The CI scripts set `CUA_TEST_REQUIRE_FIXTURES=1`, so missing required fixtures -are reported as failures rather than silently becoming skips. +| Expected | Observed | Required oracles | Status | +| --- | --- | --- | --- | +| Deliver | Delivered | Passed | Pass | +| Deliver | Refused | Any | Fail | +| Deliver | No effect or error | Any | Fail | +| Refuse | Allowed structured refusal | Passed | Pass | +| Refuse | Different refusal code | Any | Fail | +| Refuse | Delivered | Any | Fail pending contract review | -## Running locally +The structured refusal enum currently recognizes +`background_unavailable`, `background_occluded`, and +`background_uipi_blocked`. Each refusal case declares the exact subset allowed +by its controlled setup. String-prefix or message-substring matching is not +accepted. -Windows: +A refusal contract also requires focus, z-order, and leaked-input observations. +An honest refusal does not satisfy a case whose contract requires delivery. -```powershell -.\scripts\ci\windows\run-rust-e2e.ps1 -Suite shared -RequireGui -.\scripts\ci\windows\run-rust-e2e.ps1 -Suite native -RequireGui -``` +## Summary Ownership -Linux: +`cua-e2e-report` is the only behavioral Markdown renderer. Shell and +PowerShell runners must not parse `test ... ok` output or create `host=cargo` +and `host=lane` rows. Cargo logs remain available for unit-test annotations and +lane diagnostics, outside the behavioral population. -```bash -xvfb-run -a --server-args="-screen 0 1920x1080x24" \ - dbus-run-session -- bash -lc \ - "scripts/ci/linux/run-rust-e2e.sh --suite shared" -``` +The summary reports delivered passes, refused passes, failures, and skips +separately. Shared and native targets emit the same typed records; a compile or +runner failure that prevents declaration remains visible as a failed job and +cannot be mistaken for a green behavioral row. + +## Evidence Links -The generated files are under `artifacts/cua-driver//` and should -not be committed. +Every canonical behavioral cell records a validated MP4 and trajectory. The +recording directory also contains its session metadata and turn-level action +and screenshot files; Cargo target logs remain lane-level diagnostics. GitHub +cannot deep-link to a file inside a multi-cell artifact archive, so each summary +row displays the exact MP4 path as a link to the owning lane archive while the +trajectory rollup provides bulk-download links and video counts. diff --git a/libs/cua-driver/docs/e2e-convergence-journal.md b/libs/cua-driver/docs/e2e-convergence-journal.md new file mode 100644 index 0000000000..fb8e148ebc --- /dev/null +++ b/libs/cua-driver/docs/e2e-convergence-journal.md @@ -0,0 +1,156 @@ +# E2E Convergence Journal + +This journal records changes and test evidence for the Rust desktop harness +convergence work. It omits machine names, credentials, and partner names. + +## 2026-07-12 + +| Commit | Change | Verification or finding | +| --- | --- | --- | +| `8296b82b` to `16fb314a` | Reused stable Sway origins, normalized occluded zero-geometry fallbacks, and retained renderer identities across accessibility and compositor queries | Shared run `29199656600` passed 29/36 Electron rows, up from 23/36. Focused run `29200827296` passed all 9 Electron PX-background rows at the replacement source, leaving foreground AX right/double click and foreground AX/PX hotkey open. All 9 focused Tauri rows remained explicitly environment-limited by the missing DRM/EGL renderer tree. | +| `88d700f4` to `a78f525a` | Made nested pointer hit-testing target-subtree-local, reused the entered child pointer resource, fell back to the owning root resource, and completed wheel frames | Full shared run `29199596935` passed 26/36 Electron rows before these focused repairs. Run `29200462634` then passed 7/9 occluded PX background rows, including raw right click, text, key, hotkey, child-window click, and drag. Dedicated run `29200854535` failed both scroll rows, so renderer scroll, two click rows, and one AX right-click timeout keep the shared environment experimental. | +| `8937209c` to `edb64a25` | Executed the Linux support-completion plan: reconciled stale fixture docs, added compositor/input-route metadata, packaged the nested compositor, restored typed shared/native/capture lanes, pinned VM source identity, and formalized Linux release validation | Automatic Linux unit, distribution, script, docs, and source-package gates remained green. X11 run `29194456173` retained all 108 accepted outcomes on the exact source. | +| `d11c026e` to `676b9ef4` | Replaced fire-and-forget nested injection with a versioned acknowledged protocol, PID targeting, exact resource errors, pointer/keyboard/scroll/hotkey/drag verbs, logical focus routing, and stale-surface cleanup | Early typed runs deliberately failed on ambiguous targets, missing logical seat focus, and stale surface resources. Each failure became a driver/compositor fix; unsupported keys and missing resources now refuse instead of returning false success. | +| `6ab4e930` to `0b0b0e2c` | Stabilized nested client identity and coordinate spaces across AT-SPI, screencopy, scene hit-testing, desktop scope, and private surface input | Run `29197643541` passed the native GTK3 matrix 31/31. Run `29197887387` passed capture/scope 5/5, including the previously failing screen-absolute GTK click and exact `LinuxCuaCompositorInject` route metadata. The shared renderer result remains recorded separately. | +| `1280021b` to `1fe2ae35` | Reconstructed Sway renderer geometry through stable compositor identity, cropped window capture by that identity, and connected renderer AT-SPI frames to the same window origin | Run `29195551765` retained native GTK3 31/31 and capture/scope 5/5. Shared run `29196452551` passed 23 Electron rows, exposed 13 renderer-coordinate/hotkey gaps without losing the 36 declared Tauri environment failures, and supplied the focused replacement target. | +| Representative GNOME/KDE audit | Reused the unchanged desktop entrypoint against real compositor sessions | GNOME's accepted 31 native GTK, three capture, and two desktop-scope contracts remain valid. Plasma 6 reached session startup, GTK AT-SPI discovery, and portal-interface preflight, but no KDE behavioral row was accepted because target-addressable activation and interactive portal consent remain absent. Real-Xorg MPX and DRM/EGL WebKitGTK also remain explicit environment gaps. | +| `9736e19b` to `ff1071af` | Removed nested AT-SPI runtimes, repaired Wayland trajectory target resolution, preserved 64-bit action identities, and routed foreground renderer typing through real keyboard events | Focused Sway run `29180903640` proved foreground/background Electron AX clicks delivered after the runtime repair. Run `29181552459` proved complete before/after AX state, screenshots, and click markers after the recorder fix. Run `29182130646` then passed Electron foreground AX type text with fixture-state evidence. AX hotkey now reaches the target but has no DOM effect, and PX hotkey still exposes a hosted-Sway renderer-origin gap; both remain explicit failures. | +| `f8e4b37b` and `74a29a20` | Completed the Linux X11, Sway, and GNOME Wayland convergence pass and published WinRects helper metadata v3 | Exact X11 run `29179817332` passed all 108 rows: 71 delivered and 37 exact refusals. The split Sway run proved all 31 native GTK3 rows and all five capture/scope rows with playable per-cell videos. A real GNOME 46 session passed 31 GTK3 rows, three capture contracts, and two desktop-scope contracts. | +| `346f6f73` | Added the direct xkbcommon link dependency to the Nix release package | The final source gate exposed `-lxkbcommon` after the portal/libei feature became part of the release build. `libxkbcommon` now belongs to the package's build inputs rather than only the interactive Wayland shell. | +| `fabf4a20` | Stabilized GNOME Wayland capture and foreground input through the WinRects helper and portal/libei | A real GNOME 46 session passed all 31 GTK3 rows. The fix separates Mutter frame and surface-buffer origins, waits for the complete EIS device announcement, selects the newest resumed pointer, uses monotonic frame timestamps, restores target activation after consent, and persists the grant until the user revokes it. AX background rows passed with the target fully occluded; unsupported background PX gestures returned their declared exact refusals. | +| `79e5ac2a` to `8161cf56` | Repaired Linux fixture scrolling, Wayland event flushing, accessibility enablement, metadata, portal-input feature packaging, and Linux CI compilation | Exact X11 run `29173767528` passed the expanded GTK3 matrix, while the prior complete X11 run `29148643166` passed all 80 declared outcomes. The portable release build now includes GNOME/KDE RemoteDesktop/libei input without requiring the newer PipeWire capture dependency. | +| `8719e292` | Made native Wayland pointer coordinates, scroll-at-point, keyboard priming, libei key sequences, and cropped window capture target-aware | Sway GTK3 run `29175153270` passed all 23 rows: 17 delivered outcomes and 6 exact background refusals, with no failures or skips. | +| `9c65c153` and `7b985276` | Added native GNOME observation, targeted Chromium accessibility, generic `ext_foreign_toplevel_list_v1`, stable identity enrichment, and Chromium frame-origin reconstruction | Linux unit and distro compatibility checks passed. A controlled GNOME probe showed driver-launched Electron in both the Shell window list and AT-SPI without globally enabling the screen reader. | +| `c9c82908` | Split pure-Wayland shared, native, and capture owners onto independent clean runners | Run `29175548524` no longer exhausted one runner disk. It isolated one Sway hotkey failure, one obsolete standalone-activation failure, and the hosted WebKitGTK renderer limitation instead of hiding later lanes. | +| `a07ba769` | Routed hybrid GNOME/KDE sessions through native Wayland, enumerated GNOME windows from authoritative Shell metadata, translated every pointer gesture from window-local pixels, and required verified target activation before portal/libei input | Clean Linux unit and six-distro compatibility runs passed at the exact SHA. GNOME now has a verified Shell activation adapter; unsupported KDE foreground injection fails closed rather than acting on the user's currently focused application. Exact X11 and split Wayland E2E reruns are active. | + +Native GNOME validation also exposed a verification-host issue: Nix `path:` +inputs copy untracked `target/` directories when a synced VM checkout has no +Git metadata. Stale source snapshots consumed the VM disk and were removed; +this is not a driver failure. Clean GitHub runners are unaffected. + +## 2026-07-11 + +| Commit | Change | Verification or finding | +| --- | --- | --- | +| `7623f7be` to `55307b38` | Added canonical native WebView2 and WPF PX background single-left-click rows with fixture-owned state and foreground geometry probes | An active Windows RDP replay passed both rows through UIA Invoke. Each result proved FixtureState, Focus, ZOrder, NoLeakedInput, and Cursor and produced complete before/after evidence plus video. WebView2 derives its real DOM screen point through CDP, proves it with a foreground PX click, then dispatches the occluded PX action. WPF reads an atomic JSON state file written by the fixture. All 31 testkit tests, strict testkit Clippy, and host compile checks passed. | +| `6aaa914b` | Restored direct Chromium/Electron single-left background clicks through `PostMessage` for both AX-resolved and PX-resolved targets | In an active Azure RDP session, raw UIA delivered the click but raised Electron and displaced the occlusion sentinel. A VM-only `PostMessage` diagnostic then delivered both AX and PX cells in 3/3 seeded runs while FixtureState, Focus, ZOrder, NoLeakedInput, and Cursor all passed. The production build passed all 12 Electron left/right/double-click rows; only the two single-left background rows changed from refusal to delivery. | +| `768f0650` to `6e05424a` | Completed the macOS convergence pass and bound local evidence to a clean exact source SHA | The full local matrix passed 83/83 contractual outcomes: 79 deliveries and 4 exact refusals. Fixes cover PX background hit-testing, foreground typing's first character, structured scroll/drag refusals, desktop HID scope, AppKit fixture stability, and per-connection daemon sessions. | +| `d39c71c9` | Made the Windows sentinel physically acquire foreground before each background action | Hosted runs no longer fail during sentinel posture. WinUI3, WebView2, WPF background actions, and the shared catalog all reach their behavior boundary with the original focus, z-order, cursor, and leaked-input checks intact. | +| `0c2331b4` to `3f6fbd53` | Completed foreground PX activation, removed volatile UIA metadata from the WPF drag-refusal oracle, and aligned WPF combo selection with the background UIA recipe | The replay passed 19/20 WPF rows, all WinUI3 and WebView2 rows, and all capture/scope rows; its only native failure was the separately owned minimized-launch contract. Electron child-window delivery and keyboard refusal codes were updated from empirical side-effect-clean results. | +| `00a29c49` and `e43c2946` | Tested `CreateProcessW` startup show-state control, then made minimized launch fail closed when Windows cannot guarantee no activation | Electron can override `STARTF_USESHOWWINDOW`, so startup hints alone are insufficient. The driver now returns exact `background_unavailable` before spawning when the foreground lock is denied; the canonical refusal proves no new window and no desktop side effect. | +| `8d28beec` | Added the native WKWebView shared catalog, fixed AppKit AX scroll, separated SwiftUI popover activation from transient AX discovery, and expanded native AppKit/SwiftUI action rows | The exact local run passed all 130 rows: 124 deliveries and 6 structured refusals. WKWebView passed 35 deliveries plus its background-drag refusal. AppKit foreground/background AX scroll, foreground/background PX right and double click, foreground slider drag, SwiftUI background click/value, and SwiftUI foreground popover activation passed fixture-state checks. AppKit background drag was a silent no-op, so the driver now returns exact `background_unavailable`; the refusal row proves unchanged fixture state, focus, z-order, cursor, and input journal. | + +## 2026-07-10 + +| Commit | Change | Verification or finding | +| --- | --- | --- | +| `0fda31bf` | Added typed case declarations, results, and lane preflight | Testkit unit tests passed locally. The first Windows and Linux runs exposed fixture and recording failures before behavioral classification. | +| `f2e5804c` | Added full result dimensions to the reporter | Reporter tests reject changed contracts and missing rows. | +| `6fa9311e` | Removed the synthetic calculator fixture | The canonical shared target now uses only controls in the repo-local web harness. | +| `891d4c94` | Added the platform-independent desktop observer core and Windows backend | Unit tests prove that transient focus changes, z-order changes, cursor movement, and input leaks fail independently. | +| `3a48ba0d` | Made the Linux desktop preflight reject blank recordings | Run `29131453240` stopped with `94/2073600` non-dark pixels. This was an environment failure, so it produced no behavioral verdicts. | +| `28ce18b6` | Added fail-closed cell filtering | An unmatched filter now fails instead of reporting a green zero-cell run. | +| `303c7bf3` to `056728ec` | Added the occluding Electron sentinel and staged its preload | A local sentinel smoke test wrote direct ready and focus events to its JSONL journal. | +| `640fcc9a` | Replaced the default route label with explicit backend routes | Unit coverage keeps Windows Chromium PX background delivery on `WindowsTargetedInjection`; it remains a required capability. | +| `580708f7` | Added the direct macOS observer | Fifteen testkit tests and strict testkit Clippy passed locally. A native snapshot smoke test read the foreground app and real cursor. | +| `00a1d31f` | Added the direct X11 observer and attached desktop checks to background shared cells | Independent review found and corrected XWayland, reparenting, raw input-focus, and occlusion traps. Sixteen testkit tests passed locally. | +| `bcc53fe7` | Expanded the shared catalog from a 19-cell diagonal to 32 cells per host | Route tests cover every declared Win32, Quartz, X11, and Wayland combination. | +| `21f3f6e0` | Fixed workflow dispatch checkout semantics | Earlier branch dispatches had selected the branch workflow but checked out the default `inputs.ref=main`. Their behavioral results were discarded. | +| `69c7c2c1` | Made each OS runner select only the typed shared catalog | Legacy tests remain in source until their assertions pass the deletion ledger, but they no longer define the canonical shared run. | +| `6733efe9` to `b52d239f` | Removed duplicate Windows modality-input probes, normalized desktop-scope ownership, and corrected Electron's X11 show path | Local testkit checks stayed green. Linux run `29133849724` still rejected a blank root-desktop recording, so it produced no behavioral verdict. | +| `c3454d1b` to `e54088f3` | Shared the Electron foreground sentinel and attached its Focus, ZOrder, Cursor, and NoLeakedInput contract to WPF background actions | Windows run `29134124533` passed all 18 WPF tests. Its lane failed later on an independent WebView2 CDP-readiness race. | +| `48afc293` to `842f3c86` | Moved background capture, minimized launch, and agent-cursor behavior into their action-owned tests | The replacements assert application or pixel state plus the desktop observer instead of a focus-only guard. | +| `c49dd59d` | Rejected Chromium UIA Invoke's false-positive and tested coordinate injection as a replacement route | Focused run `29134320325` proved the route still refused a fully occluded target. That route was not accepted as a fix. | +| `521508aa` to `99897fed` | Added the reusable typed executor and typed WPF, launch, capture, and cursor records | Sixteen testkit tests and host compile checks passed. Fresh native and capture runs are validating the Windows-only bodies. | +| `ad891b48` | Made CDP page discovery wait for the first usable target | The earlier native run's only failure was WebView2 exposing a listener before `/json` contained a page. The DOM-state assertion remains unchanged. | +| `10a1a749` | Preserved the old guard's anti-overclaim check in canonical background text/key rows and aligned action-owned video labels | An independent deletion audit identified protocol honesty as the only unique old-guard assertion. Chromium AX background click now probes a target-bound LegacyIAccessible action rather than a coordinate route. | +| `5bca25be` | Mapped the normal Electron harness at construction | Linux's deferred BrowserWindow was enumerable and capturable but absent from the X11 root recording. A focused preflight run is validating this lifecycle correction. | +| `c83c99ac` | Limited each CI lane to the fixtures it consumes | Windows and Linux focused runs built only their internal shared, native, or capture partition while the contributor command remained selector-free. | +| `8e41859f` | Published WebView2 readiness after DOM navigation, gave typed WPF rows exact recording labels, removed the duplicate modality-background target, and classified strict Chromium AX background click as a refusal | Run `29135420638` passed all 18 WPF, 7 WinUI3, and 3 WebView2 tests plus all four typed WPF background rows. Run `29135420139` returned the exact `background_occluded` refusal with all desktop-side-effect oracles passing. | +| `0a5352c5` | Reduced workflow dispatch to one canonical matrix and added row-level evidence links | Shared, native, and capture remain internal parallel jobs. The link renderer converts each exact MP4 path into a link to its owning lane artifact. | +| `ac1f19cc` | Added scoped Windows foreground locking and X11 alpha-shaped cursor-overlay bounds | A placement-only Linux rerun (`29135421864`) retained the exact blank-root signature, confirming placement was not the cause. The X11 fix addresses the full-screen ARGB overlay on non-composited sessions; Windows now prevents activation before launch instead of restoring it afterward. | +| `c190228e` | Tightened the controlled Chromium refusal declaration | The fully occluded test setup allows only `background_occluded`; generic or message-matched refusals remain rejected. | +| `a9953533` to `2c960eaf` | Corrected Windows foreground dispatch and made unsupported background routes report structured refusals instead of silent success | Run `29137293235` passed every shared foreground row. Its remaining background failures were used to define exact Chromium and embedded-WebView capability boundaries rather than weakening fixture-state assertions. | +| `f50b7170` to `64377935` | Required real fixture launch, split keyboard actions into independent cells, and targeted visible native controls | The shared catalog is now 36 cells per host. GTK3 and Linux desktop-scope lanes passed in run `29137481789`; Windows fixture geometry failures from run `29137143798` were removed without hard-coded replacement coordinates. | +| `6d973d2c` and `309740a6` | Awaited the first minimized-launch sweep and routed explicit Windows expand requests through UIA ExpandCollapse | Run `29137992025` caught a Windows-only compile error in the new pattern route before any behavioral verdict; `465f4a2f` unwraps the pattern result and the replacement run compiles. | +| `b284b521` | Exposed complete Linux renderer targets and removed the last Python accessibility write path | Shared lookup now searches the raw accessible name, fixture-state assertions poll fresh trees, AT-SPI text writes stay native Rust, and the silent 150-node bounds cutoff is gone while the 20-second budget remains. | +| `c473e045` | Reduced each OS to one contributor-facing E2E command | CI partitions remain internal through `CUA_E2E_INTERNAL_LANE`; `shared`, `native`, and `capture` are no longer public runner selectors. | +| `a86f8208` to `94a0509e` | Allocated CDP ports per process and replaced Windows page-test sleeps with DOM polling | WebView2 and Electron page owners no longer share ports `9222/9223` or assume fixed startup/action delays. | +| `cd922be9` | Preserved the macOS signing identity during install-local | The installer now signs and verifies a staged bundle, refuses to downgrade a certificate-signed live app to ad-hoc, and keeps a rollback copy until installation succeeds. | +| `d835c15e` and `e820aaf5` | Rejected known Windows Tk keyboard drops and activated desktop-scope click targets before SendInput | Run `29138176800` passed all 29 native rows and 71 of 72 shared rows. Its capture video proved the WPF desktop click coordinates were correct but the target was non-activating; the branch now activates the resolved root window before dispatch. | +| `541b06ba` and `7a369cfa` | Kept one process-lifetime AT-SPI registry listener and initialized it outside the daemon Tokio runtime | Run `29138635957` first exposed the nested-runtime startup panic before any cells. The replacement run `29138842890` passed GTK3 native and capture/desktop-scope preflight and behavior; the shared renderer lane remains the active validation. | +| `2ef2d9bd` | Looked for keyboard focus across every descendant UI thread of a Windows target | The previous top-level-thread-only lookup posted Tauri/WebView2 keys to the native frame while the renderer lived on another input thread. The existing external fixture-state cell remains unchanged and is validating the route. | +| `e8b8aee5` and `c3ef25ab` | Exposed the Windows foreground helper and made strict desktop fixture launch/readiness fail closed | Run `29138842358` caught the missing helper export at compile time. Canonical desktop-scope helpers now reject launch errors and readiness timeouts instead of returning a reduced green run. | +| `676fce44` | Cleaned interrupted macOS signing state before fallback signing | A bounded certificate signing attempt left `cua-driver.cstemp`; signing over it produced a resource seal for a transient file. The installer now removes partial seals and temp files, and a local reinstall produced a bundle that passes `codesign --verify --deep --strict`. | +| `f7a010c9` to `3c7d03a3` | Corrected foreground pointer ordering, renderer focus lookup, fixture isolation, targeted input, and Linux CI ownership | Windows replays reduced the matrix to one fixture-side WPF false negative; Linux native GTK3 and capture/scope owners stayed green while the shared renderer failures became individually classified. | +| `16275cb0`, `c18a8142`, and `85ed9d25` | Made the fixture journal retry, observed WPF's handled routed event correctly, and removed speculative driver delays | Windows run `29143137292` reported all 110 typed cells passing. The later physically foreground sentinel invalidated its background focus verdicts, but the WPF routed-event finding remains valid and no timing workaround remains in the driver. | +| `793def12` to `cf900891` | Built a pure-Wayland Nix session with an explicit D-Bus/AT-SPI host, compositor-aware preflight, honest oracle availability, and startup diagnostics | The original hard-coded `/run/current-system/sw/bin/dbus-daemon` failure is gone. Run `29143078987` reached a real Sway session but mapped no fixture surface; later runs now preserve Electron stderr, detect early process exit, and save the Sway tree without weakening the AX, capture, or non-blank-video gates. | +| `689f732a` and `628239fd` | Made source tests select the profile-local driver and aligned Linux tool discovery | The Nix schema test now executes the binary built in the same derivation. Linux no longer advertises the deprecated `type_text_chars` alias, matching macOS and Windows while retaining invoke-time compatibility. | +| `71c07881` to `26be3644` | Made Linux text, keyboard, pointer, and scroll routes target-aware and refusal-safe | Exact AT-SPI editable writes no longer choose another process field; editable-only nodes enter the index space; focus-dependent fallbacks refuse background dispatch; foreground keyboard and wheel input execute inside the foreground transaction. The latest Nix and X11 runs are validating these source changes. | +| `ab12e6a7` | Kept Linux accessibility geometry in the same captured tree snapshot | Chromium AT-SPI traversal order can change between walks. The driver now returns element metadata and bounds from one traversal, so an element index cannot be paired with another node's rectangle. Existing PX cells remain the regression test and contain no fixture-layout coordinates. | +| `ecffaf34` | Added a compact declared-coverage grid to the typed summary | The grid groups cells by harness and action across AX/PX and foreground/background delivery while preserving the detailed result and evidence tables. Undeclared combinations stay visibly distinct from pass, refusal, failure, and skip. | +| `3a5eec8b` | Isolated the prebuilt Linux Electron fixture from Nix's glibc library path | Wayland run `29143812672` proved that the fixture exited before mapping because `LD_LIBRARY_PATH` mixed Nix glibc 2.42 libraries with the host Electron binary. The staged launcher removes those variables for Electron only; the native Wayland, accessibility, capture, and video gates remain strict. | +| `afb16821` to `8eed7730` | Tightened source identity, fixture ownership, readiness, and incomplete-result checks | The Nix source gate now tests the platform schema it actually advertises, required shared fixtures fail closed, and cold WebView2 startup has a bounded 30-second readiness budget instead of disappearing from the matrix. | +| `349e308c` | Stabilized Windows background key targeting | Known silent-drop classes are rejected before UIA focus, while embedded renderer keys wait for an observable focused descendant rather than posting to the native frame. | +| `8b1c7357` and `19e2eaab` | Corrected X11 window discovery, double-click timing, and Linux cursor motion | Empty EWMH client lists no longer fall back to stale root children, no-window-manager discovery keeps only viewable windows, XTest double clicks flush between pairs, and Linux accepts the shared Bezier/timing cursor schema. | +| `c4214672` | Made the Windows sentinel physically own foreground before every background cell | The next run invalidated the earlier all-green Windows result: a visible but unfocused sentinel had not exercised the no-activation contract. With real foreground ownership, Electron remained clean while WPF F5 and six Tauri routes exposed honest product failures. | +| `cb91c556` and `f075f3c9` | Required a live pure-Wayland AT-SPI registry and fixed the Linux source build | The exact Nix AT-SPI launcher starts and activates `org.a11y.atspi.Registry`; the subsequent preflight now fails later because no application registers in the accessibility tree. The Nix source check passes at `f075f3c9`. | +| `4fe1047f` | Classified background WPF key drops without removing foreground delivery | WPF pointer messages remain class-level refusals. Keystrokes and chords refuse only while another native window owns foreground; the canonical F5 cell asserts exact `background_unavailable`, no fixture mutation, and all desktop-side-effect oracles. | +| `470d4827` | Hardened background WebView input mechanics | Posted clicks hold a non-activation guard, preserve unrelated window styles, restore the user's foreground if a renderer self-activates, and emit `WM_*BUTTONDBLCLK` only for classes that advertise `CS_DBLCLKS`. Embedded WebView AX keys focus through the proven posted-click route and fail closed on focus timeout; UIA scroll is guarded through handler settlement. | +| `0cef4992` to `557a59c4` | Completed the supported X11 matrix without weakening background contracts | Linux WebKit foreground typing now uses XTest, exact AT-SPI writes retry transient registry races, and unsupported background WebKit text and save actions return `background_unavailable`. Exact run `29147126435` passed all 80 X11 rows. | +| `e65f49f4` and `6f8d4368` | Matched addressed WebView key delivery to the proven pixel route | Run `29147294209` proved that releasing the outer no-activation guard did not fix the remaining Tauri AX key cell. Artifact comparison showed that `GUITHREADINFO` focus metadata was absent even when the equivalent PX route delivered. The AX route now posts its focus click, settles, and returns an explicitly unverifiable result while fixture state remains the delivery oracle. | +| `eaa59957` | Made backend-reported video failure visible to callers and manifests | A recorder backend `Err` now fails session shutdown, persists `video.error`, and has a focused core regression test. Commit `022dd537` closes the separate invalid-success metadata path. | +| `18005272` | Changed pure-Wayland sentinel discovery and screenshot scaling | The sentinel uses its unique CDP title and launched PID instead of comparing connection-local Wayland IDs. Run `29148643790` validated discovery and native screenshot scaling; remaining backend gaps stay required failures under issue `#1922`. | +| `022dd537` | Rejected invalid successful video metadata | Core now treats `finalized:false`, a missing file, or an empty file as a recording-stop error even when a backend returned `Ok`. Unit tests cover unfinalized, empty, and valid metadata. | +| `0a28495a` | Closed desktop-oracle false passes | Window-scope gates require the exact structured `desktop_scope_disabled` error, background actions require the target to be fully occluded before dispatch, and sentinel journal read errors fail the cell. | +| `8abf0950` | Pinned every interactive lane to one exact source SHA | Dispatch accepts only a full commit SHA or the immutable dispatch SHA. Every lane verifies `git rev-parse HEAD`, records the SHA in `environment.jsonl`, and prints it in the typed summary. | +| `9c76eb5f` | Routed addressed WebView key focus through the complete pixel actuator | Run `29147987757` proved that a direct posted focus click still did not deliver the Tauri AX key. The AX center now converts back to screenshot coordinates and calls the same layered ClickTool route as the passing PX cell, including targeted-injection fallback. | +| `69d3538c` | Checked occlusion at the authoritative dispatch snapshot | The sentinel rejects a cell when the target rises between the initial precondition check and the desktop observer's before-action snapshot. | +| `900bfe8d` | Separated desktop-wide sentinel observations from background-target observations | Run `29148642577` passed all 72 shared rows and all 8 capture rows, proving the WebView key and strict scope fixes. Its two native failures showed that launch and agent-cursor cells intentionally observe the foreground sentinel because no background target exists. They now use an explicit desktop-observation API; real background actions keep both occlusion checks. | +| `7fafe2c0` | Ordered pure-Wayland native and renderer focus checks | The Sway-only sentinel path waits for native readiness, explicitly focuses the launched PID, verifies the compositor foreground state, and only then waits for the renderer focus journal. Windows and X11 ordering is unchanged. | +| `516cddb2` | Replaced WinUI3's cold-start sleep with external readiness polling | Run `29149160683` passed all 72 shared rows and all 8 capture rows. Its sole native failure was the first WinUI3 UIA lookup; six later WinUI3 rows passed. Native setup now polls the external UIA tree for the app marker and canonical checkbox before executing any WinUI3 cell. | +| `3b67ee30` | Made compositor focus authoritative for pure-Wayland sentinel startup | Run `29149710714` proved the sentinel was natively foreground and stable, but Electron emitted no renderer focus event because the window could be focused before preload listeners attached. The startup gate now uses the compositor snapshot; action-time native focus, z-order, and journal input-leak checks remain mandatory. | + +The earlier conclusion that fully occluded Chromium clicks required a z-order +change was incomplete. A VM-only diagnostic confirmed that UIA Invoke delivers +but activates Electron, so UIA still violates the background contract. The +same diagnostic also proved that `post_click_screen` reaches the renderer while +it remains fully occluded: both AX-resolved and PX-resolved left clicks +delivered in three consecutive runs without changing focus, z-order, or the +real cursor. Commit `6aaa914b` restores that route only for the proven +single-left-click shape; right-click, double-click, drag, and other unproven +gestures retain their existing refusal behavior. + +## Active Validation + +| Platform | Run | Purpose | State | +| --- | --- | --- | --- | +| Linux X11 | `29179817332` | Final expanded shared, native, capture, and desktop-scope matrix at `f8e4b37b` | Passed all 108 rows: 71 delivered and 37 exact refusals, with 0 failures or skips. The three lane artifacts contain 108 cell videos plus three preflight videos, and the combined summary links every row to its owning artifact. | +| Linux Wayland | `29179817787`, `29182130646` | Split Sway shared, native, capture, and focused Electron validation | Native passed all 31 rows: 24 delivered and 7 exact refusals. Capture and desktop scope passed all 5 rows. The shared lane executed all 72 Electron/Tauri rows: Electron exposed a small keyboard cluster and Tauri was environment-limited by WebKitGTK's missing DRM/EGL renderer tree. Focused repairs now prove Electron foreground/background AX left click and foreground AX type text; foreground hotkey and hosted renderer PX-origin reconstruction remain explicit failures. | +| Linux GNOME | Real GNOME 46 session | Validate native Mutter discovery, capture, AX/PX dispatch, occlusion, and persistent portal/libei input | Passed 31 GTK3 rows, 3 capture contracts, and 2 desktop-scope contracts. A second driver process reused the explicitly revocable portal grant without another consent dialog. | +| Windows | Azure RDP focused replay | Recheck occluded Electron AX/PX background left-click delivery and neighboring pointer contracts at `6aaa914b` | The background-safe `PostMessage` diagnostic delivered AX and PX in 3/3 runs with all five side-effect oracles. The production build then passed 12/12 Electron left/right/double-click rows and emitted a video for every row. | +| Windows | `29149710089` | Final exact full matrix at `516cddb2` with strengthened desktop oracles and external WinUI3 readiness | Passed all 110 rows: 87 delivered and 23 exact refusals, with 0 failures or skips. All three lane summaries identify the full source SHA; 110 cell videos plus 3 preflight videos were uploaded and parsed successfully. | +| Windows | `29166454913` | Exact converged matrix at `e43c2946` after macOS parity, native PX, sentinel, and launch-contract work | Passed all 113 rows: 92 delivered and 21 exact refusals, with 0 failures or skips. Shared passed 54/18, native passed 30/3, and capture/scope passed 8/0. The three lane artifacts contain 113 cell videos plus 3 preflight videos; every row links its exact trajectory path. | +| Nix Linux source | `29150690892` | Package build, Linux unit/protocol gate, and source identity at `3b67ee30` | Passed. The package and source-owned Rust tests completed in the pinned Nix environment at the final code SHA. | +| Linux X11 | `29148643166` | Exact supported Linux matrix at `69d3538c` with strengthened occlusion and scope gates | Passed all 80 rows: 48 delivered and 32 exact refusals, with 0 failures or skips. All three summaries and environment records identify the full source SHA; 80 cell videos plus 3 preflight videos were uploaded. | +| macOS | Local complete matrix at `6e05424a` | Validate the converged shared, native, capture, and desktop-scope catalog with stable TCC identity | Passed all 83 declared rows: 79 delivered and 4 exact refusals, with 0 failures or skips. Every row produced a typed result and video, and both the summary and environment record contain the full source SHA. | +| macOS | Local complete matrix at `8d28beec` | Validate native WKWebView and the expanded AppKit/SwiftUI catalog at one exact source SHA | Passed all 130 declared rows: 124 delivered and 6 exact refusals, with 0 failures or skips. The 108 shared rows cover Electron, Tauri, and WKWebView; all native, capture, and desktop-scope rows passed. Every row produced a typed result and validated video. | + +The earlier macOS replay fixed PX background click focus steals, foreground PX typing's +first-character loss, silent web-content background scroll/drag no-effects, +desktop-scope HID delivery, AppKit native PX background click coverage, and +daemon transport loss of structured refusal codes. Electron background scroll +and Electron/Tauri background drag now return exact `background_unavailable` +refusals. The current pass proves SwiftUI popover activation through fixture +state; only transient-panel AX enumeration remains unproven. A targeted CDP +wheel experiment could mutate Electron while visible or ordinarily +backgrounded, but had no effect under the canonical full-occlusion sentinel. +The experiment was removed, and Electron background scroll keeps its exact +`background_unavailable` contract. + +Repeated local runs also exposed two evidence-harness defects. The macOS +runner now refuses a dirty tracked tree and derives a full source SHA when CI +does not supply one. Desktop-scope tests no longer reuse a fixed daemon session +ID after that session has ended; each proxy connection keeps its own minted +session identity. diff --git a/libs/cua-driver/docs/e2e-results-summary-design.md b/libs/cua-driver/docs/e2e-results-summary-design.md new file mode 100644 index 0000000000..21e82060e9 --- /dev/null +++ b/libs/cua-driver/docs/e2e-results-summary-design.md @@ -0,0 +1,120 @@ +# E2E Results Summary Design + +This document specifies the GitHub Actions presentation built from the Rust v2 +case and result contract. Field semantics live in `e2e-ci-reporting.md`; the +convergence sequence lives in `test-harness-convergence-plan.md`. + +## Goals + +The typed artifacts and run summary together must answer: + +- Was the desktop environment ready? +- Which declared behavioral cells delivered, refused, failed, or did not run? +- Which harness, action, targeting mode, delivery mode, and driver route did + each cell cover? +- Which external and desktop oracles passed? +- Which lane archive owns that cell's recording evidence? +- Are any declared cells missing or duplicated? + +Cargo test names and lane exit codes do not answer these questions and are not +behavioral rows. + +## Summary Layout + +The generated Markdown starts with aggregate behavioral totals, followed by a +declared-coverage grid and the detailed rows: + +```markdown +# CUA Driver E2E + +**Result:** 42 delivered, 3 refused, 2 failed, 0 skipped + +**Source SHA:** `0123456789abcdef0123456789abcdef01234567` + +## Declared Coverage + +## Detailed Results +``` + +Environment readiness and source identity live in `environment.jsonl`, the +generated summary, and the workflow metadata. An environment failure prevents +behavioral cells and makes the reporter fail; it is not converted into a +partial green summary. + +## Behavioral Table + +One row represents one declared `cell_id`: + +```markdown +| Cell | Harness | Action | Targeting | Delivery | Route | Expected | Observed | Oracles | Status | Time | Evidence | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | ---: | --- | +| linux-electron-left-click-ax-background | Electron | Left click | AX | Background | AT-SPI action | Refuse | Refused | Focus, z-order, no leak | PASS | 1.5s | [recordings/.../recording.mp4] | +| linux-electron-left-click-ax-foreground | Electron | Left click | AX | Foreground | AT-SPI action | Deliver | Delivered | Fixture | PASS | 2.1s | [recordings/.../recording.mp4] | +``` + +The row does not label a refusal as pass unless the case declaration expects +refusal and every required no-side-effect oracle passed. + +## Coverage Table + +The reporter also renders declared coverage. A dash means that the Rust catalog +does not declare that combination; it is not a pass or an inferred result: + +```markdown +| Harness | Action | AX/BG | AX/FG | PX/BG | PX/FG | Page | NotApplicable | +| --- | --- | --- | --- | --- | --- | --- | --- | +| electron | left_click | REFUSED | PASS | REFUSED | PASS | - | - | +| electron | drag | - | - | REFUSED | PASS | - | - | +``` + +The detailed table remains the authority for route, contract, oracle, and +failure information for every declared cell. + +## Validation + +The reporter fails before publishing a green summary when it finds: + +- duplicate declarations or results; +- a declared cell with no result; +- a result with no declaration; +- a serialized status that contradicts expected and observed behavior; +- an unknown refusal code; +- a passing cell missing a required oracle; +- unfinalized, missing, or empty required video evidence; +- a behavioral video phase that never reached `finalized` after setup/posture; +- an invalid hosted-runner console cleanup status; +- more or fewer than one environment record. + +The reporter validates the exact video it renders. It records the trajectory +path but does not currently validate that file independently. Workflow summary +jobs may concatenate validated platform summaries, but they do not recalculate +results. + +## Evidence Packaging + +Each internal workflow lane uploads one archive. Every cell owns a stable +subdirectory derived from `cell_id` inside that archive: + +```text +recordings/-pid-/ +|-- recording.mp4 +|-- trajectory.json +|-- session.json +|-- cursor.jsonl +`-- turn-*/ + |-- action.json + `-- screenshot.png +``` + +GitHub artifact archives do not provide stable URLs to individual files. The +row therefore links its exact MP4 path text to the owning lane archive; the +trajectory rollup links the same archive and reports its video count. A future +static report may provide inline playback; it is not required for the GitHub +summary. Cargo target logs sit at the lane root and diagnose runner or test +failures; they are not invented as per-cell evidence. + +## Unit And Protocol Results + +Unit, schema, transport, and CLI tests remain separate workflow output. They do +not share the behavioral case schema, appear as invented behavioral rows, or +require desktop video. diff --git a/libs/cua-driver/docs/linux-desktop-validation.md b/libs/cua-driver/docs/linux-desktop-validation.md new file mode 100644 index 0000000000..23fb3adaf2 --- /dev/null +++ b/libs/cua-driver/docs/linux-desktop-validation.md @@ -0,0 +1,54 @@ +# Linux representative desktop validation + +Hosted CI owns the canonical Xvfb/Openbox and headless Sway environments. Some +contracts require a real user desktop and therefore run only when a maintainer +explicitly provisions the corresponding environment. + +## Environments + +| Environment | Required evidence | Current status | Preflight | +| --- | --- | --- | --- | +| GNOME/Mutter | WinRects geometry and activation, portal/libei input, portal recording, and shared renderer apps | Native GTK behavior, capture, and desktop scope are accepted; shared renderers and portal video remain open | Wayland user session, enabled WinRects helper, and portal grant | +| KDE/KWin | KWin-specific activation and portal behavior | Plasma 6 session startup, GTK AT-SPI discovery, and portal interfaces were observed; no behavioral matrix is accepted | Plasma/KWin 6 Wayland session; Plasma 5.27 is rejected | +| Real Xorg | MPX/uinput behavior that Xvfb cannot provide | Not yet validated | Non-Wayland Xorg session with `/dev/uinput` access | +| DRM/EGL renderer | Representative WebKitGTK/Tauri accessibility tree | Not available on the hosted Sway or representative desktop hosts used so far | A real `/dev/dri/renderD128`; software-only headless rendering is rejected | + +## Source ownership + +The host checkout is the only checkout that commits or pushes. Sync it with: + +```bash +libs/cua-driver/scripts/sync-vm-worktree.sh push user@host '~/cua' +``` + +The sync writes `.cua-e2e-source-sha` after transferring the worktree. The +canonical preflight validates that marker when the VM intentionally has no +`.git` directory, so reports still identify the exact host commit. + +## Run + +Start the command from the graphical user's systemd user manager or an +equivalent terminal inside that user's session. The wrapper rejects the wrong +desktop generation before building fixtures. + +```bash +scripts/ci/linux/run-rust-e2e-desktop.sh gnome +scripts/ci/linux/run-rust-e2e-desktop.sh kde +scripts/ci/linux/run-rust-e2e-desktop.sh xorg +``` + +The default is the complete canonical matrix. `CUA_E2E_INTERNAL_LANE` and +`CUA_E2E_HARNESS_FILTER` remain diagnostic/maintainer controls; they do not +define a second catalog. + +## Evidence acceptance + +Behavioral acceptance requires typed rows, the exact source SHA, independent +fixture-state or refusal evidence, and every action-specific desktop oracle. A +representative result without reporter-owned per-cell video may establish that +behavior, but it is recorded as lacking full evidence parity. + +Full hosted parity also requires the Markdown summary, screenshots, +trajectories, and per-cell videos enforced by the reporter. Record accepted +behavior and any evidence-parity gap in `action-support.md`. A setup failure is +an environment error, never a smaller green matrix. diff --git a/libs/cua-driver/docs/linux-support-completion-plan.md b/libs/cua-driver/docs/linux-support-completion-plan.md new file mode 100644 index 0000000000..e6c7a55a58 --- /dev/null +++ b/libs/cua-driver/docs/linux-support-completion-plan.md @@ -0,0 +1,305 @@ +# Linux support completion plan + +**Status:** Historical implementation plan. Current outcomes live in +`action-support.md`; remaining work lives in the +[public platform roadmap](https://cua.ai/docs/reference/cua-driver/platform-roadmap). + +**Scope:** Linux driver behavior, canonical Rust E2E evidence, and release validation + +**Principle:** Standard Wayland and the optional compositor-owned injection environment are separate products with separate claims. + +## Goal + +Close the remaining Linux gaps without restoring the deleted Python, shell, GIF, +or real-app fixture suites. Reuse the repo-local Rust harness catalogs and their +external state, focus, z-order, cursor, input-leak, screenshot, trajectory, and +video oracles. + +The work is complete when each supported action either: + +- delivers and changes fixture-owned state under every required side-effect oracle; +- returns the declared structured refusal before dispatch; or +- is recorded as an environment gap with a named owner and representative test lane. + +A successful driver response is never delivery evidence. + +## Execution status + +| Phase | Result | +| --- | --- | +| Truth and route metadata | Complete. Current docs distinguish X11, stock compositor routes, portal/libei, and the nested private protocol. Typed results record compositor and input backend. | +| Nested protocol hardening | Complete. The protocol has a version handshake, acknowledgements, stable PID targeting, focus/z-order queries, geometry queries, strict key/text validation, and explicit errors for missing resources. | +| Nested typed environment | Complete as an experimental lane. Nix packages the compositor, the canonical Rust catalogs run inside it, and every lane retains typed evidence and video. Promotion still depends on a complete accepted run. | +| Protocol action coverage | Implemented for left/right/double click, printable ASCII text, named keys, hotkeys, scroll, and single/parallel drag. Unicode text and a canonical parallel-drag behavior row remain unproven. | +| Stock Sway repairs | Validated. Shared run `29199656600` improved Electron from 23/36 to 29/36. Focused replacement run `29200827296` passed all 9 PX-background rows. Four foreground Electron delivery shapes and representative WebKitGTK remain open. | +| Representative desktops | Partial. GNOME native GTK behavior is accepted; Plasma 6 reached session, AT-SPI, and portal preflight only. Shared GNOME renderers, portal video, real Xorg MPX, and DRM/EGL WebKitGTK remain named gaps. | +| Release policy | Complete. Linux unit/source/distribution gates run automatically; X11, Sway, and nested GUI matrices are exact-SHA maintainer dispatches with row-level artifacts. | + +## Current baseline + +| Environment | Current evidence | Main gaps | +| --- | --- | --- | +| X11/Openbox on Xvfb | 108/108 declared outcomes: 71 deliveries and 37 exact refusals | Real-Xorg MPX/uinput and parallel pointer behavior are not proven by Xvfb | +| Sway/wlroots, native Wayland | GTK3 31/31 plus capture/scope; Electron shared 29/36 plus focused PX-background 9/9 | Foreground Electron AX right/double click and AX/PX hotkey; WebKitGTK without DRM/EGL | +| GNOME 46/Mutter | GTK3 31/31 using WinRects, AT-SPI, and portal/libei | Shared Electron/Tauri matrix and portal MP4 evidence | +| KDE/KWin | Plasma 6 session startup, GTK AT-SPI discovery, and portal interfaces | Target-addressable activation adapter and an accepted behavioral lane | +| Nested `cua-compositor` | Packaged experimental environment, route metadata, observer, native GTK3 31/31, capture/scope 5/5, and Electron shared 26/36 before focused repairs | Stable renderer click/scroll coverage, a complete accepted shared matrix, Unicode text, and a canonical parallel-drag row | + +Standard Wayland cannot generally route raw pointer or keyboard events to an +arbitrary occluded, unfocused surface. The nested `cua-compositor` is different: +it owns the compositor and can route directly to a client surface through its +private control socket. Evidence from that environment must never be presented +as a stock Sway, GNOME, KDE, or general Wayland capability. + +## Target environment matrix + +Keep the matrix small. An environment earns a lane only when it proves a +contract that another lane cannot. + +| ID | Environment | Purpose | Ownership | +| --- | --- | --- | --- | +| E1 | X11/Openbox on Xvfb | XSendEvent, XTest, AT-SPI, capture, scope, complete shared/native catalogs | Hosted GitHub runner; canonical | +| E2 | Sway on native Wayland | wlroots virtual pointer, screencopy, foreign toplevel, and standard-Wayland refusals | Hosted GitHub runner; canonical | +| E3 | Nested `cua-compositor` | Raw focus-free per-surface input and multi-pointer behavior | Hosted GitHub runner; experimental until proven | +| E4 | GNOME/Mutter real session | WinRects, portal/libei, shared renderer apps, portal recording | Maintainer VM; representative | +| E5 | Real Xorg session | MPX/uinput behavior unavailable under Xvfb | Maintainer VM; optional | +| E6 | KDE Plasma 6/KWin | KWin activation and portal behavior | Maintainer VM; experimental | +| E7 | Real DRM/EGL renderer session | WebKitGTK/Tauri accessibility tree and native renderer geometry | Prefer E4; use a separate VM only if needed | + +Labwc, Hyprland, and other wlroots compositors remain expected-compatible but +unproven unless a user report demonstrates a meaningful divergence. XWayland is +not a separate lane. + +## Phase 0: Reconcile the truth + +The phase sections below preserve the implementation and acceptance criteria +used for this work. Current outcomes are summarized above; unchecked evidence +gaps remain follow-up work rather than implied support. + +Update stale internal references before changing behavior. + +**Work** + +- Mark stale sections of `libs/cua-driver/docs/test-harness-convergence-plan.md` + as superseded by current evidence. +- Rewrite `nix/cua-driver/tests/README.md`; it currently names deleted fixtures. +- Remove the empty `nix/cua-driver/tests/wayland/` directory if it is still empty. +- Add the experimental nested-compositor row to + `libs/cua-driver/docs/action-support.md` and link it from the public platform + support page. +- Replace misleading "EIS compositor" wording. The current route is a private + Unix socket protocol, not libei/EIS. + +**Acceptance** + +- No documentation points to a deleted fixture or workflow. +- Every public capability has current evidence or an explicit experimental gap. + +## Phase 1: Make results identify the real route + +The result schema currently says only `Wayland` and can make nested injection +look like a standard virtual-pointer or AT-SPI route. + +**Work** + +- Add `LinuxCuaCompositorInject` to `DriverRoute` in + `cua-driver-testkit/src/e2e.rs`. +- Add environment fields for `compositor` and available `input_backends`. +- Detect values such as `openbox-x11`, `sway`, `gnome-mutter`, `kwin`, and + `cua-compositor-nested` in the environment preflight. +- Update `cua-e2e-report` and the Markdown summary to display the compositor and + input route without changing test-family or action naming. +- Keep backward compatibility with existing result artifacts. + +**Acceptance** + +- E1 and E2 retain identical cell outcomes. +- Every delivered cell identifies both its display server and actual route. +- E2 and E3 artifacts cannot be mistaken for each other. + +## Phase 2: Audit and harden nested injection + +This is a proof gate, not an assumption. The existing path has two correctness +risks: + +1. `app_id_for_window` appears to compare a toplevel object ID obtained from one + Wayland connection with objects from a new connection. Protocol IDs are + connection-scoped, so target resolution must be tested and likely replaced + with stable identity resolution. +2. The compositor silently ignores unsupported Unicode and keys and has no + scroll or hotkey verb. The driver must never return success for those cases. + +**Work** + +- Add protocol handshake/version and acknowledgements. +- Resolve targets through stable app/window identity, including duplicate app IDs. +- Pre-validate all text and key shapes before sending. +- Return exact `background_unavailable` refusals for unsupported text, key, + hotkey, scroll, or drag shapes until their protocol verb exists. +- Add focused unit tests with a mock Unix socket for encoding, acknowledgements, + validation, target ambiguity, and error mapping. +- Verify or correct the stale `LIBEI_SOCKET` comment in `wayland/libei.rs`. + +**Acceptance** + +- No unsupported command can be silently dropped. +- No route depends on connection-local Wayland object IDs as stable identities. +- A compositor protocol mismatch fails preflight once, before the matrix starts. + +## Phase 3: Restore the compositor as a typed environment + +Do not restore the old Nix/Python fixture suite. Package only the environment, +then run the existing Rust catalogs inside it. + +**Work** + +- Export `cua-compositor` as a flake package and build check. +- Add a `cua-driver-inject-e2e` development shell containing the compositor and + existing Rust harness dependencies. +- Add `scripts/ci/linux/run-rust-e2e-inject.sh` to start the nested session, + perform capability preflight, and delegate to the canonical + `scripts/ci/linux/run-rust-e2e.sh` command. +- Add an independent `CuaCompositor` observer in the testkit. Read-only control + queries must report focus and z-order directly; the driver response cannot be + the observer. +- Reuse the GTK3 and shared `CaseSpec` catalogs. Key expectations by detected + environment capability, not by weakening an oracle. +- Add a repo-local parallel-drag journal action because the old test asserted + only command completion and therefore supplied insufficient evidence. +- Record every cell with before/after screenshots, trajectory data, and video. + +**Minimum Step A matrix** + +| Background PX action | Initial contract | +| --- | --- | +| Left, right, and double click | Deliver | +| ASCII type text | Deliver | +| Supported named key | Deliver | +| Single drag | Deliver after socket routing exists; otherwise exact refusal | +| Two simultaneous drags | Deliver with fixture state for both paths | +| Scroll | Exact refusal until an axis verb exists | +| Hotkey | Exact refusal until a chord verb exists | +| Unsupported Unicode or key | Exact refusal before socket dispatch | + +All delivery rows require fixture state, unchanged sentinel focus and z-order, +and no leaked input. Cursor preservation is not an oracle when the compositor +route intentionally has no physical cursor. + +**Acceptance** + +- `nix build .#cua-compositor` is a CI check. +- The hosted experimental lane completes the Step A matrix three consecutive + times on one source SHA with owned videos and no orphan evidence. +- The two useful historical claims, focus-free typing and simultaneous drags, + are superseded by stronger typed fixture evidence. + +**Deletion gate** + +If the compositor cannot build, target stable identities, or support truthful +focus/z-order observation after two focused implementation cycles, remove the +entire dormant subsystem and its public capability claim in one PR. Do not leave +advertised but untested production branches indefinitely. + +## Phase 4: Complete the injection protocol + +After Step A is green, add protocol verbs for axis/scroll, modifier chords, +Unicode text, broader named keys, and single drag. Flip a test from refusal to +delivery only when the unchanged fixture and side-effect oracles pass. + +**Acceptance** + +- Background left/right/double click, Unicode typing, named keys, hotkeys, + scroll, single drag, and parallel drag all have typed delivery evidence. +- Remaining refusals describe genuinely unsupported public shapes. + +## Phase 5: Close stock Sway gaps + +Work independently of the custom-compositor lane. + +**Work** + +- Repair Electron foreground AX/PX hotkey delivery in the driver. +- Repair renderer-to-compositor PX origin reconstruction. +- Keep Tauri/WebKitGTK environment-limited on hosted pixman Sway; do not use + fixture changes or renderer-disabling flags as representative proof. + +**Acceptance** + +- Existing failing cells become delivered through external fixture state, or + remain named gaps with exact diagnostics and a linked issue. + +## Phase 6: Representative desktop coverage + +### GNOME and WebKitGTK + +- Run the unchanged shared Electron catalog on the existing GNOME VM. +- Add portal MP4 capture with the same ownership and nonblank checks as Sway. +- Check `/dev/dri` first and run the Tauri/WebKitGTK catalog there when a real + render node is available. +- If it is unavailable, provision a dedicated DRM/EGL VM rather than adapting + the fixture to software-renderer behavior. + +### KDE + +- Implement a target-addressable KWin activation adapter using supported KWin + scripting or D-Bus interfaces. +- Validate activation through an independent observer before enabling portal + input. +- Use Plasma 6; do not accept the broken Plasma 5.27 cloud image as evidence. + +### Real Xorg + +- Add an optional maintainer lane for MPX/uinput and parallel pointer behavior. +- Do not describe Xvfb refusals as proof that real-Xorg delivery is impossible. + +## Phase 7: Release validation policy + +- Keep E1 X11 and E2 Sway dispatch-only or maintainer-triggered, but require + green run links on the exact release SHA in the release checklist. +- Keep E3-E7 optional until each environment has an owner and three stable runs. +- Store results, summary, screenshots, trajectories, logs, and videos per lane. +- Fail preflight once for a missing session capability; do not turn all cells + into skips. + +## Suggested PR sequence + +1. Documentation reconciliation. +2. Result route and environment metadata. +3. Nested-injection refusal hardening and mock-socket tests. +4. Sway hotkey repair and PX geometry repair as independent changes. +5. Compositor package, build check, protocol query support, and stable identity. +6. Testkit observer, typed Step A matrix, runner, and experimental workflow. +7. Protocol completion and expectation flips. +8. GNOME shared/video and representative Tauri renderer evidence. +9. KWin adapter and Plasma 6 evidence. +10. Release checklist gating. + +Each PR must leave existing canonical lanes green. A new environment may fail +only in a clearly labeled experimental job and may not change the public +contract of E1 or E2. + +## Anti-overfitting rules + +- No Python behavioral runner, second matrix file, terminal/GIF fixture, or old + Nix test restoration. +- No command-return-only oracle. +- No AT-SPI delivery described as raw PX injection. +- No raw-background claim without physical occlusion and focus/z-order guards. +- No fixture expectation change made solely to accommodate driver behavior. +- No compositor proliferation without a unique contract and named owner. +- No software-renderer workaround accepted as representative WebKitGTK evidence. + +## Definition of done + +- E1 remains 108/108 on the final source SHA. +- E2 has no silent no-effect outcome; hotkey and PX geometry are delivered or + recorded as precise gaps. +- E3 has the complete typed raw-injection matrix and truthful route metadata, or + the dormant subsystem and its claim are removed. +- GNOME has shared Electron, portal video, and representative WebKitGTK evidence. +- KDE has verified target-addressable activation on Plasma 6 or remains clearly + experimental with that adapter as the named blocker. +- Every supported or refused action is represented by a typed Rust row and + external evidence across the environments where its contract differs. +- Public and contributor documentation link current evidence and never conflate + standard Wayland with compositor-owned raw injection. diff --git a/libs/cua-driver/docs/pr-2161-landing-plan.md b/libs/cua-driver/docs/pr-2161-landing-plan.md new file mode 100644 index 0000000000..a7358a01c6 --- /dev/null +++ b/libs/cua-driver/docs/pr-2161-landing-plan.md @@ -0,0 +1,640 @@ +# PR #2161 Landing Plan - Second-Wave E2E Convergence + +**Status:** proposed - **Author:** convergence audit follow-up - **Date:** 2026-07-12 +**Source PR:** #2161 (`codex/link-e2e-videos`, draft) - 229 files, +24,821 / -17,033, 310 commits. +**Merge state vs `main`:** conflicts only in `docs/content/docs/concepts/meta.json`. + +--- + +## 1. Decision and rationale + +**Decision: do not merge #2161. Re-land its delta as a stack of 7 fresh PRs cut from +current `main`, each built from a scoped final-state patch for its owned path set, not +by cherry-picking commits or overwriting current `main` with whole files blindly.** + +Rationale: + +- **The first-wave split already landed.** The #2135 split plan was executed in merged + PRs #2138-#2154. #2161 is the _second-wave convergence delta_ that accumulated on the + long-lived branch afterwards. Only that delta needs to land; the branch history is + unreviewable as-is (310 commits, many of which rework each other). +- **Cherry-picks are the wrong tool here.** The 310 commits repeatedly rewrite the same + files (e.g. `recording.rs`, the e2e workflows). Cherry-pick sequences would conflict + with each other and reproduce dead intermediate states. The branch's _final tree_ is + the reviewed artifact; scoped binary patches preserve its final add, modify, delete, + and rename intent while still forcing conflicts against newer `main` to be resolved. +- **The branch also contains intentional work outside the Cua Driver convergence + scope.** The TypeScript tooling migration and Docker cleanup need their own review + and rollback boundaries. The Lume 0.3.14 regeneration has already been superseded + by newer 0.3.15 generated output on `main`. A preservation queue keeps this work + visible without mixing it into the seven Cua Driver PRs. +- **7 PRs matches the natural seams**: one shared foundation (typed evidence testkit) + that everything compiles against, three platform silos with disjoint crates, two + Linux-display follow-ons with a real dependency chain, and one docs/cleanup capstone. + Fewer PRs would recreate the unreviewable blob; more would multiply shared-file + splits (Cargo manifests, `platform-linux/src/lib.rs`) for no review benefit. + +### Historical evidence vs fresh gates - the governing rule + +- **Historical evidence** = CI runs on #2161 / the convergence branch, and the run + records in `libs/cua-driver/docs/e2e-convergence-journal.md`. It proves the _final + tree_ once worked as a whole. Cite it in PR descriptions for reviewer context. +- **Fresh required gates** = commands/workflow runs executed **on each new stack + branch at its head SHA**. Only these gate merging. Historical evidence never + substitutes for a fresh gate, because each stack PR is a tree state that never + existed on the convergence branch (foundation without platforms, X11 without + Wayland, etc.). + +--- + +## 2. Global allowlist and preservation queue + +### 2.1 Strict global allowlist (in scope for the 7-PR stack) + +Everything in the #2161 delta under: + +- `libs/cua-driver/rust/**` (crates, tests, Skills, `Cargo.toml`, `Cargo.lock`, READMEs) +- `libs/cua-driver/tests/**` (fixtures, runners - including the deletions) +- `libs/cua-driver/docs/**`, `libs/cua-driver/README.md`, `libs/cua-driver/scripts/**` +- `libs/cua-driver/wayland-helper/**` +- `scripts/ci/**` (`README.md`, `link-e2e-evidence.sh`, `linux/`, `macos/`, `windows/`) +- `nix/cua-driver/**`, `flake.nix` +- `.github/workflows/`: `e2e-rust-windows.yml`, `e2e-rust-linux.yml`, + `e2e-rust-linux-wayland.yml`, `ci-rust-linux.yml`, `ci-rust-windows.yml`, + `ci-nix-linux.yml`, `cd-rust-cua-driver.yml`, `ci-release-reminder.yml`, + and the deletions `ci-cua-driver-interactive-linux.yml`, `nix-build.yml`, + `nix-screenshot.yml`, `nix-wayland.yml` +- `.github/scripts/tests/test_cua_driver_release_wiring.py` +- `docs/content/docs/concepts/{how-cua-driver-is-validated.mdx,index.mdx,meta.json}` +- `docs/content/docs/reference/cua-driver/**` +- Root docs: `README.md`, `CONTRIBUTING.md`, `Development.md`, `TESTING.md` + +### 2.2 Preservation queue (excluded from the seven Cua Driver PRs) + +| Owner | Paths | Intent | Disposition | +| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| Sidecar A: Docker cleanup | `Dockerfile`, `.dockerignore` | Remove the obsolete root development image and its ignore file | Fresh two-file PR from current `main` | +| Sidecar B: TypeScript tooling | `package-lock.json`, `libs/typescript/package.json`, `libs/typescript/.prettierignore`, `.github/workflows/ci-lint-typescript.yml`, `.pre-commit-config.yaml`, `.github/workflows/claude-auto-fix.yml` | Make the TypeScript workspace own pnpm setup, build core before typecheck, and keep generated output out of formatting | Fresh six-file PR from current `main` | +| Superseded by `main` | `docs/content/docs/reference/lume/cli-reference.mdx`, `docs/content/docs/reference/lume/http-api.mdx` | Regenerate Lume references at 0.3.14 | Record as satisfied by the newer generated 0.3.15 files already on `main`; do not replay the older output | + +Separation enforcement: every Cua Driver stack branch must satisfy +`git diff --name-only origin/main...HEAD | grep -E -x '(Dockerfile|\.dockerignore|package-lock\.json|\.pre-commit-config\.yaml|libs/typescript/.*|\.github/workflows/(ci-lint-typescript|claude-auto-fix)\.yml|docs/content/docs/reference/lume/.*)'` +-> **empty output** (add as a pre-push checklist item on each PR). + +#### Sidecar A - `codex/conv2-sidecar-docker-cleanup` + +Suggested title: `chore(repo): remove obsolete development Docker image`. + +Fresh gates: + +- [ ] Repo-wide reference search proves no maintained command, workflow, or guide uses + the root `Dockerfile` or `.dockerignore` +- [ ] Existing container workflows still point to their owned Dockerfiles +- [ ] The PR contains exactly the two deletions + +#### Sidecar B - `codex/conv2-sidecar-typescript-tooling` + +Suggested title: `ci(typescript): use workspace-local pnpm tooling`. + +Fresh gates: + +- [ ] `pnpm -C libs/typescript install --frozen-lockfile` +- [ ] `pnpm -C libs/typescript typecheck` +- [ ] `pnpm -C libs/typescript format:check` +- [ ] The TypeScript lint workflow passes on the sidecar head SHA +- [ ] The pre-commit TypeScript hook invokes the same workspace command + +The sidecars are independent of the seven-PR stack and may land while PR 1 is under +review. They still belong to the #2161 preservation record. + +### 2.3 Coverage invariant + +Every file in `git diff --name-status origin/main...codex/link-e2e-videos` is owned by +**exactly one** stack PR, sidecar PR, or superseded-by-main row. The audit in Section 7 +verifies this mechanically; +if a file turns out unowned or double-owned during execution, stop and amend this plan +first. + +--- + +## 3. The 7-PR stack + +Branch prefix: `codex/conv2-0N-`. PR titles: conventional-commit style with a +`(conv2 N/7)` suffix. All PRs squash-merge. All PR bodies link #2161 and this plan. + +Common transfer recipe (per PR, from a clean worktree). Replace `` with +the explicit files for that PR; do not use a broad directory when the directory also +contains sidecar-owned or shared files: + +```bash +git fetch origin +git switch -c codex/conv2-0N- origin/main +# Preserve the final #2161 add/modify/delete/rename intent and three-way it +# against the current main tree. +SNAPSHOT_BASE=$(git merge-base origin/main codex/link-e2e-videos) +git diff --binary --find-renames "$SNAPSHOT_BASE" codex/link-e2e-videos -- \ + | git apply --3way --index +# shared-file hunks (see Section 4): hand-edit, do NOT checkout the whole file +cd libs/cua-driver/rust && cargo check --workspace # regenerates Cargo.lock honestly +``` + +Common fresh gates (every PR that touches `libs/cua-driver/rust`): + +```bash +cargo fmt --all --check +cargo clippy --workspace --all-targets -- -D warnings +cargo test --workspace # host-runnable subset +cargo test --workspace --no-run # proves e2e/platform tests still compile +``` + +--- + +### PR 1 - `codex/conv2-01-typed-evidence` - "test(cua-driver): typed e2e evidence foundation (conv2 1/7)" + +**Purpose:** land the typed testkit (journal/observer/sentinel/e2e result types), the +`cua-e2e-report` binary, core recording rework, the shared scenario catalog and +cross-platform fixture updates, and the evidence-linking script. Everything later PRs +compile against. + +**Owns (final state):** + +- `libs/cua-driver/rust/crates/cua-driver-testkit/**` - whole crate: `Cargo.toml`, + `src/lib.rs`, `src/bin/cua-e2e-report.rs` (A), `driver.rs`, `e2e.rs` (A), + `journal.rs` (A), `mcp.rs`, `observer.rs` (A), `paths.rs`, `reaper.rs`, + `response.rs`, `sentinel.rs` (A), `windows_setup.rs` (A - cfg-gated, ships here so + `lib.rs` can be final-state) +- `crates/cua-driver-core/src/`: `cdp.rs`, `recording.rs`, `recording_tools.rs`, `tool.rs` +- `crates/cua-driver/src/`: `main.rs`, `serve.rs`; `crates/cua-driver/tests/README.md` +- Cross-platform tests: `capture_contract_test.rs` (rename of + `modality_capture_mode_test.rs`), `cross_platform_behavior_test.rs`, + `e2e_environment_preflight_test.rs` (A), `protocol_schema_test.rs` +- **Deletes** superseded cross-platform modality tests (they consume the pre-typed + testkit API and won't compile after this PR): `modality_background_test.rs`, + `modality_desktop_scope_test.rs`, `modality_dispatch_test.rs`, + `modality_focus_test.rs`, `modality_input_e2e_test.rs` +- Shared fixtures: `tests/fixtures/shared/scenarios.json`, + `tests/fixtures/shared/web/index.html`, electron `build.sh`, `main.js`, + `preload.js` (A), `tauri/src-tauri/src/main.rs` +- `scripts/ci/link-e2e-evidence.sh` (A) +- Dev tooling: `libs/cua-driver/scripts/_install-local-rust.sh`, `sync-vm-worktree.sh` +- `rust/Skills/cua-driver/RECORDING.md` (documents recording behavior changed here) +- `Cargo.lock` (regenerated via `cargo check`, not copied) + +**Excludes:** all platform crate changes, all workflows, all `libs/cua-driver/docs/**`, +electron `build.ps1` / `launcher-linux.sh` (platform PRs). + +**Dependencies:** none (base = `main`). + +**Fresh gates:** + +- [ ] Common cargo gates (above), on Linux CI at minimum; `cargo test --no-run` on a + Windows and macOS host or cross-check to prove cfg-gated testkit compiles +- [ ] `shellcheck scripts/ci/link-e2e-evidence.sh libs/cua-driver/scripts/*.sh` +- [ ] `cargo run -p cua-driver-testkit --bin cua-e2e-report -- --help` (or its + self-test invocation) exits 0 +- [ ] Existing `ci-rust-*` workflows green on the PR (unchanged workflow files must + still pass against the new testkit) + +**Historical evidence:** #2161 CI runs; `e2e-convergence-journal.md` (lands in PR 7 - +link to the branch copy). + +**Rollback boundary:** revert restores the pre-typed testkit and the five deleted +modality tests; nothing on `main` yet depends on the new API. Must be reverted only +if PRs 2-4 haven't merged (they consume `e2e::*`/`journal::*`). + +--- + +### PR 2 - `codex/conv2-02-windows` - "fix(cua-driver): Windows e2e convergence (conv2 2/7)" + +**Purpose:** Windows platform fixes (input injection, WGC capture, UIA cache, +recording hooks), typed Windows e2e tests, sandbox/runner + CI wiring; retire +`focus-monitor-win`. + +**Owns:** + +- `crates/platform-windows/src/`: `capture.rs`, `input/{delivery,inject,keyboard,mod,mouse}.rs`, + `recording_hooks.rs`, `tools/impl_.rs`, `uia/{cache,windows_enum}.rs`, `wgc.rs`, + `win32/windows.rs` +- **Deletes** `crates/focus-monitor-win/{Cargo.toml,src/main.rs}`; owns the one-line + member removal in workspace `rust/Cargo.toml` (hand-edit, see Section 4) +- Tests: `agent_cursor_windows_test.rs` (A), `desktop_scope_windows_test.rs` (A), + `launch_windows_test.rs` (A), `harness_winui3_test.rs`, `harness_wpf_test.rs`; + `harness_web_test.rs` (Windows-only WPF + WebView2/Electron CDP coverage); + **deletes** `guard_ux_test.rs` (superseded by the typed agent-cursor/launch suites) +- Fixtures: `windows/webview2/MainWindow.xaml{,.cs}`, `windows/wpf/MainWindow.xaml{,.cs}`, + `fixtures/build/windows.ps1`, electron `build.ps1` +- Runners: `tests/runners/windows-sandbox/{run-tests-in-sandbox,sandbox-runner}.ps1`, + `tests/runners/windows/{README.md,run-all.ps1}` +- CI: `scripts/ci/windows/{build-harnesses,run-rust-e2e}.ps1`, + `.github/workflows/{e2e-rust-windows.yml,ci-rust-windows.yml}` +- `Cargo.lock` regen + +**Excludes:** testkit sources (PR 1), all docs. + +**Dependencies:** PR 1 (typed testkit API, `windows_setup` module). + +**Fresh gates:** + +- [ ] Common cargo gates on a Windows runner (`cargo test -p platform-windows`, + `cargo test --no-run` for the e2e suites) +- [ ] `workflow_dispatch` of `e2e-rust-windows.yml` **on this branch head** -> green, + with `cua-e2e-report` summary artifact and evidence links present +- [ ] `ci-rust-windows.yml` green on the PR +- [ ] Grep proves no residual `focus-monitor-win` references: + `git grep -n focus-monitor-win` -> empty + +**Historical evidence:** #2161 `e2e-rust-windows` runs; `windows-hosted-e2e` / +`windows-px-background-clicks` branch history. Context only. + +**Rollback boundary:** revert restores old Windows behavior + `guard_ux_test.rs` + +`focus-monitor-win`; independent of PRs 3-6. Safe to revert alone before PR 7 (whose +docs describe the new Windows surface). + +--- + +### PR 3 - `codex/conv2-03-macos` - "fix(cua-driver): macOS e2e convergence (conv2 3/7)" + +**Purpose:** macOS AX/input/tool fixes, typed macOS suites (desktop scope, installed +app launch, TextEdit), fixture and smoke updates, hosted-runner e2e script. + +**Owns:** + +- `crates/platform-macos/src/`: `ax/bindings.rs`, `input/mouse.rs`, + `tools/{click,drag,mod,scroll,type_text}.rs`, `window_change_detector.rs` +- Tests: renames `modality_desktop_scope_macos_test.rs -> desktop_scope_macos_test.rs` + and `modality_launch_focus_macos_test.rs -> installed_app_launch_macos_test.rs`; + `installed_app_textedit_macos_test.rs` (A), `harness_appkit_test.rs`, + `harness_swiftui_test.rs` +- Fixtures: `macos/appkit/main.swift`, `macos/swiftui/main.swift`, + `macos/wkwebview/main.swift`, `fixtures/build/macos.sh`, `fixtures/smoke/macos.sh` +- `scripts/ci/macos/run-rust-e2e.sh` (A) +- `Cargo.lock` regen (if any) + +**Excludes:** testkit, docs, everything Windows/Linux. Note: there is no macOS e2e +GitHub workflow in the delta - the script is invoked on self-hosted/local runners. + +**Dependencies:** PR 1 only. Parallel with PR 2 and PR 4. + +**Fresh gates:** + +- [ ] Common cargo gates on a macOS host +- [ ] `bash libs/cua-driver/tests/fixtures/build/macos.sh` + + `bash libs/cua-driver/tests/fixtures/smoke/macos.sh` pass locally +- [ ] `bash scripts/ci/macos/run-rust-e2e.sh` full run on a macOS host with + Accessibility/Screen Recording grants -> summary posted to the PR (paste the + `cua-e2e-report` output; this is the fresh gate standing in for hosted CI) + +**Historical evidence:** convergence-journal macOS entries; #2161 local run logs. + +**Rollback boundary:** revert restores old macOS tool behavior and the `modality_*` +test names. Independent of PRs 2/4-6. + +--- + +### PR 4 - `codex/conv2-04-linux-x11` - "fix(cua-driver): Linux core and X11 e2e convergence (conv2 4/7)" + +**Purpose:** Linux platform core (AT-SPI, input delivery, overlay, health report, +recording hooks), X11 path, typed Linux desktop-scope suite, X11 e2e scripts and +workflows; retire the interactive-Linux workflow. + +**Owns:** + +- `crates/platform-linux/src/`: `a11y.rs`, `atspi/{mod,native}.rs`, `health_report.rs`, + `input/{delivery,mod}.rs`, `overlay.rs`, `recording_hooks.rs` (A), + `tools/{impl_,stubs}.rs`, `x11/mod.rs` +- `platform-linux/src/lib.rs` - **hunk only**: `pub mod recording_hooks;` (see Section 4) +- Tests: `desktop_scope_linux_test.rs` (A), `harness_gtk3_test.rs`; **deletes** + `modality_desktop_scope_linux_test.rs`, `modality_dispatch_linux_test.rs` +- Fixtures: `linux/gtk3/main.py`, `fixtures/build/linux.sh`, + electron `launcher-linux.sh` (A) +- CI: `scripts/ci/linux/run-rust-e2e.sh`, `run-rust-e2e-desktop.sh` (A), + `.github/workflows/{e2e-rust-linux.yml,ci-rust-linux.yml}`; + **deletes** `.github/workflows/ci-cua-driver-interactive-linux.yml` +- `rust/Skills/cua-driver/LINUX.md` +- `Cargo.lock` regen + +**Excludes:** everything under `platform-linux/src/wayland/`, `video_wayland.rs`, +`platform-linux/Cargo.toml` (all PR 5). + +**Compile caveat:** if `recording_hooks.rs` in final state imports `video_wayland` or +`wayland::sway_ipc`, `cargo check` fails here - in that case move the offending file(s) +to PR 5 and note the move in both PR bodies. The `--no-run` gate catches this. + +**Dependencies:** PR 1. Parallel with PRs 2-3. + +**Fresh gates:** + +- [ ] Common cargo gates on Linux +- [ ] `workflow_dispatch` of `e2e-rust-linux.yml` on this branch head -> green X11 + canonical matrix, evidence artifacts present +- [ ] `ci-rust-linux.yml` green on the PR +- [ ] `shellcheck scripts/ci/linux/run-rust-e2e*.sh` +- [ ] Confirm nothing still references the deleted interactive workflow: + `git grep -n ci-cua-driver-interactive-linux` -> empty + +**Historical evidence:** "record Linux completion evidence" commits; journal entries. + +**Rollback boundary:** revert restores old Linux core + interactive workflow. Must +revert PR 5 first if PR 5 has merged (Wayland modules reference core changes). + +--- + +### PR 5 - `codex/conv2-05-wayland` - "fix(cua-driver): Wayland backend convergence (conv2 5/7)" + +**Purpose:** Wayland backend work (ext-toplevel, sway IPC, portal screenshot, +persistent virtual pointer, libei, and the shared nested-injection protocol code), +Wayland video, the `portal-libei -> portal-input + portal-capture` feature split, +Wayland e2e workflow/script, GNOME helper extension, and shipping `portal-input` in +release binaries. The nested route remains dormant/experimental until PR 6 packages +and validates its owned compositor environment. + +**Owns:** + +- `crates/platform-linux/src/wayland/` - everything: `ext_toplevel.rs` (A), + `libei.rs`, `mod.rs`, `persistent_vptr.rs`, `portal_screenshot.rs`, + `shell_helper.rs`, `sway_ipc.rs` (A) +- `platform-linux/src/video_wayland.rs` (A); `platform-linux/src/lib.rs` - + **hunk only**: `pub mod video_wayland;` +- Manifests (whole-file final state - the delta in each is entirely this PR's): + `crates/platform-linux/Cargo.toml` (feature split), `crates/cua-driver/Cargo.toml` + (feature forwarding) +- CI: `scripts/ci/linux/run-rust-e2e-wayland.sh` (A), + `.github/workflows/e2e-rust-linux-wayland.yml`, + `.github/workflows/cd-rust-cua-driver.yml` (release builds gain + `--features portal-input` + `libxkbcommon-dev`) +- `libs/cua-driver/wayland-helper/{README.md,winrects@cua/extension.js,winrects@cua/metadata.json}` +- `Cargo.lock` regen + +**Excludes:** X11/core files (PR 4), Nix (PR 6). + +**Dependencies:** PR 4 (shares `platform-linux`; core/input changes land first). + +**Fresh gates:** + +- [ ] Common cargo gates, plus feature-matrix compiles: + `cargo check -p platform-linux`, `... --features portal-input`, + `... --features portal-capture`, `... --features portal-libei` +- [ ] `workflow_dispatch` of `e2e-rust-linux-wayland.yml` on this branch head with + environment `sway` -> green canonical matrix +- [ ] `cd-rust-cua-driver.yml` build job dry-run (workflow_dispatch or `act`/container + rehearsal) proves the Debian 11 container builds with `--features portal-input` +- [ ] `.github/scripts/tests/` release-wiring tests still pass **unmodified** + (`python -m unittest` - the reminder-text update itself lands in PR 7) + +**Historical evidence:** `wayland-e2e-dbus`/`wayland-e2e-dispatch` branch runs; +journal Wayland entries. + +**Rollback boundary:** revert restores `portal-libei`-only gating and the old release +build. Must revert PR 6 first if merged (Nix consumes the new feature names). + +--- + +### PR 6 - `codex/conv2-06-nix` - "ci(cua-driver): nested compositor Nix consolidation (conv2 6/7)" + +**Purpose:** consolidate the Nix story - rewritten `flake.nix` and +`nix/cua-driver/package.nix` (new feature names), nested-compositor patch, delete the +21 superseded `.nix` test files and the three standalone nix workflows in favor of +`ci-nix-linux.yml`. + +**Owns:** + +- `flake.nix`, `nix/cua-driver/{README.md,package.nix,compositor/cua_compositor_patch.py,tests/README.md}` +- `scripts/ci/linux/run-rust-e2e-inject.sh` (A), which activates the nested + environment packaged by this PR +- **Deletes** `nix/cua-driver/tests/*.nix` (integration, linux-background-gui, + linux-background-terminal-gif, linux-cursor-click-gif, linux-parallel-drag-gif, + linux-parallel-drag-xserver, openbox-rc, record-x11-gif, screenshot, set-config) + and `nix/cua-driver/tests/wayland/**` (README.md + 10 `.nix` files) +- CI: `.github/workflows/ci-nix-linux.yml`; **deletes** + `.github/workflows/{nix-build.yml,nix-screenshot.yml,nix-wayland.yml}` + +**Excludes:** all Rust sources (PRs 1-5), docs. + +**Dependencies:** PR 5 (`package.nix` builds with the split features). + +**Fresh gates:** + +- [ ] `nix flake check` and `nix build .#cua-driver` (or the flake's package attr) + succeed locally/CI +- [ ] `ci-nix-linux.yml` green on this branch head (workflow_dispatch), covering what + the three deleted workflows covered +- [ ] Dispatch `e2e-rust-linux-wayland.yml` with the nested-compositor environment on + this branch head. Native and capture owners must remain green; any experimental + shared-row failures must match the named PR description exactly, with no new + setup, protocol, or evidence regression. This run is evidence, not a promotion + of the experimental lane. +- [ ] `git grep -nE 'nix-(build|screenshot|wayland)\.yml'` -> empty (no dangling refs) + +**Historical evidence:** nested-compositor GIF/screenshot runs recorded in the journal. + +**Rollback boundary:** revert restores the old nix tests/workflows; independent of +Rust code paths (Nix is packaging/CI only). Safe to revert alone. + +--- + +### PR 7 - `codex/conv2-07-docs-cleanup` - "docs(cua-driver): convergence docs and replacement cleanup (conv2 7/7)" + +**Purpose:** land all documentation describing the converged state, delete the +superseded recording/fixture corpora that the typed evidence pipeline replaces, and +wire the release-reminder validation checklist. **Resolves the sole `main` conflict.** + +**Owns:** + +- `docs/content/docs/concepts/`: `how-cua-driver-is-validated.mdx` (A), `index.mdx`; + `meta.json` - **hand-merge, never final-state** (see Section 4): insert the + `how-cua-driver-is-validated` entry into _current main's_ `meta.json` +- `docs/content/docs/reference/cua-driver/`: `development.mdx`, `limits.mdx`, + `mcp-tools.mdx`, `meta.json`, `platform-roadmap.mdx` (A), `platform-support.mdx` (A) +- `libs/cua-driver/docs/`: adds `action-support.md`, `e2e-convergence-journal.md`, + `e2e-results-summary-design.md`, `linux-desktop-validation.md`, + `linux-support-completion-plan.md`, `test-harness-convergence-plan.md`, + `test-harnesses-guide.md`, `test-matrix.md`; modifies `e2e-ci-reporting.md`; + **deletes** `cross-platform-ci-test-plan.md`, `pr-split-plan.md`; includes this + file (`pr-2161-landing-plan.md`) for the record +- Root docs: `README.md`, `CONTRIBUTING.md`, `Development.md`, `TESTING.md`, + `libs/cua-driver/README.md`, `libs/cua-driver/rust/README.md` +- Replacement cleanup - **deletes whole directories**: + `libs/cua-driver/tests/fixtures/linux-container/**` (5 files), + `libs/cua-driver/tests/fixtures/modality-recordings/**` (15 files), + `libs/cua-driver/tests/fixtures/vision-agent-test/**` (2 files); + modifies `tests/fixtures/README.md` to match +- `scripts/ci/README.md` (final state - documents scripts landed in PRs 1-5) +- Release wiring: `.github/workflows/ci-release-reminder.yml` + + `.github/scripts/tests/test_cua_driver_release_wiring.py` (checklist links the + workflows finalized in PRs 4-5) + +**Excludes:** the superseded Lume generated docs (Section 2.2), even though they sit +next to owned docs paths. Current `main` already contains their newer generated form. + +**Dependencies:** PRs 2-6 all merged (docs describe every platform; cleanup deletes +corpora only once every replacement suite is live; reminder links final workflows). + +**Fresh gates:** + +- [ ] Docs site builds: `pnpm install && pnpm build` in `docs/` (or the repo's docs + build command) with zero broken-link errors for the new pages +- [ ] `python -m unittest .github/scripts/tests/test_cua_driver_release_wiring.py` passes +- [ ] `git grep -n 'modality-recordings\|linux-container\|vision-agent-test' -- ':!libs/cua-driver/docs'` + -> empty (nothing references deleted fixtures) +- [ ] Rendered-page spot check of `platform-support.mdx` / `platform-roadmap.mdx` / + `how-cua-driver-is-validated.mdx` + +**Historical evidence:** n/a - docs are the evidence record itself; verify links resolve. + +**Rollback boundary:** pure docs + inert fixture deletions + reminder text; safe to +revert alone at any time. + +--- + +## 4. Shared-file overlap and conflict handling + +Disjoint path ownership eliminates most conflicts. The exceptions, with their rule: + +| Shared file | Touched by | Rule | +| ------------------------------------------------------------------ | ---------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `rust/Cargo.lock` | PRs 1, 2, 4, 5 (+3 if deps shift) | **Never final-state-copied.** Each PR hand-edits its manifests and regenerates via `cargo check --workspace`. Parity audit compares it semantically (`cargo tree`), not byte-wise. | +| `rust/Cargo.toml` (workspace) | PR 2 only | Whole delta is the `focus-monitor-win` member removal -> PR 2 hand-edits that one line. | +| `crates/cua-driver-testkit/**` | PR 1 only | Entire crate (incl. cfg-gated `windows_setup.rs`) lands in PR 1 so `lib.rs` is final-state and compiles everywhere. | +| `crates/platform-linux/src/lib.rs` | PRs 4, 5 | Hunk-split: PR 4 adds `pub mod recording_hooks;`, PR 5 adds `pub mod video_wayland;`. Neither checks out the whole file. | +| `crates/platform-linux/Cargo.toml`, `crates/cua-driver/Cargo.toml` | PR 5 only | Verified: the entire delta in both is the portal feature split -> PR 5 takes final state. | +| `platform-linux/src/wayland/mod.rs` | PR 5 only | All Wayland module wiring (incl. `sway_ipc`, `ext_toplevel`, feature-gate renames) is PR 5's. | +| Shared scenario catalog `tests/fixtures/shared/scenarios.json` | PR 1 only | Final-state superset lands in PR 1; entries for platform suites not yet landed are inert data. Platform PRs must not touch it - if a platform needs a catalog fix, it amends PR 1 before landing, not its own PR. | +| `tests/fixtures/README.md` | PR 7 only | Final state; documents both adds (PRs 1-5) and deletions (PR 7). Acceptable to be momentarily ahead/behind between merges. | +| `scripts/ci/README.md` | PR 7 only | Same rule as above. | +| Workflows | one owner each (table in Section 2.1 / per-PR lists) | A workflow file never spans two PRs. `cd-rust-cua-driver.yml` -> PR 5; `ci-release-reminder.yml` -> PR 7 (its wiring test rides with it). | +| `docs/content/docs/concepts/meta.json` | PR 7 only | **The only `main` conflict.** Hand-merge: apply the #2161 _intent_ (add the new page entry) onto current main's file; do not `git checkout` the branch version. | + +Conflict-with-main policy for the whole stack: because every branch is cut fresh from +current `origin/main` and transfers final state per path, the only possible conflicts +are files that changed on `main` after #2161 diverged. Before each transfer, run +`git diff origin/main...codex/link-e2e-videos --name-only -- ` vs +`git log origin/main --since= -- `; any path with fresh +`main` history gets the `meta.json` treatment (hand-merge, note in PR body). + +## 5. Dependency graph + +```mermaid +graph TD + PR1["PR 1: typed evidence foundation"] + PR2["PR 2: Windows"] + PR3["PR 3: macOS"] + PR4["PR 4: Linux core / X11"] + PR5["PR 5: Wayland backends"] + PR6["PR 6: nested compositor / Nix"] + PR7["PR 7: docs + replacement cleanup"] + SA["Sidecar A: Docker cleanup"] + SB["Sidecar B: TypeScript tooling"] + PR1 --> PR2 + PR1 --> PR3 + PR1 --> PR4 + PR4 --> PR5 + PR5 --> PR6 + PR2 --> PR7 + PR3 --> PR7 + PR6 --> PR7 +``` + +PRs 2, 3, 4 may be prepared and reviewed in parallel once PR 1 merges. PRs 5-7 are +strictly sequential behind their parents. Sidecars A and B are independent and may +land at any point before #2161 closes. + +## 6. Sequential landing procedure + +For each PR, in dependency order: + +- [ ] Cut `codex/conv2-0N-` from **current** `origin/main` (never from a sibling + stack branch) and run the transfer recipe (Section 3) +- [ ] Run the sidecar-separation grep (Section 2.2) -> empty +- [ ] Run the PR's fresh gates; attach workflow-run links / command output to the PR body +- [ ] PR body: purpose, owned-path list, link to #2161 + this plan, historical-evidence + links clearly labeled "historical - superseded by fresh runs above" +- [ ] Review -> squash-merge +- [ ] After merge: the next PR in line re-cuts from the new `main` (re-running the + final-state transfer is cheap and conflict-free by construction - prefer re-cut + over rebase) +- [ ] Post a progress comment on #2161: "conv2 N/7 landed as #NNNN" + +If a fresh gate fails on a stack PR: fix **on that stack branch** (these are real +review findings - the point of the split); if the fix belongs to an earlier layer, +land a follow-up to that layer first. Never "fix" a Cua Driver PR by pulling in a +sidecar-owned file. + +## 7. Final allowlisted tree-parity audit + +After PR 7 merges, from an up-to-date checkout: + +```bash +git fetch origin +# 1) Allowlisted parity: for every Section 2.1 path, main must equal the #2161 tree. +git diff origin/main codex/link-e2e-videos -- \ + libs/cua-driver nix/cua-driver flake.nix scripts/ci \ + .github/workflows/e2e-rust-windows.yml .github/workflows/e2e-rust-linux.yml \ + .github/workflows/e2e-rust-linux-wayland.yml .github/workflows/ci-rust-linux.yml \ + .github/workflows/ci-rust-windows.yml .github/workflows/ci-nix-linux.yml \ + .github/workflows/cd-rust-cua-driver.yml .github/workflows/ci-release-reminder.yml \ + .github/scripts/tests/test_cua_driver_release_wiring.py \ + docs/content/docs/concepts docs/content/docs/reference/cua-driver \ + README.md CONTRIBUTING.md Development.md TESTING.md +``` + +Expected residual diff - exactly three explainable classes, anything else is a missed +file and blocks closing #2161. Maintain an **adaptation ledger** in the final #2161 +comment: one row per intentionally divergent path, with the stack PR/commit and exact +reason. A generic "main advanced" exception is not acceptable. + +1. `libs/cua-driver/rust/Cargo.lock` - regenerated, semantically compared via + `cargo tree --workspace` on both trees +2. `docs/content/docs/concepts/meta.json` - hand-merged against newer `main` +3. Files intentionally adapted while landing against newer `main`, each listed in the + adaptation ledger with its owning stack PR and commit + +```bash +# 2) Preservation parity: after both sidecars merge, their paths should match +# the intended #2161 state unless the adaptation ledger says otherwise. +git diff --name-only origin/main codex/link-e2e-videos -- \ + Dockerfile .dockerignore package-lock.json .pre-commit-config.yaml \ + libs/typescript .github/workflows/ci-lint-typescript.yml \ + .github/workflows/claude-auto-fix.yml +# expected: empty after Sidecars A and B merge, except ledgered adaptations. + +# The Lume references are intentionally different: main must remain newer than +# the snapshot's 0.3.14 generated output. +rg -n 'Version: 0\.3\.15|Documented against Lume \*\*0\.3\.15\*\*' \ + docs/content/docs/reference/lume/{cli-reference,http-api}.mdx + +# 3) Coverage invariant: every delta file is stack-owned, sidecar-owned, or +# superseded by the two newer Lume files. +git diff --name-only origin/main...codex/link-e2e-videos | sort > /tmp/delta.txt +# compare against the union of Section 3 ownership lists + Section 2.2 - must match 1:1. +``` + +Record the audit output as a final comment on #2161. + +## 8. Disposition of PR #2161 and its branch + +**Link:** each stack PR body carries `Part of the #2161 second-wave landing +(conv2 N/7, plan: libs/cua-driver/docs/pr-2161-landing-plan.md)`. #2161's description +gets a checklist of the seven PRs, updated as they merge. + +**Close #2161 when all of:** + +- [ ] All 7 stack PRs merged to `main` +- [ ] Sidecars A and B merged to `main` +- [ ] Section 7 parity audit run and posted, residual diff limited to the three explained classes +- [ ] The two Lume paths are recorded as superseded by newer generated output on `main` +- [ ] Closing comment maps every #2161 theme -> landing PR, sidecar PR, or + superseded-by-main record + +Close as **closed, not merged**. + +**Delete `codex/link-e2e-videos` when all of:** + +- [ ] #2161 closed for at least 30 days (not before 2026-08-11) with no parity regressions reported +- [ ] The first post-landing cua-driver release (per the `ci-release-reminder` + validation checklist) has shipped green on the converged workflows +- [ ] A final `git diff` against `main` over the allowlist is re-run and still clean - + confirming the branch holds nothing unlanded outside the two explicitly + superseded Lume files + +Until then the branch stays as the forensic reference for the parity audit and the +historical-evidence links in the stack PRs. diff --git a/libs/cua-driver/docs/pr-split-plan.md b/libs/cua-driver/docs/pr-split-plan.md deleted file mode 100644 index 9293a3ad3e..0000000000 --- a/libs/cua-driver/docs/pr-split-plan.md +++ /dev/null @@ -1,297 +0,0 @@ -# Cua-driver PR Split Plan - -Status: draft for review - -## Decision - -Do not merge PR #2135 as one change. It currently contains 204 files and -roughly 11.5k additions/deletions across fixture migration, Rust test -ownership, Windows behavior, Linux/macOS behavior, and CI/Nix architecture. - -Keep the current branch as an untouched snapshot while smaller PRs are built -from `main`. The historical commits are useful provenance, but several of them -mix unrelated file families. Cherry-picking those commits wholesale would -reintroduce the same review problem. - -## Target PRs - -### PR 0: Documentation lockfile housekeeping - -Suggested title: - -```text -chore(docs): sync pnpm lockfile with pnpm 9 -``` - -Scope: - -- `docs/pnpm-lock.yaml` -- Current provenance: `8fee84dd` - -This is independent and can merge first. It should not be coupled to desktop -behavior or CI changes. - -Acceptance: - -- Docs dependency install/check passes. -- No `libs/cua-driver` files change. - -### PR 1: Repo-local Rust harness foundation - -Suggested title: - -```text -test(cua-driver): make repo-local Rust harnesses canonical -``` - -Purpose: establish where fixtures, Rust integration tests, and runner scripts -live. This PR should make the source tree coherent without changing the -platform input algorithms. - -Scope by file family: - -- Move `libs/cua-driver/test-harness` content to - `libs/cua-driver/tests/fixtures`. -- Add the repo-local Electron and Tauri source/build scripts for each host. -- Move the Windows sandbox runner to `tests/runners/windows-sandbox`. -- Add the Rust-only Windows `tests/runners/windows/run-all.ps1` entrypoint. -- Remove the old Python e2e harness and downloaded desktop binaries. -- Port the optional LibreOffice and macOS launch-focus tests to Rust where - needed, keeping them outside the default run-all path. -- Update path references, `.gitignore` files, fixture READMEs, Rust test README, - and the small contributor-facing Fumadocs page. -- Include `libs/cua-driver/scripts/sync-vm-worktree.sh` because it belongs to - the host-owned fixture workflow. - -Historical provenance: - -- `811ffa6d`, `d44971c`, `98e59d75`, `d9e4311f`, `9ab85981`, - `5e8759e1`, and `4e1636b9`. - -Do not take the platform implementation portions of `a759840b` or -`d01c7a14` into this PR. If a test path needs a behavior change to compile, -make the smallest path-only adjustment and leave the behavioral assertion -change for the relevant behavior PR. - -Acceptance: - -- No old downloaded Electron/Tauri application is tracked. -- No Python behavioral suite remains under the cua-driver test tree. -- Fixture build scripts pass shell/PowerShell parsing on their native hosts. -- `cargo test -p cua-driver --tests --no-run` passes on macOS, Linux, and - Windows. -- Staged outputs remain under ignored `rust/test-apps`. -- Optional external-app tests are visibly excluded from the default runner. - -### PR 2: Shared Rust behavioral matrix - -Suggested title: - -```text -test(cua-driver): add shared external-oracle behavior matrix -``` - -Purpose: make the trusted user-behavior scenarios explicit before wiring them -into expensive CI. The same scenario IDs, fixture DOM, delivery modes, and -external-state oracles should be used on every supported host. - -Scope: - -- Add `cross_platform_behavior_test.rs`. -- Add/update the shared web fixture and scenario catalog. -- Add only the generic testkit response/tree helpers required by the matrix. -- Keep the matrix ignored by default and runnable through one Rust command. -- Keep assertions based on fresh application state, not driver success alone. - -Historical provenance: primarily `ca60e0ba`, but extract only the matrix, -fixture, and generic testkit pieces. The platform algorithm changes from that -commit belong in PRs 3 and 4. - -Acceptance: - -- Matrix compiles on all three hosts. -- Electron and Tauri are selected from repo-local source-built fixtures. -- Every scenario has an external oracle and a documented outcome rule. -- Missing fixtures are visible as a runner/setup failure on dedicated e2e - lanes, not a green silent skip. - -### PR 3: Windows delivery and interactive UX fixes - -Suggested title: - -```text -fix(cua-driver): harden Windows delivery and GUI validation -``` - -Scope: - -- Windows Session 0 and interactive-desktop diagnostics. -- Focus sentinel and guard UX behavior. -- Background keyboard/click refusal reporting and preserved actuator causes. -- UIA focus, scroll, WebView2/Tauri, WPF, and WinUI3 behavior changes. -- Windows-specific test assertions and fixture details that validate those - behaviors. - -Historical provenance: Windows portions of `a759840b`, `98e59d75`, -`d01c7a14`, and `ca60e0ba`. - -Acceptance: - -- Windows unit and compile checks pass on `windows-latest`. -- SSH/Session 0 runs skip GUI-only assertions with an explicit reason. -- An active user-session run passes the guard and native harness suites. -- W1-style background input never reports verified delivery without an - external state change. -- W8-style unsupported background delivery returns a structured refusal with - the concrete cause. - -### PR 4: Linux delivery fixes - -Suggested title: - -```text -fix(cua-driver): harden Linux AT-SPI and input delivery -``` - -Scope: - -- Linux AT-SPI reference handling, action delivery, input capability errors, - and related external-oracle test updates. -- Linux focused tests and strict known-gap assertions. - -Historical provenance: Linux portions of `a759840b`, `ca60e0ba`, and -`d848e365`. The VM sync helper from `d848e365` belongs in PR 1, not here. - -Acceptance: - -- Linux unit and interactive Xvfb/AT-SPI checks pass. -- Known gaps remain strict and visible; no assertion is weakened to fit the - current driver behavior. -- Linux behavior changes are covered by the shared matrix where the platform - supports the scenario. - -### PR 5: macOS input and capture delivery - -Suggested title: - -```text -fix(cua-driver): harden macOS input and capture delivery -``` - -Scope: - -- macOS background click, drag, targeted wheel, and accessibility scrolling. -- macOS Swift runtime/build fixes required for Rust test binaries. -- macOS focused tests and strict nested-scroll known-gap assertions. - -Historical provenance: macOS portions of `a759840b`, `ca60e0ba`, and -`9c3c719a`, plus the macOS portions of `0447bd52`. - -Acceptance: - -- macOS default tests and installed-daemon harness checks pass with TCC - already authorized. -- Electron and Tauri click/drag rows use external application-state oracles. -- The nested web scrolling gap remains strict and visible. - -### PR 6: OS-scoped CI and Nix e2e architecture - -Suggested title: - -```text -ci(cua-driver): add OS-scoped Rust and manual e2e lanes -``` - -Scope: - -- Testkit path overrides and their unit test: - `CUA_TEST_DRIVER_BIN`, `CUA_TEST_APPS_ROOT`, and - `CUA_TEST_WORKSPACE_ROOT`. -- Strict fixture preflight for canonical e2e runs. -- Linux and Windows OS-scoped unit workflows. -- Maintainer-dispatched Linux and Windows interactive e2e workflows. -- Source-built Linux Rust check in Nix. -- Nix and workflow READMEs describing canonical versus supporting coverage. -- Move the legacy GUI/toolkit and compositor matrices to maintainer dispatch; - preserve them as supporting diagnostics. -- Keep new canonical Linux tests free of GIF requirements. - -Historical provenance: `eef79f2a` and `0447bd52`, after extracting the Linux -and macOS driver edits from `0447bd52` into PRs 4 and 5. - -Acceptance: - -- Shared Rust changes trigger both OS unit workflows. -- Platform-specific changes trigger only the relevant OS unit workflow. -- Linux Rust source check passes in Nix. -- Manual Linux e2e builds repo-local fixtures from source. -- Windows e2e verifies an interactive user session and supports an Azure - active-RDP runner label. -- No VM or GitHub runner can push source changes. -- The PR description says clearly that fully sandboxed Nix Electron/Tauri - packaging remains a follow-up requiring committed dependency lockfiles. - -## Dependency Graph - -```mermaid -graph LR - P0[Docs lockfile] --> Merge - P1[Repo-local harness foundation] --> P2[Shared Rust behavior matrix] - P2 --> P3[Windows behavior fixes] - P2 --> P4[Linux behavior fixes] - P2 --> P5[macOS behavior fixes] - P3 --> P6[OS-scoped CI and Nix] -``` - -PR 0 is independent. PR 6 is based on PR 3 because its Windows unit workflow -compiles the Windows guard tests and therefore needs the Windows desktop-state -diagnostics. It can merge after PR 3; PRs 4 and 5 remain independent platform -slices and should merge before enabling the full cross-platform e2e gate. - -## Branching Procedure - -1. Freeze PR #2135. Do not add more mixed commits to it. -2. Preserve the current branch as the snapshot containing all work. The - private `cross-platform-fix-journal.md` stays untracked and must not be - included in any PR. -3. Create each split branch from `main`, then apply the final file families - from the snapshot. Use `git diff --find-renames main...HEAD -- ` to - preserve moves and deletions. -4. Do not cherry-pick `a759840b`, `ca60e0ba`, `d848e365`, or `0447bd52` - wholesale. Each mixes concerns that belong in different PRs. -5. Cherry-picking `8fee84dd` is safe for PR 0. `eef79f2a` is safe only after - confirming the old interactive workflow remains compatible with the new CI - split. -6. Run the acceptance checks for each branch before opening its PR. Keep PR - descriptions narrowly scoped and list known failures instead of hiding - them behind broad “cross-platform fixes” language. -7. Open the PRs in dependency order. Stack PR 2 on PR 1 if needed, then - retarget it to `main` after PR 1 merges. Repeat for later PRs. -8. Close PR #2135 only after the smaller PRs are open and its body links to - them. Keep the snapshot branch available until all split PRs merge. - -## Review and Rollback Rules - -- A PR must have one primary reason to change and one obvious owner. -- Fixture moves must be reviewed with rename detection enabled. -- Deleting the Python suite and downloaded binaries is only part of PR 1 when - the Rust replacement and build scripts are present in the same diff. -- Platform behavior PRs must not change CI trigger policy. -- CI/Nix PRs must not contain opportunistic platform algorithm changes. -- If a split branch cannot compile independently, either add the missing - foundation to its parent PR or make the dependency explicit; do not silently - retain unrelated files from PR #2135. -- Revert the smallest PR that caused a regression. The dependency order keeps - the harness foundation and behavior fixes independently revertible from CI. - -## Final State - -After all PRs merge, the repository should have: - -- Rust harnesses as the canonical behavioral source of truth. -- One shared scenario matrix with platform-specific implementations behind it. -- Cheap OS-scoped unit checks on normal PRs. -- Manual, real-session e2e gates for expensive desktop behavior. -- Nix retained as the Linux environment, with legacy visual checks clearly - labeled as supporting coverage. -- No opaque downloaded desktop fixture in the canonical test path. diff --git a/libs/cua-driver/docs/test-harness-convergence-plan.md b/libs/cua-driver/docs/test-harness-convergence-plan.md new file mode 100644 index 0000000000..32780a90e3 --- /dev/null +++ b/libs/cua-driver/docs/test-harness-convergence-plan.md @@ -0,0 +1,455 @@ +# Rust E2E Harness Convergence Plan + +## Re-review Verdict + +The original direction remains sound: + +- Rust owns scenarios, assertions, and result records. +- Repo-local harness applications are the canonical E2E targets. +- AX/PX targeting and foreground/background delivery are dimensions on an + action. They are not test families. +- Focus, z-order, cursor, and desktop checks are cross-cutting observations. +- The selector-free contributor invocation runs the complete matrix on every + OS. CI may fan it into internal jobs for failure isolation. + +This review changes five parts of the earlier plan: + +1. Do not make a historical cell count a target. The shared matrix must cover + supported route combinations, while every omission needs a route + equivalence or unsupported-capability reason. +2. Separate test status from driver behavior. A test may pass because a + required action was delivered or because a declared unsupported route was + refused correctly. Those outcomes must remain distinct in reports. +3. Treat environment readiness as a lane preflight. A missing desktop, TCC + grant, fixture, AT-SPI bus, recorder, or interactive Windows session must + fail once before behavioral cells run. +4. Use a typed Rust case catalog as the matrix source. Do not add a second + `matrix.yaml`, Python collector, or shell-owned scenario list. +5. Preserve one evidence bundle per cell, but reuse the driver and harness + process where explicit state reset proves isolation. + +The target is a smaller set of tests with named coverage reasons and strong +external oracles. Test count is not a success metric. + +## Non-negotiable Rules + +1. A successful tool response never proves delivery. A delivered action must + change fixture or desktop state that the test reads independently. +2. A background action must also prove that it did not steal focus, raise the + target, move the real cursor, or leak partial input when those invariants + apply. +3. A refusal is valid only when the cell contract expects refusal, the driver + returns an allowed structured refusal code, and the desktop observer sees no + side effect. +4. A cell that requires delivery fails when the driver refuses, even if the + refusal is honest. +5. A required canonical fixture or desktop capability cannot become a passing + early return. Optional tests must be declared optional before execution. +6. Known gaps do not use `#[should_panic]` and do not count as green coverage. + They run in a named optional lane with a linked issue until fixed. +7. Shell and PowerShell runners build the environment and collect artifacts. + They do not infer behavioral results from Cargo output. +8. The Rust source build under test must be recorded in every run. macOS may + proxy through the installed app bundle for TCC, but that bundle must come + from the same source revision. + +## Current Branch State + +The branch implements the target ownership and reporting model. Platform +validation below replaces the older 60-cell proposal and records the remaining +environment or backend gaps separately. + +### Implemented + +- `cross_platform_behavior_test.rs` has one typed 36-cell catalog per shared + harness application: 72 shared cells across Electron and Tauri per platform. + It covers AX and PX with foreground and background delivery for click, + text, keyboard, scroll, and child-window actions; PX drag and AX editor-save + each cover both delivery modes. +- Declaration and result schema v2 record action, `targeting`, delivery, scope, + backend route, expected and observed behavior, independent test status, + required oracles, duration, and evidence. +- Background refusal matching uses the explicit structured set: + `background_unavailable`, `background_occluded`, and + `background_uipi_blocked`. +- Every catalog route is explicit for Win32, Quartz, X11, and Wayland. Windows + Chromium PX background keeps its targeted-injection route metadata, while a + fully occluded target expects the exact `background_occluded` refusal because + temporarily raising the window would violate the desktop-side-effect contract. +- Every background shared cell attaches independent focus, z-order, real + cursor, leaked-input, and fixture-state observations. Windows, macOS, and X11 + use direct testkit observers; an occluding Electron sentinel supplies the + focus and leaked-input journal. Unsupported Wayland observations fail closed. +- Electron and Tauri use repo-local builds of the same shared web fixture. +- Windows, Linux, and macOS runners have strict source, fixture, AX, capture, + and video preflights. Shell runners no longer synthesize behavioral rows. +- Rust validates duplicate, missing, contradictory, and evidence-less results + before rendering the GitHub summary. +- Per-cell source-driver recording support exists in `cua-driver-testkit`. +- The typed catalog is the sole shared behavior owner; the three older shared + tests and duplicate guard/modality targets were removed after their unique + assertions moved to typed shared, native, launch, capture, cursor, and + desktop-scope rows. +- WPF, WinUI3, WebView2, AppKit, SwiftUI, GTK3, capture, launch, cursor, and + desktop-scope owners emit the same typed result records as the shared matrix. + +### Validation and known gaps + +> **Evidence update (2026-07-12):** The counts below preserve the evidence that +> was available when this convergence plan was executed. Current Linux support +> and run IDs live in `action-support.md`; the original 80-row X11 and early +> 6/80 Wayland snapshots are superseded by the expanded 108-outcome X11 run, +> the 31-row Sway GTK3 run, focused Electron evidence, and the GNOME 46 run. + +- The stable installed macOS app identity retains Accessibility and Screen + Recording grants across `install-local`. The complete local matrix passes + strict daemon identity, fixture, AX, capture, and video preflight. +- AppKit scroll remains an optional failing gap outside the canonical run. +- Linux X11 was complete for the original catalog: exact run `29148643166` + passed all 80 typed rows, with 48 delivered actions and 32 exact refusals. + The current expanded catalog is tracked in `action-support.md`. +- Windows is complete: exact run `29149710089` passed all 110 typed rows, with + 87 delivered actions and 23 exact refusals. Every cell and lane preflight + produced parseable MP4 evidence. +- The first pure-Wayland baseline was experimental. Exact run `29150698432` passed strict + preflight and executed all 80 rows: 6 delivered and 74 failed, with no skips. + Sentinel startup noise is gone; issue `#1922` owns the remaining target + posture metadata, Tauri AX tree, Electron input, AT-SPI geometry, and GTK + mapping gaps present at that point. Later Sway and GNOME evidence supersedes + this baseline. Some native targets still use fixed waits; shared and Windows + web targets poll external state and allocate CDP ports per process. + +## Target Test Model + +### Typed case catalog + +Every behavioral cell is declared in Rust with one structure shared by web and +native harness tests: + +```rust +struct CaseSpec { + id: &'static str, + platform: Platform, + display_server: DisplayServer, + harness: Harness, + action: Action, + targeting: Targeting, + delivery: Delivery, + scope: Scope, + expectation: ContractExpectation, + oracles: &'static [OracleKind], + route: DriverRoute, +} + +enum ContractExpectation { + Deliver, + Refuse { allowed_codes: &'static [RefusalCode] }, +} +``` + +`RefusalCode` is an enum, not a string-prefix check. Its initial Windows set is +`BackgroundUnavailable`, `BackgroundOccluded`, and `BackgroundUipiBlocked`; +Linux currently declares only `BackgroundUnavailable`. A cell lists the exact +codes allowed by its controlled setup. + +`Targeting` uses `Ax`, `Px`, `Page`, or `NotApplicable`. Use `targeting` in the +schema instead of `capture_mode`; capture is a separate read contract. + +`DriverRoute` names the implementation path that justifies coverage, such as +UIA Invoke, PostMessage, coordinate injection, CGEvent, AT-SPI action, libei, +or CDP. It is test metadata, not a request parameter. + +The catalog is the machine-readable inventory. Contributor documentation and +the coverage table are generated from it or checked against it. There is no +second matrix file to keep in sync. + +### Result record + +Use one schema version and these independent fields: + +| Field | Meaning | +| --- | --- | +| `cell_id` | Stable case id | +| `platform`, `display_server` | OS and Win32/Quartz/X11/Wayland environment | +| `harness`, `toolkit` | Electron, Tauri, WPF, WinUI3, WebView2, AppKit, SwiftUI, GTK3 | +| `action`, `targeting`, `delivery`, `scope` | Contract dimensions | +| `driver_route` | Backend path covered by the cell | +| `expected_behavior` | `DELIVER` or `REFUSE` | +| `test_status` | `PASS`, `FAIL`, `SKIP`, or `ENVIRONMENT_ERROR` | +| `observed_behavior` | `DELIVERED`, `REFUSED`, `NO_EFFECT`, `ERROR`, or `NOT_RUN` | +| `refusal_code` | Structured code when observed behavior is `REFUSED` | +| `oracles` | App state and attached desktop observations | +| `known_issue` | Optional issue id; it never changes a failure to a pass | +| `evidence` | Video, trajectory, screenshots, structured state, and log paths | + +This removes the ambiguous `EXPECTED_REFUSAL` status. A refusal contract passes +only when `expected_behavior=REFUSE`, `observed_behavior=REFUSED`, the code is +allowed, and all no-side-effect oracles pass. Reports count delivered and +refused passes separately. + +### Coverage selection + +Do not restore the full Cartesian product automatically. Select cells by +driver route: + +1. Every supported action has foreground and background coverage when the tool + exposes both delivery modes. +2. Every distinct targeting path used by that action has at least one cell. +3. Every distinct OS backend route has at least one cell. +4. Renderer or toolkit duplication is kept only when it changes the route or + has produced a real compatibility defect. +5. Every omitted combination names an `equivalent_to` cell or an unsupported + contract reason. + +The shared catalog currently declares 36 cells per harness application, or 72 +cells across Electron and Tauri per platform. Add a missing combination when it +reaches a different driver route. Remove a combination only when another cell +proves the same route with an equal or stronger oracle. + +## Cross-cutting Desktop Observer + +Add one testkit interface that snapshots desktop state before and after an +operation: + +```text +DesktopObservation + foreground window + target z-order and minimized state + focus-change journal + real cursor position + optional leaked-input journal +``` + +The testkit `DesktopObserver` provides this interface with native Windows, +macOS, and Linux backends. + +Attach the observer to: + +- every background delivery cell; +- refusal cells; +- launch/minimize cells; +- screenshot and capture cells that promise no focus change; +- cursor evidence cells. + +For successful background delivery, the cell requires both target-state change +and unchanged desktop invariants. For refusal, the cell requires no target +change and unchanged desktop invariants. A focus-only pass never proves input +delivery. + +## Process And Evidence Lifecycle + +Use these boundaries: + +- one Rust source build per lane; +- one driver daemon or MCP process per lane; +- one harness process per harness group when the fixture has a verified reset + operation; +- one recording session and result record per cell; +- a harness restart after crash, reset failure, or window identity change. + +Each fixture reset must return a generation token. The next cell verifies the +new token and clean marker state before acting. Until a harness has this reset +contract, keep process-per-cell isolation. + +Video remains required for every canonical E2E cell. Test video capture once in +the lane preflight. A recorder failure aborts the lane before the case catalog +runs, rather than generating the same permission failure for every cell. + +## Environment Preflight + +Each OS runner performs one preflight and emits one environment record. + +Common checks: + +- source revision and driver version match; +- required fixture binaries exist; +- the display/user session is interactive; +- the driver can list and inspect a preflight fixture; +- accessibility and capture permissions work; +- a short video starts, stops, and passes `ffprobe`; +- artifact directories are writable. + +Platform checks: + +| Platform | Required preflight | +| --- | --- | +| Windows | Non-Session-0 interactive desktop, input desktop, foreground sentinel, FFmpeg, UIA visibility | +| macOS | App-bundle daemon identity, live socket, Accessibility, Screen Recording, fixture window visibility | +| Linux X11 | X server, DBus, AT-SPI, window manager, capture, input backend | +| Linux Wayland | Compositor, DBus, AT-SPI, portal/capture path, libei or declared refusal path | + +Canonical invocations set strict mode. Missing required capabilities produce +`ENVIRONMENT_ERROR`; they never return from a test as a pass. + +## File Ownership And Disposition + +| Current file | Final owner or action | +| --- | --- | +| `cross_platform_behavior_test.rs` | Shared Electron/Tauri case catalog and external fixture-state and desktop-side-effect oracles | +| `harness_wpf_test.rs` | WPF-specific rows using the common case/result runner | +| `harness_winui3_test.rs` | WinUI3-specific rows; keep only toolkit-distinct behavior | +| `harness_web_test.rs` | WebView2 and Page/CDP behavior; do not mix Page targeting with AX/PX labels | +| `harness_appkit_test.rs` | AppKit rows; scroll is an honest failing optional test outside the canonical run | +| `harness_swiftui_test.rs` | SwiftUI AX tree/capture; keep the unverified popover action in an explicit optional test | +| `harness_gtk3_test.rs` | Minimal GTK3/AT-SPI rows for X11 and Wayland | +| Legacy Windows UX guard target | Deleted after typed launch, cursor, shared, capture, and desktop-scope owners passed the replacement audit | +| `modality_input_e2e_test.rs` | Deleted; shared cells own web actions and the Notepad row had no delivery oracle | +| `modality_background_test.rs` | Deleted; typed WPF background action rows and capture ownership passed the replacement audit | +| `capture_contract_test.rs` | Sole owner for tree/image inclusion behavior; canonical prerequisites fail instead of skipping | +| `desktop_scope__test.rs` | Platform-specific window/desktop scope contracts | +| `modality_focus_test.rs` | Deleted; shared click/type cells own focus preservation and launch focus has a separate optional owner | +| `installed_app_launch_macos_test.rs` | Optional real-app lane with issue ownership; never part of canonical harness counts | +| `installed_app_textedit_macos_test.rs` | Optional real TextEdit AX integration; schema assertions remain in protocol tests | +| `harness_libreoffice_test.rs` | Optional installed-app lane; exclude from the canonical run and counts | +| `protocol_*`, schema, transport tests | Unit/protocol gate; no desktop video and no behavioral matrix rows | +| `tests/fixtures/shared/scenarios.json` | Prune only after selector and marker-reference audit | + +## CI Shape + +The contributor invocation stays selector-free. Lane selection remains private +CI plumbing and diagnostic state. + +### Pull requests + +- Run unit/protocol jobs by affected OS paths. +- Shared core or schema changes trigger Linux and Windows unit jobs. +- Platform-only changes trigger that platform's unit job. +- E2E is maintainer-dispatched and may become an optional pre-merge gate. + +### Maintainer E2E + +- Windows GitHub-hosted runners are canonical when the preflight proves an + interactive desktop. An Azure RDP runner is an optional environment-parity + replay, not a second source of behavioral truth. +- Linux GitHub-hosted runners run X11 under Xvfb with hosted packages. The Nix + source gate is separate; the pure-Wayland maintainer lane uses the Nix dev + shell. Linux produces no GIF requirement. +- macOS runs on a logged-in, TCC-authorized host through the canonical macOS + runner. A future self-hosted runner must use the same preflight. + +CI may fan the complete matrix into shared, native, and capture/scope jobs. +Those are execution partitions, not alternate public test suites. + +## Reporting And Evidence + +Rust emits one record for every declared cell. A shared Rust reporter then: + +1. rejects duplicate or missing cell ids; +2. verifies the result against the case contract; +3. verifies required video evidence exists and is non-empty; +4. renders the behavioral table and the declared coverage table; +5. fails when a declared cell produced no result. + +Do not parse `test ... ok` lines to create behavioral rows. Cargo/JUnit output +may still provide failure annotations for unit tests. + +Upload one artifact archive per internal lane, with one stable evidence +subdirectory per behavioral cell. Each GitHub summary row links its exact +video path text to the owning lane archive. The adjacent `trajectory.json` +path remains in `results.jsonl` and the archive. GitHub cannot deep-link to a +file inside an archive; the exact path keeps the row unambiguous without +multiplying artifact uploads by the cell count. + +## Implementation Slices + +### Slice 1: Contract and preflight + +- Finalize `CaseSpec`, result enums, refusal-code enums, and one schema version. +- Add reporter validation for duplicate, missing, and contradictory records. +- Add strict environment preflight to Windows, Linux, and macOS runners. +- Stop shell runners from inventing behavioral rows. + +Exit: a missing session, fixture, permission, or recorder fails once as an +environment error, and a synthetic CaseSpec set renders a valid report. + +### Slice 2: Shared matrix integrity + +- Map every shared cell to an explicit driver route. +- Add any missing route cells; document every omitted equivalent. +- Add the desktop observer to background and refusal cells. +- Preserve editor-save and all existing external markers. +- Remove the three legacy shared tests after mapping every assertion. + +Exit: no fake drag pass, no arbitrary error accepted as refusal, no shared +legacy test, and one result/evidence bundle per shared cell. This source +convergence is complete; platform delivery defects remain visible as cell +failures. + +### Slice 3: Windows convergence + +- Move guard, modality-input, and modality-background assertions into shared, + WPF, WinUI3, capture, launch, or desktop-scope owners. +- Preserve every current failing action as a failing required-delivery cell or + an issue-linked optional cell. Do not convert it to a green refusal. +- Add Windows desktop-scope to the canonical run. +- Delete the three transitional files only after cell-by-cell parity. + +Exit: Windows has no guard/modality family, and every background cell has both +target-state and desktop-side-effect evidence. + +### Slice 4: macOS and Linux convergence + +- Adopt the common case/result runner in AppKit, SwiftUI, and GTK3 tests. +- Replace canonical early-return skips with preflight failures. +- Split Linux schema checks from real desktop behavior. +- Rename capture and desktop-scope ownership files. +- Move AppKit scroll and real-app checks to explicit optional issue lanes. + +Exit: native cells emit the same records as shared cells, X11 and Wayland are +separate dimensions, and macOS failures distinguish TCC from driver behavior. + +### Slice 5: Flake and fixture cleanup + +- Replace fixed coordinates with discovered geometry. +- Allocate CDP ports per process. +- Replace sleeps with deadline polling on external markers. +- Add fixture generation-token reset and reuse harness processes only after it + proves clean state. +- Prune fixture controls and markers with no live selector or oracle reference. + +Exit: no canonical cell depends on VM-specific coordinates, fixed shared ports, +or an unexplained sleep. + +### Slice 6: CI validation and deletion + +- Keep the complete macOS matrix green after `install-local` and TCC preflight. +- Keep the accepted GitHub-hosted Windows matrix as the canonical result; use + an optional RDP runner only for environment-parity investigations. +- Keep the accepted Linux X11 and Nix source runs as the supported Linux gates. +- Run pure Wayland as an experimental issue-linked lane until `#1922` closes. +- Compare old/new cells before each transitional file deletion. +- Update contributor docs and the PR description from the generated catalog. + +Exit: every declared cell is delivered, refused according to contract, or +fails with a linked unresolved bug. Environment failures are separate. + +## Deletion Gates + +A test or fixture path may be deleted only when: + +1. every assertion maps to a CaseSpec or unit/optional owner; +2. the replacement oracle is equal or stronger; +3. old and replacement outcomes were compared on each affected OS; +4. a required-delivery failure remains visible; +5. refusal cells use an explicit allowed code and desktop-side-effect proof; +6. runners, docs, and artifact labels stop referencing the old target in the + same change; +7. the reporter finds no missing declared cells. + +## Definition Of Done + +- Rust has one typed behavioral catalog and one result schema. +- Required GUI prerequisites cannot pass through early returns. +- Test status and observed driver behavior are separate fields. +- Every supported action has foreground/background coverage by distinct driver + route, with reasons for omitted combinations. +- Every delivered action has an external target-state oracle. +- Every background or refusal cell has desktop-side-effect evidence. +- No permanent guard, modality, or delivery test family remains. +- No `#[should_panic]` known-gap E2E test remains. +- No orphaned target is counted as coverage. +- Shared and native harnesses emit the same result/evidence shape. +- Unit/protocol tests stay desktop-independent and video-free. +- Windows, macOS, and supported Linux X11 complete runs produce classified + outcomes and per-cell evidence links. Experimental Wayland and environment + failures stop or fail explicitly without producing a false behavioral pass. diff --git a/libs/cua-driver/docs/test-harnesses-guide.md b/libs/cua-driver/docs/test-harnesses-guide.md new file mode 100644 index 0000000000..61ab95e9ae --- /dev/null +++ b/libs/cua-driver/docs/test-harnesses-guide.md @@ -0,0 +1,399 @@ +# CUA Driver Test Harnesses Guide + +This document explains how the CUA Driver tests are organized, what each layer +proves, and what still needs to be finished. It is written as a starting point +for contributors who are unfamiliar with the repository. + +## The Short Version + +There are two main test layers: + +1. **Unit and protocol tests** exercise Rust code and the public MCP/CLI + contract without launching a real application. +2. **Harness E2E tests** launch a small application built from this repository, + drive it through the Rust driver, and verify an external application or + desktop state. + +The Rust tests are the source of truth. Python tests, old shell runners, and +historical recording scripts are not part of the canonical E2E path. + +The canonical E2E command on every OS runs the complete matrix and takes no +suite selector: + +```text +Linux: scripts/ci/linux/run-rust-e2e.sh +Windows: .\scripts\ci\windows\run-rust-e2e.ps1 -RequireGui +macOS: scripts/ci/macos/run-rust-e2e.sh +``` + +The OS workflow may fan the complete matrix out into independent jobs for +reporting and failure isolation. That is an execution detail; contributors +should think of it as one canonical suite. + +## Repository Map + +```text +cua/ +|-- libs/cua-driver/ +| |-- rust/ +| | |-- crates/ +| | | |-- cua-driver/ Rust driver and integration tests +| | | |-- cua-driver-core/ Shared driver logic and unit tests +| | | |-- cua-driver-testkit/ Shared Rust E2E helpers and evidence capture +| | | |-- platform-linux/ Linux backend +| | | |-- platform-macos/ macOS backend +| | | |-- platform-windows/ Windows backend +| | | `-- cursor-overlay/ Cursor evidence helper +| | |-- test-apps/ Ignored staged harness binaries +| | `-- Cargo.toml Rust workspace +| |-- tests/ +| | |-- fixtures/ +| | | |-- shared/web/ Shared web page and external markers +| | | |-- apps/ Repo-local fixture sources +| | | `-- build/ macOS/Linux/Windows fixture builders +| | `-- runners/ Auxiliary VM and sandbox entrypoints +| `-- docs/ Test matrix, reporting, and contributor docs +`-- scripts/ci/ + |-- linux/run-rust-e2e.sh Linux canonical runner + |-- windows/run-rust-e2e.ps1 Windows canonical runner + `-- macos/run-rust-e2e.sh macOS canonical runner +``` + +The important separation is: + +| Layer | Owns | Does not own | +| --- | --- | --- | +| Rust integration test | Scenarios, driver calls, assertions, action metadata | OS setup and fixture compilation | +| `cua-driver-testkit` | Session helpers, fixture launching, screenshots, recordings, trajectories | The scenario list | +| Fixture app | Visible controls and externally observable state markers | Driver correctness assertions | +| OS runner | Build environment, user session, test selection, artifact collection | Test behavior definitions | + +There is deliberately no second Python E2E implementation that the Rust suite +has to mirror. + +## How One E2E Test Works + +Every canonical E2E cell follows this shape: + +```text +OS runner + -> builds the Rust driver and repo-local fixture + -> starts or connects to a real desktop session + -> Rust testkit starts one harness application + -> Rust test discovers the target window + -> get_window_state provides accessibility tree and screenshot + -> action addresses a target by AX element index or PX coordinates + -> delivery is foreground or background where the action supports it + -> fixture state, focus, pixels, or protocol response is checked + -> testkit writes video, screenshots, trajectory, logs, and result data +``` + +A tool returning `ok` is not enough to pass an E2E cell. The fixture must show +that the action happened, or the test must verify a documented structured +refusal and the absence of focus or input side effects. + +## The Test Layers + +### Unit and Protocol Tests + +These run without a repo-local GUI application and normally run without +`--ignored`: + +| Location or prefix | What it proves | +| --- | --- | +| `rust/crates/*/src/**` | Core driver, platform-independent logic, schemas, and helpers | +| `protocol_*_test.rs` | MCP handshake, tool calls, sessions, media, and errors | +| `schema_*_test.rs` | Shared schema and backend consistency | +| `transport_config_persistence_test.rs` | CLI/MCP configuration persistence | +| `protocol_element_token_test.rs` | Element-token protocol behavior | + +These tests should be fast, deterministic, and safe to run on ordinary CI +workers. They do not prove that a real click, key, scroll, or background input +reached an application. + +### Harness E2E Tests + +These are Rust integration tests under: + +```text +libs/cua-driver/rust/crates/cua-driver/tests/ +``` + +Most are marked `#[ignore]` because they require a desktop, built fixtures, and +platform permissions. They are selected by the OS runner rather than the +ordinary unit command. + +The canonical E2E suite has two behavior owners: + +| Owner | Purpose | +| --- | --- | +| Shared app | Same web behavior tested through Electron and Tauri | +| Native harness | Toolkit-specific controls and native window behavior | + +WebView, CDP, and page-tool integration stays inside the shared or native +owner that exercises it. It is not a third public test family or command. + +Delivery is not a test family. It is a dimension on each action row: an action +is tested in foreground and background modes whenever the driver and OS support +both. Capture and desktop scope are separate environment checks, while focus +preservation is a cross-cutting oracle that can be attached to any action row. +Focus, z-order, cursor, and desktop-state checks are cross-cutting invariants, +not a separate family. These names describe responsibilities, not separate +sources of truth or commands to run instead of the selector-free canonical +invocation. + +## What The Complete Run Includes + +### Windows + +Runner: `scripts/ci/windows/run-rust-e2e.ps1` + +| Runner area | Rust test | Real harness or app | +| --- | --- | --- | +| Shared app matrix | `cross_platform_behavior_test.rs` | Electron and Tauri | +| Native controls | `harness_wpf_test.rs` | Repo-local WPF app | +| Native controls | `harness_winui3_test.rs` | Repo-local WinUI3 app | +| Web integration | `harness_web_test.rs` | WebView2 and Electron | +| Capture contract | `capture_contract_test.rs` | WPF plus driver tree/image output | +| Launch contract | `launch_windows_test.rs` | Repo-local Electron launch and focus behavior | +| Agent cursor | `agent_cursor_windows_test.rs` | Source-built cursor overlay and pixel evidence | +| Desktop scope | `desktop_scope_windows_test.rs` | Windowless desktop input and scope rejection | + +Cross-cutting instrumentation used by these rows includes the testkit +`DesktopObserver`, capture validation, cursor evidence, and desktop-scope +checks. These invariants are attached to their owning action rows rather than +run as a separate test family. + +Windows is currently the broadest native matrix. It covers UIA controls, web +integration, background focus checks, and Windows-specific input routes. The +desktop observer is attached to shared and native action rows wherever +background delivery is tested. + +### macOS + +Runner: `scripts/ci/macos/run-rust-e2e.sh` + +| Runner area | Rust test | Real harness or app | +| --- | --- | --- | +| Shared app matrix | `cross_platform_behavior_test.rs` | Electron and Tauri | +| Native web matrix | `cross_platform_behavior_test.rs` | Repo-local WKWebView host | +| Native controls | `harness_appkit_test.rs` | Repo-local AppKit app | +| Native controls | `harness_swiftui_test.rs` | Repo-local SwiftUI app | +| Capture contract | `capture_contract_test.rs` | Installed driver and macOS capture APIs | +| Desktop scope | `desktop_scope_macos_test.rs` | macOS window and desktop scope | + +The WKWebView host runs the same 36 typed shared-web cells as Electron and +Tauri. `installed_app_launch_macos_test.rs` and +`installed_app_textedit_macos_test.rs` are optional real-app checks for +Calculator/TextEdit and are not part of the canonical run. + +### Linux + +Runner: `scripts/ci/linux/run-rust-e2e.sh` + +| Runner area | Rust test | Real harness or app | +| --- | --- | --- | +| Shared app matrix | `cross_platform_behavior_test.rs` | Electron and Tauri | +| Native controls | `harness_gtk3_test.rs` | Repo-local GTK3 app | +| Capture contract | `capture_contract_test.rs` | Linux capture backend | +| Desktop scope | `desktop_scope_linux_test.rs` | X11/Wayland desktop scope | + +Linux has separate X11 and Wayland concerns. Nix supplies the reproducible +build and desktop environment, but the E2E test still needs an actual X11 or +Wayland session. Linux does not need GIF output; MP4, screenshots, accessibility +trees, trajectories, and logs are the useful evidence. + +Wayland results are compositor-specific. The hosted lane uses Sway to prove +wlroots protocols. GNOME requires the optional WinRects Shell helper for +authoritative frame and buffer geometry, observation, capture, and verified +target activation. A portal/libei grant persists until the user revokes it, so +subsequent driver processes do not reopen the consent dialog. +KDE requires a future target-addressable KWin adapter; portal availability by +itself is not evidence that input can be sent safely to a named window. + +## AX, PX, and Delivery + +See [`action-support.md`](action-support.md) for the current Windows, macOS, and Linux +delivery, refusal, and unproven-action ledger. + +These terms describe different dimensions: + +| Term | Meaning | +| --- | --- | +| AX | Address a target through its accessibility/UI automation element | +| PX | Address a target by screen coordinates or pointer geometry | +| Foreground | The target may be brought to the foreground for delivery | +| Background | The target should receive the action without being raised or stealing focus | +| Window scope | Capture or action is limited to one target window | +| Desktop scope | Capture or action covers the full desktop | + +The shared and native action matrices should test left click, right click, +double click, typing, keys, hotkeys, scroll, child windows, and drag across +AX/PX and foreground/background combinations where the driver supports them. +Unsupported background routes require an explicit refusal contract with an +allowed structured code and desktop-side-effect oracles. A refusal fails a +cell that requires delivery. There should not be a separate "delivery" family +whose only purpose is to repeat those same actions in the background. + +Native harness rows use the same typed case/result contract as the shared +matrix. Current native `set_value` rows declare background delivery because +their contract includes no-focus and no-raise observations; actions without a +delivery concept use `not_applicable` explicitly. + +## Cross-Cutting Invariants + +The desktop observer is cross-cutting test instrumentation. It answers the same +question for any action, harness, or catalog area: + +> Did the driver perform or reject the operation without disturbing the user's +> foreground application or desktop? + +`cua-driver-testkit::DesktopObserver` owns the shared interface. Native Windows, +macOS, and Linux backends snapshot foreground-window, target z-order, cursor, +and leaked-input state before and after an action. Background rows opt into the +observer directly; there is no special guard suite. + +| Invariant or scenario | What it checks | +| --- | --- | +| Background click/type/key | The target action does not move focus away from the user's foreground window | +| Minimized app launch | `launch_app(start_minimized=true)` does not raise the new app | +| Background hotkey | A keyboard chord does not steal focus | +| Child-window click | A target-created window does not unexpectedly become foreground | +| Background screenshot | Reading the target does not change focus or z-order | +| Agent cursor visibility | The cursor appears in the captured pixels when enabled and moved | + +A focus assertion can prove "no focus steal" while failing to prove that a +click changed the target application state. An action row must therefore check +both the target's external state and, when background delivery is under test, +the cross-cutting desktop observer. + +These tests require a real interactive Windows user desktop. They reject +Session 0, locked desktops, and disconnected RDP sessions. Without +`CUA_REQUIRE_GUI=1`, an unusable desktop can self-skip for local development; +the canonical Windows runner enables the hard-failure behavior. + +## Evidence + +Canonical GUI runs are expected to produce evidence per test cell: + +```text +artifacts/cua-driver// +|-- recordings/-pid-/recording.mp4 +|-- recordings/-pid-/trajectory.json +|-- recordings/-pid-/turn-*/before_state.json +|-- recordings/-pid-/turn-*/before.png +|-- recordings/-pid-/turn-*/after_state.json +|-- recordings/-pid-/turn-*/after.png +|-- cases.jsonl +|-- environment.jsonl +|-- results.jsonl +|-- summary.md +`-- .log +``` + +The GitHub Actions summary contains one row per meaningful behavioral cell, +including its OS, harness, action, AX/PX targeting, delivery mode, driver route, +expected and observed behavior, oracles, and one evidence link. The link uses +the exact video path as its label and opens the owning lane archive. Unit tests +need normal test output and logs; they do not need desktop video. + +## What Is Implemented Today + +- Rust owns the canonical scenario definitions and external-state assertions. +- Electron and Tauri use the same shared web fixture across supported OSs. +- Native Windows, macOS, and Linux harnesses are repo-local applications built + from source. +- The three OS runners use a selector-free command for the complete matrix. +- Shared and native harness owners emit the same typed v2 result records. +- Canonical GUI rows collect a trajectory and MP4, validated before reporting. +- Canonical GUI rows require parseable pre/post state and non-empty pre/post + target-window images for each targeted turn. Missing expected evidence fails + the report even when the turn records an unavailable-capture classification. +- Per-cell video starts after fixture readiness and foreground/background + posture. A 300 ms baseline precedes dispatch, and capture continues through + external oracle collection. `trajectory.json` must finish with + `behavior_video.status = "finalized"`. +- Windows hosted runs use `GetConsoleWindow` to select the inherited + HostedComputeAgent/runner console, verify its identity, and minimize it + through `ShowWindow(SW_MINIMIZE)` before fixture or sentinel posture is + established. The sentinel remains a separate test fixture and is reasserted + after console cleanup. +- Strict lane preflights fail on missing fixtures, desktop access, permissions, + accessibility, capture, or recording support instead of silently skipping. +- GitHub summaries link every evidence-bearing row to its lane archive and + display the exact recording path. The trajectory path remains in the typed + evidence and archive. +- Unit/protocol tests remain separate from interactive E2E tests. + +## What Still Needs Implementation + +The remaining work is platform coverage and validation, not another test +hierarchy: + +1. **Broaden native action rows.** The shared web matrix covers every declared + AX/PX and foreground/background cell. AppKit, SwiftUI, WPF, WinUI3, + WebView2, and non-GTK3 Linux toolkits still have unproven native combinations listed in + [`action-support.md`](action-support.md). +2. **Preserve exact-source validation.** Accepted Windows and macOS runs must + record one immutable source SHA and retain the typed evidence contract. +3. **Close compositor-specific Wayland gaps.** Sway and a real GNOME 46 session + each pass all 31 GTK3 outcomes. Complete the shared Electron/Tauri lane on + a representative renderer, add a Plasma 6 lane, and implement a verified + KWin activation adapter. Issue `#1922` tracks the grouped backend work. +4. **Add representative toolkit surfaces.** GTK4, Qt5/Qt6, VTE, VCL, and GL + canvases remain optional real-app gaps; shared Electron/Tauri coverage does + not substitute for those native stacks. +5. **Flake cleanup.** Replace remaining fixed native waits with external-state + polling and add fixture reset tokens before reusing a harness process. + +## File Convergence Plan + +The goal is not to put every assertion into one enormous test file. The goal is +to give each behavior one clear owner and make cross-cutting evidence reusable. + +### Target Ownership + +```text +rust/crates/cua-driver-testkit/src/ +`-- observer.rs Cross-OS desktop-side-effect interface + +rust/crates/cua-driver/tests/ +|-- cross_platform_behavior_test.rs Shared Electron/Tauri action matrix +|-- harness_wpf_test.rs Windows WPF action rows +|-- harness_winui3_test.rs Windows WinUI3 action rows +|-- harness_web_test.rs WebView2/Electron page and CDP rows +|-- harness_appkit_test.rs macOS AppKit action rows +|-- harness_swiftui_test.rs macOS SwiftUI action rows +|-- harness_gtk3_test.rs Linux GTK3 action rows +|-- capture_contract_test.rs Tree and screenshot read contract +|-- desktop_scope__test.rs Window/desktop scope invariants +`-- protocol_*_test.rs Protocol and schema tests +``` + +The desktop observer is a helper, not a test family. An action row invokes it when +the row is testing background delivery. The row then records both outcomes: + +1. Did the target application state change, or did the driver return the + documented structured refusal? +2. Did focus, z-order, cursor, and desktop state remain within the contract? + +The deleted guard and modality files remain owned by the typed shared/native +rows and the launch, cursor, capture, and desktop-scope contracts. The +canonical runner is the only user-facing command; lane selectors are +internal diagnostics. + +## Contributor Workflow + +When adding a new scenario: + +1. Add or update the repo-local fixture and its external state marker. +2. Add the Rust scenario under `rust/crates/cua-driver/tests/`. +3. Declare AX/PX addressing, foreground/background delivery, scope, and oracle. +4. Add the scenario to `docs/test-matrix.md` and this guide when it changes the + cross-OS structure. +5. Update only the OS runner selection when the test is platform-specific. +6. Run the smallest Rust test locally, then run the OS command before + calling the matrix complete. + +The goal is one understandable Rust E2E model across platforms, with +platform-specific harnesses where the OS genuinely differs. diff --git a/libs/cua-driver/docs/test-matrix.md b/libs/cua-driver/docs/test-matrix.md new file mode 100644 index 0000000000..e1e6719533 --- /dev/null +++ b/libs/cua-driver/docs/test-matrix.md @@ -0,0 +1,202 @@ +# CUA Driver Test Matrix + +This is the source map for the Rust test suites in `libs/cua-driver`. It uses +two top-level test classes: + +1. **Unit and deterministic protocol tests.** These do not depend on a repo-local + application or an interactive desktop. +2. **Harness E2E tests.** These build and launch a repo-local application, drive + it through the Rust driver, and verify an external application or desktop + oracle. + +A test is E2E because it crosses the driver, OS, window system, and application +boundary. The Rust harness catalog and its external oracles are the source of +truth. + +## Matrix Dimensions + +Every harness E2E result should identify these dimensions: + +| Dimension | Values | +| --- | --- | +| OS | `windows`, `macos`, `linux` | +| Window system | Win32/UIA, AppKit/AX, X11/AT-SPI, Wayland/AT-SPI, WebView/CDP | +| Harness | Electron, Tauri, WPF, WinUI3, WebView2, AppKit, SwiftUI, WKWebView, GTK3 | +| Action targeting | `ax`, `px`, `page`, `not_applicable` | +| Delivery | `background`, `foreground`, `N/A` | +| Scope | `window`, `desktop`, `N/A` | +| Oracle | App state, accessibility state, focus state, pixel state, protocol state | +| Test status | `pass`, `fail`, `skip`, `environment_error` | +| Observed behavior | `delivered`, `refused`, `no_effect`, `error`, `not_run` | + +Focus preservation is a cross-cutting oracle, not a separate matrix family. +When a background action is tested, the row should attach the platform focus +observer in addition to checking the target application's external state. + +`AX` and `PX` describe how an action addresses a target. They are not capture +modes. `get_window_state` returns the tree and screenshot together; the action +uses an element index or coordinates. + +## Unit And Deterministic Tests + +These run without the repo-local GUI applications: + +| Area | Location | Coverage | +| --- | --- | --- | +| Core driver logic | `rust/crates/*/src/**` | Protocol values, sessions, schemas, image helpers, input helpers, configuration, telemetry, CLI behavior | +| MCP and CLI boundary | `rust/crates/cua-driver/tests/protocol_*` | Handshake, tool registration, tool calls, media, sessions, and errors | +| Schema gate | `schema_consistency_test.rs` | Shared tool schema parity across OS backends | +| Configuration transport | `transport_config_persistence_test.rs` | CLI and MCP configuration persistence | +| Token and protocol surfaces | `protocol_element_token_test.rs`, related tests | JSON-RPC-visible contract behavior | + +Some protocol tests spawn the driver process. They remain deterministic because +they do not launch a real target application or require a desktop. They should +be reported with the unit gate, separately from Harness E2E. + +The ordinary unit gate must not run `#[ignore]` GUI tests and must not turn +missing desktop fixtures into silent skips. + +## Harness E2E: Shared Web Applications + +Electron and Tauri are separate repo-local applications that load the shared +web harness. The calculator fixture was removed because it added a synthetic +task without adding useful application diversity. + +Source: + +- `tests/fixtures/shared/web/index.html` +- `tests/fixtures/apps/cross-platform/electron/` +- `tests/fixtures/apps/cross-platform/tauri/` +- `rust/crates/cua-driver/tests/cross_platform_behavior_test.rs` + +The shared harness exposes deterministic external markers for these actions: + +| Action family | Actions | Addressing | Delivery | +| --- | --- | --- | --- | +| Pointer | Left click, right click, double click | AX and PX | Background and foreground | +| Keyboard and text | Type text, Return, hotkey | AX and PX where supported | Background and foreground | +| Scroll | Scroll | AX and PX where supported | Background and foreground | +| Child windows | Open child window | AX and PX | Background and foreground | +| Drag | Drag source to drop target | PX | Background and foreground | +| State controls | Checkbox, radio, combo, slider | AX or PX by control | Mode declared per action | +| Editor | Type, save, saved-state readback | AX and PX where supported | Mode declared per action | + +The Rust shared catalog declares 36 evidence-bearing cells per harness +application. Windows and Linux run 72 shared cells across Electron and Tauri; +macOS also runs the native WKWebView host for 108 shared cells. Each host covers +the full AX/PX and foreground/background cross-product for click, text, +keyboard, scroll, and child-window actions, plus both delivery modes for PX +drag and AX editor-save. A +background capability refusal is a valid result only when the test verifies the +declared structured refusal and the no-focus/no-z-order/no-input-leak side +effect. A refusal fails a cell whose contract requires delivery. + +## Harness E2E: Native Windows + +Windows native harnesses are repo-local applications built from source: + +| Harness | Source test | Coverage | +| --- | --- | --- | +| WPF | `harness_wpf_test.rs` | UIA controls, text, keys, pointer actions, scroll, drag, popups, menus, modal windows | +| WinUI3 | `harness_winui3_test.rs` | XAML controls, text, checkbox/radio, slider, combo, popup | +| WebView2 | `harness_web_test.rs` | Window discovery, CDP page access, JavaScript, DOM click path | +| Desktop invariants | Testkit `DesktopObserver` plus typed launch/capture/cursor owners | Cross-cutting focus, z-order, minimized-launch, screenshot, cursor, and desktop checks | + +Native controls use AX/UIA state as their oracle. Pointer actions also use PX +where the tool contract requires coordinates. Current `set_value` rows declare +background delivery and attach desktop-side-effect oracles. + +## Harness E2E: macOS + +| Harness | Source test | Coverage | +| --- | --- | --- | +| AppKit | `harness_appkit_test.rs` | AX tree/capture, AX value/text, AX scroll, PX clicks, and foreground slider drag across the proven delivery modes; background drag is an exact refusal | +| SwiftUI | `harness_swiftui_test.rs` | AX tree/capture, background click/value, and foreground popover-trigger state | +| WKWebView | `cross_platform_behavior_test.rs` | Dedicated native host running the full 36-cell shared web catalog | +| Installed-app launch/focus | `installed_app_launch_macos_test.rs` | Real Calculator/TextEdit launch and focus behavior, when explicitly enabled | +| Installed-app text | `installed_app_textedit_macos_test.rs` | Real TextEdit AX background write and verification, when explicitly enabled | + +macOS uses the installed ScreenCaptureKit/AX permissions for GUI runs. The +repo-local harnesses are canonical; Calculator and TextEdit are supporting +real-app checks. SwiftUI's popover trigger is proven independently from the +remaining transient-panel AX discovery gap. + +## Harness E2E: Linux + +| Harness | Source test | Window system | Coverage | +| --- | --- | --- | --- | +| Electron | `cross_platform_behavior_test.rs` | X11 and supported AT-SPI sessions | Shared web action matrix | +| Tauri | `cross_platform_behavior_test.rs` | X11 and native Wayland sessions | Shared web action matrix | +| GTK3 | `harness_gtk3_test.rs` | X11/AT-SPI and Wayland/AT-SPI where configured | Native GTK controls and input | +| Desktop scope | `desktop_scope_linux_test.rs` | X11/Wayland | Desktop versus window scope | + +Nix provides the Linux build and desktop environment. X11 and Wayland are +separate matrix dimensions because their capture and input contracts differ. +Linux runs do not produce GIF output. Every canonical GUI cell retains an MP4 +and trajectory alongside screenshots, AX trees, structured results, and driver +logs for E2E evidence. + +## Action Delivery Matrix + +The current per-OS delivery/refusal ledger is maintained in +[`action-support.md`](action-support.md). This section defines the coverage +policy; the ledger records empirical status. + +For each OS and harness where the action is supported, the canonical E2E suite +should cover both delivery modes: + +| Action | Background | Foreground | Addressing | +| --- | --- | --- | --- | +| Left click | Required delivery or a declared refusal contract | Required | AX, PX | +| Right click | Required delivery or a declared refusal contract | Required | AX, PX | +| Double click | Required delivery or a declared refusal contract | Required | AX, PX | +| Drag | Required delivery or a declared refusal contract | Required | PX, with AX discovery | +| Scroll | Required delivery or a declared refusal contract | Required | AX, PX where supported | +| Type text | Required delivery or a declared refusal contract | Required | AX, PX where supported | +| Press key | Required delivery or a declared refusal contract | Required | AX, PX where supported | +| Hotkey | Required delivery or a declared refusal contract | Required | AX, PX where supported | +| Set value | Required in current native rows | Not separately declared | AX/UIA/AXValue | +| Screenshot | N/A | N/A | Window or desktop capture | +| Page/CDP | N/A | N/A | Page selector/JavaScript | + +`N/A` means the API has no delivery mode for that operation. A refusal is a +passing observation only when the Rust case expects refusal, the exact code is +allowed, and the desktop-side-effect oracles pass. + +## CI And Local Entry Points + +| Gate | Environment | Entry point | +| --- | --- | --- | +| Linux unit/source | Nix Linux CI | `nix build .#checks.x86_64-linux.cua-driver-build .#checks.x86_64-linux.cua-driver-linux-rust-unit` | +| Windows unit/compile | `windows-latest` | Package-scoped `cargo test --all-targets --no-run --locked` | +| Windows Harness E2E | Active Windows user session | `scripts/ci/windows/run-rust-e2e.ps1 -RequireGui` | +| Linux X11 Harness E2E | Nix X11 session | `scripts/ci/linux/run-rust-e2e.sh` | +| Linux Sway Harness E2E | Controlled wlroots session | `scripts/ci/linux/run-rust-e2e-wayland.sh` | +| Linux nested-compositor E2E | Controlled experimental session | `scripts/ci/linux/run-rust-e2e-inject.sh` | +| Linux representative desktop E2E | Existing GNOME, KDE, or Xorg login | `scripts/ci/linux/run-rust-e2e-desktop.sh ` | +| macOS Harness E2E | Logged-in macOS session with permissions | `scripts/ci/macos/run-rust-e2e.sh` | + +Workflows select private execution lanes. Rust source owns scenario definitions, +fixture oracles, and result records. OS runners only build the driver, stage the +local harnesses, establish the desktop session, collect evidence, and publish +the shared report. + +## Evidence And Ownership + +Every Harness E2E cell should produce: + +```text +recordings/-pid-/recording.mp4 +recordings/-pid-/trajectory.json +results.jsonl +.log +``` + +The GitHub summary links the exact recording path from the corresponding row to +its lane archive. Target logs remain lane-level diagnostics. Unit tests need +logs and test-result output, but do not need desktop video. + +When a new action or modality is added, update this document, the shared fixture +oracle, the Rust test, and the OS-specific runner selection together. This is +the cross-OS checklist that prevents a Windows-only test from being mistaken +for cross-platform coverage. diff --git a/libs/cua-driver/rust/Cargo.lock b/libs/cua-driver/rust/Cargo.lock index 1f5737aaba..fda7abab4f 100644 --- a/libs/cua-driver/rust/Cargo.lock +++ b/libs/cua-driver/rust/Cargo.lock @@ -651,8 +651,13 @@ dependencies = [ name = "cua-driver-testkit" version = "0.7.1" dependencies = [ + "core-foundation", + "objc2-app-kit", + "serde", "serde_json", + "tempfile", "windows 0.61.3", + "x11rb", ] [[package]] @@ -937,13 +942,6 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" -[[package]] -name = "focus-monitor-win" -version = "0.7.1" -dependencies = [ - "windows 0.58.0", -] - [[package]] name = "foldhash" version = "0.1.5" diff --git a/libs/cua-driver/rust/Cargo.toml b/libs/cua-driver/rust/Cargo.toml index 72419af9f5..a1aead5b1b 100644 --- a/libs/cua-driver/rust/Cargo.toml +++ b/libs/cua-driver/rust/Cargo.toml @@ -9,7 +9,6 @@ members = [ "crates/platform-windows", "crates/platform-linux", "crates/cursor-overlay", - "crates/focus-monitor-win", "crates/pip-preview", ] diff --git a/libs/cua-driver/rust/README.md b/libs/cua-driver/rust/README.md index 11570f7a52..96b339b73b 100644 --- a/libs/cua-driver/rust/README.md +++ b/libs/cua-driver/rust/README.md @@ -14,7 +14,6 @@ implementations, testkit, and helper crates. | `platform-linux` | Linux AT-SPI, X11/Wayland, capture, and input support | | `cua-driver-testkit` | Test-only helpers for spawning the daemon and parsing responses | | `cua-driver-uia` | Windows UIAccess worker | -| `focus-monitor-win` | Windows focus sentinel used by UX guard tests | | `cursor-overlay` | Cursor overlay support | | `pip-preview` | Packaging preview helper | diff --git a/libs/cua-driver/rust/Skills/cua-driver/LINUX.md b/libs/cua-driver/rust/Skills/cua-driver/LINUX.md index 24cd6906b1..ed0d3ef37c 100644 --- a/libs/cua-driver/rust/Skills/cua-driver/LINUX.md +++ b/libs/cua-driver/rust/Skills/cua-driver/LINUX.md @@ -67,8 +67,9 @@ keystroke / XSendEvent / XTest / foreground rungs are not read-back-confirmed **`effect` / `escalation`** — alongside `verified`, action responses carry the cross-platform `effect` (`confirmed` / `unverifiable` / `suspected_noop`) and, when you should change rung, `escalation:{recommended, reason}`. See `SKILL.md` -→ behavior matrix. The Linux-specific value of `recommended` is **`foreground` -on Wayland** (background pixel can't target an unfocused window) and +→ behavior matrix. On a standard Wayland compositor the Linux-specific value +of `recommended` is **`foreground`** (raw background pixels cannot target an +unfocused window); the opt-in nested compositor is a separate environment. Use **`px` on X11** (an element px action — background pixel click — lands via AT-SPI `do_action`-at-point off the screenshot already in the snapshot — the matrix below). @@ -101,8 +102,9 @@ Modality is chosen at **action time**, by how you address the target: `get_window_state` returning `degraded:true` (empty AT-SPI walk) is the cue to do an **element px action** off that same screenshot (X11) or escalate to -`delivery_mode:"foreground"` (Wayland — background pixel can't target an -unfocused window there). +`delivery_mode:"foreground"` (standard Wayland: raw background pixels cannot +target an unfocused window there). The nested compositor has its own +experimental per-surface routes. ## Cross-platform schema residuals (Linux) @@ -180,42 +182,37 @@ uinput, escalate to `delivery_mode:"foreground"`. (`type_text` in the `background` rung is focus-dependent for non-editable widgets; that's the one genuine background limitation, and `foreground` is the documented escalation.) -## Wayland (opt-in — still preview) - -Native Wayland support is behind an opt-in flag and not yet at the same -maturity as X11: - -- `CUA_DRIVER_RS_ENABLE_WAYLAND=1` enables the native-Wayland backend. - On wlroots compositors (labwc/sway) it uses foreign-toplevel + - screencopy; on GNOME-Mutter / KDE-KWin, which expose no client - protocols for cross-window enumeration, cua-driver brings its own - nested compositor (`CUA_WAYLAND_NEST=1`). -- Without the flag, on a Wayland session the driver falls back to - XWayland where available. -- Screenshots work via `grim` / wlr-screencopy. **Video recording is - not yet available on Wayland** — the recorder is `x11grab`, which is - X11-only. -- **`delivery_mode` on Wayland is constrained by the protocol**, honestly: - input goes through libei + xdg-desktop-portal, which injects to the - **compositor's input focus** — there is no per-window background targeting - like X11/macOS/Windows. So `background` can't aim at a specific non-focused - window, and `bring_to_front` has no standalone external activate (the - compositor bundles activation into the virtual-pointer/click path) — it - returns `bring_to_front_wayland_bundled` and points you at - `delivery_mode:"foreground"` on the input call instead. The delivery - contract also defines a structured `background_unavailable` error for the - no-libei-backend case (built without `portal-libei` or a denied portal - session); when input has no actuator the tools surface an error rather - than silently succeeding. -- **Wayland escalation is `foreground`, NOT `px`.** This is the - one place the cross-platform escalation hint flips. Everywhere else - (macOS, X11, most Windows surfaces) a background pixel click can - land on an unfocused window, so `escalation.recommended` is `px`. - On Wayland an unfocused window **cannot** be pixel-targeted in the - background (libei injects only to compositor focus → `background_unavailable`), - so the hint on action responses and on a `degraded` `get_window_state` is - `foreground`: re-call the action with `delivery_mode:"foreground"`, don't - expect an element px action off the screenshot to land in the background. +## Wayland (opt-in) + +Set `CUA_DRIVER_RS_ENABLE_WAYLAND=1` to enable native Wayland support. The +driver selects a backend from compositor capabilities: + +- Sway and other wlroots compositors use foreign-toplevel discovery, + wlr-screencopy, virtual pointer, and virtual keyboard protocols. +- GNOME/Mutter uses the bundled WinRects Shell helper for target geometry and + activation, plus portal/libei for foreground raw input. +- KDE/KWin uses AT-SPI and portal facilities where available. Target-specific + foreground activation remains experimental, so unsafe raw input refuses. +- The optional `cua-compositor` is a separate nested session enabled + explicitly for controlled automation. GNOME and KDE never switch into it. + +Sway recording works through the wlroots recorder path and is exercised by the +canonical harness runner. Portal-backed GNOME recording is still an evidence +gap. Capture and recording availability therefore depend on the compositor, +installed helpers, and portal grant. + +Standard Wayland has no general client protocol for raw input to an arbitrary +occluded surface. Background AX actions can still deliver through AT-SPI, and +a PX left click can deliver when hit-testing resolves to an actionable AT-SPI +control. Other focus-bound background pointer and keyboard shapes return an +exact `background_unavailable` result. They do not report success after a +silent drop. + +Use `delivery_mode:"foreground"` for raw Wayland input. The driver activates +the selected target through a verified compositor adapter before dispatch. If +the compositor has no target-addressable activation or input backend, the call +refuses before sending input. Reconstructing coordinates alone does not make +raw background PX possible on a standard compositor. ## Quick triage @@ -251,17 +248,13 @@ ask the user. ## What to expect -| Intent | Status | -|---|---| -| Snapshot AT-SPI tree | ✅ GTK3/4, Qt5/6, wxWidgets, Electron (GTK4/Qt6 can be partial — re-snapshot) | -| Pixel click | ✅ background `XSendEvent`, no focus steal, no pointer move | -| Element-indexed click | ✅ AT-SPI `do_action` | -| Type text | ✅ AT-SPI focus-free, with XTest / pty fallback for the focused widget | -| Hotkey / `press_key` | ✅ | -| Screenshot full-display | ✅ X11; ✅ Wayland via `grim` | -| Screenshot per-window | ✅ X11 | -| `launch_app` | ✅ env-scrubbed launch (no workspace steal) | -| Recording (video) | ✅ X11 (`x11grab`); ⚠️ Wayland not yet supported (preview) | +| Environment | Proven baseline | Main limits | +|---|---|---| +| X11/Openbox | AT-SPI trees and actions, foreground pointer and keyboard input, window and desktop capture, and video | Raw background delivery remains toolkit-specific; unsupported shapes refuse | +| Sway/wlroots | AT-SPI, native discovery, full-display and cropped-window screencopy, foreground input, semantic background actions, and video | Raw background pointer and keyboard input remains focus-bound | +| GNOME/Mutter | AT-SPI, WinRects geometry and activation, capture, and portal/libei foreground input | Requires the helper and portal grant; portal video parity remains open | +| KDE/KWin | AT-SPI and generic discovery where exposed | Target-specific activation and behavioral coverage remain experimental | +| Nested `cua-compositor` | Versioned direct per-surface input, native GTK 31/31, capture/scope 5/5, and partial Electron coverage | The complete shared matrix remains experimental; do not infer standard-Wayland support | See `SKILL.md` for the cross-platform loop (snapshot-before-AND-after, pixel-click contract, failure modes) and `RECORDING.md` for session diff --git a/libs/cua-driver/rust/Skills/cua-driver/RECORDING.md b/libs/cua-driver/rust/Skills/cua-driver/RECORDING.md index a7b2e55a98..d205613fc0 100644 --- a/libs/cua-driver/rust/Skills/cua-driver/RECORDING.md +++ b/libs/cua-driver/rust/Skills/cua-driver/RECORDING.md @@ -67,26 +67,26 @@ daemon restart resets to disabled. Each action writes to `turn-NNNNN/` (five-digit zero-padded counter): -- `app_state.json` — post-action AX/UIA snapshot for the target - `(pid, window_id)` carrying the same `tree_markdown` + - `element_count` shape `get_window_state` returns (minus the - screenshot fields — those live in `screenshot.png`). On macOS the - recorder resolves a frontmost window internally when the action's - args don't carry one; on Windows it uses the first window of the - target pid. **Omitted on Linux** — ATSPI doesn't expose a cheap - whole-tree snapshot, and the file is left out rather than faked. -- `screenshot.png` — post-action capture of the target window. - Omitted when the pid has no visible window. +- `before_state.json` and `after_state.json` — application accessibility + state immediately before and after the action. They carry the same + `tree_markdown` and `element_count` shape as `get_window_state`. +- `before.png` and `after.png` — target-window images immediately before + and after the action. Window capture remains scoped to the target when + another window covers it. +- `evidence.json` — capture status for each phase. Missing expected capture + has an explicit classification instead of disappearing from the turn. +- `app_state.json` and `screenshot.png` — compatibility aliases for + `after_state.json` and `after.png`. - `action.json` — the tool name, full input arguments, result summary, pid, click point (when applicable), ISO-8601 timestamp. - `click.png` — for click-family actions (`click`, `double_click`, - `right_click`): a copy of `screenshot.png` with a red dot drawn at + `right_click`): a copy of `before.png` with a red marker drawn at the click point. **Both addressing modes are covered:** explicit `x, y` clicks use the supplied coordinates directly, and `element_index`-addressed clicks resolve to the element's center via the live AX/UIA cache, then convert to window-local screenshot - pixels. Absent for non-click tools and for clicks whose resolved - point falls outside the captured window. + pixels. Absent for non-click tools and classified as unavailable when + a click point was expected but could not be resolved or rendered. ## When to use it diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/cdp.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/cdp.rs index 6ba3dda49d..2bc7d570d4 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/cdp.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/cdp.rs @@ -49,19 +49,14 @@ async fn cdp_evaluate( await_promise: bool, target_url_contains: Option<&str>, ) -> anyhow::Result { - // Wrap the /json discovery in its own timeout — `cdp_list_pages` does an - // unbounded `read_to_end`, and a half-open localhost socket (browser - // mid-shutdown, firewall mismatch, port stolen by another process) would - // otherwise hang us forever. 10 s is generous for a localhost HTTP roundtrip. + // A listener can be reachable before Chromium publishes its first page + // target. Bound both that readiness interval and the HTTP roundtrip. let pages = tokio::time::timeout( std::time::Duration::from_secs(10), - cdp_list_pages(port), + cdp_wait_for_page(port), ) .await .map_err(|_| anyhow::anyhow!("CDP /json discovery on port {port} timed out after 10 s"))??; - if pages.is_empty() { - anyhow::bail!("No CDP page tabs found on port {port}"); - } let page = pick_page(&pages, target_url_contains) .ok_or_else(|| match target_url_contains { @@ -141,6 +136,16 @@ fn format_cdp_result(response: &Value) -> String { // ── HTTP page discovery ─────────────────────────────────────────────────── +async fn cdp_wait_for_page(port: u16) -> anyhow::Result> { + loop { + let pages = cdp_list_pages(port).await?; + if !pages.is_empty() { + return Ok(pages); + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } +} + async fn cdp_list_pages(port: u16) -> anyhow::Result> { use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs index 602d079bf0..150a07a2fc 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/recording.rs @@ -1,9 +1,9 @@ //! Trajectory recording session. //! //! When enabled, every non-read-only, non-recording tool call writes a -//! `turn-NNNNN/action.json` file to the configured output directory. -//! When a screenshot callback is registered via `set_screenshot_fn`, it also -//! writes `screenshot.png` (extracted from `pid`/`window_id` in the args). +//! `turn-NNNNN/action.json` file to the configured output directory. Targeted +//! turns also persist explicit before/after state and image evidence. The +//! legacy `app_state.json` and `screenshot.png` names remain post-action aliases. //! //! Schema mirrors the Swift/Windows reference `action.json`: //! { tool, arguments, result_summary, timestamp, t_ms_from_session_start, @@ -26,12 +26,48 @@ use crate::video::{self, VideoBackend, VideoMetadata}; // and returns raw PNG bytes, or None if capture fails. The callback is called // synchronously from write_turn (a blocking context). -type ScreenshotFnBox = Box, Option) -> Option> + Send + Sync>; +pub struct ScreenshotCapture { + pub png: Option>, + pub classification: Option<&'static str>, +} + +impl ScreenshotCapture { + pub fn captured(png: Vec) -> Self { + Self { + png: Some(png), + classification: None, + } + } + + pub fn unavailable(classification: &'static str) -> Self { + Self { + png: None, + classification: Some(classification), + } + } +} + +type ScreenshotFnBox = + Box, Option) -> ScreenshotCapture + Send + Sync>; static SCREENSHOT_FN: OnceLock = OnceLock::new(); /// Register the platform-specific screenshot callback. Call once at startup /// before any tool invocations. Subsequent calls are silently ignored. -pub fn set_screenshot_fn(f: impl Fn(Option, Option) -> Option> + Send + Sync + 'static) { +pub fn set_screenshot_fn( + f: impl Fn(Option, Option) -> Option> + Send + Sync + 'static, +) { + set_classified_screenshot_fn(move |window_id, pid| { + f(window_id, pid) + .map(ScreenshotCapture::captured) + .unwrap_or_else(|| ScreenshotCapture::unavailable("capture_failed")) + }); +} + +/// Register a screenshot callback that preserves a stable unavailable-capture +/// classification for the turn evidence manifest. +pub fn set_classified_screenshot_fn( + f: impl Fn(Option, Option) -> ScreenshotCapture + Send + Sync + 'static, +) { let _ = SCREENSHOT_FN.set(Box::new(f)); } @@ -40,14 +76,16 @@ pub fn set_screenshot_fn(f: impl Fn(Option, Option) -> Option> /// by the PiP push hook (and by anything else that wants to share the /// per-turn screenshot pipeline without duplicating the platform glue). pub fn screenshot_for(window_id: Option, pid: Option) -> Option> { - SCREENSHOT_FN.get().and_then(|f| f(window_id, pid)) + SCREENSHOT_FN + .get() + .and_then(|capture| capture(window_id, pid).png) } // ── Platform click-marker callback ─────────────────────────────────────────── // // Takes (png_bytes, cx, cy) and returns modified PNG bytes with a red crosshair // at (cx, cy), or None if drawing fails. Used to produce click.png alongside -// screenshot.png when a click-family tool is recorded. +// before.png when a click-family tool is recorded, producing click.png. type ClickMarkerFnBox = Box Option> + Send + Sync>; static CLICK_MARKER_FN: OnceLock = OnceLock::new(); @@ -59,15 +97,16 @@ pub fn set_click_marker_fn(f: impl Fn(&[u8], f64, f64) -> Option> + Send // ── Platform AX-snapshot callback ──────────────────────────────────────────── // -// Takes (window_id, pid) and returns JSON bytes for `app_state.json` (the -// post-action AX/UIA snapshot), or None if no snapshot is available on this -// platform. +// Takes (window_id, pid) and returns JSON bytes for the phase's application +// state. The post-action bytes are also kept as legacy `app_state.json`. type AxSnapshotFnBox = Box, Option) -> Option> + Send + Sync>; static AX_SNAPSHOT_FN: OnceLock = OnceLock::new(); /// Register the platform-specific AX/UIA snapshot callback. Call once at startup. -pub fn set_ax_snapshot_fn(f: impl Fn(Option, Option) -> Option> + Send + Sync + 'static) { +pub fn set_ax_snapshot_fn( + f: impl Fn(Option, Option) -> Option> + Send + Sync + 'static, +) { let _ = AX_SNAPSHOT_FN.set(Box::new(f)); } @@ -82,10 +121,35 @@ type ElementBoundsFnBox = Box Option<(f64, f64)> + Send static ELEMENT_BOUNDS_FN: OnceLock = OnceLock::new(); /// Register the platform-specific element-bounds resolver. Args: (window_id, pid, element_index). -pub fn set_element_bounds_fn(f: impl Fn(u64, i64, u32) -> Option<(f64, f64)> + Send + Sync + 'static) { +pub fn set_element_bounds_fn( + f: impl Fn(u64, i64, u32) -> Option<(f64, f64)> + Send + Sync + 'static, +) { let _ = ELEMENT_BOUNDS_FN.set(Box::new(f)); } +#[derive(Default)] +struct TurnCapture { + state: Option>, + screenshot: Option>, + screenshot_classification: Option<&'static str>, +} + +/// A reserved recording turn captured immediately before tool dispatch. +/// `ToolRegistry` passes this token back after dispatch so both phases share +/// one stable `turn-NNNNN` directory even when calls complete out of order. +pub struct PendingTurn { + generation: u64, + turn_dir: PathBuf, + tool_name: String, + args: Value, + start_ms: u64, + session_start_ms: u64, + window_id: Option, + pid: Option, + click_point: Option<(f64, f64)>, + before: TurnCapture, +} + /// Persistent recording session state (singleton per process). pub struct RecordingSession { inner: Mutex, @@ -93,6 +157,7 @@ pub struct RecordingSession { struct RecordingInner { enabled: bool, + generation: u64, /// Session that owns the live recording, stamped on every successful /// `start()` from the daemon-injected `_session_id`. The daemon-global /// recorder is a singleton, so when session A starts a recording and @@ -156,6 +221,7 @@ impl RecordingSession { Self { inner: Mutex::new(RecordingInner { enabled: false, + generation: 0, owner: None, output_dir: None, next_turn: 1, @@ -179,7 +245,7 @@ impl RecordingSession { /// `recording_tools.rs` — so video only records when explicitly requested. /// The legacy CLI `recording start` path via `configure()` still forces /// video on. If ffmpeg isn't on PATH the start still succeeds — - /// the per-turn capture (action.json + screenshot.png) is independent + /// the per-turn capture (action.json + pre/post evidence) is independent /// of video — but the structured state carries the ffmpeg error so /// the caller can surface it. /// @@ -187,7 +253,12 @@ impl RecordingSession { /// `_session_id`). `None` marks an anonymous start (CLI one-shot / legacy /// `configure()` shim) owned by nobody. See `stop_owner()` for how this /// gates teardown. - pub fn start(&self, output_dir: &str, record_video: bool, owner: Option<&str>) -> anyhow::Result<()> { + pub fn start( + &self, + output_dir: &str, + record_video: bool, + owner: Option<&str>, + ) -> anyhow::Result<()> { let mut inner = self.inner.lock().unwrap(); // Write-boundary resurrection guard — checked INSIDE the lock so the // is_session_ended test is atomic with the enabled/owner write below. @@ -249,7 +320,9 @@ impl RecordingSession { // background thread + a small jsonl file. let cursor_path = dir.join("cursor.jsonl"); match CursorSampler::start(cursor_path, monotonic_start) { - Ok(s) => { inner.cursor = Some(s); } + Ok(s) => { + inner.cursor = Some(s); + } Err(e) => { tracing::warn!(target: "recording", "Cursor sampler failed to start: {e}"); @@ -265,10 +338,7 @@ impl RecordingSession { "video": video_session_payload(video_present, video_error.as_deref(), None), "cursor": { "present": inner.cursor.is_some(), "sample_count": 0 } }); - let _ = write_json_atomic( - &dir.join("session.json"), - &session_payload, - ); + let _ = write_json_atomic(&dir.join("session.json"), &session_payload); // Stamp the owning session on every successful start (reached only on // the success path — start() returns early via `?` on `create_dir_all` @@ -276,6 +346,7 @@ impl RecordingSession { // the daemon-global recorder is a singleton, so the latest start() owns // it. The previous owner's disconnect then no-ops in stop_owner(). inner.owner = owner.map(str::to_owned); + inner.generation = inner.generation.wrapping_add(1); inner.enabled = true; inner.output_dir = Some(dir); inner.next_turn = 1; @@ -320,7 +391,14 @@ impl RecordingSession { } inner.owner = None; let dir = inner.output_dir.clone(); - let video_meta = inner.video.take().and_then(|rec| rec.stop().ok()); + let (video_meta, stop_error) = match inner.video.take().map(|rec| rec.stop()) { + Some(Ok(meta)) => match validate_video_metadata(meta) { + Ok(meta) => (Some(meta), None), + Err(error) => (None, Some(error.to_string())), + }, + Some(Err(error)) => (None, Some(error.to_string())), + None => (None, None), + }; let cursor_samples = inner.cursor.take().map(|c| c.stop()).unwrap_or(0); inner.enabled = false; @@ -328,11 +406,14 @@ impl RecordingSession { inner.next_turn = 1; inner.session_start_ms = 0; inner.session_monotonic_start = None; - if video_meta.is_some() { + if let Some(error) = &stop_error { + inner.last_error = Some(error.clone()); + } else if video_meta.is_some() { inner.last_error = None; } inner.last_video = video_meta.clone(); inner.last_cursor_samples = cursor_samples; + let final_video_error = inner.last_error.clone(); // Rewrite session.json with final video metadata + cursor count // so the renderer (and any external analysis) sees what actually @@ -341,7 +422,7 @@ impl RecordingSession { let video_block = if let Some(ref m) = video_meta { video_session_payload(true, None, Some(m)) } else { - video_session_payload(false, None, None) + video_session_payload(false, final_video_error.as_deref(), None) }; let session_payload = serde_json::json!({ "schema_version": 1, @@ -349,10 +430,10 @@ impl RecordingSession { "video": video_block, "cursor": { "present": cursor_samples > 0, "sample_count": cursor_samples } }); - let _ = write_json_atomic( - &dir.join("session.json"), - &session_payload, - ); + let _ = write_json_atomic(&dir.join("session.json"), &session_payload); + } + if let Some(error) = stop_error { + anyhow::bail!("video finalization failed: {error}"); } Ok(()) } @@ -377,37 +458,40 @@ impl RecordingSession { let inner = self.inner.lock().unwrap(); RecordingState { enabled: inner.enabled, - output_dir: inner.output_dir.as_ref().map(|p| p.to_string_lossy().into_owned()), + output_dir: inner + .output_dir + .as_ref() + .map(|p| p.to_string_lossy().into_owned()), next_turn: inner.next_turn, last_error: inner.last_error.clone(), video_active: inner.video.is_some(), - last_video_path: inner.last_video.as_ref() + last_video_path: inner + .last_video + .as_ref() .map(|m| m.path.to_string_lossy().into_owned()), owner: inner.owner.clone(), } } - /// Record a completed tool call. No-op when recording is disabled. - /// `start_ms` — wall-clock ms at invocation start (use `now_ms()` before calling the tool). - pub fn record( - &self, - tool_name: &str, - args: &Value, - result_text: &str, - start_ms: u64, - ) { - let (turn_dir, session_start_ms) = { + /// Reserve a turn and capture its target immediately before tool dispatch. + /// No-op when recording is disabled. + pub fn begin_turn(&self, tool_name: &str, args: &Value, start_ms: u64) -> Option { + let (turn_dir, session_start_ms, generation) = { let mut inner = self.inner.lock().unwrap(); if !inner.enabled { - return; + return None; } let out = match inner.output_dir.clone() { Some(o) => o, - None => return, + None => return None, }; let idx = inner.next_turn; inner.next_turn += 1; - (out.join(format!("turn-{idx:05}")), inner.session_start_ms) + ( + out.join(format!("turn-{idx:05}")), + inner.session_start_ms, + inner.generation, + ) }; // Strip the daemon-injected `_session_id` (and any other reserved @@ -415,28 +499,184 @@ impl RecordingSession { // in action.json's `arguments`. The injection point is the daemon // `call` branch (serve.rs); recording is the single chokepoint where // those internal keys must not leak into the persisted trajectory. - let args = strip_internal_keys(args); + let args = strip_internal_keys(args).into_owned(); + use crate::tool_args::ArgsExt; + let mut window_id = args.opt_u64("window_id"); + let pid = args.opt_i64("pid"); + let mut element_index = args.opt_u64("element_index"); + if let (Some(pid), Some(token)) = ( + pid.and_then(|pid| i32::try_from(pid).ok()), + args.get("element_token").and_then(Value::as_str), + ) { + if let Ok((resolved_window, resolved_index)) = + crate::element_token::global().resolve(pid, token) + { + window_id = Some(u64::from(resolved_window)); + element_index = u64::try_from(resolved_index).ok(); + } + } + let click_point = resolve_click_point(tool_name, &args, window_id, pid, element_index); + let before = capture_turn(window_id, pid); + + let mut inner = self.inner.lock().unwrap(); + if !inner.enabled || inner.generation != generation { + return None; + } + if let Err(error) = write_phase_artifacts(&turn_dir, "before", &before) { + inner.last_error = Some(error.to_string()); + } + drop(inner); - if let Err(e) = write_turn( - &turn_dir, - tool_name, - args.as_ref(), - result_text, + Some(PendingTurn { + generation, + turn_dir, + tool_name: tool_name.to_owned(), + args, start_ms, session_start_ms, - ) { - let mut inner = self.inner.lock().unwrap(); - inner.last_error = Some(e.to_string()); + window_id, + pid, + click_point, + before, + }) + } + + /// Finalize a previously reserved turn after tool dispatch. + pub fn finish_turn(&self, pending: PendingTurn, result_text: &str) { + let mut inner = self.inner.lock().unwrap(); + if !inner.enabled || inner.generation != pending.generation { + tracing::warn!( + target: "recording", + "discarding a turn from an inactive recording generation" + ); + return; + } + if let Err(error) = write_turn(pending, result_text) { + inner.last_error = Some(error.to_string()); } } + + /// Compatibility helper for callers that only report completed calls. + /// New dispatch paths should use `begin_turn` and `finish_turn` so the + /// before phase is captured before the action changes application state. + pub fn record(&self, tool_name: &str, args: &Value, result_text: &str, start_ms: u64) { + let Some(pending) = self.begin_turn(tool_name, args, start_ms) else { + return; + }; + self.finish_turn(pending, result_text); + } } impl Default for RecordingSession { - fn default() -> Self { Self::new() } + fn default() -> Self { + Self::new() + } } // ── helpers ─────────────────────────────────────────────────────────────────── +fn capture_turn(window_id: Option, pid: Option) -> TurnCapture { + let screenshot = SCREENSHOT_FN + .get() + .map(|capture| capture(window_id, pid)) + .unwrap_or_else(|| ScreenshotCapture::unavailable("capture_hook_unavailable")); + TurnCapture { + state: AX_SNAPSHOT_FN + .get() + .and_then(|capture| capture(window_id, pid)), + screenshot: screenshot.png, + screenshot_classification: screenshot.classification, + } +} + +fn resolve_click_point( + tool_name: &str, + args: &Value, + window_id: Option, + pid: Option, + element_index: Option, +) -> Option<(f64, f64)> { + use crate::tool_args::ArgsExt; + if !matches!(tool_name, "click" | "double_click" | "right_click") { + return None; + } + match (args.opt_f64("x"), args.opt_f64("y")) { + (Some(x), Some(y)) => Some((x, y)), + _ => match (window_id, pid, element_index, ELEMENT_BOUNDS_FN.get()) { + (Some(wid), Some(pid), Some(index), Some(resolve)) => u32::try_from(index) + .ok() + .and_then(|index| resolve(wid, pid, index)), + _ => None, + }, + } +} + +fn write_phase_artifacts( + turn_dir: &Path, + phase: &str, + capture: &TurnCapture, +) -> anyhow::Result<()> { + std::fs::create_dir_all(turn_dir)?; + if let Some(state) = &capture.state { + std::fs::write(turn_dir.join(format!("{phase}_state.json")), state)?; + } + if let Some(screenshot) = &capture.screenshot { + std::fs::write(turn_dir.join(format!("{phase}.png")), screenshot)?; + } + Ok(()) +} + +fn capture_status( + captured: bool, + expected: bool, + classification: Option<&'static str>, +) -> Value { + if captured { + serde_json::json!({ "status": "captured" }) + } else if expected { + serde_json::json!({ + "status": "unavailable", + "classification": classification.unwrap_or("capture_failed") + }) + } else { + serde_json::json!({ + "status": "not_applicable", + "classification": "no_target_pid" + }) + } +} + +fn write_evidence_manifest( + turn_dir: &Path, + before: &TurnCapture, + after: &TurnCapture, + state_expected: bool, + click_expected: bool, + click_captured: bool, +) -> anyhow::Result<()> { + let manifest = serde_json::json!({ + "schema": "cua-turn-evidence/v1", + "before": { + "state": capture_status(before.state.is_some(), state_expected, None), + "screenshot": capture_status( + before.screenshot.is_some(), + true, + before.screenshot_classification, + ), + }, + "after": { + "state": capture_status(after.state.is_some(), state_expected, None), + "screenshot": capture_status( + after.screenshot.is_some(), + true, + after.screenshot_classification, + ), + }, + "click": capture_status(click_captured, click_expected, None), + }); + write_json_atomic(&turn_dir.join("evidence.json"), &manifest) +} + /// Drop reserved internal keys (any `_`-prefixed key, e.g. the daemon-injected /// `_session_id`) from a tool-call args object so they never persist into a /// recorded `action.json`. Returns the value unchanged when it isn't an object @@ -455,42 +695,26 @@ fn strip_internal_keys(args: &Value) -> std::borrow::Cow<'_, Value> { } } -fn write_turn( - turn_dir: &Path, - tool_name: &str, - args: &Value, - result_text: &str, - start_ms: u64, - session_start_ms: u64, -) -> anyhow::Result<()> { - std::fs::create_dir_all(turn_dir)?; +fn write_turn(pending: PendingTurn, result_text: &str) -> anyhow::Result<()> { + let PendingTurn { + generation: _, + turn_dir, + tool_name, + args, + start_ms, + session_start_ms, + window_id, + pid, + click_point, + before, + } = pending; + std::fs::create_dir_all(&turn_dir)?; let now = now_ms(); - - use crate::tool_args::ArgsExt; - // Extract window_id and pid from args for screenshot capture. - let window_id = args.opt_u64("window_id"); - let pid = args.opt_i64("pid"); - let element_index = args.opt_u64("element_index"); - - // Extract click point for click-family tools. Falls back to the - // platform element_index → window-local-pixels resolver when the call - // used `element_index` instead of explicit `x, y`, so click.png is - // written for AX-indexed clicks too. - let click_point: Option<(f64, f64)> = if matches!( - tool_name, "click" | "double_click" | "right_click" - ) { - match (args.opt_f64("x"), args.opt_f64("y")) { - (Some(x), Some(y)) => Some((x, y)), - _ => match (window_id, pid, element_index, ELEMENT_BOUNDS_FN.get()) { - (Some(wid), Some(p), Some(idx), Some(f)) => { - u32::try_from(idx).ok().and_then(|idx32| f(wid, p, idx32)) - } - _ => None, - }, - } - } else { - None - }; + let after = capture_turn(window_id, pid); + let click_expected = matches!( + tool_name.as_str(), + "click" | "double_click" | "right_click" + ); let mut payload = serde_json::json!({ "tool": tool_name, @@ -504,29 +728,38 @@ fn write_turn( payload["click_point"] = serde_json::json!({"x": cx, "y": cy}); } write_json_atomic(&turn_dir.join("action.json"), &payload)?; + write_phase_artifacts(&turn_dir, "after", &after)?; - // Post-action AX/UIA snapshot — omitted on platforms that don't expose - // a cheap snapshot helper (today: Linux ATSPI). - if let Some(ax_fn) = AX_SNAPSHOT_FN.get() { - if let Some(json_bytes) = ax_fn(window_id, pid) { - let _ = std::fs::write(turn_dir.join("app_state.json"), &json_bytes); - } + // Preserve the original post-action names for existing trajectory readers. + if let Some(state) = &after.state { + std::fs::write(turn_dir.join("app_state.json"), state)?; + } + if let Some(screenshot) = &after.screenshot { + std::fs::write(turn_dir.join("screenshot.png"), screenshot)?; } - // Capture screenshot if a callback is registered. - if let Some(screenshot_fn) = SCREENSHOT_FN.get() { - if let Some(png_bytes) = screenshot_fn(window_id, pid) { - let _ = std::fs::write(turn_dir.join("screenshot.png"), &png_bytes); - // Write click.png (screenshot + red crosshair) for click-family tools. - if let Some((cx, cy)) = click_point { - if let Some(marker_fn) = CLICK_MARKER_FN.get() { - if let Some(click_png) = marker_fn(&png_bytes, cx, cy) { - let _ = std::fs::write(turn_dir.join("click.png"), &click_png); - } - } - } + // A click marker describes where the action was aimed, so ground it on + // the pre-action image. This also keeps modal-dismiss evidence available + // after the modal HWND has closed. + let mut click_captured = false; + if let (Some((cx, cy)), Some(screenshot), Some(marker)) = ( + click_point, + before.screenshot.as_deref(), + CLICK_MARKER_FN.get(), + ) { + if let Some(click_png) = marker(screenshot, cx, cy) { + std::fs::write(turn_dir.join("click.png"), click_png)?; + click_captured = true; } } + write_evidence_manifest( + &turn_dir, + &before, + &after, + pid.is_some(), + click_expected, + click_captured, + )?; Ok(()) } @@ -593,3 +826,239 @@ fn expand_tilde(path: &str) -> PathBuf { } PathBuf::from(path) } + +fn validate_video_metadata(meta: VideoMetadata) -> anyhow::Result { + if !meta.finalized { + anyhow::bail!("video backend did not finalize {}", meta.path.display()); + } + let output = std::fs::metadata(&meta.path).map_err(|error| { + anyhow::anyhow!( + "finalized video is missing at {}: {error}", + meta.path.display() + ) + })?; + if output.len() == 0 { + anyhow::bail!("finalized video is empty at {}", meta.path.display()); + } + Ok(meta) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct FailingVideo; + + impl VideoBackend for FailingVideo { + fn stop(self: Box) -> anyhow::Result { + anyhow::bail!("recorder did not finalize") + } + } + + #[test] + fn turn_capture_brackets_action_and_preserves_post_action_aliases() { + static SCREENSHOTS: AtomicUsize = AtomicUsize::new(0); + static STATES: AtomicUsize = AtomicUsize::new(0); + set_screenshot_fn(|window_id, pid| { + if (window_id, pid) == (Some(2), Some(1)) { + let phase = SCREENSHOTS.fetch_add(1, Ordering::SeqCst); + return Some(if phase == 0 { + b"before".to_vec() + } else { + b"after".to_vec() + }); + } + // Other recording tests share this process-global hook and may run + // concurrently. Give them stable bytes without advancing this + // test's before/after phase counter. + Some(b"after".to_vec()) + }); + set_ax_snapshot_fn(|_, _| { + let phase = STATES.fetch_add(1, Ordering::SeqCst); + Some(format!(r#"{{"phase":{phase}}}"#).into_bytes()) + }); + set_click_marker_fn(|_, _, _| Some(b"click".to_vec())); + set_element_bounds_fn(|window_id, pid, element_index| { + Some((window_id as f64 + element_index as f64, pid as f64)) + }); + + let output_dir = std::env::temp_dir().join(format!( + "cua-recording-turn-evidence-{}-{}", + std::process::id(), + now_ms() + )); + let session = RecordingSession::new(); + { + let mut inner = session.inner.lock().expect("recording lock"); + inner.enabled = true; + inner.output_dir = Some(output_dir.clone()); + inner.session_start_ms = now_ms(); + } + let pending = session + .begin_turn( + "click", + &serde_json::json!({"pid": 1, "window_id": 2, "x": 3, "y": 4}), + now_ms(), + ) + .expect("recording should reserve a turn"); + let turn = output_dir.join("turn-00001"); + assert_eq!(std::fs::read(turn.join("before.png")).unwrap(), b"before"); + assert!(!turn.join("after.png").exists()); + + session.finish_turn(pending, "clicked"); + assert_eq!(std::fs::read(turn.join("after.png")).unwrap(), b"after"); + assert_eq!( + std::fs::read(turn.join("screenshot.png")).unwrap(), + std::fs::read(turn.join("after.png")).unwrap() + ); + assert_eq!( + std::fs::read(turn.join("app_state.json")).unwrap(), + std::fs::read(turn.join("after_state.json")).unwrap() + ); + assert_eq!(std::fs::read(turn.join("click.png")).unwrap(), b"click"); + + let snapshot_id = crate::element_token::global().register_snapshot(1, 77, 1); + let token = crate::element_token::token_for(snapshot_id, 0); + let pending = session + .begin_turn( + "click", + &serde_json::json!({"pid": 1, "element_token": token}), + now_ms(), + ) + .expect("token-only click should reserve a targeted turn"); + session.finish_turn(pending, "token click"); + let token_turn = output_dir.join("turn-00002"); + let token_action: Value = serde_json::from_slice( + &std::fs::read(token_turn.join("action.json")).expect("read token action"), + ) + .expect("parse token action"); + assert_eq!(token_action["click_point"]["x"], 77.0); + assert_eq!(token_action["click_point"]["y"], 1.0); + assert!(token_turn.join("click.png").exists()); + + let files = [ + "action.json", + "app_state.json", + "screenshot.png", + "click.png", + "before_state.json", + "before.png", + "after_state.json", + "after.png", + "evidence.json", + ]; + for directory in [&turn, &token_turn] { + for file in files { + std::fs::remove_file(directory.join(file)).expect("remove turn fixture file"); + } + std::fs::remove_dir(directory).expect("remove turn fixture directory"); + } + std::fs::remove_dir(&output_dir).expect("remove recording fixture directory"); + } + + #[test] + fn stale_recording_generation_cannot_finalize_a_reserved_turn() { + let output_dir = std::env::temp_dir().join(format!( + "cua-recording-generation-{}-{}", + std::process::id(), + now_ms() + )); + let session = RecordingSession::new(); + { + let mut inner = session.inner.lock().expect("recording lock"); + inner.enabled = true; + inner.generation = 1; + inner.output_dir = Some(output_dir.clone()); + inner.session_start_ms = now_ms(); + } + let pending = session + .begin_turn("click", &serde_json::json!({"x": 1, "y": 2}), now_ms()) + .expect("reserve first generation turn"); + session.inner.lock().unwrap().generation = 2; + session.finish_turn(pending, "must be discarded"); + + let turn = output_dir.join("turn-00001"); + assert!(!turn.join("action.json").exists()); + for entry in std::fs::read_dir(&turn).expect("read partial turn") { + std::fs::remove_file(entry.expect("turn entry").path()).expect("remove partial file"); + } + std::fs::remove_dir(&turn).expect("remove partial turn"); + std::fs::remove_dir(&output_dir).expect("remove recording directory"); + } + + #[test] + fn stop_owner_surfaces_video_finalization_failure() { + let output_dir = std::env::temp_dir().join(format!( + "cua-recording-stop-failure-{}-{}", + std::process::id(), + now_ms() + )); + std::fs::create_dir_all(&output_dir).expect("create recording test directory"); + let session = RecordingSession::new(); + { + let mut inner = session.inner.lock().expect("recording lock"); + inner.enabled = true; + inner.output_dir = Some(output_dir.clone()); + inner.video = Some(Box::new(FailingVideo)); + } + + let error = session + .stop_owner(None) + .expect_err("video finalization failure must reach the caller"); + assert!(error.to_string().contains("recorder did not finalize")); + let state = session.current_state(); + assert!(!state.enabled); + assert!(state.last_video_path.is_none()); + assert_eq!( + state.last_error.as_deref(), + Some("recorder did not finalize") + ); + + let manifest: Value = serde_json::from_slice( + &std::fs::read(output_dir.join("session.json")).expect("read session manifest"), + ) + .expect("parse session manifest"); + assert_eq!(manifest["video"]["present"], false); + assert_eq!(manifest["video"]["error"], "recorder did not finalize"); + let _ = std::fs::remove_dir_all(output_dir); + } + + #[test] + fn video_metadata_requires_finalized_nonempty_output() { + let output_dir = std::env::temp_dir().join(format!( + "cua-recording-metadata-{}-{}", + std::process::id(), + now_ms() + )); + std::fs::create_dir_all(&output_dir).expect("create video metadata test directory"); + let path = output_dir.join("recording.mp4"); + std::fs::write(&path, b"video").expect("write video fixture"); + + let error = validate_video_metadata(VideoMetadata { + path: path.clone(), + duration_ms: 1, + finalized: false, + }) + .expect_err("unfinalized output must fail"); + assert!(error.to_string().contains("did not finalize")); + + std::fs::write(&path, []).expect("empty video fixture"); + let error = validate_video_metadata(VideoMetadata { + path: path.clone(), + duration_ms: 1, + finalized: true, + }) + .expect_err("empty finalized output must fail"); + assert!(error.to_string().contains("is empty")); + + std::fs::write(&path, b"video").expect("restore video fixture"); + validate_video_metadata(VideoMetadata { + path, + duration_ms: 1, + finalized: true, + }) + .expect("finalized nonempty output must pass"); + let _ = std::fs::remove_dir_all(output_dir); + } +} diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/recording_tools.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/recording_tools.rs index 0860954f7c..09612b3035 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/recording_tools.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/recording_tools.rs @@ -59,13 +59,18 @@ impl Tool for StartRecordingTool { description: "Start trajectory recording. Every subsequent action-tool \ invocation (click, right_click, scroll, type_text, press_key, hotkey, \ set_value) writes a turn folder under `output_dir`:\n\n\ + - `before_state.json` / `after_state.json` — application AX/UIA/AT-SPI \ + state immediately before and after the action.\n\ + - `before.png` / `after.png` — target-window screenshots immediately \ + before and after the action.\n\ + - `evidence.json` — capture status and a stable classification when an \ + expected artifact could not be captured.\n\ - `app_state.json` — post-action AX/UIA snapshot for the target pid.\n\ - - `screenshot.png` — post-action per-window screenshot of the target's \ - frontmost on-screen window.\n\ + - `screenshot.png` — compatibility alias of `after.png`.\n\ - `action.json` — tool name, full input arguments, result summary, pid, \ click point (when applicable), ISO-8601 timestamp.\n\ - - `click.png` — for click-family actions only, `screenshot.png` with a \ - red dot drawn at the click point.\n\n\ + - `click.png` — for click-family actions only, `before.png` with a red \ + marker at the click point.\n\n\ Turn folders are named `turn-00001/`, `turn-00002/`, etc. Turn \ numbering restarts at 1 each time recording is (re-)started.\n\n\ **Video is off by default.** Pass `record_video: true` to also \ diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs index 77c53d14d7..b31c7fae17 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/tool.rs @@ -392,6 +392,19 @@ impl ToolRegistry { other => other, }; + // Reserve and capture the turn before dispatch so recorded evidence + // shows the application immediately before the action changed it. + let should_record = self.tools.get(resolved_name) + .map(|tool| !tool.def().read_only) + .unwrap_or(false) + && !matches!( + resolved_name, + "start_recording" | "stop_recording" | "get_recording_state" | "replay_trajectory" + ); + let pending_turn = should_record + .then(|| self.recording.begin_turn(resolved_name, &args, start_ms)) + .flatten(); + let result = match self.tools.get(resolved_name) { Some(tool) => tool.invoke(args.clone()).await, None => return ToolResult::error(format!("Unknown tool: {name}")), @@ -405,22 +418,14 @@ impl ToolRegistry { // control tools themselves are excluded so the recorded turn // stream stays the actual user-action sequence (not the meta // start/stop frames). - let should_record = self.tools.get(name) - .map(|t| !t.def().read_only) - .unwrap_or(false) - && !matches!( - name, - "start_recording" | "stop_recording" | "get_recording_state" | "replay_trajectory" - ); - - if should_record { + if let Some(pending_turn) = pending_turn { let result_text = result.content.iter() .find_map(|c| { if let Content::Text { text, .. } = c { Some(text.as_str()) } else { None } }) .unwrap_or(""); - self.recording.record(name, &args, result_text, start_ms); + self.recording.finish_turn(pending_turn, result_text); } // Experimental PiP push — only when --experimental-pip is on argv diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/Cargo.toml b/libs/cua-driver/rust/crates/cua-driver-testkit/Cargo.toml index b030305eb6..c5264661c9 100644 --- a/libs/cua-driver/rust/crates/cua-driver-testkit/Cargo.toml +++ b/libs/cua-driver/rust/crates/cua-driver-testkit/Cargo.toml @@ -9,7 +9,21 @@ description = "Shared harness for cua-driver integration tests: MCP/CLI transpor publish = false [dependencies] +serde = { workspace = true } serde_json = { workspace = true } +tempfile = "3" + +[target.'cfg(target_os = "linux")'.dependencies] +x11rb = "0.13" + +[target.'cfg(target_os = "macos")'.dependencies] +core-foundation = "0.10" +objc2-app-kit = { version = "0.2", features = [ + "NSApplication", + "NSRunningApplication", + "NSWorkspace", + "libc", +] } [target.'cfg(target_os = "windows")'.dependencies] windows = { version = "0.61", features = [ @@ -17,6 +31,9 @@ windows = { version = "0.61", features = [ # Job Object: assign every spawned test child to a KILL_ON_JOB_CLOSE job so # the OS terminates the whole tree when the test process dies for ANY reason. "Win32_System_JobObjects", + "Win32_System_Console", "Win32_Security", "Win32_System_Threading", + "Win32_UI_Input_KeyboardAndMouse", + "Win32_UI_WindowsAndMessaging", ] } diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/bin/cua-e2e-report.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/bin/cua-e2e-report.rs new file mode 100644 index 0000000000..0c9e1db22a --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/bin/cua-e2e-report.rs @@ -0,0 +1,89 @@ +use std::path::PathBuf; + +use cua_driver_testkit::e2e::{ + environment_schema_supported, read_json_lines, validate_catalog, CaseDeclaration, CaseResult, + EnvironmentRecord, EnvironmentStatus, DECLARATION_SCHEMA, +}; + +fn value(args: &[String], name: &str) -> Option { + args.windows(2) + .find(|pair| pair[0] == name) + .map(|pair| pair[1].clone()) +} + +fn emit(markdown: String, output: Option<&PathBuf>) { + if let Some(path) = output { + std::fs::write(path, markdown) + .unwrap_or_else(|error| panic!("{}: {error}", path.display())); + } else { + print!("{markdown}"); + } +} + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + let declarations = value(&args, "--declarations") + .map(PathBuf::from) + .unwrap_or_else(|| panic!("missing --declarations")); + let results = value(&args, "--results") + .map(PathBuf::from) + .unwrap_or_else(|| panic!("missing --results")); + let output = value(&args, "--output").map(PathBuf::from); + let artifact_root = value(&args, "--artifact-root").map(PathBuf::from); + let environment = value(&args, "--environment").map(PathBuf::from); + let require_video = args.iter().any(|arg| arg == "--require-video"); + + let declarations: Vec = read_json_lines(&declarations) + .unwrap_or_else(|errors| panic!("invalid declarations:\n{}", errors.join("\n"))); + let declarations = declarations + .into_iter() + .map(|declaration| { + assert_eq!( + declaration.schema, DECLARATION_SCHEMA, + "unsupported declaration schema" + ); + declaration.case + }) + .collect::>(); + let results: Vec = read_json_lines(&results) + .unwrap_or_else(|errors| panic!("invalid results:\n{}", errors.join("\n"))); + let mut source_sha = None; + let mut environment_record = None; + if let Some(path) = environment { + let records: Vec = read_json_lines(&path) + .unwrap_or_else(|errors| panic!("invalid environment:\n{}", errors.join("\n"))); + assert_eq!(records.len(), 1, "expected one environment record"); + assert!( + environment_schema_supported(&records[0].schema), + "unsupported environment schema: {}", + records[0].schema, + ); + source_sha = records[0].source_sha.clone(); + if records[0].status == EnvironmentStatus::Error { + emit( + format!( + "# CUA Driver E2E\n\n**Environment:** ERROR\n\n{}\n", + records[0].message + ), + output.as_ref(), + ); + eprintln!("E2E environment is not ready: {}", records[0].message); + std::process::exit(2); + } + environment_record = records.into_iter().next(); + } + let summary = validate_catalog( + &declarations, + &results, + artifact_root.as_deref(), + require_video, + ) + .unwrap_or_else(|errors| panic!("invalid E2E report:\n{}", errors.join("\n"))); + let markdown = summary.markdown_with_declarations_source_and_environment( + &declarations, + &results, + source_sha.as_deref(), + environment_record.as_ref(), + ); + emit(markdown, output.as_ref()); +} diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/driver.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/driver.rs index 758ae9bf4b..eae7a18d48 100644 --- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/driver.rs +++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/driver.rs @@ -13,3 +13,8 @@ pub trait Driver { /// Invoke `tool` with `args`, returning the normalized response. fn call(&mut self, tool: &str, args: Value) -> ToolResponse; } + +/// Explicit lifecycle capability for canonical per-cell behavioral clips. +pub trait BehaviorRecording { + fn start_behavior_recording(&mut self); +} diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs new file mode 100644 index 0000000000..f6d47c7c10 --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/e2e.rs @@ -0,0 +1,2261 @@ +//! Typed contracts and result reporting for desktop E2E cells. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs::{File, OpenOptions}; +use std::io::{self, BufRead, BufReader, Write}; +use std::panic::{self, AssertUnwindSafe}; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use serde_json::Value; + +pub const DECLARATION_SCHEMA: &str = "cua-e2e-case/v2"; +pub const ENVIRONMENT_SCHEMA_V2: &str = "cua-e2e-environment/v2"; +pub const ENVIRONMENT_SCHEMA: &str = "cua-e2e-environment/v3"; +pub const RESULT_SCHEMA: &str = "cua-e2e-result/v2"; + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Platform { + Windows, + Macos, + Linux, +} + +impl Platform { + pub fn current() -> Self { + #[cfg(target_os = "windows")] + { + Self::Windows + } + #[cfg(target_os = "macos")] + { + Self::Macos + } + #[cfg(target_os = "linux")] + { + Self::Linux + } + #[cfg(not(any(target_os = "windows", target_os = "macos", target_os = "linux")))] + { + panic!("unsupported E2E platform") + } + } +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum DisplayServer { + Win32, + Quartz, + X11, + Wayland, +} + +impl DisplayServer { + pub fn current() -> Self { + #[cfg(target_os = "windows")] + { + Self::Win32 + } + #[cfg(target_os = "macos")] + { + Self::Quartz + } + #[cfg(target_os = "linux")] + { + match std::env::var("XDG_SESSION_TYPE") + .unwrap_or_else(|_| "x11".to_owned()) + .to_ascii_lowercase() + .as_str() + { + "wayland" => Self::Wayland, + _ => Self::X11, + } + } + #[cfg(not(any(target_os = "windows", target_os = "macos", target_os = "linux")))] + { + panic!("unsupported E2E display server") + } + } +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Targeting { + Ax, + Px, + Page, + NotApplicable, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Delivery { + Background, + Foreground, + NotApplicable, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Scope { + Window, + Desktop, + NotApplicable, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum DriverRoute { + AxRead, + WindowState, + UiaInvoke, + UiaToggle, + UiaSelection, + UiaExpandCollapse, + UiaValue, + UiaRangeValue, + UiaScroll, + PostMessage, + WindowsTargetedInjection, + WindowsSendInput, + WindowsShellExecute, + WindowsPrintWindow, + WindowsOverlay, + MacosAxAction, + MacosAxValue, + MacosCgEventPid, + MacosCgEventHid, + LinuxAtSpiAction, + LinuxAtSpiValue, + LinuxXSendEvent, + LinuxXTest, + LinuxLibei, + LinuxWaylandVirtualPointer, + LinuxCuaCompositorInject, + Cdp, + Composite, +} + +pub fn shared_web_route( + platform: Platform, + display_server: DisplayServer, + action: &str, + targeting: Targeting, + delivery: Delivery, +) -> Result { + shared_web_route_for_environment( + platform, + display_server, + action, + targeting, + delivery, + nested_inject_from_env(), + ) +} + +fn shared_web_route_for_environment( + platform: Platform, + display_server: DisplayServer, + action: &str, + targeting: Targeting, + delivery: Delivery, + nested_inject: bool, +) -> Result { + use DriverRoute as Route; + + let pointer_or_key_route = |background, foreground| match delivery { + Delivery::Background => Ok(background), + Delivery::Foreground => Ok(foreground), + Delivery::NotApplicable => Err(format!("{action}: delivery mode is required")), + }; + match (platform, display_server, targeting, action) { + (Platform::Windows, DisplayServer::Win32, Targeting::Px, _) => { + pointer_or_key_route(Route::WindowsTargetedInjection, Route::WindowsSendInput) + } + (Platform::Windows, DisplayServer::Win32, Targeting::Ax, "left_click") + | (Platform::Windows, DisplayServer::Win32, Targeting::Ax, "child_window") => { + Ok(Route::UiaInvoke) + } + (Platform::Windows, DisplayServer::Win32, Targeting::Ax, "type_text") => { + Ok(Route::UiaValue) + } + (Platform::Windows, DisplayServer::Win32, Targeting::Ax, "scroll") => { + Ok(Route::UiaScroll) + } + ( + Platform::Windows, + DisplayServer::Win32, + Targeting::Ax, + "right_click" | "double_click" | "press_key" | "hotkey", + ) => pointer_or_key_route(Route::PostMessage, Route::WindowsSendInput), + (Platform::Windows, DisplayServer::Win32, Targeting::Ax, "editor_save") => { + Ok(Route::Composite) + } + + (Platform::Macos, DisplayServer::Quartz, Targeting::Px, _) => { + pointer_or_key_route(Route::MacosCgEventPid, Route::MacosCgEventHid) + } + (Platform::Macos, DisplayServer::Quartz, Targeting::Ax, "left_click") + | (Platform::Macos, DisplayServer::Quartz, Targeting::Ax, "child_window") => { + Ok(Route::MacosAxAction) + } + (Platform::Macos, DisplayServer::Quartz, Targeting::Ax, "scroll") => { + Ok(Route::Composite) + } + (Platform::Macos, DisplayServer::Quartz, Targeting::Ax, "type_text") => { + Ok(Route::MacosAxValue) + } + ( + Platform::Macos, + DisplayServer::Quartz, + Targeting::Ax, + "right_click" | "double_click" | "press_key" | "hotkey", + ) => pointer_or_key_route(Route::MacosCgEventPid, Route::MacosCgEventHid), + (Platform::Macos, DisplayServer::Quartz, Targeting::Ax, "editor_save") => { + Ok(Route::Composite) + } + + (Platform::Linux, DisplayServer::Wayland, Targeting::Px, _) if nested_inject => + { + Ok(Route::LinuxCuaCompositorInject) + } + (Platform::Linux, DisplayServer::Wayland, Targeting::Px, _) => { + Ok(Route::LinuxWaylandVirtualPointer) + } + (Platform::Linux, DisplayServer::X11, Targeting::Px, _) => { + pointer_or_key_route(Route::LinuxXSendEvent, Route::LinuxXTest) + } + (Platform::Linux, DisplayServer::Wayland, Targeting::Ax, "type_text") + if nested_inject => + { + Ok(Route::LinuxCuaCompositorInject) + } + ( + Platform::Linux, + DisplayServer::X11 | DisplayServer::Wayland, + Targeting::Ax, + "left_click" | "child_window" | "scroll", + ) => Ok(Route::LinuxAtSpiAction), + ( + Platform::Linux, + DisplayServer::X11 | DisplayServer::Wayland, + Targeting::Ax, + "type_text", + ) => Ok(Route::LinuxAtSpiValue), + ( + Platform::Linux, + DisplayServer::X11, + Targeting::Ax, + "right_click" | "double_click" | "press_key" | "hotkey", + ) => pointer_or_key_route(Route::LinuxXSendEvent, Route::LinuxXTest), + ( + Platform::Linux, + DisplayServer::Wayland, + Targeting::Ax, + "right_click" | "double_click" | "press_key" | "hotkey", + ) if nested_inject => Ok(Route::LinuxCuaCompositorInject), + ( + Platform::Linux, + DisplayServer::Wayland, + Targeting::Ax, + "right_click" | "double_click" | "press_key" | "hotkey", + ) => Ok(Route::LinuxWaylandVirtualPointer), + ( + Platform::Linux, + DisplayServer::X11 | DisplayServer::Wayland, + Targeting::Ax, + "editor_save", + ) => Ok(Route::Composite), + _ => Err(format!( + "no shared route for {platform:?}/{display_server:?}/{action}/{targeting:?}/{delivery:?}" + )), + } +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum OracleKind { + FixtureState, + AxState, + Pixels, + Focus, + ZOrder, + Cursor, + NoLeakedInput, + Protocol, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum RefusalCode { + BackgroundUnavailable, + BackgroundOccluded, + BackgroundUipiBlocked, +} + +impl RefusalCode { + pub fn from_driver_code(code: &str) -> Option { + match code { + "background_unavailable" => Some(Self::BackgroundUnavailable), + "background_occluded" => Some(Self::BackgroundOccluded), + "background_uipi_blocked" => Some(Self::BackgroundUipiBlocked), + _ => None, + } + } +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case", tag = "kind")] +pub enum ContractExpectation { + Deliver, + Refuse { allowed_codes: Vec }, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum TestStatus { + Pass, + Fail, + Skip, + EnvironmentError, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum EnvironmentStatus { + Ready, + Error, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct EnvironmentRecord { + pub schema: String, + pub platform: Platform, + pub display_server: DisplayServer, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub compositor: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub input_backends: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_sha: Option, + pub status: EnvironmentStatus, + pub duration_ms: u128, + pub message: String, +} + +impl EnvironmentRecord { + pub fn ready(duration: Duration) -> Self { + Self { + schema: ENVIRONMENT_SCHEMA.to_owned(), + platform: Platform::current(), + display_server: DisplayServer::current(), + compositor: compositor_from_env(), + input_backends: input_backends_from_env(), + source_sha: source_sha_from_env(), + status: EnvironmentStatus::Ready, + duration_ms: duration.as_millis(), + message: String::new(), + } + } + + pub fn error(duration: Duration, message: impl Into) -> Self { + Self { + schema: ENVIRONMENT_SCHEMA.to_owned(), + platform: Platform::current(), + display_server: DisplayServer::current(), + compositor: compositor_from_env(), + input_backends: input_backends_from_env(), + source_sha: source_sha_from_env(), + status: EnvironmentStatus::Error, + duration_ms: duration.as_millis(), + message: message.into(), + } + } +} + +pub fn environment_schema_supported(schema: &str) -> bool { + matches!(schema, ENVIRONMENT_SCHEMA | ENVIRONMENT_SCHEMA_V2) +} + +fn nested_inject_from_env() -> bool { + std::env::var_os("CUA_INJECT_SOCKET").is_some() +} + +fn compositor_from_env() -> Option { + if let Ok(value) = std::env::var("CUA_E2E_COMPOSITOR") { + if !value.trim().is_empty() { + return Some(value); + } + } + match (Platform::current(), DisplayServer::current()) { + (Platform::Windows, DisplayServer::Win32) => Some("windows-desktop".to_owned()), + (Platform::Macos, DisplayServer::Quartz) => Some("windowserver".to_owned()), + (Platform::Linux, DisplayServer::X11) => Some("openbox-x11".to_owned()), + (Platform::Linux, DisplayServer::Wayland) if nested_inject_from_env() => { + Some("cua-compositor-nested".to_owned()) + } + (Platform::Linux, DisplayServer::Wayland) => { + let desktop = std::env::var("XDG_CURRENT_DESKTOP") + .unwrap_or_default() + .to_ascii_lowercase(); + if desktop.contains("sway") { + Some("sway".to_owned()) + } else if desktop.contains("gnome") { + Some("gnome-mutter".to_owned()) + } else if desktop.contains("kde") || desktop.contains("plasma") { + Some("kwin".to_owned()) + } else { + Some("wayland-unknown".to_owned()) + } + } + _ => None, + } +} + +fn input_backends_from_env() -> Vec { + if let Ok(value) = std::env::var("CUA_E2E_INPUT_BACKENDS") { + let mut backends = value + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_owned) + .collect::>(); + backends.sort(); + backends.dedup(); + return backends; + } + match (Platform::current(), DisplayServer::current()) { + (Platform::Windows, DisplayServer::Win32) => vec!["win32".to_owned(), "uia".to_owned()], + (Platform::Macos, DisplayServer::Quartz) => { + vec!["accessibility".to_owned(), "cg-event".to_owned()] + } + (Platform::Linux, DisplayServer::X11) => vec![ + "atspi".to_owned(), + "xsend-event".to_owned(), + "xtest".to_owned(), + ], + (Platform::Linux, DisplayServer::Wayland) if nested_inject_from_env() => { + vec!["atspi".to_owned(), "cua-compositor-inject".to_owned()] + } + (Platform::Linux, DisplayServer::Wayland) => vec!["atspi".to_owned()], + _ => Vec::new(), + } +} + +fn source_sha_from_env() -> Option { + std::env::var("CUA_E2E_SOURCE_SHA") + .ok() + .filter(|sha| !sha.is_empty()) +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, Hash, Ord, PartialEq, PartialOrd, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ObservedBehavior { + Delivered, + Refused, + NoEffect, + Error, + NotRun, +} + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +pub struct Evidence { + #[serde(skip_serializing_if = "Option::is_none")] + pub video: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub trajectory: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub screenshot: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub log: Option, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct CaseSpec { + pub cell_id: String, + pub platform: Platform, + pub display_server: DisplayServer, + pub harness: String, + pub toolkit: String, + pub action: String, + pub targeting: Targeting, + pub delivery: Delivery, + pub scope: Scope, + pub driver_route: DriverRoute, + pub expected_behavior: ContractExpectation, + pub oracles: Vec, +} + +impl CaseSpec { + #[allow(clippy::too_many_arguments)] + pub fn delivered( + cell_id: impl Into, + harness: impl Into, + toolkit: impl Into, + action: impl Into, + targeting: Targeting, + delivery: Delivery, + scope: Scope, + driver_route: DriverRoute, + oracles: Vec, + ) -> Self { + Self { + cell_id: cell_id.into(), + platform: Platform::current(), + display_server: DisplayServer::current(), + harness: harness.into(), + toolkit: toolkit.into(), + action: action.into(), + targeting, + delivery, + scope, + driver_route, + expected_behavior: ContractExpectation::Deliver, + oracles, + } + } + + pub fn expecting_refusal(mut self, allowed_codes: Vec) -> Self { + self.expected_behavior = ContractExpectation::Refuse { allowed_codes }; + self + } + + pub fn validate(&self) -> Result<(), String> { + if self.cell_id.is_empty() + || !self + .cell_id + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.')) + { + return Err(format!("{}: cell id is not artifact-safe", self.cell_id)); + } + if self.oracles.is_empty() { + return Err(format!("{}: no external oracle declared", self.cell_id)); + } + if let ContractExpectation::Refuse { allowed_codes } = &self.expected_behavior { + if self.delivery != Delivery::Background { + return Err(format!( + "{}: only background delivery may declare refusal", + self.cell_id + )); + } + if allowed_codes.is_empty() { + return Err(format!("{}: refusal has no allowed code", self.cell_id)); + } + for required in [ + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::NoLeakedInput, + ] { + if !self.oracles.contains(&required) { + return Err(format!( + "{}: refusal is missing {:?} oracle", + self.cell_id, required + )); + } + } + } + Ok(()) + } +} + +fn native_cell_id(toolkit: &str, action: &str, targeting: Targeting, delivery: Delivery) -> String { + let targeting = match targeting { + Targeting::Ax => "ax", + Targeting::Px => "px", + Targeting::Page => "page", + Targeting::NotApplicable => "not-applicable", + }; + let delivery = match delivery { + Delivery::Background => "background", + Delivery::Foreground => "foreground", + Delivery::NotApplicable => "not-applicable", + }; + format!( + "{}-{toolkit}-{action}-{targeting}-{delivery}", + std::env::consts::OS + ) + .replace('_', "-") +} + +pub fn native_background_case( + toolkit: &str, + action: &str, + targeting: Targeting, + route: DriverRoute, +) -> CaseSpec { + let mut oracles = vec![ + OracleKind::FixtureState, + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::NoLeakedInput, + ]; + if DisplayServer::current() != DisplayServer::Wayland { + oracles.push(OracleKind::Cursor); + } + CaseSpec::delivered( + native_cell_id(toolkit, action, targeting, Delivery::Background), + toolkit, + toolkit, + action, + targeting, + Delivery::Background, + Scope::Window, + route, + oracles, + ) +} + +pub fn native_foreground_case( + toolkit: &str, + action: &str, + targeting: Targeting, + route: DriverRoute, +) -> CaseSpec { + CaseSpec::delivered( + native_cell_id(toolkit, action, targeting, Delivery::Foreground), + toolkit, + toolkit, + action, + targeting, + Delivery::Foreground, + Scope::Window, + route, + vec![OracleKind::FixtureState], + ) +} + +pub fn native_readonly_case( + toolkit: &str, + action: &str, + targeting: Targeting, + route: DriverRoute, + oracles: Vec, +) -> CaseSpec { + CaseSpec::delivered( + native_cell_id(toolkit, action, targeting, Delivery::NotApplicable), + toolkit, + toolkit, + action, + targeting, + Delivery::NotApplicable, + Scope::Window, + route, + oracles, + ) +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct CaseDeclaration { + pub schema: String, + #[serde(flatten)] + pub case: CaseSpec, +} + +impl From for CaseDeclaration { + fn from(case: CaseSpec) -> Self { + Self { + schema: DECLARATION_SCHEMA.to_owned(), + case, + } + } +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct Observation { + pub behavior: ObservedBehavior, + #[serde(skip_serializing_if = "Option::is_none")] + pub refusal_code: Option, + pub passed_oracles: Vec, + pub message: String, + pub evidence: Evidence, +} + +impl Observation { + pub fn delivered(passed_oracles: Vec, evidence: Evidence) -> Self { + Self { + behavior: ObservedBehavior::Delivered, + refusal_code: None, + passed_oracles, + message: String::new(), + evidence, + } + } + + pub fn delivered_with_fixture_state(mut passed_oracles: Vec) -> Self { + passed_oracles.push(OracleKind::FixtureState); + passed_oracles.sort(); + passed_oracles.dedup(); + Self::delivered(passed_oracles, Evidence::default()) + } + + pub fn refused( + code: RefusalCode, + passed_oracles: Vec, + message: impl Into, + evidence: Evidence, + ) -> Self { + Self { + behavior: ObservedBehavior::Refused, + refusal_code: Some(code), + passed_oracles, + message: message.into(), + evidence, + } + } + + pub fn error(message: impl Into, evidence: Evidence) -> Self { + Self { + behavior: ObservedBehavior::Error, + refusal_code: None, + passed_oracles: Vec::new(), + message: message.into(), + evidence, + } + } +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct CaseResult { + pub schema: String, + #[serde(flatten)] + pub case: CaseSpec, + pub test_status: TestStatus, + pub observed_behavior: ObservedBehavior, + #[serde(skip_serializing_if = "Option::is_none")] + pub refusal_code: Option, + pub passed_oracles: Vec, + pub duration_ms: u128, + pub message: String, + pub evidence: Evidence, +} + +impl CaseResult { + pub fn evaluate(case: CaseSpec, observation: Observation, duration: Duration) -> Self { + let mut failures = Vec::new(); + if let Err(error) = case.validate() { + failures.push(error); + } + for oracle in &case.oracles { + if !observation.passed_oracles.contains(oracle) { + failures.push(format!("missing {:?} oracle", oracle)); + } + } + match (&case.expected_behavior, observation.behavior) { + (ContractExpectation::Deliver, ObservedBehavior::Delivered) => {} + (ContractExpectation::Deliver, ObservedBehavior::Refused) => { + failures.push("required delivery was refused".to_owned()); + } + (ContractExpectation::Refuse { allowed_codes }, ObservedBehavior::Refused) => { + match observation.refusal_code { + Some(code) if allowed_codes.contains(&code) => {} + Some(code) => failures.push(format!("unexpected refusal code: {code:?}")), + None => failures.push("refusal has no structured code".to_owned()), + } + } + (ContractExpectation::Refuse { .. }, ObservedBehavior::Delivered) => { + failures.push("unexpected delivery requires contract review".to_owned()); + } + (_, other) => failures.push(format!("observed behavior was {other:?}")), + } + + let status = if failures.is_empty() { + TestStatus::Pass + } else { + TestStatus::Fail + }; + let message = [observation.message, failures.join("; ")] + .into_iter() + .filter(|part| !part.is_empty()) + .collect::>() + .join("; "); + Self { + schema: RESULT_SCHEMA.to_owned(), + case, + test_status: status, + observed_behavior: observation.behavior, + refusal_code: observation.refusal_code, + passed_oracles: observation.passed_oracles, + duration_ms: duration.as_millis(), + message, + evidence: observation.evidence, + } + } +} + +#[derive(Debug, Default, Eq, PartialEq)] +pub struct ValidationSummary { + pub delivered: usize, + pub refused: usize, + pub failed: usize, + pub skipped: usize, +} + +impl ValidationSummary { + pub fn markdown(&self, results: &[CaseResult]) -> String { + let declarations = results + .iter() + .map(|result| result.case.clone()) + .collect::>(); + self.markdown_with_declarations(&declarations, results) + } + + pub fn markdown_with_declarations( + &self, + declarations: &[CaseSpec], + results: &[CaseResult], + ) -> String { + self.markdown_with_declarations_and_source(declarations, results, None) + } + + pub fn markdown_with_declarations_and_source( + &self, + declarations: &[CaseSpec], + results: &[CaseResult], + source_sha: Option<&str>, + ) -> String { + self.markdown_with_declarations_source_and_environment( + declarations, + results, + source_sha, + None, + ) + } + + pub fn markdown_with_declarations_source_and_environment( + &self, + declarations: &[CaseSpec], + results: &[CaseResult], + source_sha: Option<&str>, + environment: Option<&EnvironmentRecord>, + ) -> String { + let mut output = format!( + "# CUA Driver E2E\n\n**Result:** {} delivered, {} refused, {} failed, {} skipped\n\n", + self.delivered, self.refused, self.failed, self.skipped + ); + if let Some(source_sha) = source_sha { + output.push_str(&format!("**Source SHA:** `{source_sha}`\n\n")); + } + if let Some(environment) = environment { + let compositor = environment.compositor.as_deref().unwrap_or("unknown"); + let input_backends = if environment.input_backends.is_empty() { + "unknown".to_owned() + } else { + environment.input_backends.join(", ") + }; + output.push_str(&format!( + "**Environment:** `{:?}/{:?}` · compositor `{}` · input backends `{}`\n\n", + environment.platform, + environment.display_server, + markdown_inline_code(compositor), + markdown_inline_code(&input_backends), + )); + } + output.push_str("## Declared Coverage\n\n"); + output.push_str(&declared_coverage_markdown(declarations, results)); + output.push_str("\n## Detailed Results\n\n"); + output.push_str( + "| Cell | Platform | Harness | Action | Targeting | Delivery | Scope | Route | Oracles | Expected | Observed | Status | Duration | Evidence | Details |\n", + ); + output.push_str("| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | ---: | --- | --- |\n"); + for result in results { + let evidence = result + .evidence + .video + .as_deref() + .or(result.evidence.trajectory.as_deref()) + .unwrap_or("-"); + let oracles = result + .case + .oracles + .iter() + .map(|oracle| format!("{oracle:?}")) + .collect::>() + .join(", "); + let details = if result.message.is_empty() { + "-".to_owned() + } else { + result.message.replace('|', "\\|").replace('\n', " ") + }; + output.push_str(&format!( + "| {} | {:?}/{:?} | {} | {} | {:?} | {:?} | {:?} | {:?} | {} | {:?} | {:?} | {:?} | {} ms | {} | {} |\n", + result.case.cell_id, + result.case.platform, + result.case.display_server, + result.case.harness, + result.case.action, + result.case.targeting, + result.case.delivery, + result.case.scope, + result.case.driver_route, + oracles, + result.case.expected_behavior, + result.observed_behavior, + result.test_status, + result.duration_ms, + evidence, + details, + )); + } + output + } +} + +fn markdown_inline_code(value: &str) -> String { + value.replace('`', "\\`").replace('\n', " ") +} + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +enum CoverageColumn { + AxBackground, + AxForeground, + PxBackground, + PxForeground, + Page, + NotApplicable, +} + +const COVERAGE_COLUMNS: [CoverageColumn; 6] = [ + CoverageColumn::AxBackground, + CoverageColumn::AxForeground, + CoverageColumn::PxBackground, + CoverageColumn::PxForeground, + CoverageColumn::Page, + CoverageColumn::NotApplicable, +]; + +fn declared_coverage_markdown(declarations: &[CaseSpec], results: &[CaseResult]) -> String { + let results_by_cell = results + .iter() + .map(|result| (result.case.cell_id.as_str(), result)) + .collect::>(); + let mut rows = BTreeMap::<(String, String), BTreeMap>>::new(); + + for declaration in declarations { + let column = coverage_column(declaration); + let status = results_by_cell + .get(declaration.cell_id.as_str()) + .map(|result| coverage_status(result)) + .unwrap_or("MISSING"); + let label = match column { + CoverageColumn::Page => format!("{:?}: {status}", declaration.delivery), + CoverageColumn::NotApplicable => format!( + "{:?}/{:?}: {status}", + declaration.targeting, declaration.delivery + ), + _ => status.to_owned(), + }; + rows.entry((declaration.harness.clone(), declaration.action.clone())) + .or_default() + .entry(column) + .or_default() + .push(label); + } + + let mut output = String::from( + "| Harness | Action | AX/BG | AX/FG | PX/BG | PX/FG | Page | NotApplicable |\n", + ); + output.push_str("| --- | --- | --- | --- | --- | --- | --- | --- |\n"); + for ((harness, action), cells) in rows { + output.push_str(&format!( + "| {} | {} | {} |\n", + markdown_table_text(&harness), + markdown_table_text(&action), + COVERAGE_COLUMNS + .iter() + .map(|column| render_coverage_cell(cells.get(column))) + .collect::>() + .join(" | ") + )); + } + output +} + +fn coverage_column(case: &CaseSpec) -> CoverageColumn { + match (case.targeting, case.delivery) { + (Targeting::Page, _) => CoverageColumn::Page, + (Targeting::Ax, Delivery::Background) => CoverageColumn::AxBackground, + (Targeting::Ax, Delivery::Foreground) => CoverageColumn::AxForeground, + (Targeting::Px, Delivery::Background) => CoverageColumn::PxBackground, + (Targeting::Px, Delivery::Foreground) => CoverageColumn::PxForeground, + _ => CoverageColumn::NotApplicable, + } +} + +fn coverage_status(result: &CaseResult) -> &'static str { + match (result.test_status, result.observed_behavior) { + (TestStatus::Pass, ObservedBehavior::Delivered) => "PASS", + (TestStatus::Pass, ObservedBehavior::Refused) => "REFUSED", + (TestStatus::Pass, _) => "INVALID", + (TestStatus::Fail | TestStatus::EnvironmentError, _) => "FAIL", + (TestStatus::Skip, _) => "SKIP", + } +} + +fn render_coverage_cell(labels: Option<&Vec>) -> String { + let Some(labels) = labels else { + return "-".to_owned(); + }; + let mut counts = BTreeMap::new(); + for label in labels { + *counts.entry(label).or_insert(0usize) += 1; + } + counts + .into_iter() + .map(|(label, count)| { + if count == 1 { + label.clone() + } else { + format!("{label} ({count})") + } + }) + .collect::>() + .join("
") +} + +fn markdown_table_text(value: &str) -> String { + value.replace('|', "\\|").replace('\n', " ") +} + +pub fn append_json_line(path: &Path, value: &T) -> io::Result<()> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent)?; + } + let mut file = OpenOptions::new().create(true).append(true).open(path)?; + serde_json::to_writer(&mut file, value)?; + file.write_all(b"\n") +} + +pub fn write_declaration_from_env(case: &CaseSpec) -> io::Result<()> { + let Some(path) = std::env::var_os("CUA_E2E_DECLARATIONS_FILE") else { + return Ok(()); + }; + append_json_line(&PathBuf::from(path), &CaseDeclaration::from(case.clone())) +} + +pub fn write_result_from_env(result: &CaseResult) -> io::Result<()> { + let Some(path) = std::env::var_os("CUA_E2E_RESULTS_FILE") else { + return Ok(()); + }; + append_json_line(&PathBuf::from(path), result) +} + +/// Execute one declared E2E cell, persist its typed result even when the body +/// panics, and preserve Cargo's failing-test signal. +pub fn execute_case(case: CaseSpec, test: impl FnOnce(&mut Evidence) -> Observation) -> CaseResult { + write_declaration_from_env(&case).expect("write E2E case declaration"); + let started = Instant::now(); + let mut evidence = Evidence::default(); + let outcome = panic::catch_unwind(AssertUnwindSafe(|| test(&mut evidence))); + let observation = match outcome { + Ok(mut observation) => { + if observation.evidence == Evidence::default() { + observation.evidence = evidence; + } + observation + } + Err(payload) => Observation::error(panic_message(&payload), evidence), + }; + let result = CaseResult::evaluate(case, observation, started.elapsed()); + write_result_from_env(&result).expect("write E2E case result"); + assert_eq!( + result.test_status, + TestStatus::Pass, + "{}: {}", + result.case.cell_id, + result.message + ); + result +} + +pub fn recording_evidence(recording_dir: Option<&Path>) -> Evidence { + let Some(recording_dir) = recording_dir else { + return Evidence::default(); + }; + let relative_dir = std::env::var_os("CUA_E2E_RECORDINGS_ROOT") + .map(PathBuf::from) + .and_then(|root| recording_dir.strip_prefix(root).ok().map(PathBuf::from)) + .unwrap_or_else(|| { + recording_dir + .file_name() + .map(PathBuf::from) + .unwrap_or_default() + }); + let artifact_dir = PathBuf::from("recordings").join(relative_dir); + let path = |name: &str| artifact_dir.join(name).to_string_lossy().replace('\\', "/"); + Evidence { + video: Some(path("recording.mp4")), + trajectory: Some(path("trajectory.json")), + screenshot: None, + log: None, + } +} + +fn panic_message(payload: &Box) -> String { + payload + .downcast_ref::() + .cloned() + .or_else(|| { + payload + .downcast_ref::<&str>() + .map(|message| (*message).to_owned()) + }) + .unwrap_or_else(|| "E2E cell panicked without a string payload".to_owned()) +} + +pub fn write_environment_from_env(record: &EnvironmentRecord) -> io::Result<()> { + let Some(path) = std::env::var_os("CUA_E2E_ENVIRONMENT_FILE") else { + return Ok(()); + }; + append_json_line(&PathBuf::from(path), record) +} + +pub fn read_json_lines(path: &Path) -> Result, Vec> { + let file = File::open(path).map_err(|error| vec![format!("{}: {error}", path.display())])?; + let mut values = Vec::new(); + let mut errors = Vec::new(); + for (line_index, line) in BufReader::new(file).lines().enumerate() { + match line { + Ok(line) if line.trim().is_empty() => {} + Ok(line) => match serde_json::from_str(&line) { + Ok(value) => values.push(value), + Err(error) => { + errors.push(format!("{}:{}: {error}", path.display(), line_index + 1)) + } + }, + Err(error) => errors.push(format!("{}:{}: {error}", path.display(), line_index + 1)), + } + } + if errors.is_empty() { + Ok(values) + } else { + Err(errors) + } +} + +pub fn validate_catalog( + declarations: &[CaseSpec], + results: &[CaseResult], + artifact_root: Option<&Path>, + require_video: bool, +) -> Result> { + validate_catalog_with_evidence( + declarations, + results, + artifact_root, + require_video, + require_video, + ) +} + +pub fn validate_catalog_with_evidence( + declarations: &[CaseSpec], + results: &[CaseResult], + artifact_root: Option<&Path>, + require_video: bool, + require_turn_evidence: bool, +) -> Result> { + let mut errors = Vec::new(); + if declarations.is_empty() { + errors.push("E2E catalog has no declarations".to_owned()); + } + let mut declared = BTreeMap::new(); + for case in declarations { + if let Err(error) = case.validate() { + errors.push(error); + } + if declared.insert(case.cell_id.clone(), case).is_some() { + errors.push(format!("duplicate declaration: {}", case.cell_id)); + } + } + + let mut observed = BTreeSet::new(); + let mut summary = ValidationSummary::default(); + for result in results { + let cell_id = &result.case.cell_id; + if result.schema != RESULT_SCHEMA { + errors.push(format!( + "unsupported result schema for {cell_id}: {}", + result.schema + )); + } + if !observed.insert(cell_id.clone()) { + errors.push(format!("duplicate result: {cell_id}")); + continue; + } + match declared.get(cell_id) { + Some(case) if **case == result.case => {} + Some(_) => errors.push(format!("result contract changed: {cell_id}")), + None => errors.push(format!("undeclared result: {cell_id}")), + } + + let rebuilt = CaseResult::evaluate( + result.case.clone(), + Observation { + behavior: result.observed_behavior, + refusal_code: result.refusal_code, + passed_oracles: result.passed_oracles.clone(), + message: String::new(), + evidence: result.evidence.clone(), + }, + Duration::from_millis(result.duration_ms.min(u64::MAX as u128) as u64), + ); + if rebuilt.test_status != result.test_status { + errors.push(format!("invalid status for {cell_id}")); + } + match result.test_status { + TestStatus::Pass => match result.observed_behavior { + ObservedBehavior::Delivered => summary.delivered += 1, + ObservedBehavior::Refused => summary.refused += 1, + _ => errors.push(format!("passing cell has no valid behavior: {cell_id}")), + }, + TestStatus::Fail | TestStatus::EnvironmentError => summary.failed += 1, + TestStatus::Skip => summary.skipped += 1, + } + + if require_video { + let Some(video) = result.evidence.video.as_deref() else { + errors.push(format!("missing video evidence: {cell_id}")); + continue; + }; + if let Some(root) = artifact_root { + let path = root.join(video); + if std::fs::metadata(&path) + .map(|metadata| metadata.len() == 0) + .unwrap_or(true) + { + errors.push(format!( + "video evidence is missing or empty: {}", + path.display() + )); + } + } + } + if require_turn_evidence { + match (artifact_root, recording_directory(&result.evidence)) { + (Some(root), Some(relative_dir)) => { + validate_turn_evidence( + &root.join(relative_dir), + cell_id, + case_requires_action_turn(&result.case), + &mut errors, + ); + } + (None, _) => errors.push(format!( + "cannot validate turn evidence without an artifact root: {cell_id}" + )), + (_, None) => errors.push(format!( + "missing recording path for turn evidence: {cell_id}" + )), + } + } + } + + for cell_id in declared.keys() { + if !observed.contains(cell_id) { + errors.push(format!("missing result: {cell_id}")); + } + } + + if errors.is_empty() { + Ok(summary) + } else { + Err(errors) + } +} + +fn case_requires_action_turn(case: &CaseSpec) -> bool { + !matches!(case.driver_route, DriverRoute::AxRead | DriverRoute::WindowState) + && case.action != "screenshot" +} + +fn recording_directory(evidence: &Evidence) -> Option<&Path> { + evidence + .video + .as_deref() + .or(evidence.trajectory.as_deref()) + .and_then(|path| Path::new(path).parent()) +} + +fn validate_turn_evidence( + recording_dir: &Path, + cell_id: &str, + require_turn: bool, + errors: &mut Vec, +) { + let trajectory = recording_dir.join("trajectory.json"); + match read_json_value(&trajectory) { + Ok(manifest) => { + if manifest["behavior_video"]["status"] != "finalized" { + errors.push(format!( + "behavioral video phase is not finalized for {cell_id}: {}", + manifest["behavior_video"]["status"] + )); + } + let started = manifest["behavior_video"]["started_at_unix_ms"].as_u64(); + let baseline = manifest["behavior_video"]["baseline_ready_at_unix_ms"].as_u64(); + let finalized = manifest["behavior_video"]["finalized_at_unix_ms"].as_u64(); + if !matches!( + (started, baseline, finalized), + (Some(started), Some(baseline), Some(finalized)) + if baseline.saturating_sub(started) >= 250 && finalized >= baseline + ) { + errors.push(format!( + "behavioral video boundary timestamps are invalid for {cell_id}" + )); + } + if !matches!( + manifest["hosted_runner_console"]["status"].as_str(), + Some("minimized" | "already_minimized" | "not_applicable") + ) { + errors.push(format!( + "hosted-runner console cleanup status is invalid for {cell_id}: {}", + manifest["hosted_runner_console"]["status"] + )); + } + } + Err(error) => errors.push(format!("invalid trajectory evidence for {cell_id}: {error}")), + } + + let mut turns = match std::fs::read_dir(recording_dir) { + Ok(entries) => entries + .filter_map(Result::ok) + .filter(|entry| { + entry.file_type().is_ok_and(|kind| kind.is_dir()) + && entry.file_name().to_string_lossy().starts_with("turn-") + }) + .map(|entry| entry.path()) + .collect::>(), + Err(error) => { + errors.push(format!( + "turn evidence directory is unavailable for {cell_id}: {}: {error}", + recording_dir.display() + )); + return; + } + }; + turns.sort(); + if turns.is_empty() { + if require_turn { + errors.push(format!("missing turn evidence: {cell_id}")); + } + return; + } + + for turn in turns { + validate_one_turn(&turn, cell_id, errors); + } +} + +fn validate_one_turn(turn: &Path, cell_id: &str, errors: &mut Vec) { + let turn_name = turn + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or("turn-unknown"); + let action_path = turn.join("action.json"); + let action = match read_json_value(&action_path) { + Ok(action) => Some(action), + Err(error) => { + errors.push(format!( + "invalid action evidence for {cell_id}/{turn_name}: {error}" + )); + None + } + }; + let manifest_path = turn.join("evidence.json"); + let manifest = match read_json_value(&manifest_path) { + Ok(manifest) if manifest["schema"] == "cua-turn-evidence/v1" => Some(manifest), + Ok(_) => { + errors.push(format!( + "unsupported turn evidence manifest for {cell_id}/{turn_name}: {}", + manifest_path.display() + )); + None + } + Err(error) => { + errors.push(format!( + "invalid turn evidence manifest for {cell_id}/{turn_name}: {error}" + )); + None + } + }; + let classification = |phase: &str, kind: &str| { + manifest + .as_ref() + .and_then(|value| value[phase][kind]["classification"].as_str()) + }; + + for (phase, kind) in [("before", "screenshot"), ("after", "screenshot")] { + validate_capture_status( + manifest.as_ref(), + &[phase, kind], + cell_id, + &format!("{turn_name}/{phase} {kind}"), + errors, + ); + } + + for (file, phase, kind) in [ + ("before.png", "before", "screenshot"), + ("after.png", "after", "screenshot"), + ("screenshot.png", "after", "screenshot"), + ] { + validate_nonempty_file( + &turn.join(file), + cell_id, + &format!("{turn_name}/{file}"), + classification(phase, kind), + errors, + ); + } + + let state_expected = action + .as_ref() + .and_then(|value| value["arguments"]["pid"].as_i64()) + .is_some(); + if state_expected { + for phase in ["before", "after"] { + validate_capture_status( + manifest.as_ref(), + &[phase, "state"], + cell_id, + &format!("{turn_name}/{phase} state"), + errors, + ); + } + for (file, phase) in [ + ("before_state.json", "before"), + ("after_state.json", "after"), + ("app_state.json", "after"), + ] { + validate_json_file( + &turn.join(file), + cell_id, + &format!("{turn_name}/{file}"), + classification(phase, "state"), + errors, + ); + } + } + + if action.as_ref().is_some_and(|value| { + matches!( + value["tool"].as_str(), + Some("click" | "double_click" | "right_click") + ) + }) { + validate_capture_status( + manifest.as_ref(), + &["click"], + cell_id, + &format!("{turn_name}/click"), + errors, + ); + validate_nonempty_file( + &turn.join("click.png"), + cell_id, + &format!("{turn_name}/click.png"), + manifest + .as_ref() + .and_then(|value| value["click"]["classification"].as_str()), + errors, + ); + } +} + +fn validate_capture_status( + manifest: Option<&Value>, + path: &[&str], + cell_id: &str, + label: &str, + errors: &mut Vec, +) { + let Some(manifest) = manifest else { + return; + }; + let value = path.iter().fold(manifest, |value, key| &value[*key]); + if value["status"] == "captured" { + return; + } + let classification = value["classification"] + .as_str() + .unwrap_or("missing_classification"); + errors.push(format!( + "required evidence capture failed for {cell_id}: {label} (classified {classification})" + )); +} + +fn validate_nonempty_file( + path: &Path, + cell_id: &str, + label: &str, + classification: Option<&str>, + errors: &mut Vec, +) { + if std::fs::metadata(path).is_ok_and(|metadata| metadata.len() > 0) { + return; + } + let classified = classification + .map(|value| format!(" (classified {value})")) + .unwrap_or_default(); + errors.push(format!( + "required evidence is missing or empty for {cell_id}: {label}{classified}" + )); +} + +fn validate_json_file( + path: &Path, + cell_id: &str, + label: &str, + classification: Option<&str>, + errors: &mut Vec, +) { + if read_json_value(path).is_ok() { + return; + } + let classified = classification + .map(|value| format!(" (classified {value})")) + .unwrap_or_default(); + errors.push(format!( + "required JSON evidence is missing or invalid for {cell_id}: {label}{classified}" + )); +} + +fn read_json_value(path: &Path) -> Result { + let bytes = std::fs::read(path).map_err(|error| format!("{}: {error}", path.display()))?; + if bytes.is_empty() { + return Err(format!("{} is empty", path.display())); + } + serde_json::from_slice(&bytes).map_err(|error| format!("{}: {error}", path.display())) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn delivered_case(id: &str) -> CaseSpec { + CaseSpec::delivered( + id, + "electron", + "chromium", + "click", + Targeting::Ax, + Delivery::Background, + Scope::Window, + DriverRoute::UiaInvoke, + vec![OracleKind::FixtureState], + ) + } + + fn coverage_case( + id: &str, + harness: &str, + action: &str, + targeting: Targeting, + delivery: Delivery, + ) -> CaseSpec { + CaseSpec::delivered( + id, + harness, + harness, + action, + targeting, + delivery, + Scope::Window, + DriverRoute::Composite, + vec![OracleKind::FixtureState], + ) + } + + fn complete_turn_fixture() -> (TempDir, CaseSpec, CaseResult, PathBuf) { + let root = tempfile::tempdir().expect("create artifact root"); + let recording = root.path().join("recordings/cell-pid1-001"); + let turn = recording.join("turn-00001"); + std::fs::create_dir_all(&turn).expect("create turn fixture"); + std::fs::write(recording.join("recording.mp4"), b"video").expect("write video"); + std::fs::write( + recording.join("trajectory.json"), + br#"{ + "behavior_video":{ + "status":"finalized", + "started_at_unix_ms":100, + "baseline_ready_at_unix_ms":400, + "finalized_at_unix_ms":800 + }, + "hosted_runner_console":{"status":"not_applicable"} + }"#, + ) + .expect("write trajectory"); + std::fs::write( + turn.join("action.json"), + br#"{ + "tool":"click", + "arguments":{"pid":1,"window_id":2}, + "click_point":{"x":10,"y":20} + }"#, + ) + .expect("write action"); + std::fs::write( + turn.join("evidence.json"), + br#"{ + "schema":"cua-turn-evidence/v1", + "before":{"state":{"status":"captured"},"screenshot":{"status":"captured"}}, + "after":{"state":{"status":"captured"},"screenshot":{"status":"captured"}}, + "click":{"status":"captured"} + }"#, + ) + .expect("write manifest"); + for file in ["before_state.json", "after_state.json", "app_state.json"] { + std::fs::write(turn.join(file), b"{}").expect("write state evidence"); + } + for file in ["before.png", "after.png", "screenshot.png", "click.png"] { + std::fs::write(turn.join(file), b"png").expect("write image evidence"); + } + + let case = delivered_case("cell"); + let evidence = Evidence { + video: Some("recordings/cell-pid1-001/recording.mp4".to_owned()), + trajectory: Some("recordings/cell-pid1-001/trajectory.json".to_owned()), + ..Evidence::default() + }; + let result = CaseResult::evaluate( + case.clone(), + Observation::delivered(vec![OracleKind::FixtureState], evidence), + Duration::from_millis(1), + ); + (root, case, result, turn) + } + + #[test] + fn required_delivery_rejects_honest_refusal() { + let case = delivered_case("required-delivery"); + let result = CaseResult::evaluate( + case, + Observation::refused( + RefusalCode::BackgroundUnavailable, + vec![OracleKind::FixtureState], + "honest refusal", + Evidence::default(), + ), + Duration::from_millis(1), + ); + assert_eq!(result.test_status, TestStatus::Fail); + assert!(result.message.contains("required delivery was refused")); + } + + #[test] + fn refusal_requires_explicit_code_and_side_effect_oracles() { + let case = delivered_case("expected-refusal") + .expecting_refusal(vec![RefusalCode::BackgroundUnavailable]); + assert!(case.validate().is_err()); + + let mut case = case; + case.oracles = vec![ + OracleKind::FixtureState, + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::NoLeakedInput, + ]; + let result = CaseResult::evaluate( + case, + Observation::refused( + RefusalCode::BackgroundUnavailable, + vec![ + OracleKind::FixtureState, + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::NoLeakedInput, + ], + "", + Evidence::default(), + ), + Duration::from_millis(1), + ); + assert_eq!(result.test_status, TestStatus::Pass); + assert_eq!(result.observed_behavior, ObservedBehavior::Refused); + } + + #[test] + fn unknown_background_error_is_not_a_refusal_code() { + assert_eq!( + RefusalCode::from_driver_code("background_unavailable"), + Some(RefusalCode::BackgroundUnavailable) + ); + assert_eq!(RefusalCode::from_driver_code("background_timeout"), None); + } + + #[test] + fn validator_rejects_missing_and_duplicate_results() { + let case = delivered_case("one"); + let result = CaseResult::evaluate( + case.clone(), + Observation::delivered(vec![OracleKind::FixtureState], Evidence::default()), + Duration::from_millis(1), + ); + let errors = validate_catalog( + std::slice::from_ref(&case), + &[result.clone(), result], + None, + false, + ) + .expect_err("duplicate result should fail"); + assert!(errors + .iter() + .any(|error| error.contains("duplicate result"))); + + let errors = + validate_catalog(&[case], &[], None, false).expect_err("missing result should fail"); + assert!(errors.iter().any(|error| error.contains("missing result"))); + } + + #[test] + fn validator_rejects_empty_catalog() { + let errors = validate_catalog(&[], &[], None, false) + .expect_err("an empty E2E catalog must not pass"); + assert!(errors.iter().any(|error| error.contains("no declarations"))); + } + + #[test] + fn validator_rejects_contradictory_status_and_missing_video() { + let case = delivered_case("contradiction"); + let mut result = CaseResult::evaluate( + case.clone(), + Observation::error("driver failed", Evidence::default()), + Duration::from_millis(1), + ); + result.test_status = TestStatus::Pass; + let errors = validate_catalog(std::slice::from_ref(&case), &[result], None, false) + .expect_err("contradictory status should fail"); + assert!(errors.iter().any(|error| error.contains("invalid status"))); + + let result = CaseResult::evaluate( + case.clone(), + Observation::delivered(vec![OracleKind::FixtureState], Evidence::default()), + Duration::from_millis(1), + ); + let errors = validate_catalog(&[case], &[result], None, true) + .expect_err("required video should fail"); + assert!(errors + .iter() + .any(|error| error.contains("missing video evidence"))); + } + + #[test] + fn validator_accepts_complete_pre_and_post_turn_evidence() { + let (root, case, result, _) = complete_turn_fixture(); + validate_catalog(&[case], &[result], Some(root.path()), true) + .expect("complete turn evidence should pass strict validation"); + } + + #[test] + fn validator_rejects_unreached_behavior_video_boundary() { + let (root, case, result, turn) = complete_turn_fixture(); + let recording = turn.parent().expect("turn has recording parent"); + std::fs::write( + recording.join("trajectory.json"), + br#"{ + "behavior_video":{"status":"pending"}, + "hosted_runner_console":{"status":"minimized"} + }"#, + ) + .expect("write pending trajectory"); + + let errors = validate_catalog(&[case], &[result], Some(root.path()), true) + .expect_err("pending behavior phase must fail strict validation"); + assert!(errors + .iter() + .any(|error| error.contains("behavioral video phase is not finalized"))); + } + + #[test] + fn strict_readonly_cell_does_not_invent_an_action_turn() { + let root = tempfile::tempdir().expect("create artifact root"); + let recording = root.path().join("recordings/readonly-pid1-001"); + std::fs::create_dir_all(&recording).expect("create recording fixture"); + std::fs::write(recording.join("recording.mp4"), b"video").expect("write video"); + std::fs::write( + recording.join("trajectory.json"), + br#"{ + "behavior_video":{ + "status":"finalized", + "started_at_unix_ms":100, + "baseline_ready_at_unix_ms":400, + "finalized_at_unix_ms":800 + }, + "hosted_runner_console":{"status":"not_applicable"} + }"#, + ) + .expect("write trajectory"); + let case = native_readonly_case( + "wpf", + "ax_tree", + Targeting::Ax, + DriverRoute::AxRead, + vec![OracleKind::FixtureState], + ); + let evidence = Evidence { + video: Some("recordings/readonly-pid1-001/recording.mp4".to_owned()), + trajectory: Some("recordings/readonly-pid1-001/trajectory.json".to_owned()), + ..Evidence::default() + }; + let result = CaseResult::evaluate( + case.clone(), + Observation::delivered(vec![OracleKind::FixtureState], evidence), + Duration::from_millis(1), + ); + + validate_catalog(&[case], &[result], Some(root.path()), true) + .expect("readonly cells have no action turn to capture"); + } + + #[test] + fn strict_background_screenshot_does_not_invent_an_action_turn() { + let case = CaseSpec::delivered( + "windows-wpf-screenshot-px-background", + "wpf", + "wpf", + "screenshot", + Targeting::Px, + Delivery::Background, + Scope::Window, + DriverRoute::WindowsPrintWindow, + vec![OracleKind::Pixels], + ); + assert!(!case_requires_action_turn(&case)); + } + + #[test] + fn strict_not_applicable_mutation_still_requires_an_action_turn() { + let root = tempfile::tempdir().expect("create artifact root"); + let recording = root.path().join("recordings/cursor-pid1-001"); + std::fs::create_dir_all(&recording).expect("create recording fixture"); + std::fs::write(recording.join("recording.mp4"), b"video").expect("write video"); + std::fs::write( + recording.join("trajectory.json"), + br#"{ + "behavior_video":{ + "status":"finalized", + "started_at_unix_ms":100, + "baseline_ready_at_unix_ms":400, + "finalized_at_unix_ms":800 + }, + "hosted_runner_console":{"status":"not_applicable"} + }"#, + ) + .expect("write trajectory"); + let case = CaseSpec::delivered( + "cursor", + "desktop", + "win32", + "agent_cursor", + Targeting::Px, + Delivery::NotApplicable, + Scope::Desktop, + DriverRoute::WindowsOverlay, + vec![OracleKind::FixtureState], + ); + let evidence = Evidence { + video: Some("recordings/cursor-pid1-001/recording.mp4".to_owned()), + trajectory: Some("recordings/cursor-pid1-001/trajectory.json".to_owned()), + ..Evidence::default() + }; + let result = CaseResult::evaluate( + case.clone(), + Observation::delivered(vec![OracleKind::FixtureState], evidence), + Duration::from_millis(1), + ); + + let errors = validate_catalog(&[case], &[result], Some(root.path()), true) + .expect_err("mutable not-applicable cells still need a recorded turn"); + assert!(errors.iter().any(|error| error.contains("missing turn evidence"))); + } + + #[test] + fn validator_exposes_missing_legacy_modal_images() { + let (root, case, result, turn) = complete_turn_fixture(); + std::fs::remove_file(turn.join("screenshot.png")).expect("remove screenshot fixture"); + std::fs::remove_file(turn.join("click.png")).expect("remove click fixture"); + + let errors = validate_catalog(&[case], &[result], Some(root.path()), true) + .expect_err("missing expected images must fail closed"); + assert!(errors + .iter() + .any(|error| error.contains("turn-00001/screenshot.png"))); + assert!(errors + .iter() + .any(|error| error.contains("turn-00001/click.png"))); + } + + #[test] + fn validator_reports_capture_classification_for_missing_phase() { + let (root, case, result, turn) = complete_turn_fixture(); + std::fs::remove_file(turn.join("after.png")).expect("remove after image fixture"); + std::fs::write( + turn.join("evidence.json"), + br#"{ + "schema":"cua-turn-evidence/v1", + "before":{"state":{"status":"captured"},"screenshot":{"status":"captured"}}, + "after":{"state":{"status":"captured"},"screenshot":{"status":"unavailable","classification":"capture_failed"}}, + "click":{"status":"captured"} + }"#, + ) + .expect("write classified manifest"); + + let errors = validate_catalog(&[case], &[result], Some(root.path()), true) + .expect_err("classified unavailability must remain a strict failure"); + assert!(errors.iter().any(|error| { + error.contains("turn-00001/after.png") && error.contains("classified capture_failed") + })); + } + + #[test] + fn non_strict_validation_keeps_legacy_results_compatible() { + let case = delivered_case("legacy"); + let result = CaseResult::evaluate( + case.clone(), + Observation::delivered(vec![OracleKind::FixtureState], Evidence::default()), + Duration::from_millis(1), + ); + validate_catalog(&[case], &[result], None, false) + .expect("non-canonical legacy results remain valid"); + } + + #[test] + fn validator_rejects_unknown_result_schema() { + let case = delivered_case("unknown-schema"); + let mut result = CaseResult::evaluate( + case.clone(), + Observation::delivered(vec![OracleKind::FixtureState], Evidence::default()), + Duration::from_millis(1), + ); + result.schema = "cua-e2e-result/v999".to_owned(); + let errors = validate_catalog(&[case], &[result], None, false) + .expect_err("unknown result schema should fail"); + assert!(errors + .iter() + .any(|error| error.contains("unsupported result schema"))); + } + + #[test] + fn valid_catalog_renders_delivered_rollup_and_flat_schema() { + let case = delivered_case("rendered"); + let result = CaseResult::evaluate( + case.clone(), + Observation::delivered(vec![OracleKind::FixtureState], Evidence::default()), + Duration::from_millis(7), + ); + let summary = validate_catalog( + std::slice::from_ref(&case), + std::slice::from_ref(&result), + None, + false, + ) + .expect("valid catalog"); + assert_eq!(summary.delivered, 1); + assert!(summary + .markdown_with_declarations(std::slice::from_ref(&case), std::slice::from_ref(&result),) + .contains("1 delivered")); + + let value = serde_json::to_value(result).expect("serialize result"); + assert_eq!(value["schema"], RESULT_SCHEMA); + assert_eq!(value["cell_id"], "rendered"); + assert!(value.get("case").is_none()); + } + + #[test] + fn declared_coverage_distinguishes_statuses_and_groups_harness_actions() { + let delivered = coverage_case( + "electron-click-ax-background", + "electron", + "click", + Targeting::Ax, + Delivery::Background, + ); + let mut refused = coverage_case( + "electron-click-px-background", + "electron", + "click", + Targeting::Px, + Delivery::Background, + ) + .expecting_refusal(vec![RefusalCode::BackgroundUnavailable]); + refused.oracles = vec![ + OracleKind::FixtureState, + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::NoLeakedInput, + ]; + let failed = coverage_case( + "electron-click-px-foreground", + "electron", + "click", + Targeting::Px, + Delivery::Foreground, + ); + let grouped = coverage_case( + "tauri-type-text-ax-background", + "tauri", + "type_text", + Targeting::Ax, + Delivery::Background, + ); + + let results = vec![ + CaseResult::evaluate( + delivered.clone(), + Observation::delivered(vec![OracleKind::FixtureState], Evidence::default()), + Duration::from_millis(1), + ), + CaseResult::evaluate( + refused.clone(), + Observation::refused( + RefusalCode::BackgroundUnavailable, + vec![ + OracleKind::FixtureState, + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::NoLeakedInput, + ], + "", + Evidence::default(), + ), + Duration::from_millis(1), + ), + CaseResult::evaluate( + failed.clone(), + Observation::error("driver error", Evidence::default()), + Duration::from_millis(1), + ), + CaseResult::evaluate( + grouped.clone(), + Observation::delivered(vec![OracleKind::FixtureState], Evidence::default()), + Duration::from_millis(1), + ), + ]; + let declarations = vec![delivered, refused, failed, grouped]; + let summary = + validate_catalog(&declarations, &results, None, false).expect("valid catalog"); + let markdown = summary.markdown_with_declarations(&declarations, &results); + + assert!(markdown.contains("| electron | click | PASS | - | REFUSED | FAIL | - | - |")); + assert!(markdown.contains("| tauri | type_text | PASS | - | - | - | - | - |")); + } + + #[test] + fn summary_records_the_exact_source_sha() { + let case = delivered_case("source-sha"); + let result = CaseResult::evaluate( + case.clone(), + Observation::delivered(vec![OracleKind::FixtureState], Evidence::default()), + Duration::from_millis(1), + ); + let summary = validate_catalog( + std::slice::from_ref(&case), + std::slice::from_ref(&result), + None, + false, + ) + .expect("valid catalog"); + let sha = "0123456789abcdef0123456789abcdef01234567"; + let markdown = summary.markdown_with_declarations_and_source( + std::slice::from_ref(&case), + std::slice::from_ref(&result), + Some(sha), + ); + + assert!(markdown.contains(&format!("**Source SHA:** `{sha}`"))); + } + + #[test] + fn declared_coverage_keeps_page_and_not_applicable_out_of_the_ax_px_grid() { + let page = coverage_case( + "web-evaluate-page-background", + "web", + "evaluate", + Targeting::Page, + Delivery::Background, + ); + let not_applicable = coverage_case( + "web-evaluate-not-applicable-background", + "web", + "evaluate", + Targeting::NotApplicable, + Delivery::Background, + ); + let results = [page.clone(), not_applicable.clone()] + .into_iter() + .map(|case| { + CaseResult::evaluate( + case, + Observation::delivered(vec![OracleKind::FixtureState], Evidence::default()), + Duration::from_millis(1), + ) + }) + .collect::>(); + let declarations = vec![page, not_applicable]; + let summary = + validate_catalog(&declarations, &results, None, false).expect("valid catalog"); + let markdown = summary.markdown_with_declarations(&declarations, &results); + + assert!(markdown.contains( + "| web | evaluate | - | - | - | - | Background: PASS | NotApplicable/Background: PASS |" + )); + } + + #[test] + fn native_case_builders_encode_delivery_and_oracle_contracts() { + let background = + native_background_case("wpf", "left_click", Targeting::Ax, DriverRoute::UiaInvoke); + assert_eq!(background.delivery, Delivery::Background); + assert_eq!( + background.cell_id, + format!("{}-wpf-left-click-ax-background", std::env::consts::OS) + ); + for oracle in [ + OracleKind::FixtureState, + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::Cursor, + OracleKind::NoLeakedInput, + ] { + assert!(background.oracles.contains(&oracle)); + } + background.validate().expect("background case is valid"); + + let foreground = native_foreground_case( + "wpf", + "right_click", + Targeting::Ax, + DriverRoute::WindowsSendInput, + ); + assert_eq!(foreground.delivery, Delivery::Foreground); + assert_eq!(foreground.oracles, vec![OracleKind::FixtureState]); + + let readonly = native_readonly_case( + "wpf", + "ax_tree", + Targeting::Ax, + DriverRoute::AxRead, + vec![OracleKind::AxState], + ); + assert_eq!(readonly.delivery, Delivery::NotApplicable); + assert_eq!(readonly.oracles, vec![OracleKind::AxState]); + readonly.validate().expect("read-only case is valid"); + } + + #[test] + fn delivered_with_fixture_state_deduplicates_the_oracle() { + let observation = Observation::delivered_with_fixture_state(vec![ + OracleKind::Focus, + OracleKind::FixtureState, + ]); + assert_eq!( + observation.passed_oracles, + vec![OracleKind::FixtureState, OracleKind::Focus] + ); + } + + #[test] + fn shared_routes_are_explicit_for_the_current_matrix() { + let mut cells = Vec::new(); + for action in [ + "left_click", + "right_click", + "double_click", + "type_text", + "press_key", + "hotkey", + "scroll", + "child_window", + ] { + for targeting in [Targeting::Ax, Targeting::Px] { + for delivery in [Delivery::Background, Delivery::Foreground] { + cells.push((action, targeting, delivery)); + } + } + } + for delivery in [Delivery::Background, Delivery::Foreground] { + cells.push(("drag", Targeting::Px, delivery)); + cells.push(("editor_save", Targeting::Ax, delivery)); + } + assert_eq!(cells.len(), 36); + for (platform, display_server) in [ + (Platform::Windows, DisplayServer::Win32), + (Platform::Macos, DisplayServer::Quartz), + (Platform::Linux, DisplayServer::X11), + (Platform::Linux, DisplayServer::Wayland), + ] { + for (action, targeting, delivery) in cells.iter().copied() { + shared_web_route(platform, display_server, action, targeting, delivery) + .unwrap_or_else(|error| panic!("{error}")); + } + } + } + + #[test] + fn windows_pixel_background_route_remains_targeted_injection() { + assert_eq!( + shared_web_route( + Platform::Windows, + DisplayServer::Win32, + "left_click", + Targeting::Px, + Delivery::Background, + ), + Ok(DriverRoute::WindowsTargetedInjection) + ); + } + + #[test] + fn nested_wayland_pixel_route_is_distinct_from_stock_wayland() { + let stock = shared_web_route_for_environment( + Platform::Linux, + DisplayServer::Wayland, + "left_click", + Targeting::Px, + Delivery::Background, + false, + ); + let nested = shared_web_route_for_environment( + Platform::Linux, + DisplayServer::Wayland, + "left_click", + Targeting::Px, + Delivery::Background, + true, + ); + assert_eq!(stock, Ok(DriverRoute::LinuxWaylandVirtualPointer)); + assert_eq!(nested, Ok(DriverRoute::LinuxCuaCompositorInject)); + + let nested_text = shared_web_route_for_environment( + Platform::Linux, + DisplayServer::Wayland, + "type_text", + Targeting::Ax, + Delivery::Background, + true, + ); + assert_eq!(nested_text, Ok(DriverRoute::LinuxCuaCompositorInject)); + } + + #[test] + fn environment_v2_artifacts_remain_readable() { + let record: EnvironmentRecord = serde_json::from_value(serde_json::json!({ + "schema": ENVIRONMENT_SCHEMA_V2, + "platform": "linux", + "display_server": "wayland", + "source_sha": null, + "status": "ready", + "duration_ms": 1, + "message": "" + })) + .expect("v2 environment record should deserialize"); + assert!(environment_schema_supported(&record.schema)); + assert_eq!(record.compositor, None); + assert!(record.input_backends.is_empty()); + } +} diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/journal.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/journal.rs new file mode 100644 index 0000000000..b9486f1dc9 --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/journal.rs @@ -0,0 +1,174 @@ +use std::io::{Read, Write}; +use std::net::TcpListener; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::Duration; + +/// Fixture-owned state receiver used as an oracle independently of cua-driver. +/// +/// Web fixtures POST their current DOM state to this loopback endpoint. Tests +/// may still use AX or pixels to target an action, but delivery is judged from +/// this state rather than by reading the driver's own accessibility snapshot. +pub struct FixtureJournal { + url: String, + latest: Arc>, + stop: Arc, + worker: Option>, +} + +impl FixtureJournal { + pub fn start() -> Self { + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("bind fixture journal"); + listener + .set_nonblocking(true) + .expect("make fixture journal nonblocking"); + let address = listener.local_addr().expect("read fixture journal address"); + let latest = Arc::new(Mutex::new(serde_json::json!({}))); + let stop = Arc::new(AtomicBool::new(false)); + let latest_for_worker = Arc::clone(&latest); + let stop_for_worker = Arc::clone(&stop); + let worker = thread::spawn(move || { + while !stop_for_worker.load(Ordering::Relaxed) { + match listener.accept() { + Ok((mut stream, _)) => { + let _ = stream.set_read_timeout(Some(Duration::from_secs(1))); + let mut request = Vec::new(); + let mut chunk = [0u8; 8192]; + let mut body_range = None; + loop { + match stream.read(&mut chunk) { + Ok(0) => break, + Ok(n) => request.extend_from_slice(&chunk[..n]), + Err(error) + if matches!( + error.kind(), + std::io::ErrorKind::WouldBlock + | std::io::ErrorKind::TimedOut + ) => + { + break; + } + Err(_) => break, + } + if body_range.is_none() { + body_range = request + .windows(4) + .position(|window| window == b"\r\n\r\n") + .map(|header_end| { + let headers = + String::from_utf8_lossy(&request[..header_end]); + let content_len = headers + .lines() + .find_map(|line| { + line.split_once(':').and_then(|(name, value)| { + name.eq_ignore_ascii_case("content-length") + .then_some(value) + }) + }) + .and_then(|value| value.trim().parse::().ok()) + .unwrap_or(0); + (header_end + 4, content_len) + }); + } + if let Some((body_start, content_len)) = body_range { + if request.len() >= body_start + content_len { + break; + } + } + } + if let Some((body_start, content_len)) = body_range { + if let Some(body) = request.get(body_start..body_start + content_len) { + if let Ok(state) = serde_json::from_slice(body) { + *latest_for_worker.lock().expect("lock fixture journal") = + state; + } + } + } + let _ = stream.write_all( + b"HTTP/1.1 204 No Content\r\nAccess-Control-Allow-Origin: *\r\nConnection: close\r\n\r\n", + ); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(10)); + } + Err(_) => break, + } + } + }); + Self { + url: format!("http://{address}/state"), + latest, + stop, + worker: Some(worker), + } + } + + pub fn url(&self) -> &str { + &self.url + } + + pub fn contains(&self, marker: &str) -> bool { + self.latest + .lock() + .expect("lock fixture journal") + .to_string() + .contains(marker) + } + + pub fn text(&self, id: &str) -> Option { + self.latest + .lock() + .expect("lock fixture journal") + .get(id) + .and_then(|entry| entry.get("text")) + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + } + + pub fn snapshot(&self) -> serde_json::Value { + self.latest.lock().expect("lock fixture journal").clone() + } +} + +impl Drop for FixtureJournal { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +#[cfg(test)] +mod tests { + use super::FixtureJournal; + use std::io::Write; + use std::net::TcpStream; + use std::time::{Duration, Instant}; + + #[test] + fn receives_fixture_state_over_loopback() { + let journal = FixtureJournal::start(); + let body = r#"{"status":{"text":"last_action=left_click"}}"#; + let address = journal + .url() + .strip_prefix("http://") + .and_then(|url| url.strip_suffix("/state")) + .expect("journal URL shape"); + let mut stream = TcpStream::connect(address).expect("connect fixture journal"); + write!( + stream, + "POST /state HTTP/1.1\r\nHost: {address}\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ) + .expect("post fixture state"); + drop(stream); + + let deadline = Instant::now() + Duration::from_secs(1); + while !journal.contains("last_action=left_click") && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(10)); + } + assert!(journal.contains("last_action=left_click")); + } +} diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/lib.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/lib.rs index 504eb22c50..547133d985 100644 --- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/lib.rs +++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/lib.rs @@ -35,14 +35,20 @@ pub mod ax; mod driver; +pub mod e2e; +mod journal; mod mcp; mod cli; +pub mod observer; mod paths; mod raw; mod reaper; mod response; +pub mod sentinel; +mod windows_setup; -pub use driver::Driver; +pub use driver::{BehaviorRecording, Driver}; +pub use journal::FixtureJournal; pub use mcp::McpDriver; pub use raw::RawDriver; pub use cli::CliDriver; diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/mcp.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/mcp.rs index 50f2d1441a..9eb4243853 100644 --- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/mcp.rs +++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/mcp.rs @@ -9,7 +9,7 @@ use std::time::{Duration, Instant}; use serde_json::Value; -use crate::driver::Driver; +use crate::driver::{BehaviorRecording, Driver}; use crate::paths::driver_binary; use crate::reaper::{spawn_in_job, ChildReaper}; use crate::response::ToolResponse; @@ -27,6 +27,7 @@ pub struct McpDriver { rx: Receiver, next_id: u32, recording_dir: Option, + recording_started: bool, } static RECORDING_SEQUENCE: AtomicU64 = AtomicU64::new(1); @@ -44,6 +45,11 @@ impl McpDriver { Self::spawn_internal(&[], Some(recording_label)) } + /// Spawn a named driver with environment variables scoped to this child. + pub fn spawn_named_with_env(recording_label: &str, env: &[(&str, &str)]) -> Option { + Self::spawn_internal(env, Some(recording_label)) + } + /// Spawn the driver with extra environment variables set on the child. pub fn spawn_with_env(env: &[(&str, &str)]) -> Option { Self::spawn_internal(env, None) @@ -58,9 +64,14 @@ impl McpDriver { let mut reaper = ChildReaper::new(); let mut cmd = Command::new(&bin); + let stderr = if std::env::var_os("CUA_TEST_DRIVER_STDERR").is_some() { + Stdio::inherit() + } else { + Stdio::null() + }; cmd.stdin(Stdio::piped()) .stdout(Stdio::piped()) - .stderr(Stdio::null()); + .stderr(stderr); for (key, value) in env { cmd.env(key, value); } @@ -94,9 +105,10 @@ impl McpDriver { rx, next_id: 2, recording_dir: None, + recording_started: false, }; d.initialize(); - d.start_e2e_recording(recording_label); + d.prepare_e2e_recording(recording_label); Some(d) } @@ -144,7 +156,7 @@ impl McpDriver { let _ = self.stdin.flush(); } - fn start_e2e_recording(&mut self, explicit_label: Option<&str>) { + fn prepare_e2e_recording(&mut self, explicit_label: Option<&str>) { let Some(root) = std::env::var_os("CUA_E2E_RECORDINGS_ROOT") else { return; }; @@ -156,6 +168,50 @@ impl McpDriver { let label = recording_label(explicit_label.unwrap_or(&thread_name)); let output_dir = PathBuf::from(root).join(format!("{label}-pid{}-{sequence:03}", std::process::id())); + std::fs::create_dir_all(&output_dir).unwrap_or_else(|error| { + panic!("could not prepare E2E recording directory for {thread_name}: {error}") + }); + let runner_console = crate::windows_setup::minimize_hosted_runner_console(); + let runner_console_status = runner_console.as_deref().unwrap_or("error"); + let manifest = serde_json::json!({ + "schema": "cua-e2e-trajectory/v1", + "label": label, + "rust_test_thread": thread_name, + "process_id": std::process::id(), + "sequence": sequence, + "behavior_video": { + "status": "pending", + "baseline_settle_ms": 300 + }, + "hosted_runner_console": { + "status": runner_console_status + } + }); + std::fs::write( + output_dir.join("trajectory.json"), + serde_json::to_vec_pretty(&manifest).unwrap_or_default(), + ) + .expect("write prepared E2E trajectory manifest"); + eprintln!( + "[testkit] prepared E2E evidence directory {}", + output_dir.display() + ); + self.recording_dir = Some(output_dir); + if let Err(error) = runner_console { + panic!("hosted-runner console cleanup failed before fixture setup: {error}"); + } + } + + /// Begin the behavioral clip after fixture readiness and foreground or + /// background posture have been established. The settle interval gives + /// the video backend time to encode a visible baseline before dispatch. + pub fn start_behavior_recording(&mut self) { + let Some(output_dir) = self.recording_dir.clone() else { + return; + }; + if self.recording_started { + return; + } let response = self.call( "start_recording", serde_json::json!({ @@ -167,30 +223,33 @@ impl McpDriver { .as_bool() .unwrap_or(false); if response.is_error() || !video_active { + update_behavior_video_status(&output_dir, "error"); panic!( - "E2E video recording did not start for {thread_name}: {}", - response.text() + "E2E behavioral video did not start after setup: {}; structured={}", + response.text(), + response.structured() ); } - let manifest = serde_json::json!({ - "schema": "cua-e2e-trajectory/v1", - "label": label, - "rust_test_thread": thread_name, - "process_id": std::process::id(), - "sequence": sequence - }); - let _ = std::fs::write( - output_dir.join("trajectory.json"), - serde_json::to_vec_pretty(&manifest).unwrap_or_default(), + self.recording_started = true; + update_behavior_video_status(&output_dir, "started"); + std::thread::sleep(Duration::from_millis(300)); + mark_behavior_video_baseline_ready(&output_dir); + eprintln!( + "[testkit] behavioral E2E video started at {}", + output_dir.display() ); - eprintln!("[testkit] recording E2E video to {}", output_dir.display()); - self.recording_dir = Some(output_dir); } fn stop_e2e_recording(&mut self) { let Some(output_dir) = self.recording_dir.take() else { return; }; + if !self.recording_started { + let message = "E2E behavioral video boundary was never reached; fixture setup or posture failed before capture"; + eprintln!("[testkit] {message}"); + let _ = std::fs::write(output_dir.join("recording-error.txt"), message); + return; + } let response = self.call("stop_recording", serde_json::json!({})); let video_path = output_dir.join("recording.mp4"); let valid_video = !response.is_error() @@ -199,6 +258,7 @@ impl McpDriver { .map(|metadata| metadata.len() > 0) .unwrap_or(false); if valid_video { + update_behavior_video_status(&output_dir, "finalized"); eprintln!("[testkit] finalized E2E video at {}", video_path.display()); return; } @@ -209,6 +269,7 @@ impl McpDriver { response.text() ); eprintln!("[testkit] {message}"); + update_behavior_video_status(&output_dir, "error"); let _ = std::fs::create_dir_all(&output_dir); let _ = std::fs::write(output_dir.join("recording-error.txt"), message); } @@ -219,6 +280,11 @@ impl McpDriver { &mut self.reaper } + /// Directory for this driver's active per-cell recording, when enabled. + pub fn recording_dir(&self) -> Option<&std::path::Path> { + self.recording_dir.as_deref() + } + /// Poll `list_windows` until a window of `pid` whose title contains /// `title_substr` appears (up to ~12s). Returns `(window_id, title)`. /// Replaces the per-file `find_harness_window` helper. @@ -287,6 +353,58 @@ fn recording_label(name: &str) -> String { } } +impl Driver for McpDriver { + fn call(&mut self, tool: &str, args: Value) -> ToolResponse { + ToolResponse::from_mcp(self.call_raw(tool, args)) + } +} + +impl BehaviorRecording for McpDriver { + fn start_behavior_recording(&mut self) { + McpDriver::start_behavior_recording(self); + } +} + +fn update_behavior_video_status(output_dir: &std::path::Path, status: &str) { + let path = output_dir.join("trajectory.json"); + let Ok(bytes) = std::fs::read(&path) else { + return; + }; + let Ok(mut manifest) = serde_json::from_slice::(&bytes) else { + return; + }; + manifest["behavior_video"]["status"] = Value::String(status.to_owned()); + let timestamp_field = match status { + "started" => Some("started_at_unix_ms"), + "finalized" => Some("finalized_at_unix_ms"), + "error" => Some("error_at_unix_ms"), + _ => None, + }; + if let Some(field) = timestamp_field { + manifest["behavior_video"][field] = Value::from(unix_ms()); + } + let _ = std::fs::write(path, serde_json::to_vec_pretty(&manifest).unwrap_or_default()); +} + +fn mark_behavior_video_baseline_ready(output_dir: &std::path::Path) { + let path = output_dir.join("trajectory.json"); + let Ok(bytes) = std::fs::read(&path) else { + return; + }; + let Ok(mut manifest) = serde_json::from_slice::(&bytes) else { + return; + }; + manifest["behavior_video"]["baseline_ready_at_unix_ms"] = Value::from(unix_ms()); + let _ = std::fs::write(path, serde_json::to_vec_pretty(&manifest).unwrap_or_default()); +} + +fn unix_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64 +} + #[cfg(test)] mod tests { use super::recording_label; @@ -300,9 +418,3 @@ mod tests { assert_eq!(recording_label("///"), "unnamed-test"); } } - -impl Driver for McpDriver { - fn call(&mut self, tool: &str, args: Value) -> ToolResponse { - ToolResponse::from_mcp(self.call_raw(tool, args)) - } -} diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/observer.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/observer.rs new file mode 100644 index 0000000000..90f2ce9fc7 --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/observer.rs @@ -0,0 +1,1736 @@ +//! Driver-independent desktop side-effect observations for E2E tests. + +use std::fmt; +use std::time::Duration; + +use crate::e2e::OracleKind; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct TargetWindow { + pub pid: u32, + pub native_id: u64, +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct ObserverCapabilities { + pub focus: bool, + pub z_order: bool, + pub cursor: bool, + pub leaked_input: bool, +} + +impl ObserverCapabilities { + fn supports(self, oracle: OracleKind) -> bool { + match oracle { + OracleKind::Focus => self.focus, + OracleKind::ZOrder => self.z_order, + OracleKind::Cursor => self.cursor, + OracleKind::NoLeakedInput => self.leaked_input, + _ => true, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TargetZ { + BackgroundOccluded, + BackgroundVisible, + Foreground, + Minimized, + NotFound, +} + +impl TargetZ { + fn rank(self) -> Option { + match self { + Self::BackgroundOccluded => Some(0), + Self::BackgroundVisible => Some(1), + Self::Foreground => Some(2), + Self::Minimized | Self::NotFound => None, + } + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct DesktopSnapshot { + pub foreground: Option, + pub input_focus: Option, + pub target_z: TargetZ, + pub cursor_pos: Option<(f64, f64)>, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct FocusEvent { + pub from: Option, + pub to: Option, +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct DesktopJournal { + pub focus_events: Vec, + pub leaked_input_events: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ObserverError { + message: String, +} + +impl ObserverError { + pub fn new(message: impl Into) -> Self { + Self { + message: message.into(), + } + } +} + +impl fmt::Display for ObserverError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.message) + } +} + +impl std::error::Error for ObserverError {} + +pub trait ObserverBackend { + fn capabilities(&self) -> ObserverCapabilities; + fn snapshot(&self, target: TargetWindow) -> Result; + fn start_journal(&mut self) -> Result<(), ObserverError>; + fn drain_journal(&mut self) -> Result; +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ObservationDelta { + pub before: DesktopSnapshot, + pub after: DesktopSnapshot, + pub journal: DesktopJournal, + passed: Vec, + unsupported: Vec, + violations: Vec, +} + +impl ObservationDelta { + pub fn passed(&self) -> &[OracleKind] { + &self.passed + } + + pub fn unsupported(&self) -> &[OracleKind] { + &self.unsupported + } + + pub fn violations(&self) -> &[String] { + &self.violations + } + + pub fn ensure_supported(&self) -> Result<(), ObserverError> { + if self.unsupported.is_empty() { + Ok(()) + } else { + Err(ObserverError::new(format!( + "desktop observer does not support required oracles: {:?}", + self.unsupported + ))) + } + } +} + +pub struct DesktopObserver { + backend: B, + target: TargetWindow, + settle: Duration, +} + +impl DesktopObserver { + pub fn new(backend: B, target: TargetWindow) -> Self { + Self { + backend, + target, + settle: Duration::from_millis(150), + } + } + + pub fn with_settle(mut self, settle: Duration) -> Self { + self.settle = settle; + self + } + + pub fn snapshot(&self) -> Result { + self.backend.snapshot(self.target) + } + + pub fn observe( + &mut self, + requested: &[OracleKind], + action: impl FnOnce() -> R, + ) -> Result<(R, ObservationDelta), ObserverError> { + let before = self.backend.snapshot(self.target)?; + self.backend.start_journal()?; + let result = action(); + if !self.settle.is_zero() { + std::thread::sleep(self.settle); + } + let journal = self.backend.drain_journal()?; + let after = self.backend.snapshot(self.target)?; + let delta = evaluate( + self.backend.capabilities(), + requested, + before, + after, + journal, + ); + Ok((result, delta)) + } +} + +fn evaluate( + capabilities: ObserverCapabilities, + requested: &[OracleKind], + before: DesktopSnapshot, + after: DesktopSnapshot, + journal: DesktopJournal, +) -> ObservationDelta { + let mut passed = Vec::new(); + let mut unsupported = Vec::new(); + let mut violations = Vec::new(); + + for oracle in requested.iter().copied() { + if !capabilities.supports(oracle) { + unsupported.push(oracle); + continue; + } + let violation = match oracle { + OracleKind::Focus => { + if before.foreground != after.foreground { + Some(format!( + "foreground changed from {:?} to {:?}", + before.foreground, after.foreground + )) + } else if before.input_focus != after.input_focus { + Some(format!( + "input focus changed from {:?} to {:?}", + before.input_focus, after.input_focus + )) + } else if !journal.focus_events.is_empty() { + Some(format!( + "foreground changed transiently: {:?}", + journal.focus_events + )) + } else { + None + } + } + OracleKind::ZOrder => match (before.target_z.rank(), after.target_z.rank()) { + (Some(before_rank), Some(after_rank)) if after_rank <= before_rank => None, + (Some(_), Some(_)) => Some(format!( + "target rose from {:?} to {:?}", + before.target_z, after.target_z + )), + _ => Some(format!( + "target z-order could not be compared: {:?} -> {:?}", + before.target_z, after.target_z + )), + }, + OracleKind::Cursor => match (before.cursor_pos, after.cursor_pos) { + (Some((before_x, before_y)), Some((after_x, after_y))) + if (before_x - after_x).abs() <= 1.0 && (before_y - after_y).abs() <= 1.0 => + { + None + } + (Some(before_pos), Some(after_pos)) => Some(format!( + "real cursor moved from {before_pos:?} to {after_pos:?}" + )), + _ => Some("real cursor position was unavailable".to_owned()), + }, + OracleKind::NoLeakedInput => { + if journal.leaked_input_events.is_empty() { + None + } else { + Some(format!( + "foreground sentinel received input: {:?}", + journal.leaked_input_events + )) + } + } + _ => continue, + }; + if let Some(violation) = violation { + violations.push(violation); + } else { + passed.push(oracle); + } + } + + ObservationDelta { + before, + after, + journal, + passed, + unsupported, + violations, + } +} + +#[cfg(target_os = "windows")] +pub mod windows { + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; + use std::thread::JoinHandle; + use std::time::Duration; + + use windows::Win32::Foundation::{HWND, POINT, RECT}; + use windows::Win32::UI::WindowsAndMessaging::{ + GetAncestor, GetCursorPos, GetForegroundWindow, GetWindowRect, IsIconic, IsWindow, + WindowFromPoint, GA_ROOT, + }; + + use super::{ + DesktopJournal, DesktopSnapshot, FocusEvent, ObserverBackend, ObserverCapabilities, + ObserverError, TargetWindow, TargetZ, + }; + + pub struct WindowsObserver { + stop: Arc, + events: Arc>>, + sampler: Option>, + } + + impl WindowsObserver { + pub fn new() -> Self { + Self { + stop: Arc::new(AtomicBool::new(false)), + events: Arc::new(Mutex::new(Vec::new())), + sampler: None, + } + } + } + + impl Default for WindowsObserver { + fn default() -> Self { + Self::new() + } + } + + impl ObserverBackend for WindowsObserver { + fn capabilities(&self) -> ObserverCapabilities { + ObserverCapabilities { + focus: true, + z_order: true, + cursor: true, + leaked_input: false, + } + } + + fn snapshot(&self, target: TargetWindow) -> Result { + let hwnd = HWND(target.native_id as *mut _); + let target_z = unsafe { + if !IsWindow(Some(hwnd)).as_bool() { + TargetZ::NotFound + } else if IsIconic(hwnd).as_bool() { + TargetZ::Minimized + } else if root(GetForegroundWindow()) == root(hwnd) { + TargetZ::Foreground + } else if is_occluded(hwnd)? { + TargetZ::BackgroundOccluded + } else { + TargetZ::BackgroundVisible + } + }; + let foreground = unsafe { raw(root(GetForegroundWindow())) }; + let mut cursor = POINT::default(); + let cursor_pos = unsafe { + GetCursorPos(&mut cursor) + .ok() + .map(|_| (f64::from(cursor.x), f64::from(cursor.y))) + }; + Ok(DesktopSnapshot { + foreground, + input_focus: foreground, + target_z, + cursor_pos, + }) + } + + fn start_journal(&mut self) -> Result<(), ObserverError> { + if self.sampler.is_some() { + return Err(ObserverError::new("Windows focus journal already active")); + } + self.stop.store(false, Ordering::Release); + self.events.lock().expect("focus journal lock").clear(); + let stop = Arc::clone(&self.stop); + let events = Arc::clone(&self.events); + self.sampler = Some(std::thread::spawn(move || { + let mut previous = unsafe { raw(root(GetForegroundWindow())) }; + while !stop.load(Ordering::Acquire) { + let current = unsafe { raw(root(GetForegroundWindow())) }; + if current != previous { + events.lock().expect("focus journal lock").push(FocusEvent { + from: previous, + to: current, + }); + previous = current; + } + std::thread::sleep(Duration::from_millis(10)); + } + })); + Ok(()) + } + + fn drain_journal(&mut self) -> Result { + self.stop.store(true, Ordering::Release); + if let Some(sampler) = self.sampler.take() { + sampler + .join() + .map_err(|_| ObserverError::new("Windows focus journal panicked"))?; + } + Ok(DesktopJournal { + focus_events: self.events.lock().expect("focus journal lock").clone(), + leaked_input_events: Vec::new(), + }) + } + } + + impl Drop for WindowsObserver { + fn drop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(sampler) = self.sampler.take() { + let _ = sampler.join(); + } + } + } + + unsafe fn root(hwnd: HWND) -> HWND { + if hwnd.0.is_null() { + hwnd + } else { + unsafe { GetAncestor(hwnd, GA_ROOT) } + } + } + + fn raw(hwnd: HWND) -> Option { + (!hwnd.0.is_null()).then_some(hwnd.0 as usize as u64) + } + + unsafe fn is_occluded(hwnd: HWND) -> Result { + let mut rect = RECT::default(); + unsafe { GetWindowRect(hwnd, &mut rect) } + .map_err(|error| ObserverError::new(format!("GetWindowRect failed: {error}")))?; + if rect.right - rect.left <= 4 || rect.bottom - rect.top <= 4 { + return Ok(false); + } + let points = [ + POINT { + x: rect.left + 2, + y: rect.top + 2, + }, + POINT { + x: rect.right - 3, + y: rect.top + 2, + }, + POINT { + x: rect.left + 2, + y: rect.bottom - 3, + }, + POINT { + x: rect.right - 3, + y: rect.bottom - 3, + }, + POINT { + x: (rect.left + rect.right) / 2, + y: (rect.top + rect.bottom) / 2, + }, + ]; + let target_root = unsafe { root(hwnd) }; + let covered = points + .into_iter() + .filter(|point| { + let owner = unsafe { WindowFromPoint(*point) }; + !owner.0.is_null() && unsafe { root(owner) } != target_root + }) + .count(); + Ok(covered >= 2) + } +} + +#[cfg(target_os = "macos")] +pub mod macos { + use std::ffi::c_void; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; + use std::thread::JoinHandle; + use std::time::Duration; + + use core_foundation::array::{CFArray, CFArrayRef}; + use core_foundation::base::{CFGetTypeID, CFTypeRef, TCFType}; + use core_foundation::boolean::CFBoolean; + use core_foundation::dictionary::CFDictionary; + use core_foundation::number::CFNumber; + use core_foundation::string::CFString; + use objc2_app_kit::NSWorkspace; + + use super::{ + DesktopJournal, DesktopSnapshot, FocusEvent, ObserverBackend, ObserverCapabilities, + ObserverError, TargetWindow, TargetZ, + }; + + const WINDOW_LIST_ON_SCREEN: u32 = 1; + const WINDOW_LIST_EXCLUDE_DESKTOP: u32 = 16; + + #[link(name = "CoreGraphics", kind = "framework")] + unsafe extern "C" { + fn CGWindowListCopyWindowInfo(option: u32, relative_to_window: u32) -> CFArrayRef; + fn CGEventCreate(source: *mut c_void) -> *mut c_void; + fn CGEventGetLocation(event: *mut c_void) -> CGPoint; + } + + #[link(name = "CoreFoundation", kind = "framework")] + unsafe extern "C" { + fn CFRelease(value: *const c_void); + } + + #[repr(C)] + struct CGPoint { + x: f64, + y: f64, + } + + #[derive(Clone, Copy)] + struct Bounds { + x: f64, + y: f64, + width: f64, + height: f64, + } + + impl Bounds { + fn area(self) -> f64 { + self.width.max(0.0) * self.height.max(0.0) + } + + fn intersection_area(self, other: Self) -> f64 { + let left = self.x.max(other.x); + let top = self.y.max(other.y); + let right = (self.x + self.width).min(other.x + other.width); + let bottom = (self.y + self.height).min(other.y + other.height); + (right - left).max(0.0) * (bottom - top).max(0.0) + } + } + + struct WindowRow { + id: u64, + pid: u32, + on_screen: bool, + bounds: Bounds, + } + + pub struct MacosObserver { + stop: Arc, + events: Arc>>, + sampler: Option>, + } + + impl MacosObserver { + pub fn new() -> Self { + Self { + stop: Arc::new(AtomicBool::new(false)), + events: Arc::new(Mutex::new(Vec::new())), + sampler: None, + } + } + } + + impl Default for MacosObserver { + fn default() -> Self { + Self::new() + } + } + + impl ObserverBackend for MacosObserver { + fn capabilities(&self) -> ObserverCapabilities { + ObserverCapabilities { + focus: true, + z_order: true, + cursor: true, + leaked_input: false, + } + } + + fn snapshot(&self, target: TargetWindow) -> Result { + let rows = window_rows(); + let target_index = rows.iter().position(|row| row.id == target.native_id); + let target_z = match target_index { + None => TargetZ::NotFound, + Some(index) if !rows[index].on_screen => TargetZ::Minimized, + Some(_) if frontmost_pid() == Some(target.pid as u64) => TargetZ::Foreground, + Some(index) => { + let target_bounds = rows[index].bounds; + let meaningful_cover = rows[..index].iter().any(|row| { + row.pid != target.pid + && target_bounds.intersection_area(row.bounds) + >= target_bounds.area() * 0.10 + }); + if meaningful_cover { + TargetZ::BackgroundOccluded + } else { + TargetZ::BackgroundVisible + } + } + }; + let foreground = frontmost_pid(); + Ok(DesktopSnapshot { + foreground, + input_focus: foreground, + target_z, + cursor_pos: cursor_position(), + }) + } + + fn start_journal(&mut self) -> Result<(), ObserverError> { + if self.sampler.is_some() { + return Err(ObserverError::new("macOS focus journal already active")); + } + self.stop.store(false, Ordering::Release); + self.events.lock().expect("focus journal lock").clear(); + let stop = Arc::clone(&self.stop); + let events = Arc::clone(&self.events); + self.sampler = Some(std::thread::spawn(move || { + let mut previous = frontmost_pid(); + while !stop.load(Ordering::Acquire) { + let current = frontmost_pid(); + if current != previous { + events.lock().expect("focus journal lock").push(FocusEvent { + from: previous, + to: current, + }); + previous = current; + } + std::thread::sleep(Duration::from_millis(10)); + } + })); + Ok(()) + } + + fn drain_journal(&mut self) -> Result { + self.stop.store(true, Ordering::Release); + if let Some(sampler) = self.sampler.take() { + sampler + .join() + .map_err(|_| ObserverError::new("macOS focus journal panicked"))?; + } + Ok(DesktopJournal { + focus_events: self.events.lock().expect("focus journal lock").clone(), + leaked_input_events: Vec::new(), + }) + } + } + + impl Drop for MacosObserver { + fn drop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(sampler) = self.sampler.take() { + let _ = sampler.join(); + } + } + } + + fn frontmost_pid() -> Option { + unsafe { + Some( + NSWorkspace::sharedWorkspace() + .frontmostApplication()? + .processIdentifier() as u64, + ) + } + } + + fn cursor_position() -> Option<(f64, f64)> { + unsafe { + let event = CGEventCreate(std::ptr::null_mut()); + if event.is_null() { + return None; + } + let point = CGEventGetLocation(event); + CFRelease(event); + Some((point.x, point.y)) + } + } + + fn window_rows() -> Vec { + let raw = unsafe { + CGWindowListCopyWindowInfo(WINDOW_LIST_ON_SCREEN | WINDOW_LIST_EXCLUDE_DESKTOP, 0) + }; + if raw.is_null() { + return Vec::new(); + } + let array: CFArray = unsafe { CFArray::wrap_under_create_rule(raw.cast()) }; + array + .iter() + .filter_map(|item| parse_window_row(*item)) + .collect() + } + + fn parse_window_row(item: CFTypeRef) -> Option { + let dictionary_type = CFDictionary::<*const c_void, *const c_void>::type_id(); + if unsafe { CFGetTypeID(item) } != dictionary_type { + return None; + } + let dictionary: CFDictionary<*const c_void, *const c_void> = + unsafe { CFDictionary::wrap_under_get_rule(item.cast()) }; + let id = number(&dictionary, "kCGWindowNumber")? as u64; + let pid = number(&dictionary, "kCGWindowOwnerPID")? as u32; + let on_screen = boolean(&dictionary, "kCGWindowIsOnscreen").unwrap_or(false); + let bounds = dictionary_value(&dictionary, "kCGWindowBounds") + .and_then(|value| { + if unsafe { CFGetTypeID(value) } != dictionary_type { + return None; + } + let bounds: CFDictionary<*const c_void, *const c_void> = + unsafe { CFDictionary::wrap_under_get_rule(value.cast()) }; + Some(Bounds { + x: number(&bounds, "X").unwrap_or(0) as f64, + y: number(&bounds, "Y").unwrap_or(0) as f64, + width: number(&bounds, "Width").unwrap_or(0) as f64, + height: number(&bounds, "Height").unwrap_or(0) as f64, + }) + }) + .unwrap_or(Bounds { + x: 0.0, + y: 0.0, + width: 0.0, + height: 0.0, + }); + Some(WindowRow { + id, + pid, + on_screen, + bounds, + }) + } + + fn dictionary_value( + dictionary: &CFDictionary<*const c_void, *const c_void>, + key: &str, + ) -> Option { + let key = CFString::new(key); + dictionary + .find(key.as_concrete_TypeRef().cast()) + .map(|value| (*value).cast()) + } + + fn number(dictionary: &CFDictionary<*const c_void, *const c_void>, key: &str) -> Option { + let value = dictionary_value(dictionary, key)?; + if unsafe { CFGetTypeID(value) } != CFNumber::type_id() { + return None; + } + unsafe { CFNumber::wrap_under_get_rule(value.cast()) }.to_i64() + } + + fn boolean(dictionary: &CFDictionary<*const c_void, *const c_void>, key: &str) -> Option { + let value = dictionary_value(dictionary, key)?; + if unsafe { CFGetTypeID(value) } != CFBoolean::type_id() { + return None; + } + Some(bool::from(unsafe { + CFBoolean::wrap_under_get_rule(value.cast()) + })) + } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn native_snapshot_reads_desktop_without_a_target() { + let snapshot = MacosObserver::new() + .snapshot(TargetWindow { + native_id: u64::MAX, + pid: u32::MAX, + }) + .expect("macOS desktop snapshot"); + assert_eq!(snapshot.target_z, TargetZ::NotFound); + assert!(snapshot.foreground.is_some()); + assert!(snapshot.cursor_pos.is_some()); + } + } +} + +#[cfg(target_os = "linux")] +pub mod linux { + use std::process::{Command, Stdio}; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; + use std::thread::JoinHandle; + use std::time::Duration; + + use serde::Deserialize; + use x11rb::connection::Connection; + use x11rb::protocol::xproto::{ + AtomEnum, ConnectionExt, MapState, Rectangle, Window, WindowClass, + }; + use x11rb::rust_connection::RustConnection; + + use super::{ + DesktopJournal, DesktopSnapshot, FocusEvent, ObserverBackend, ObserverCapabilities, + ObserverError, TargetWindow, TargetZ, + }; + + #[derive(Clone, Copy, Debug, Eq, PartialEq)] + enum SessionKind { + X11, + Sway, + Gnome, + CuaCompositor, + Missing, + } + + pub struct LinuxObserver { + session: SessionKind, + stop: Arc, + events: Arc>>, + sampler: Option>, + } + + impl LinuxObserver { + pub fn new() -> Self { + let explicit_wayland = std::env::var("XDG_SESSION_TYPE") + .map(|value| value.eq_ignore_ascii_case("wayland")) + .unwrap_or(false) + || std::env::var_os("WAYLAND_DISPLAY").is_some(); + let session = if explicit_wayland { + if cua_compositor_available() { + SessionKind::CuaCompositor + } else if sway_available() { + SessionKind::Sway + } else if gnome_windows().is_ok() { + SessionKind::Gnome + } else { + SessionKind::Missing + } + } else if std::env::var_os("DISPLAY").is_some() { + if x11_window_manager_ready() { + SessionKind::X11 + } else { + SessionKind::Missing + } + } else { + SessionKind::Missing + }; + Self { + session, + stop: Arc::new(AtomicBool::new(false)), + events: Arc::new(Mutex::new(Vec::new())), + sampler: None, + } + } + } + + impl Default for LinuxObserver { + fn default() -> Self { + Self::new() + } + } + + impl ObserverBackend for LinuxObserver { + fn capabilities(&self) -> ObserverCapabilities { + match self.session { + SessionKind::X11 => ObserverCapabilities { + focus: true, + z_order: true, + cursor: true, + leaked_input: false, + }, + SessionKind::Sway | SessionKind::Gnome | SessionKind::CuaCompositor => { + ObserverCapabilities { + focus: true, + z_order: true, + cursor: false, + leaked_input: false, + } + } + SessionKind::Missing => ObserverCapabilities::default(), + } + } + + fn snapshot(&self, target: TargetWindow) -> Result { + match self.session { + SessionKind::X11 => x11_snapshot(target), + SessionKind::Sway => sway_snapshot(target), + SessionKind::Gnome => gnome_snapshot(target), + SessionKind::CuaCompositor => cua_compositor_snapshot(target), + SessionKind::Missing => Ok(DesktopSnapshot { + foreground: None, + input_focus: None, + target_z: TargetZ::NotFound, + cursor_pos: None, + }), + } + } + + fn start_journal(&mut self) -> Result<(), ObserverError> { + if self.sampler.is_some() { + return Err(ObserverError::new("Linux focus journal already active")); + } + self.stop.store(false, Ordering::Release); + self.events.lock().expect("focus journal lock").clear(); + if self.session == SessionKind::Missing { + return Ok(()); + } + let session = self.session; + let stop = Arc::clone(&self.stop); + let events = Arc::clone(&self.events); + self.sampler = Some(std::thread::spawn(move || { + let focus_identity = || match session { + SessionKind::X11 => x11_focus_identity(), + SessionKind::Sway => sway_focus_identity(), + SessionKind::Gnome => gnome_focus_identity(), + SessionKind::CuaCompositor => cua_compositor_focus_identity(), + SessionKind::Missing => Ok(None), + }; + let mut previous = focus_identity().ok().flatten(); + while !stop.load(Ordering::Acquire) { + if let Ok(current) = focus_identity() { + if current != previous { + events.lock().expect("focus journal lock").push(FocusEvent { + from: previous, + to: current, + }); + previous = current; + } + } + std::thread::sleep(Duration::from_millis(10)); + } + })); + Ok(()) + } + + fn drain_journal(&mut self) -> Result { + self.stop.store(true, Ordering::Release); + if let Some(sampler) = self.sampler.take() { + sampler + .join() + .map_err(|_| ObserverError::new("Linux focus journal panicked"))?; + } + Ok(DesktopJournal { + focus_events: self.events.lock().expect("focus journal lock").clone(), + leaked_input_events: Vec::new(), + }) + } + } + + impl Drop for LinuxObserver { + fn drop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(sampler) = self.sampler.take() { + let _ = sampler.join(); + } + } + } + + fn x11_connection() -> Result<(RustConnection, usize), ObserverError> { + x11rb::connect(None) + .map_err(|error| ObserverError::new(format!("X11 connect failed: {error}"))) + } + + #[derive(Default)] + struct SwayTreeState { + focused: Option, + focused_workspace: Option, + focused_fullscreen: Option, + target: Option<(u64, bool, Option)>, + } + + fn sway_tree() -> Result { + let output = Command::new("swaymsg") + .args(["-r", "-t", "get_tree"]) + .output() + .map_err(|error| ObserverError::new(format!("swaymsg get_tree failed: {error}")))?; + if !output.status.success() { + return Err(ObserverError::new(format!( + "swaymsg get_tree exited with {}: {}", + output.status, + String::from_utf8_lossy(&output.stderr) + ))); + } + serde_json::from_slice(&output.stdout) + .map_err(|error| ObserverError::new(format!("invalid sway tree JSON: {error}"))) + } + + fn sway_available() -> bool { + std::env::var_os("SWAYSOCK").is_some_and(|value| !value.is_empty()) && sway_tree().is_ok() + } + + #[derive(Clone, Copy, Debug, Eq, PartialEq)] + struct CuaCompositorState { + focused_pid: Option, + target_z: TargetZ, + } + + fn parse_cua_compositor_state(line: &str) -> Result { + let mut fields = line.split_whitespace(); + if fields.next() != Some("state") { + return Err(ObserverError::new(format!( + "invalid cua-compositor observer response: {line:?}" + ))); + } + let focused_pid = fields + .next() + .ok_or_else(|| ObserverError::new("cua-compositor state omitted focused pid"))? + .parse::() + .map_err(|error| ObserverError::new(format!("invalid focused pid: {error}")))?; + let target_z = match fields.next() { + Some("foreground") => TargetZ::Foreground, + Some("background_occluded") => TargetZ::BackgroundOccluded, + Some("background_visible") => TargetZ::BackgroundVisible, + Some("not_found") => TargetZ::NotFound, + Some(value) => { + return Err(ObserverError::new(format!( + "invalid cua-compositor target state: {value}" + ))) + } + None => { + return Err(ObserverError::new( + "cua-compositor state omitted target state", + )) + } + }; + if fields.next().is_some() { + return Err(ObserverError::new( + "cua-compositor state contained trailing fields", + )); + } + Ok(CuaCompositorState { + focused_pid: (focused_pid != 0).then_some(focused_pid), + target_z, + }) + } + + fn cua_compositor_state(target_pid: u32) -> Result { + use std::io::{BufRead, BufReader, Write}; + use std::os::unix::net::UnixStream; + + let socket = std::env::var("CUA_INJECT_SOCKET") + .map_err(|_| ObserverError::new("CUA_INJECT_SOCKET is not set"))?; + let stream = UnixStream::connect(&socket).map_err(|error| { + ObserverError::new(format!("connect cua-compositor observer socket: {error}")) + })?; + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .map_err(|error| ObserverError::new(format!("set observer timeout: {error}")))?; + let mut writer = stream + .try_clone() + .map_err(|error| ObserverError::new(format!("clone observer socket: {error}")))?; + let mut reader = BufReader::new(stream); + + writeln!(writer, "cua-inject v1") + .and_then(|_| writer.flush()) + .map_err(|error| ObserverError::new(format!("write observer handshake: {error}")))?; + let mut line = String::new(); + reader + .read_line(&mut line) + .map_err(|error| ObserverError::new(format!("read observer handshake: {error}")))?; + if line.trim() != "cua-inject v1" { + return Err(ObserverError::new(format!( + "cua-compositor observer handshake mismatch: {:?}", + line.trim() + ))); + } + + writeln!(writer, "q {target_pid}") + .and_then(|_| writer.flush()) + .map_err(|error| ObserverError::new(format!("write observer query: {error}")))?; + line.clear(); + let read = reader + .read_line(&mut line) + .map_err(|error| ObserverError::new(format!("read observer query: {error}")))?; + if read == 0 { + return Err(ObserverError::new( + "cua-compositor closed the observer query", + )); + } + parse_cua_compositor_state(&line) + } + + fn cua_compositor_available() -> bool { + std::env::var_os("CUA_INJECT_SOCKET").is_some() && cua_compositor_state(0).is_ok() + } + + fn cua_compositor_snapshot(target: TargetWindow) -> Result { + let state = cua_compositor_state(target.pid)?; + Ok(DesktopSnapshot { + foreground: state.focused_pid, + input_focus: state.focused_pid, + target_z: state.target_z, + cursor_pos: None, + }) + } + + fn cua_compositor_focus_identity() -> Result, ObserverError> { + Ok(cua_compositor_state(0)?.focused_pid) + } + + #[derive(Clone, Debug, Deserialize, Eq, PartialEq)] + struct GnomeWindow { + id: u64, + pid: u32, + #[serde(rename = "title")] + _title: String, + x: i64, + y: i64, + w: i64, + h: i64, + focused: bool, + minimized: bool, + visible: bool, + stacking: u64, + } + + fn gnome_windows() -> Result, ObserverError> { + let mut child = Command::new("gdbus") + .args([ + "call", + "--session", + "--dest", + "org.cua.WinRects", + "--object-path", + "/org/cua/WinRects", + "--method", + "org.cua.WinRects.GetRects", + ]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|error| ObserverError::new(format!("gdbus GetRects failed: {error}")))?; + let deadline = std::time::Instant::now() + Duration::from_millis(800); + loop { + match child.try_wait() { + Ok(Some(_)) => { + let output = child.wait_with_output().map_err(|error| { + ObserverError::new(format!("gdbus GetRects output failed: {error}")) + })?; + if !output.status.success() { + return Err(ObserverError::new(format!( + "gdbus GetRects exited with {}: {}", + output.status, + String::from_utf8_lossy(&output.stderr) + ))); + } + return parse_gnome_windows(&String::from_utf8_lossy(&output.stdout)); + } + Ok(None) if std::time::Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(15)) + } + Ok(None) => { + let _ = child.kill(); + let _ = child.wait(); + return Err(ObserverError::new("gdbus GetRects timed out")); + } + Err(error) => { + let _ = child.kill(); + let _ = child.wait(); + return Err(ObserverError::new(format!( + "gdbus GetRects wait failed: {error}" + ))); + } + } + } + } + + fn parse_gnome_windows(raw: &str) -> Result, ObserverError> { + let start = raw + .find('[') + .ok_or_else(|| ObserverError::new("GetRects response had no JSON array"))?; + let end = raw + .rfind(']') + .filter(|end| *end >= start) + .ok_or_else(|| ObserverError::new("GetRects response had no complete JSON array"))?; + serde_json::from_str(&raw[start..=end]) + .map_err(|error| ObserverError::new(format!("invalid GetRects JSON: {error}"))) + } + + fn gnome_target<'a>( + windows: &'a [GnomeWindow], + target: TargetWindow, + ) -> Option<&'a GnomeWindow> { + let matching = windows.iter().filter(|window| window.pid == target.pid); + matching + .clone() + .find(|window| window.id == target.native_id) + .or_else(|| matching.max_by_key(|window| window.stacking)) + } + + fn fully_covers(cover: &GnomeWindow, target: &GnomeWindow) -> bool { + target.w > 0 + && target.h > 0 + && cover.w > 0 + && cover.h > 0 + && cover.x <= target.x + && cover.y <= target.y + && cover.x.saturating_add(cover.w) >= target.x.saturating_add(target.w) + && cover.y.saturating_add(cover.h) >= target.y.saturating_add(target.h) + } + + fn classify_gnome_target(windows: &[GnomeWindow], target: TargetWindow) -> TargetZ { + let Some(target) = gnome_target(windows, target) else { + return TargetZ::NotFound; + }; + if target.focused { + TargetZ::Foreground + } else if target.minimized || !target.visible { + TargetZ::Minimized + } else if windows.iter().any(|window| { + window.stacking > target.stacking + && window.visible + && !window.minimized + && fully_covers(window, target) + }) { + TargetZ::BackgroundOccluded + } else { + TargetZ::BackgroundVisible + } + } + + fn gnome_focus_identity_from(windows: &[GnomeWindow]) -> Option { + windows + .iter() + .find(|window| window.focused) + .map(|window| window.id) + } + + fn gnome_snapshot(target: TargetWindow) -> Result { + let windows = gnome_windows()?; + let focused = gnome_focus_identity_from(&windows); + Ok(DesktopSnapshot { + foreground: focused, + input_focus: focused, + target_z: classify_gnome_target(&windows, target), + cursor_pos: None, + }) + } + + fn gnome_focus_identity() -> Result, ObserverError> { + Ok(gnome_focus_identity_from(&gnome_windows()?)) + } + + fn walk_sway_tree( + node: &serde_json::Value, + target_pid: u32, + workspace: Option, + state: &mut SwayTreeState, + ) { + let id = node["id"].as_u64(); + let workspace = if node["type"].as_str() == Some("workspace") { + id + } else { + workspace + }; + let focused = node["focused"].as_bool().unwrap_or(false); + if focused { + state.focused = id; + state.focused_workspace = workspace; + if node["fullscreen_mode"].as_i64().unwrap_or(0) != 0 { + state.focused_fullscreen = id; + } + } + if node["pid"].as_u64() == Some(u64::from(target_pid)) { + if let Some(id) = id { + state.target = Some((id, node["visible"].as_bool().unwrap_or(true), workspace)); + } + } + for child in ["nodes", "floating_nodes"] + .into_iter() + .flat_map(|key| node[key].as_array().into_iter().flatten()) + { + walk_sway_tree(child, target_pid, workspace, state); + } + } + + fn classify_sway_target(state: &SwayTreeState) -> TargetZ { + match state.target { + None => TargetZ::NotFound, + Some((id, _, _)) if state.focused == Some(id) => TargetZ::Foreground, + Some((_, _, target_workspace)) + if state.focused_fullscreen.is_some() + && target_workspace == state.focused_workspace => + { + // Sway reports windows hidden behind a fullscreen sibling as + // visible=false. They are occluded, not minimized. + TargetZ::BackgroundOccluded + } + Some((_, false, _)) => TargetZ::Minimized, + Some(_) => TargetZ::BackgroundVisible, + } + } + + fn sway_snapshot(target: TargetWindow) -> Result { + let tree = sway_tree()?; + let mut state = SwayTreeState::default(); + walk_sway_tree(&tree, target.pid, None, &mut state); + let target_z = classify_sway_target(&state); + Ok(DesktopSnapshot { + foreground: state.focused, + input_focus: state.focused, + target_z, + cursor_pos: None, + }) + } + + fn sway_focus_identity() -> Result, ObserverError> { + let tree = sway_tree()?; + let mut state = SwayTreeState::default(); + walk_sway_tree(&tree, 0, None, &mut state); + Ok(state.focused) + } + + fn x11_snapshot(target: TargetWindow) -> Result { + let (connection, screen_index) = x11_connection()?; + let root = connection.setup().roots[screen_index].root; + let target = u32::try_from(target.native_id) + .map_err(|_| ObserverError::new("X11 window id does not fit in u32"))?; + let target_root = match top_level(&connection, target, root) { + Ok(window) => window, + Err(_) => { + return Ok(DesktopSnapshot { + foreground: active_window(&connection, root)?.map(u64::from), + input_focus: input_focus(&connection, root)?.map(u64::from), + target_z: TargetZ::NotFound, + cursor_pos: query_pointer(&connection, root)?, + }); + } + }; + let active = active_window(&connection, root)?; + let focus = input_focus(&connection, root)?; + let target_z = if !is_viewable(&connection, target_root)? { + TargetZ::Minimized + } else if active == Some(target_root) || focus == Some(target_root) { + TargetZ::Foreground + } else if is_occluded(&connection, root, target_root)? { + TargetZ::BackgroundOccluded + } else { + TargetZ::BackgroundVisible + }; + Ok(DesktopSnapshot { + foreground: active.map(u64::from), + input_focus: focus.map(u64::from), + target_z, + cursor_pos: query_pointer(&connection, root)?, + }) + } + + fn x11_focus_identity() -> Result, ObserverError> { + let (connection, screen_index) = x11_connection()?; + let root = connection.setup().roots[screen_index].root; + Ok(active_window(&connection, root)? + .or(input_focus(&connection, root)?) + .map(u64::from)) + } + + fn x11_window_manager_ready() -> bool { + let Ok((connection, screen_index)) = x11_connection() else { + return false; + }; + let root = connection.setup().roots[screen_index].root; + let Ok(atom_cookie) = connection.intern_atom(false, b"_NET_SUPPORTING_WM_CHECK") else { + return false; + }; + let Ok(atom_reply) = atom_cookie.reply() else { + return false; + }; + let atom = atom_reply.atom; + let read_window = |window| { + connection + .get_property(false, window, atom, AtomEnum::WINDOW, 0, 1) + .ok()? + .reply() + .ok()? + .value32()? + .next() + }; + let Some(manager) = read_window(root) else { + return false; + }; + manager != 0 && read_window(manager) == Some(manager) + } + + fn active_window( + connection: &RustConnection, + root: Window, + ) -> Result, ObserverError> { + let atom = connection + .intern_atom(false, b"_NET_ACTIVE_WINDOW") + .map_err(x11_error("intern _NET_ACTIVE_WINDOW"))? + .reply() + .map_err(x11_error("read _NET_ACTIVE_WINDOW atom"))? + .atom; + let reply = connection + .get_property(false, root, atom, AtomEnum::WINDOW, 0, 1) + .map_err(x11_error("request _NET_ACTIVE_WINDOW"))? + .reply() + .map_err(x11_error("read _NET_ACTIVE_WINDOW"))?; + let active = reply + .value32() + .and_then(|mut values| values.next()) + .filter(|window| *window != 0); + active + .map(|window| top_level(connection, window, root)) + .transpose() + } + + fn input_focus( + connection: &RustConnection, + root: Window, + ) -> Result, ObserverError> { + let window = connection + .get_input_focus() + .map_err(x11_error("request input focus"))? + .reply() + .map_err(x11_error("read input focus"))? + .focus; + if window == 0 || window == 1 { + Ok(None) + } else { + top_level(connection, window, root).map(Some) + } + } + + fn top_level( + connection: &RustConnection, + mut window: Window, + root: Window, + ) -> Result { + for _ in 0..32 { + let tree = connection + .query_tree(window) + .map_err(x11_error("request X11 window tree"))? + .reply() + .map_err(x11_error("read X11 window tree"))?; + if tree.parent == root || window == root { + return Ok(window); + } + window = tree.parent; + } + Err(ObserverError::new("X11 window ancestry exceeded 32 levels")) + } + + fn is_viewable(connection: &RustConnection, window: Window) -> Result { + let attributes = connection + .get_window_attributes(window) + .map_err(x11_error("request X11 window attributes"))? + .reply() + .map_err(x11_error("read X11 window attributes"))?; + Ok(attributes.class == WindowClass::INPUT_OUTPUT + && attributes.map_state == MapState::VIEWABLE) + } + + fn absolute_bounds( + connection: &RustConnection, + window: Window, + root: Window, + ) -> Result { + let geometry = connection + .get_geometry(window) + .map_err(x11_error("request X11 window geometry"))? + .reply() + .map_err(x11_error("read X11 window geometry"))?; + let translated = connection + .translate_coordinates(window, root, 0, 0) + .map_err(x11_error("request X11 translated coordinates"))? + .reply() + .map_err(x11_error("read X11 translated coordinates"))?; + Ok(Rectangle { + x: translated.dst_x, + y: translated.dst_y, + width: geometry.width, + height: geometry.height, + }) + } + + fn is_occluded( + connection: &RustConnection, + root: Window, + target: Window, + ) -> Result { + let target_bounds = absolute_bounds(connection, target, root)?; + if target_bounds.width <= 4 || target_bounds.height <= 4 { + return Ok(false); + } + let covered = sample_points(target_bounds) + .into_iter() + .filter(|(x, y)| { + connection + .translate_coordinates(root, root, *x, *y) + .ok() + .and_then(|cookie| cookie.reply().ok()) + .map(|reply| reply.child != 0 && reply.child != target) + .unwrap_or(true) + }) + .count(); + Ok(covered >= 2) + } + + fn sample_points(bounds: Rectangle) -> [(i16, i16); 5] { + let left = bounds.x.saturating_add(2); + let top = bounds.y.saturating_add(2); + let right = i32::from(bounds.x) + .saturating_add(i32::from(bounds.width)) + .saturating_sub(3) + .clamp(i32::from(i16::MIN), i32::from(i16::MAX)) as i16; + let bottom = i32::from(bounds.y) + .saturating_add(i32::from(bounds.height)) + .saturating_sub(3) + .clamp(i32::from(i16::MIN), i32::from(i16::MAX)) as i16; + let center_x = i32::from(bounds.x) + .saturating_add(i32::from(bounds.width) / 2) + .clamp(i32::from(i16::MIN), i32::from(i16::MAX)) as i16; + let center_y = i32::from(bounds.y) + .saturating_add(i32::from(bounds.height) / 2) + .clamp(i32::from(i16::MIN), i32::from(i16::MAX)) as i16; + [ + (left, top), + (right, top), + (left, bottom), + (right, bottom), + (center_x, center_y), + ] + } + + fn query_pointer( + connection: &RustConnection, + root: Window, + ) -> Result, ObserverError> { + let pointer = connection + .query_pointer(root) + .map_err(x11_error("request X11 pointer"))? + .reply() + .map_err(x11_error("read X11 pointer"))?; + Ok(Some((f64::from(pointer.root_x), f64::from(pointer.root_y)))) + } + + fn x11_error(operation: &'static str) -> impl FnOnce(E) -> ObserverError { + move |error| ObserverError::new(format!("{operation} failed: {error}")) + } + + #[cfg(test)] + mod tests { + use super::*; + + fn gnome_payload() -> &'static str { + r#"('[{"id":10,"pid":100,"title":"target","x":10,"y":20,"w":300,"h":200,"focused":false,"minimized":false,"visible":true,"stacking":0},{"id":20,"pid":200,"title":"sentinel","x":0,"y":0,"w":800,"h":600,"focused":true,"minimized":false,"visible":true,"stacking":1}]',)"# + } + + #[test] + fn occlusion_samples_corners_and_center() { + let bounds = Rectangle { + x: -100, + y: 20, + width: 200, + height: 100, + }; + assert_eq!( + sample_points(bounds), + [(-98, 22), (97, 22), (-98, 117), (97, 117), (0, 70)] + ); + } + + #[test] + fn sway_hidden_target_behind_fullscreen_sibling_is_occluded() { + let state = SwayTreeState { + focused: Some(30), + focused_workspace: Some(10), + focused_fullscreen: Some(30), + target: Some((20, false, Some(10))), + }; + assert_eq!(classify_sway_target(&state), TargetZ::BackgroundOccluded); + } + + #[test] + fn sway_hidden_target_on_another_workspace_is_not_occluded() { + let state = SwayTreeState { + focused: Some(30), + focused_workspace: Some(10), + focused_fullscreen: Some(30), + target: Some((20, false, Some(11))), + }; + assert_eq!(classify_sway_target(&state), TargetZ::Minimized); + } + + #[test] + fn cua_compositor_state_is_strict_and_pid_based() { + assert_eq!( + parse_cua_compositor_state("state 200 background_occluded\n") + .expect("valid compositor state"), + CuaCompositorState { + focused_pid: Some(200), + target_z: TargetZ::BackgroundOccluded, + } + ); + assert_eq!( + parse_cua_compositor_state("state 0 not_found") + .expect("zero means no focused client"), + CuaCompositorState { + focused_pid: None, + target_z: TargetZ::NotFound, + } + ); + assert!(parse_cua_compositor_state("ok").is_err()); + assert!(parse_cua_compositor_state("state 1 unknown").is_err()); + assert!(parse_cua_compositor_state("state 1 foreground trailing").is_err()); + } + + #[test] + fn gnome_json_and_focus_identity_are_compositor_derived() { + let windows = parse_gnome_windows(gnome_payload()).expect("GNOME JSON"); + assert_eq!(windows.len(), 2); + assert_eq!(gnome_focus_identity_from(&windows), Some(20)); + assert!(parse_gnome_windows(r#"('[{"pid":100}]',)"#).is_err()); + } + + #[test] + fn gnome_only_full_higher_cover_occludes() { + let mut windows = parse_gnome_windows(gnome_payload()).expect("GNOME JSON"); + let target = TargetWindow { + pid: 100, + native_id: 10, + }; + assert_eq!( + classify_gnome_target(&windows, target), + TargetZ::BackgroundOccluded + ); + windows[1].w = 200; + assert_eq!( + classify_gnome_target(&windows, target), + TargetZ::BackgroundVisible + ); + windows[1].w = 800; + windows[1].visible = false; + assert_eq!( + classify_gnome_target(&windows, target), + TargetZ::BackgroundVisible + ); + } + + #[test] + fn gnome_focused_and_minimized_are_direct() { + let mut windows = parse_gnome_windows(gnome_payload()).expect("GNOME JSON"); + let target = TargetWindow { + pid: 100, + native_id: 10, + }; + windows[0].focused = true; + assert_eq!(classify_gnome_target(&windows, target), TargetZ::Foreground); + windows[0].focused = false; + windows[0].minimized = true; + assert_eq!(classify_gnome_target(&windows, target), TargetZ::Minimized); + } + } +} + +#[cfg(target_os = "linux")] +pub use linux::LinuxObserver as NativeObserver; +#[cfg(target_os = "macos")] +pub use macos::MacosObserver as NativeObserver; +#[cfg(target_os = "windows")] +pub use windows::WindowsObserver as NativeObserver; + +#[cfg(test)] +mod tests { + use super::*; + + fn snapshot(foreground: u64, target_z: TargetZ, cursor: (f64, f64)) -> DesktopSnapshot { + DesktopSnapshot { + foreground: Some(foreground), + input_focus: Some(foreground), + target_z, + cursor_pos: Some(cursor), + } + } + + #[test] + fn transient_focus_change_fails_even_when_pre_and_post_match() { + let before = snapshot(10, TargetZ::BackgroundOccluded, (100.0, 100.0)); + let after = before.clone(); + let delta = evaluate( + ObserverCapabilities { + focus: true, + ..ObserverCapabilities::default() + }, + &[OracleKind::Focus], + before, + after, + DesktopJournal { + focus_events: vec![ + FocusEvent { + from: Some(10), + to: Some(20), + }, + FocusEvent { + from: Some(20), + to: Some(10), + }, + ], + leaked_input_events: Vec::new(), + }, + ); + assert!(delta.passed().is_empty()); + assert!(delta.violations()[0].contains("transiently")); + } + + #[test] + fn input_focus_change_fails_when_foreground_is_stable() { + let before = snapshot(10, TargetZ::BackgroundOccluded, (100.0, 100.0)); + let mut after = before.clone(); + after.input_focus = Some(20); + let delta = evaluate( + ObserverCapabilities { + focus: true, + ..ObserverCapabilities::default() + }, + &[OracleKind::Focus], + before, + after, + DesktopJournal::default(), + ); + assert!(delta.passed().is_empty()); + assert!(delta.violations()[0].contains("input focus changed")); + } + + #[test] + fn target_raise_cursor_move_and_input_leak_are_independent_failures() { + let delta = evaluate( + ObserverCapabilities { + focus: true, + z_order: true, + cursor: true, + leaked_input: true, + }, + &[ + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::Cursor, + OracleKind::NoLeakedInput, + ], + snapshot(10, TargetZ::BackgroundOccluded, (100.0, 100.0)), + snapshot(10, TargetZ::BackgroundVisible, (102.0, 100.0)), + DesktopJournal { + focus_events: Vec::new(), + leaked_input_events: vec!["keydown:A".to_owned()], + }, + ); + assert_eq!(delta.passed(), &[OracleKind::Focus]); + assert_eq!(delta.violations().len(), 3); + } + + #[test] + fn unsupported_oracle_is_never_counted_as_passed() { + let state = snapshot(10, TargetZ::BackgroundOccluded, (100.0, 100.0)); + let delta = evaluate( + ObserverCapabilities::default(), + &[OracleKind::NoLeakedInput], + state.clone(), + state, + DesktopJournal::default(), + ); + assert_eq!(delta.unsupported(), &[OracleKind::NoLeakedInput]); + assert!(delta.ensure_supported().is_err()); + assert!(delta.passed().is_empty()); + } +} diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/paths.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/paths.rs index 1bb58655cb..2aafd74cbc 100644 --- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/paths.rs +++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/paths.rs @@ -34,6 +34,18 @@ pub fn driver_binary() -> PathBuf { } else { "cua-driver" }; + if let Ok(test_exe) = std::env::current_exe() { + if let Some(profile_dir) = test_exe + .parent() + .filter(|dir| dir.file_name().is_some_and(|name| name == "deps")) + .and_then(std::path::Path::parent) + { + let sibling = profile_dir.join(name); + if sibling.exists() { + return sibling; + } + } + } let root = workspace_root(); let release = root.join("target/release").join(name); if release.exists() { diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/reaper.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/reaper.rs index a184c118bb..ac38d186c7 100644 --- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/reaper.rs +++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/reaper.rs @@ -15,7 +15,10 @@ pub struct ChildReaper { impl ChildReaper { pub fn new() -> Self { - ChildReaper { children: Vec::new(), pids: Vec::new() } + ChildReaper { + children: Vec::new(), + pids: Vec::new(), + } } /// Spawn `cmd` into the kill-on-close job (Windows) and own the child. @@ -55,6 +58,7 @@ impl Drop for ChildReaper { tree_kill(pid); } for c in &mut self.children { + tree_kill(c.id()); let _ = c.kill(); let _ = c.wait(); } @@ -100,8 +104,8 @@ mod win { use std::sync::OnceLock; use windows::Win32::Foundation::{CloseHandle, HANDLE}; use windows::Win32::System::JobObjects::{ - AssignProcessToJobObject, CreateJobObjectW, SetInformationJobObject, - JobObjectExtendedLimitInformation, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, + AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation, + SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, }; use windows::Win32::System::Threading::{OpenProcess, PROCESS_SET_QUOTA, PROCESS_TERMINATE}; @@ -112,7 +116,8 @@ mod win { fn job() -> HANDLE { let raw = *JOB.get_or_init(|| unsafe { - let h = CreateJobObjectW(None, windows::core::PCWSTR::null()).expect("CreateJobObjectW"); + let h = + CreateJobObjectW(None, windows::core::PCWSTR::null()).expect("CreateJobObjectW"); let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; let _ = SetInformationJobObject( @@ -130,7 +135,12 @@ mod win { pub(super) fn assign_child(child: &Child) { unsafe { let h = HANDLE(child.as_raw_handle() as *mut c_void); - let _ = AssignProcessToJobObject(job(), h); + if let Err(error) = AssignProcessToJobObject(job(), h) { + eprintln!( + "[testkit] could not assign child {} to job: {error}", + child.id() + ); + } } } @@ -139,7 +149,9 @@ mod win { unsafe { if let Ok(h) = OpenProcess(PROCESS_SET_QUOTA | PROCESS_TERMINATE, false, pid) { if !h.is_invalid() { - let _ = AssignProcessToJobObject(job(), h); + if let Err(error) = AssignProcessToJobObject(job(), h) { + eprintln!("[testkit] could not assign pid {pid} to job: {error}"); + } let _ = CloseHandle(h); } } diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/response.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/response.rs index e828766fa6..41418727c2 100644 --- a/libs/cua-driver/rust/crates/cua-driver-testkit/src/response.rs +++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/response.rs @@ -33,6 +33,8 @@ impl ToolResponse { pub(crate) fn from_mcp(raw: Value) -> Self { let text = raw["result"]["content"][0]["text"] .as_str() + .or_else(|| raw["error"]["message"].as_str()) + .or_else(|| raw["error"].as_str()) .unwrap_or("") .to_string(); let structured = raw["result"]["structuredContent"].clone(); @@ -99,3 +101,23 @@ impl ToolResponse { .unwrap_or(false) } } + +#[cfg(test)] +mod tests { + use super::ToolResponse; + + #[test] + fn mcp_error_text_accepts_object_and_string_envelopes() { + let object = ToolResponse::from_mcp(serde_json::json!({ + "error": { "message": "object error" } + })); + assert!(object.is_error()); + assert_eq!(object.text(), "object error"); + + let string = ToolResponse::from_mcp(serde_json::json!({ + "error": "string error" + })); + assert!(string.is_error()); + assert_eq!(string.text(), "string error"); + } +} diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rs new file mode 100644 index 0000000000..ad6b9171d3 --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/sentinel.rs @@ -0,0 +1,417 @@ +//! Full-desktop foreground sentinel used by background E2E cells. + +use std::fs; +use std::net::TcpListener; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +use crate::e2e::OracleKind; +use crate::observer::{DesktopObserver, NativeObserver, TargetWindow}; +use crate::{harness_app, spawn_in_job, BehaviorRecording, ChildReaper, Driver}; + +/// A foreground Electron window that journals focus and leaked input while it +/// fully occludes the background target. +pub struct ForegroundSentinel { + journal_path: std::path::PathBuf, + target: TargetWindow, + _reaper: ChildReaper, + _user_data: tempfile::TempDir, +} + +impl ForegroundSentinel { + pub fn launch(driver: &mut impl Driver) -> Self { + let electron = electron_fixture(); + assert!( + electron.path.exists(), + "Electron sentinel fixture is missing at {}", + electron.path.display() + ); + let user_data = tempfile::Builder::new() + .prefix("cua-e2e-sentinel-") + .tempdir() + .expect("create sentinel user-data directory"); + let journal_path = user_data.path().join("sentinel-events.jsonl"); + fs::write(&journal_path, "").expect("initialize sentinel event journal"); + let cdp_port = TcpListener::bind(("127.0.0.1", 0)) + .and_then(|listener| listener.local_addr()) + .expect("allocate sentinel CDP port") + .port(); + let mut command = Command::new(&electron.path); + command + .args(&electron.args) + .env("CUA_E2E_SENTINEL", "1") + .env("CUA_E2E_SENTINEL_JOURNAL", &journal_path) + .env("CUA_E2E_USER_DATA_DIR", user_data.path()) + .env("CUA_ELECTRON_CDP_PORT", cdp_port.to_string()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + let child = spawn_in_job(&mut command).expect("launch foreground sentinel"); + let launched_pid = child.id(); + let mut reaper = ChildReaper::new(); + reaper.push(child); + let expected_title = format!("CuaTestHarness Sentinel [cdp={cdp_port}]"); + + let window_deadline = Instant::now() + Duration::from_secs(15); + let target = loop { + let windows = driver.call("list_windows", serde_json::json!({})); + if let Some(target) = windows.structured()["windows"] + .as_array() + .and_then(|windows| { + windows.iter().find_map(|window| { + let id = window["window_id"].as_u64()?; + let title = window["title"].as_str().unwrap_or(""); + title.contains(&expected_title).then(|| TargetWindow { + pid: window["pid"].as_u64().unwrap_or(launched_pid as u64) as u32, + native_id: id, + }) + }) + }) + { + assert_ne!( + target.pid, 0, + "foreground sentinel window has no process id" + ); + break target; + } + assert!( + Instant::now() < window_deadline, + "foreground sentinel window did not appear" + ); + std::thread::sleep(Duration::from_millis(100)); + }; + reaper.track_pid(target.pid); + + let focus_deadline = Instant::now() + Duration::from_secs(10); + if is_wayland_session() { + wait_for_journal(&journal_path, focus_deadline, r#""kind":"ready""#, "ready"); + activate_native_foreground(driver, target); + // Electron may already be focused before its preload listener is ready. + // The compositor observation is the authoritative Wayland focus gate. + wait_for_native_focus_stable(target); + } else { + loop { + let journal = fs::read_to_string(&journal_path).unwrap_or_default(); + if journal.contains(r#""kind":"ready""#) && journal.contains(r#""kind":"focus""#) { + break; + } + assert!( + Instant::now() < focus_deadline, + "foreground sentinel did not become ready and focused: {journal}" + ); + std::thread::sleep(Duration::from_millis(100)); + } + activate_native_foreground(driver, target); + wait_for_native_focus_stable(target); + } + fs::write(&journal_path, "").expect("reset focused sentinel journal"); + + Self { + journal_path, + target, + _reaper: reaper, + _user_data: user_data, + } + } + + pub fn observe(&self) -> (Vec, Vec) { + std::thread::sleep(Duration::from_millis(200)); + let journal = match fs::read_to_string(&self.journal_path) { + Ok(journal) => journal, + Err(error) => { + return ( + Vec::new(), + vec![format!( + "foreground sentinel journal could not be read: {error}" + )], + ) + } + }; + let mut passed = Vec::new(); + let mut violations = Vec::new(); + if journal.contains(r#""kind":"blur""#) { + violations.push("foreground sentinel lost focus".to_owned()); + } else { + passed.push(OracleKind::Focus); + } + let leaked = ["keydown", "pointerdown", "wheel", "contextmenu"] + .into_iter() + .filter(|kind| journal.contains(&format!(r#""kind":"{kind}""#))) + .collect::>(); + if leaked.is_empty() { + passed.push(OracleKind::NoLeakedInput); + } else { + violations.push(format!( + "foreground sentinel received input events: {}", + leaked.join(", ") + )); + } + (passed, violations) + } + + pub fn target(&self) -> TargetWindow { + self.target + } + + /// Confirm the target is fully behind the ready foreground sentinel before + /// the behavioral video boundary is crossed. + pub fn assert_background_posture(&self, target: TargetWindow) -> Result<(), String> { + let observer = DesktopObserver::new(NativeObserver::new(), target); + let before = observer.snapshot().map_err(|error| error.to_string())?; + if before.target_z == crate::observer::TargetZ::BackgroundOccluded { + Ok(()) + } else { + Err(format!( + "background target was not fully occluded before recording: {:?}", + before.target_z + )) + } + } + + /// Run one background action while checking the native desktop and the + /// sentinel journal. The returned oracle list is suitable for a typed E2E + /// result; any unsupported observation or side effect is an error. + pub fn observe_background( + &self, + target: TargetWindow, + action: impl FnOnce() -> R, + ) -> Result<(R, Vec), String> { + self.observe_target(target, true, action) + } + + /// Observe a desktop-wide action against the foreground sentinel itself. + /// Launch and cursor-overlay cells have no pre-existing background target, + /// so they verify desktop stability without the target-occlusion precondition. + pub fn observe_desktop( + &self, + action: impl FnOnce() -> R, + ) -> Result<(R, Vec), String> { + self.observe_target(self.target, false, action) + } + + fn observe_target( + &self, + target: TargetWindow, + require_occluded_target: bool, + action: impl FnOnce() -> R, + ) -> Result<(R, Vec), String> { + let mut observer = DesktopObserver::new(NativeObserver::new(), target); + let before = observer.snapshot().map_err(|error| error.to_string())?; + if require_occluded_target + && before.target_z != crate::observer::TargetZ::BackgroundOccluded + { + return Err(format!( + "background target was not fully occluded before dispatch: {:?}", + before.target_z + )); + } + let mut native_oracles = vec![OracleKind::Focus, OracleKind::ZOrder]; + if std::env::var("XDG_SESSION_TYPE") + .map(|session| !session.eq_ignore_ascii_case("wayland")) + .unwrap_or(true) + { + native_oracles.push(OracleKind::Cursor); + } + let (result, delta) = observer + .observe(&native_oracles, action) + .map_err(|error| error.to_string())?; + if require_occluded_target + && delta.before.target_z != crate::observer::TargetZ::BackgroundOccluded + { + return Err(format!( + "background target was not fully occluded at dispatch: {:?}", + delta.before.target_z + )); + } + delta + .ensure_supported() + .map_err(|error| error.to_string())?; + + let mut passed = delta.passed().to_vec(); + let mut violations = delta.violations().to_vec(); + let (sentinel_passed, sentinel_violations) = self.observe(); + passed.extend(sentinel_passed); + violations.extend(sentinel_violations); + passed.sort(); + passed.dedup(); + if violations.is_empty() { + Ok((result, passed)) + } else { + Err(violations.join("; ")) + } + } +} + +fn wait_for_journal(path: &std::path::Path, deadline: Instant, marker: &str, state: &str) { + loop { + let journal = fs::read_to_string(path).unwrap_or_default(); + if journal.contains(marker) { + return; + } + assert!( + Instant::now() < deadline, + "foreground sentinel did not become {state}: {journal}" + ); + std::thread::sleep(Duration::from_millis(100)); + } +} + +fn is_wayland_session() -> bool { + cfg!(target_os = "linux") + && std::env::var("XDG_SESSION_TYPE") + .is_ok_and(|session| session.eq_ignore_ascii_case("wayland")) +} + +#[cfg(any(target_os = "windows", target_os = "linux"))] +fn activate_native_foreground(driver: &mut impl Driver, target: TargetWindow) { + let response = driver.call( + "bring_to_front", + serde_json::json!({ + "pid": target.pid, + "window_id": target.native_id, + }), + ); + assert!( + !response.is_error(), + "could not activate foreground sentinel: {}", + response.text() + ); + #[cfg(target_os = "windows")] + physically_focus_windows_sentinel(target); +} + +#[cfg(target_os = "windows")] +fn physically_focus_windows_sentinel(target: TargetWindow) { + use windows::Win32::Foundation::{HWND, POINT, RECT}; + use windows::Win32::UI::Input::KeyboardAndMouse::{ + SendInput, INPUT, INPUT_0, INPUT_MOUSE, MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP, + MOUSEINPUT, + }; + use windows::Win32::UI::WindowsAndMessaging::{GetCursorPos, GetWindowRect, SetCursorPos}; + + let hwnd = HWND(target.native_id as *mut _); + let mut rect = RECT::default(); + let mut original_cursor = POINT::default(); + unsafe { + GetWindowRect(hwnd, &mut rect).expect("read foreground sentinel bounds"); + GetCursorPos(&mut original_cursor).expect("read cursor before focusing sentinel"); + } + let x = (rect.left + rect.right) / 2; + let y = (rect.top + rect.bottom) / 2; + assert!( + rect.right > rect.left && rect.bottom > rect.top, + "foreground sentinel has invalid bounds: {rect:?}" + ); + + let inputs = [ + INPUT { + r#type: INPUT_MOUSE, + Anonymous: INPUT_0 { + mi: MOUSEINPUT { + dwFlags: MOUSEEVENTF_LEFTDOWN, + ..Default::default() + }, + }, + }, + INPUT { + r#type: INPUT_MOUSE, + Anonymous: INPUT_0 { + mi: MOUSEINPUT { + dwFlags: MOUSEEVENTF_LEFTUP, + ..Default::default() + }, + }, + }, + ]; + unsafe { + SetCursorPos(x, y).expect("move cursor onto foreground sentinel"); + let sent = SendInput(&inputs, std::mem::size_of::() as i32); + assert_eq!(sent, inputs.len() as u32, "click foreground sentinel"); + SetCursorPos(original_cursor.x, original_cursor.y) + .expect("restore cursor after focusing sentinel"); + } +} + +#[cfg(not(any(target_os = "windows", target_os = "linux")))] +fn activate_native_foreground(_driver: &mut impl Driver, _target: TargetWindow) {} + +#[cfg(any(target_os = "windows", target_os = "linux"))] +fn wait_for_native_focus_stable(target: TargetWindow) { + use crate::observer::{ObserverBackend, TargetZ}; + + let backend = NativeObserver::new(); + let deadline = Instant::now() + Duration::from_secs(3); + let mut stable_since = None; + loop { + let foreground = backend + .snapshot(target) + .map(|snapshot| snapshot.target_z == TargetZ::Foreground) + .unwrap_or(false); + if foreground { + let since = stable_since.get_or_insert_with(Instant::now); + if since.elapsed() >= Duration::from_millis(300) { + return; + } + } else { + stable_since = None; + } + assert!( + Instant::now() < deadline, + "foreground sentinel did not remain natively focused" + ); + std::thread::sleep(Duration::from_millis(20)); + } +} + +#[cfg(not(any(target_os = "windows", target_os = "linux")))] +fn wait_for_native_focus_stable(_target: TargetWindow) {} + +pub fn run_with_background_oracles( + driver: &mut D, + target: TargetWindow, + action: impl FnOnce(&mut D) -> R, +) -> Result<(R, Vec), String> { + let sentinel = ForegroundSentinel::launch(driver); + sentinel.assert_background_posture(target)?; + driver.start_behavior_recording(); + sentinel.observe_background(target, || action(driver)) +} + +struct ElectronFixture { + path: std::path::PathBuf, + args: Vec<&'static str>, +} + +fn electron_fixture() -> ElectronFixture { + #[cfg(target_os = "windows")] + { + ElectronFixture { + path: harness_app("harness-electron", "CuaTestHarness.Electron.exe"), + args: vec![ + "--no-sandbox", + "--disable-gpu", + "--force-renderer-accessibility", + ], + } + } + #[cfg(target_os = "macos")] + { + ElectronFixture { + path: harness_app( + "harness-electron", + "CuaTestHarness.Electron.app/Contents/MacOS/Electron", + ), + args: vec!["--force-renderer-accessibility"], + } + } + #[cfg(target_os = "linux")] + { + ElectronFixture { + path: harness_app("harness-electron", "CuaTestHarness.Electron"), + args: vec![ + "--no-sandbox", + "--disable-gpu", + "--force-renderer-accessibility", + ], + } + } +} diff --git a/libs/cua-driver/rust/crates/cua-driver-testkit/src/windows_setup.rs b/libs/cua-driver/rust/crates/cua-driver-testkit/src/windows_setup.rs new file mode 100644 index 0000000000..3425190602 --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-testkit/src/windows_setup.rs @@ -0,0 +1,122 @@ +//! Windows hosted-runner desktop cleanup performed before behavioral capture. + +#[cfg(any(target_os = "windows", test))] +fn is_hosted_runner_console(title: &str, class_name: &str) -> bool { + let title = title.to_ascii_lowercase(); + let console_class = matches!( + class_name, + "ConsoleWindowClass" | "CASCADIA_HOSTING_WINDOW_CLASS" + ); + console_class + && [ + "hostedcomputeagent", + "hosted-compute-agent", + "runner.worker", + "github actions runner", + ] + .iter() + .any(|marker| title.contains(marker)) +} + +#[cfg(target_os = "windows")] +pub fn minimize_hosted_runner_console() -> Result<&'static str, String> { + if std::env::var("RUNNER_ENVIRONMENT").as_deref() != Ok("github-hosted") { + return Ok("not_applicable"); + } + + use windows::core::BOOL; + use windows::Win32::Foundation::{HWND, LPARAM, TRUE}; + use windows::Win32::UI::WindowsAndMessaging::{ + EnumWindows, GetClassNameW, GetWindowTextLengthW, GetWindowTextW, IsIconic, + IsWindowVisible, ShowWindow, SW_MINIMIZE, + }; + + unsafe fn identity(hwnd: HWND) -> (String, String) { + let title_len = GetWindowTextLengthW(hwnd); + let mut title = vec![0u16; title_len.max(0) as usize + 1]; + let copied = GetWindowTextW(hwnd, &mut title); + let title = String::from_utf16_lossy(&title[..copied.max(0) as usize]); + let mut class_name = [0u16; 256]; + let class_len = GetClassNameW(hwnd, &mut class_name); + let class_name = String::from_utf16_lossy(&class_name[..class_len.max(0) as usize]); + (title, class_name) + } + + unsafe extern "system" fn collect(hwnd: HWND, state: LPARAM) -> BOOL { + if IsWindowVisible(hwnd).as_bool() { + let (title, class_name) = identity(hwnd); + if is_hosted_runner_console(&title, &class_name) { + let windows = &mut *(state.0 as *mut Vec); + windows.push(hwnd); + } + } + TRUE + } + + let mut windows: Vec = Vec::new(); + unsafe { + let _ = EnumWindows( + Some(collect), + LPARAM(&mut windows as *mut Vec as isize), + ); + } + windows.sort_by_key(|hwnd| hwnd.0 as usize); + windows.dedup_by_key(|hwnd| hwnd.0 as usize); + if windows.is_empty() { + return Ok("not_present"); + } + + let mut changed = false; + for &hwnd in &windows { + if !unsafe { IsIconic(hwnd) }.as_bool() { + let _ = unsafe { ShowWindow(hwnd, SW_MINIMIZE) }; + changed = true; + } + } + std::thread::sleep(std::time::Duration::from_millis(100)); + if windows + .iter() + .any(|&hwnd| !unsafe { IsIconic(hwnd) }.as_bool()) + { + return Err("HostedComputeAgent console did not enter the minimized state".to_owned()); + } + Ok(if changed { + "minimized" + } else { + "already_minimized" + }) +} + +#[cfg(not(target_os = "windows"))] +pub fn minimize_hosted_runner_console() -> Result<&'static str, String> { + Ok("not_applicable") +} + +#[cfg(test)] +mod tests { + use super::is_hosted_runner_console; + + #[test] + fn matches_only_named_runner_console_windows() { + assert!(is_hosted_runner_console( + "Administrator: HostedComputeAgent.exe", + "ConsoleWindowClass" + )); + assert!(is_hosted_runner_console( + "GitHub Actions Runner", + "CASCADIA_HOSTING_WINDOW_CLASS" + )); + assert!(is_hosted_runner_console( + r"C:\ProgramData\GitHub\HostedComputeAgent\hosted-compute-agent", + "ConsoleWindowClass" + )); + assert!(!is_hosted_runner_console( + "CuaTestHarness Sentinel", + "Chrome_WidgetWin_1" + )); + assert!(!is_hosted_runner_console( + "HostedComputeAgent status", + "Chrome_WidgetWin_1" + )); + } +} diff --git a/libs/cua-driver/rust/crates/cua-driver/Cargo.toml b/libs/cua-driver/rust/crates/cua-driver/Cargo.toml index ecbbb9ec1f..f915a83249 100644 --- a/libs/cua-driver/rust/crates/cua-driver/Cargo.toml +++ b/libs/cua-driver/rust/crates/cua-driver/Cargo.toml @@ -61,10 +61,11 @@ platform-linux = { path = "../platform-linux" } [features] default = [] -# Forwards `platform-linux/portal-libei` so the Nix build can flip the -# GNOME/KDE portal stack (PipeWire ScreenCast + libei RemoteDesktop) on -# from a single workspace-level flag while the cross-platform release CD -# leaves it off. No-op on macOS/Windows builds. +# GNOME/KDE portal input ships in ordinary Linux releases; PipeWire capture +# remains a separate modern-desktop feature. The combined name stays as a +# compatibility alias for Nix and downstream builds. +portal-input = ["platform-linux/portal-input"] +portal-capture = ["platform-linux/portal-capture"] portal-libei = ["platform-linux/portal-libei"] [target.'cfg(target_os = "windows")'.build-dependencies] diff --git a/libs/cua-driver/rust/crates/cua-driver/src/main.rs b/libs/cua-driver/rust/crates/cua-driver/src/main.rs index 54d5c56329..8c727cd327 100644 --- a/libs/cua-driver/rust/crates/cua-driver/src/main.rs +++ b/libs/cua-driver/rust/crates/cua-driver/src/main.rs @@ -753,17 +753,8 @@ fn build_registry(cursor_cfg: cursor_overlay::CursorConfig) -> cua_driver_core:: let compat = CLAUDE_CODE_COMPAT.load(Ordering::SeqCst); #[cfg(target_os = "windows")] { - cua_driver_core::recording::set_screenshot_fn(|window_id, pid| { - if let Some(hwnd) = window_id { - platform_windows::capture::screenshot_window_bytes(hwnd).ok() - } else if let Some(p) = pid { - let wins = platform_windows::win32::list_windows(Some(p as u32)); - wins.first().and_then(|w| { - platform_windows::capture::screenshot_window_bytes(w.hwnd).ok() - }) - } else { - platform_windows::capture::screenshot_display_bytes().ok() - } + cua_driver_core::recording::set_classified_screenshot_fn(|window_id, pid| { + platform_windows::recording_hooks::screenshot_for_recording(window_id, pid) }); cua_driver_core::recording::set_click_marker_fn(|png_bytes, cx, cy| { platform_windows::capture::crosshair_png_bytes(png_bytes, cx, cy).ok() @@ -783,11 +774,11 @@ fn build_registry(cursor_cfg: cursor_overlay::CursorConfig) -> cua_driver_core:: { cua_driver_core::recording::set_screenshot_fn(|window_id, pid| { if let Some(xid) = window_id { - platform_linux::capture::screenshot_window_bytes(xid).ok() + platform_linux::wayland::screenshot_dispatch(xid).ok() } else if let Some(p) = pid { - let wins = platform_linux::x11::list_windows(Some(p as u32)); + let wins = platform_linux::wayland::list_windows_dispatch(Some(p as u32)); wins.first().and_then(|w| { - platform_linux::capture::screenshot_window_bytes(w.xid).ok() + platform_linux::wayland::screenshot_dispatch(w.xid).ok() }) } else { platform_linux::capture::screenshot_display_bytes().ok() @@ -796,9 +787,21 @@ fn build_registry(cursor_cfg: cursor_overlay::CursorConfig) -> cua_driver_core:: cua_driver_core::recording::set_click_marker_fn(|png_bytes, cx, cy| { platform_linux::capture::crosshair_png_bytes(png_bytes, cx, cy).ok() }); - cua_driver_core::video::set_video_backend_factory( - Box::new(cua_driver_core::video_ffmpeg::FfmpegVideoBackendFactory), - ); + cua_driver_core::recording::set_ax_snapshot_fn(|window_id, pid| { + platform_linux::recording_hooks::app_state_json_for(window_id, pid) + }); + cua_driver_core::recording::set_element_bounds_fn(|wid, pid, idx| { + platform_linux::recording_hooks::element_window_local_xy(wid, pid, idx) + }); + if platform_linux::wayland::is_wayland() { + cua_driver_core::video::set_video_backend_factory(Box::new( + platform_linux::video_wayland::WfRecorderVideoBackendFactory, + )); + } else { + cua_driver_core::video::set_video_backend_factory(Box::new( + cua_driver_core::video_ffmpeg::FfmpegVideoBackendFactory, + )); + } // SSH-driven Wayland+Xwayland sessions inherit DISPLAY but not // XAUTHORITY; adopt the running X server's auth cookie so X11 tools // don't all fail "Authorization required" (#1926). No-op when @@ -815,6 +818,9 @@ fn build_registry(cursor_cfg: cursor_overlay::CursorConfig) -> cua_driver_core:: // so their AT-SPI trees are visible to get_window_state. Best-effort and // idempotent; only on the serve path, not for short-lived CLI calls. platform_linux::a11y::ensure_chromium_accessibility_enabled(); + if let Err(error) = platform_linux::atspi::ensure_listener_active() { + tracing::warn!("could not activate the persistent AT-SPI listener: {error}"); + } { let mut r = platform_linux::register_tools_with_cursor(cursor_cfg, compat); check_update_tool::register_into(&mut r); r } } #[cfg(not(any(target_os = "windows", target_os = "linux")))] @@ -834,17 +840,8 @@ fn build_registry_no_cursor() -> cua_driver_core::tool::ToolRegistry { let compat = CLAUDE_CODE_COMPAT.load(Ordering::SeqCst); #[cfg(target_os = "windows")] { - cua_driver_core::recording::set_screenshot_fn(|window_id, pid| { - if let Some(hwnd) = window_id { - platform_windows::capture::screenshot_window_bytes(hwnd).ok() - } else if let Some(p) = pid { - let wins = platform_windows::win32::list_windows(Some(p as u32)); - wins.first().and_then(|w| { - platform_windows::capture::screenshot_window_bytes(w.hwnd).ok() - }) - } else { - platform_windows::capture::screenshot_display_bytes().ok() - } + cua_driver_core::recording::set_classified_screenshot_fn(|window_id, pid| { + platform_windows::recording_hooks::screenshot_for_recording(window_id, pid) }); cua_driver_core::recording::set_click_marker_fn(|png_bytes, cx, cy| { platform_windows::capture::crosshair_png_bytes(png_bytes, cx, cy).ok() @@ -869,13 +866,19 @@ fn build_registry_no_cursor() -> cua_driver_core::tool::ToolRegistry { } #[cfg(target_os = "linux")] { + platform_linux::xauth::ensure_xauthority_discovered(); + platform_linux::session_bus::ensure_session_bus_discovered(); + platform_linux::a11y::ensure_chromium_accessibility_enabled(); + if let Err(error) = platform_linux::atspi::ensure_listener_active() { + tracing::warn!("could not activate the persistent AT-SPI listener: {error}"); + } cua_driver_core::recording::set_screenshot_fn(|window_id, pid| { if let Some(xid) = window_id { - platform_linux::capture::screenshot_window_bytes(xid).ok() + platform_linux::wayland::screenshot_dispatch(xid).ok() } else if let Some(p) = pid { - let wins = platform_linux::x11::list_windows(Some(p as u32)); + let wins = platform_linux::wayland::list_windows_dispatch(Some(p as u32)); wins.first().and_then(|w| { - platform_linux::capture::screenshot_window_bytes(w.xid).ok() + platform_linux::wayland::screenshot_dispatch(w.xid).ok() }) } else { platform_linux::capture::screenshot_display_bytes().ok() @@ -884,9 +887,21 @@ fn build_registry_no_cursor() -> cua_driver_core::tool::ToolRegistry { cua_driver_core::recording::set_click_marker_fn(|png_bytes, cx, cy| { platform_linux::capture::crosshair_png_bytes(png_bytes, cx, cy).ok() }); - cua_driver_core::video::set_video_backend_factory( - Box::new(cua_driver_core::video_ffmpeg::FfmpegVideoBackendFactory), - ); + cua_driver_core::recording::set_ax_snapshot_fn(|window_id, pid| { + platform_linux::recording_hooks::app_state_json_for(window_id, pid) + }); + cua_driver_core::recording::set_element_bounds_fn(|wid, pid, idx| { + platform_linux::recording_hooks::element_window_local_xy(wid, pid, idx) + }); + if platform_linux::wayland::is_wayland() { + cua_driver_core::video::set_video_backend_factory(Box::new( + platform_linux::video_wayland::WfRecorderVideoBackendFactory, + )); + } else { + cua_driver_core::video::set_video_backend_factory(Box::new( + cua_driver_core::video_ffmpeg::FfmpegVideoBackendFactory, + )); + } { let mut r = platform_linux::register_tools_with_cursor( cursor_overlay::CursorConfig { enabled: false, ..Default::default() }, diff --git a/libs/cua-driver/rust/crates/cua-driver/src/serve.rs b/libs/cua-driver/rust/crates/cua-driver/src/serve.rs index d867f4dfdc..1cb5d0abfb 100644 --- a/libs/cua-driver/rust/crates/cua-driver/src/serve.rs +++ b/libs/cua-driver/rust/crates/cua-driver/src/serve.rs @@ -730,17 +730,9 @@ pub async fn run_serve( if let Some(sc) = result.structured_content { result_obj["structuredContent"] = sc; } - let resp = if is_err { - DaemonResponse::err( - result.content.iter() - .filter_map(|c| if let cua_driver_core::protocol::Content::Text { text, .. } = c { Some(text.as_str()) } else { None }) - .collect::>() - .join("\n"), - 1 - ) - } else { - DaemonResponse::ok(result_obj) - }; + // Preserve tool-level `isError` and structured + // content inside a successful daemon transport. + let resp = DaemonResponse::ok(result_obj); let _ = writer.write_all( (serde_json::to_string(&resp).unwrap() + "\n").as_bytes() ).await; @@ -1256,16 +1248,9 @@ pub async fn run_serve( if let Some(sc) = result.structured_content { result_obj["structuredContent"] = sc; } - let resp = if is_err { - DaemonResponse::err( - result.content.iter() - .filter_map(|c| if let cua_driver_core::protocol::Content::Text { text, .. } = c { Some(text.as_str()) } else { None }) - .collect::>().join("\n"), - 1 - ) - } else { - DaemonResponse::ok(result_obj) - }; + // Preserve tool-level `isError` and structured + // content inside a successful daemon transport. + let resp = DaemonResponse::ok(result_obj); let _ = writer.write_all( (serde_json::to_string(&resp).unwrap() + "\n").as_bytes() ).await; diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/README.md b/libs/cua-driver/rust/crates/cua-driver/tests/README.md index 58c97352fc..021cb339d8 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/README.md +++ b/libs/cua-driver/rust/crates/cua-driver/tests/README.md @@ -1,9 +1,17 @@ # cua-driver Rust integration tests Tests in this directory exercise the public driver interface. Headless protocol -tests run by default; GUI and modality tests are marked `#[ignore]` because they +tests run by default; harness E2E tests are marked `#[ignore]` because they need staged harness apps and an interactive desktop. +Start with the contributor overview in +`../../../../docs/test-harnesses-guide.md`, then use the matrix below as the +coverage map. + +The cross-OS ownership map is maintained in +`../../../../docs/test-matrix.md`. Update that matrix when adding a harness, +action, addressing mode, delivery mode, or OS-specific window-system case. + ## Naming | Prefix | Runs by default | Purpose | @@ -11,8 +19,7 @@ need staged harness apps and an interactive desktop. | `protocol_*_test.rs` | yes | MCP/CLI protocol and schema behavior | | `schema_*_test.rs` | yes | Generated schema consistency | | `harness__test.rs` | no, `#[ignore]` | Toolkit-specific harness apps | -| `modality_[_]_test.rs` | no, `#[ignore]` | Background input, capture, desktop scope | -| `guard_*_test.rs` | usually ignored or self-skipping | UX guard and interactive desktop checks | +| `desktop_scope__test.rs` | no, `#[ignore]` | Platform window/desktop scope contract | ## Harness Requirements @@ -35,32 +42,45 @@ Staged outputs are read from `../../test-apps/harness-/`. The canonical cross-platform matrix is `cross_platform_behavior_test.rs`. It runs the same external-state scenarios -against Electron and Tauri on each supported host. CI and VM runners set +against Electron and Tauri on each supported host, plus the native WKWebView +host on macOS. CI and VM runners set `CUA_TEST_DRIVER_BIN`, `CUA_TEST_APPS_ROOT`, and `CUA_TEST_WORKSPACE_ROOT` when artifacts are built outside Cargo's default workspace paths. They also set `CUA_TEST_REQUIRE_FIXTURES=1`, turning a missing fixture into a failure instead of a silent skip. -The canonical Windows and Linux runners also set +Each matrix row declares its action, AX/PX targeting, foreground or background +delivery, scope, driver route, external oracles, and required behavior in +`cases.jsonl`. `results.jsonl` records the observed behavior and derived test +status. The Rust reporter validates both files and renders `summary.md`. + +The canonical OS runners also set `CUA_E2E_RECORDINGS_ROOT`. Every testkit `McpDriver` then records its full desktop trajectory to a unique directory containing `recording.mp4`, cursor samples, action JSON, per-turn screenshots, and a `trajectory.json` test-label manifest. Windows and Linux require FFmpeg; macOS uses the installed driver's ScreenCaptureKit backend. The runner -validates each MP4 with `ffprobe` before reporting success. +validates each MP4 with `ffprobe` before reporting success. A separate Rust +preflight verifies the desktop, fixture, AX tree, screenshot, and video +lifecycle once before behavioral cells run. ## Running ```bash cargo test -p cua-driver --test protocol_handshake_test cargo test -p cua-driver --test harness_appkit_test -- --ignored --nocapture -cargo test -p cua-driver --test modality_desktop_scope_macos_test -- --ignored --nocapture +cargo test -p cua-driver --test desktop_scope_macos_test -- --ignored --nocapture ``` -Windows Rust run-all uses -`../../../../tests/runners/windows/run-all.ps1`. It builds repo-local fixtures -and runs the default, guard, harness, and modality suites. It intentionally -excludes optional external-app suites. +The canonical Windows E2E entrypoint is +`scripts/ci/windows/run-rust-e2e.ps1 -RequireGui`. +It runs the complete Rust harness matrix; internal lane selectors are retained +only for focused diagnosis. Optional +external-app suites remain separate. + +The canonical macOS E2E entrypoint is +`scripts/ci/macos/run-rust-e2e.sh`. It requires a logged-in user session and an +installed driver with Accessibility and Screen Recording grants. Legacy Windows Sandbox runs use `../../../../tests/runners/windows-sandbox/run-tests-in-sandbox.ps1`, which @@ -68,21 +88,17 @@ builds selected Windows harness apps and maps them into the sandbox. The current Windows GUI validation path should use a real user desktop session through RDP or an interactive scheduled task. -Windows GUI modality tests require a user desktop where the focus sentinel can -become the foreground window. SSH-launched commands start in Session 0 and +Windows GUI tests require a usable interactive desktop. SSH-launched commands start in Session 0 and cannot drive the user's desktop directly; launch GUI tests through an interactive scheduled task (`/IT`) or equivalent so they run in the logged-on user session. -The Windows probe distinguishes two no-foreground states: - -- `input_desktop=Default, foreground_hwnd=0`: the desktop is usable but idle. - Tests now launch `focus-monitor-win` and require that sentinel HWND to become - foreground before assertions start. -- `input_desktop` is not `Default` or cannot be opened: the session is usually - locked/disconnected, for example after an RDP client drops. Reconnect, use - `tscon /dest:console` on a disposable GUI VM, or boot the VM into an unlocked - console session before running ignored GUI tests. +The testkit's native `DesktopObserver` records foreground-window, z-order, +cursor, and leaked-input state around rows that promise no desktop side effects. +If the input desktop is not `Default` or cannot be opened, the session is +usually locked or disconnected. Reconnect, use `tscon /dest:console` on a +disposable GUI VM, or boot the VM into an unlocked console session before +running ignored GUI tests. Set `CUA_REQUIRE_GUI=1` on dedicated GUI runners to turn these desktop self-skips into hard failures with the full desktop-state diagnostic. @@ -91,11 +107,11 @@ The repository-level runners are the preferred entrypoints for the canonical matrix: ```bash -scripts/ci/linux/run-rust-e2e.sh --suite shared +scripts/ci/linux/run-rust-e2e.sh ``` ```powershell -.\scripts\ci\windows\run-rust-e2e.ps1 -Suite shared -RequireGui +.\scripts\ci\windows\run-rust-e2e.ps1 -RequireGui ``` ## Optional External Apps @@ -107,5 +123,7 @@ desktop state outside the repo-local fixtures: - `harness_libreoffice_test.rs`: Windows LibreOffice Writer/Calc. Requires LibreOffice installed, or `LO_SWRITER_EXE` / `LO_SCALC_EXE` pointing at the executables. -- `modality_launch_focus_macos_test.rs`: macOS Calculator/TextEdit launch focus +- `installed_app_launch_macos_test.rs`: macOS Calculator/TextEdit launch focus checks. Requires a logged-in GUI session and usable System Events scripting. +- `installed_app_textedit_macos_test.rs`: real TextEdit background AX write and + verification. Requires a logged-in GUI session and Accessibility permission. diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/agent_cursor_windows_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/agent_cursor_windows_test.rs new file mode 100644 index 0000000000..1e68e456cb --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver/tests/agent_cursor_windows_test.rs @@ -0,0 +1,143 @@ +//! Windows agent-cursor rendering and desktop-side-effect contract. + +#![cfg(target_os = "windows")] + +use std::time::Duration; + +use cua_driver_testkit::e2e::{ + execute_case, recording_evidence, CaseSpec, Delivery, DriverRoute, Evidence, Observation, + OracleKind, Scope, Targeting, +}; +use cua_driver_testkit::sentinel::ForegroundSentinel; +use cua_driver_testkit::{Driver, McpDriver}; + +#[test] +#[ignore] +fn agent_cursor_overlay_is_visible_without_moving_real_cursor() { + let case = CaseSpec::delivered( + "windows-desktop-agent-cursor-px", + "desktop", + "win32", + "agent_cursor", + Targeting::Px, + Delivery::NotApplicable, + Scope::Desktop, + DriverRoute::WindowsOverlay, + vec![ + OracleKind::Pixels, + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::Cursor, + OracleKind::NoLeakedInput, + ], + ); + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_named("windows-desktop-agent-cursor-px") + .expect("required source-built driver did not start"); + *evidence = recording_evidence(driver.recording_dir()); + let sentinel = ForegroundSentinel::launch(&mut driver); + driver.start_behavior_recording(); + let screen = driver.call("get_screen_size", serde_json::json!({})); + assert!( + !screen.is_error(), + "get_screen_size failed: {}", + screen.text() + ); + let width = screen.structured()["width"].as_f64().unwrap_or(0.0); + let height = screen.structured()["height"].as_f64().unwrap_or(0.0); + assert!( + width >= 80.0 && height >= 80.0, + "invalid screen size {width}x{height}" + ); + let (x, y) = (width / 2.0, height / 2.0); + let cursor_id = "windows-agent-cursor-e2e"; + + let (_, mut passed) = sentinel + .observe_desktop(|| { + for (tool, arguments) in [ + ( + "set_agent_cursor_enabled", + serde_json::json!({"enabled": true, "cursor_id": cursor_id}), + ), + ( + "set_agent_cursor_motion", + serde_json::json!({ + "cursor_id": cursor_id, + "glide_duration_ms": 100, + "idle_hide_ms": 0 + }), + ), + ( + "move_cursor", + serde_json::json!({"x": x, "y": y, "cursor_id": cursor_id}), + ), + ] { + let response = driver.call(tool, arguments); + assert!(!response.is_error(), "{tool} failed: {}", response.text()); + } + std::thread::sleep(Duration::from_millis(350)); + }) + .unwrap_or_else(|error| panic!("agent cursor disturbed the real desktop: {error}")); + assert_required_background_oracles(&passed); + + let png = platform_windows::capture::screenshot_display_bytes() + .expect("screenshot_display_bytes failed"); + let image = image::load_from_memory(&png) + .expect("decode display screenshot") + .to_rgba8(); + let (image_width, image_height) = image.dimensions(); + let half = 20u32; + let center_x = x + .round() + .clamp(0.0, f64::from(image_width.saturating_sub(1))) as u32; + let center_y = y + .round() + .clamp(0.0, f64::from(image_height.saturating_sub(1))) as u32; + let x0 = center_x.saturating_sub(half); + let x1 = center_x.saturating_add(half).min(image_width); + let y0 = center_y.saturating_sub(half); + let y1 = center_y.saturating_add(half).min(image_height); + let visible_pixels = (y0..y1) + .flat_map(|pixel_y| (x0..x1).map(move |pixel_x| (pixel_x, pixel_y))) + .filter(|(pixel_x, pixel_y)| { + let [red, green, blue, alpha] = image.get_pixel(*pixel_x, *pixel_y).0; + if alpha < 10 { + return false; + } + let brightness = u32::from(red) + u32::from(green) + u32::from(blue); + let saturation = u32::from(red.max(green).max(blue) - red.min(green).min(blue)); + brightness > 60 && (saturation > 30 || brightness > 600) + }) + .count(); + assert!( + visible_pixels >= 5, + "agent cursor not visible at ({x:.0},{y:.0}): only {visible_pixels} qualifying pixels" + ); + + let disabled = driver.call( + "set_agent_cursor_enabled", + serde_json::json!({"enabled": false, "cursor_id": cursor_id}), + ); + assert!( + !disabled.is_error(), + "failed to disable agent cursor: {}", + disabled.text() + ); + passed.push(OracleKind::Pixels); + Observation::delivered(passed, Evidence::default()) + }); +} + +fn assert_required_background_oracles(passed: &[OracleKind]) { + for required in [ + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::Cursor, + OracleKind::NoLeakedInput, + ] { + assert!( + passed.contains(&required), + "agent cursor test omitted required {required:?} oracle" + ); + } +} diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_capture_mode_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/capture_contract_test.rs similarity index 55% rename from libs/cua-driver/rust/crates/cua-driver/tests/modality_capture_mode_test.rs rename to libs/cua-driver/rust/crates/cua-driver/tests/capture_contract_test.rs index feec766cbc..d5d8b64a20 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/modality_capture_mode_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/capture_contract_test.rs @@ -1,4 +1,4 @@ -//! modality_capture_mode_test — the get_window_state **perception** contract, +//! `get_window_state` perception contract, //! asserted on each platform's native controlled-harness app. //! //! get_window_state is perception-mode-agnostic now: it ALWAYS returns BOTH the @@ -11,36 +11,66 @@ //! * `include_screenshot:false` → tree present, NO `image` (the cheap path). //! * `capture_mode:"vision"` (deprecated) → IGNORED; still returns both. //! -//! Caveats handled as graceful skips (these are `#[ignore]` interactive tests, -//! consistent with the rest of the harness suite): -//! * The screenshot needs a screen-capture grant (Screen Recording on macOS). -//! Without it the driver returns a tree but no PNG, so the image assertions -//! skip-with-note rather than false-fail. -//! * The tree needs the platform accessibility grant (TCC on macOS, AT-SPI bus -//! on Linux). An empty tree skips the tree assertion. -//! //! Run explicitly: -//! cargo test -p cua-driver --test modality_capture_mode_test -- --ignored --nocapture --test-threads=1 +//! cargo test -p cua-driver --test capture_contract_test -- --ignored --nocapture --test-threads=1 #![cfg(any(target_os = "windows", target_os = "macos", target_os = "linux"))] use std::process::{Command, Stdio}; use std::time::{Duration, Instant}; +use cua_driver_testkit::e2e::{ + execute_case, native_readonly_case, recording_evidence, DriverRoute, Evidence, Observation, + OracleKind, Targeting, +}; +#[cfg(target_os = "windows")] +use cua_driver_testkit::e2e::{CaseSpec, Delivery, Scope}; +#[cfg(target_os = "windows")] +use cua_driver_testkit::observer::TargetWindow; +#[cfg(target_os = "windows")] +use cua_driver_testkit::sentinel::ForegroundSentinel; use cua_driver_testkit::{harness_app, Driver, McpDriver, ToolResponse}; /// The aid every harness exposes on its increment button — the tree marker /// (WPF AutomationId / AppKit AX identifier / GTK3 AT-SPI accessible name). const TREE_MARKER: &str = "btn-increment"; +fn strict() -> bool { + std::env::var_os("CUA_TEST_REQUIRE_FIXTURES").is_some() +} + #[cfg(target_os = "macos")] -fn spawn_driver() -> Option { - McpDriver::spawn_macos_daemon_proxy() +fn spawn_driver(label: &str) -> Option { + McpDriver::spawn_macos_daemon_proxy_named(label) } #[cfg(not(target_os = "macos"))] -fn spawn_driver() -> Option { - McpDriver::spawn() +fn spawn_driver(label: &str) -> Option { + McpDriver::spawn_named(label) +} + +fn test_driver(label: &str) -> Option { + let driver = spawn_driver(label); + assert!( + driver.is_some() || !strict(), + "required source-built driver did not start" + ); + driver +} + +fn capture_toolkit() -> &'static str { + #[cfg(target_os = "windows")] + { + "wpf" + } + #[cfg(target_os = "macos")] + { + "appkit" + } + #[cfg(target_os = "linux")] + { + "gtk3" + } } /// Does the response carry a screenshot? Checks both the MCP `image` content @@ -171,18 +201,24 @@ fn launch_new_instance( title: &str, ) -> Option<(u32, u64)> { if !exe.exists() { - eprintln!("[capture_mode] harness not built ({exe:?}) — skipping"); + if strict() { + panic!("required capture harness is missing: {exe:?}"); + } + eprintln!("[capture] optional harness not built: {exe:?}"); return None; } let before = harness_pids(driver, title); - driver - .reaper() - .spawn( - Command::new(exe) - .stdout(Stdio::null()) - .stderr(Stdio::null()), - ) - .ok()?; + let spawned = driver.reaper().spawn( + Command::new(exe) + .stdout(Stdio::null()) + .stderr(Stdio::null()), + ); + if let Err(error) = spawned { + if strict() { + panic!("failed to launch required capture harness {exe:?}: {error}"); + } + return None; + } resolve_new_window(driver, title, &before) } @@ -227,47 +263,65 @@ fn resolve_new_window( } std::thread::sleep(Duration::from_millis(400)); } - eprintln!("[capture_mode] new harness window {title:?} never appeared — is a graphical session available? skipping"); + if strict() { + panic!("required capture harness window {title:?} never appeared"); + } + eprintln!("[capture] optional harness window {title:?} never appeared"); None } +fn run_capture_case( + action: &str, + oracles: Vec, + test: impl FnOnce(u32, u64, &mut McpDriver), +) { + let case = native_readonly_case( + capture_toolkit(), + action, + Targeting::NotApplicable, + DriverRoute::WindowState, + oracles.clone(), + ); + let cell_id = case.cell_id.clone(); + execute_case(case, |evidence| { + let mut driver = test_driver(&cell_id).expect("required capture driver did not start"); + *evidence = recording_evidence(driver.recording_dir()); + let (pid, wid) = launch(&mut driver).expect("required capture harness did not launch"); + driver.start_behavior_recording(); + test(pid, wid, &mut driver); + Observation::delivered(oracles, Evidence::default()) + }); +} + // ── tests ─────────────────────────────────────────────────────────────────────── /// DEFAULT: get_window_state returns BOTH the tree AND a screenshot — grounding -/// on both is the whole point. Each half is asserted only when its OS grant is -/// available (accessibility for the tree, screen-recording for the image), so a -/// missing grant skips-with-note rather than false-failing. +/// on both is the whole point. #[test] #[ignore] fn default_returns_tree_and_screenshot() { - let Some(mut driver) = spawn_driver() else { - return; - }; - let Some((pid, wid)) = launch(&mut driver) else { - return; - }; - - let resp = snapshot_settled_default(&mut driver, pid, wid); - assert!( - !resp.is_error(), - "get_window_state(default) errored: {}", - resp.text() - ); + run_capture_case( + "tree_and_screenshot", + vec![OracleKind::AxState, OracleKind::Pixels], + |pid, wid, driver| { + let resp = snapshot_settled_default(driver, pid, wid); + assert!( + !resp.is_error(), + "get_window_state(default) errored: {}", + resp.text() + ); - let tree = tree_has_marker(&resp); - let img = has_image(&resp); - assert!( - tree || img, - "default returned neither tree nor image — both grants missing? {}", - resp.text().chars().take(160).collect::() + assert!( + tree_has_marker(&resp), + "default response is missing the required tree marker: {}", + resp.text().chars().take(160).collect::() + ); + assert!( + has_image(&resp), + "default response is missing its screenshot" + ); + }, ); - // When BOTH grants are present, BOTH must be returned (the contract). - if tree && !img { - eprintln!("[capture_mode] default: tree present, no image (screen-capture grant likely missing) — partial check"); - } else if img && !tree { - eprintln!("[capture_mode] default: image present, no tree (accessibility grant likely missing) — partial check"); - } - println!("✅ default: tree={tree} image={img} (both expected when grants present)"); } /// `include_screenshot:false` is the perf opt-out: tree present, NO image. This @@ -275,30 +329,28 @@ fn default_returns_tree_and_screenshot() { #[test] #[ignore] fn include_screenshot_false_returns_tree_only() { - let Some(mut driver) = spawn_driver() else { - return; - }; - let Some((pid, wid)) = launch(&mut driver) else { - return; - }; + run_capture_case( + "tree_only", + vec![OracleKind::AxState], + |pid, wid, driver| { + let resp = snapshot_settled_tree_only(driver, pid, wid); + assert!( + !resp.is_error(), + "get_window_state(include_screenshot:false) errored: {}", + resp.text() + ); - let resp = snapshot_settled_tree_only(&mut driver, pid, wid); - assert!( - !resp.is_error(), - "get_window_state(include_screenshot:false) errored: {}", - resp.text() + assert!( + tree_has_marker(&resp), + "tree-only response is missing {TREE_MARKER:?}" + ); + assert!( + !has_image(&resp), + "include_screenshot:false must NOT return an image content entry: {}", + resp.text().chars().take(160).collect::() + ); + }, ); - - if !tree_has_marker(&resp) { - eprintln!("[capture_mode] tree-only: no {TREE_MARKER:?} — accessibility grant likely missing; skipping"); - return; - } - assert!( - !has_image(&resp), - "include_screenshot:false must NOT return an image content entry: {}", - resp.text().chars().take(160).collect::() - ); - println!("✅ include_screenshot:false: tree present ({TREE_MARKER}), no image"); } /// `capture_mode` is DEPRECATED and ignored: passing `vision` (which used to @@ -306,33 +358,104 @@ fn include_screenshot_false_returns_tree_only() { #[test] #[ignore] fn deprecated_capture_mode_is_ignored() { - let Some(mut driver) = spawn_driver() else { - return; - }; - let Some((pid, wid)) = launch(&mut driver) else { - return; - }; + run_capture_case( + "deprecated_mode_ignored", + vec![OracleKind::AxState, OracleKind::Pixels], + |pid, wid, driver| { + let resp = snapshot_settled(driver, pid, wid, "vision"); + assert!( + !resp.is_error(), + "get_window_state(capture_mode=vision) errored: {}", + resp.text() + ); - let resp = snapshot_settled(&mut driver, pid, wid, "vision"); - assert!( - !resp.is_error(), - "get_window_state(capture_mode=vision) errored: {}", - resp.text() + assert!( + tree_has_marker(&resp), + "deprecated capture_mode=vision suppressed the tree" + ); + assert!( + has_image(&resp), + "deprecated capture_mode=vision suppressed the screenshot" + ); + }, ); +} - // The legacy "vision" value must NOT suppress the tree anymore. - if has_image(&resp) && tree_has_marker(&resp) { - println!("✅ capture_mode=vision ignored: BOTH tree and image returned"); - return; - } - // Tolerate a missing grant, but the deprecated arg must never make the - // present half disappear: if the tree marker shows, that already proves - // "vision" didn't suppress it. - if tree_has_marker(&resp) { - println!("✅ capture_mode=vision ignored: tree still present (image grant may be missing)"); - } else { - eprintln!( - "[capture_mode] vision: no tree marker — accessibility grant likely missing; skipping" +/// Capturing an occluded background window is a read-only operation: it must +/// return pixels without disturbing the user's foreground desktop. +#[cfg(target_os = "windows")] +#[test] +#[ignore] +fn background_screenshot_preserves_desktop() { + let case = CaseSpec::delivered( + "windows-wpf-screenshot-px-background", + "wpf", + "wpf", + "screenshot", + Targeting::Px, + Delivery::Background, + Scope::Window, + DriverRoute::WindowsPrintWindow, + vec![ + OracleKind::Pixels, + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::Cursor, + OracleKind::NoLeakedInput, + ], + ); + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_named("windows-wpf-screenshot-px-background") + .expect("required source-built driver did not start"); + *evidence = recording_evidence(driver.recording_dir()); + let (pid, wid) = launch(&mut driver).expect("required WPF capture harness did not start"); + let sentinel = ForegroundSentinel::launch(&mut driver); + sentinel + .assert_background_posture(TargetWindow { + pid, + native_id: wid, + }) + .expect("establish capture background posture before recording"); + driver.start_behavior_recording(); + let (response, mut passed) = sentinel + .observe_background( + TargetWindow { + pid, + native_id: wid, + }, + || { + driver.call( + "get_window_state", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "capture_mode": "vision" + }), + ) + }, + ) + .unwrap_or_else(|error| panic!("background screenshot disturbed the desktop: {error}")); + assert!( + !response.is_error(), + "background screenshot failed: {}", + response.text() ); - } + assert!( + has_image(&response), + "background screenshot returned no image" + ); + for required in [ + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::Cursor, + OracleKind::NoLeakedInput, + ] { + assert!( + passed.contains(&required), + "background screenshot omitted required {required:?} oracle" + ); + } + passed.push(OracleKind::Pixels); + Observation::delivered(passed, Evidence::default()) + }); } diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs index 8865d1edc3..49c617b949 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs @@ -3,8 +3,9 @@ //! These are source-owned Rust tests, not a copy of a partner test runner. The //! shared web fixture is loaded by Electron and Tauri on every supported OS; //! Windows also has WebView2 coverage in `harness_web_test.rs`. Assertions read -//! the fixture's mutated application state from a fresh accessibility snapshot. -//! A successful driver response alone is never sufficient. +//! mutated application state from a fixture-owned loopback journal, independently +//! of the driver's accessibility snapshot. A successful response alone is never +//! sufficient. //! //! Run after building the shared fixtures: //! @@ -16,8 +17,6 @@ #![cfg(any(target_os = "windows", target_os = "macos", target_os = "linux"))] use std::any::Any; -use std::fs::OpenOptions; -use std::io::Write; use std::panic::{self, AssertUnwindSafe}; use std::path::PathBuf; use std::process::{Command, Stdio}; @@ -25,7 +24,16 @@ use std::thread; use std::time::{Duration, Instant}; use cua_driver_testkit::ax::{element_index_by_id, element_index_containing}; -use cua_driver_testkit::{harness_app, spawn_in_job, Driver, McpDriver, ToolResponse}; +use cua_driver_testkit::e2e::{ + recording_evidence, shared_web_route, write_declaration_from_env, write_result_from_env, + CaseResult, CaseSpec, Delivery, Evidence, Observation, OracleKind, RefusalCode, Scope, + Targeting, +}; +use cua_driver_testkit::observer::TargetWindow; +use cua_driver_testkit::sentinel::ForegroundSentinel; +use cua_driver_testkit::{ + harness_app, spawn_in_job, Driver, FixtureJournal, McpDriver, ToolResponse, +}; struct HostSpec { name: &'static str, @@ -77,6 +85,15 @@ fn host_specs() -> Vec { args: Vec::new(), title: "CuaTestHarness Tauri", }); + hosts.push(HostSpec { + name: "wkwebview", + path: harness_app( + "harness-wkwebview", + "CuaTestHarness.WKWebView.app/Contents/MacOS/CuaTestHarness.WKWebView", + ), + args: Vec::new(), + title: "CuaTestHarness WKWebView", + }); } #[cfg(target_os = "linux")] @@ -91,18 +108,22 @@ fn host_specs() -> Vec { ], title: "CuaTestHarness Electron", }); - // WebKitGTK's AT-SPI tree is exposed through a separate WebProcess; - // the Rust walker handles that reference shape, but headless Xvfb - // still does not provide a reliable input-delivery contract for the - // Tauri renderer. Keep this strict lane opt-in until that renderer - // path is fixed, while the Electron matrix remains deterministic. - if std::env::var_os("CUA_INCLUDE_TAURI_LINUX").is_some() { - hosts.push(HostSpec { - name: "tauri", - path: harness_app("harness-tauri", "CuaTestHarness.Tauri"), - args: Vec::new(), - title: "CuaTestHarness Tauri", - }); + hosts.push(HostSpec { + name: "tauri", + path: harness_app("harness-tauri", "CuaTestHarness.Tauri"), + args: Vec::new(), + title: "CuaTestHarness Tauri", + }); + } + + if let Ok(filter) = std::env::var("CUA_E2E_HARNESS_FILTER") { + let selected = filter + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + .collect::>(); + if !selected.is_empty() { + hosts.retain(|host| selected.contains(host.name)); } } @@ -121,83 +142,71 @@ fn panic_message(payload: &Box) -> String { .unwrap_or_else(|| "test panicked without a string payload".to_owned()) } -fn escape_markdown(value: &str) -> String { - value - .replace('|', "\\|") - .replace('\r', "") - .replace('\n', " ") -} - -fn append_result_line(path: &str, line: &str) { - let Some(parent) = std::path::Path::new(path).parent() else { - return; - }; - let _ = std::fs::create_dir_all(parent); - if let Ok(mut file) = OpenOptions::new().create(true).append(true).open(path) { - let _ = writeln!(file, "{line}"); - } -} - -fn record_result(scenario: &str, host: &str, status: &str, message: &str, duration: Duration) { - let message = escape_markdown(message); - if let Ok(path) = std::env::var("CUA_E2E_RESULTS_FILE") { - let line = serde_json::json!({ - "schema": "cua-e2e-result/v1", - "platform": std::env::consts::OS, - "host": host, - "scenario": scenario, - "status": status, - "message": message, - "duration_ms": duration.as_millis(), - }); - append_result_line(&path, &line.to_string()); - } - if let Ok(path) = std::env::var("CUA_E2E_SUMMARY_FILE") { - append_result_line( - &path, - &format!( - "| {} | {} | {} | {} | {} ms | {} |", - std::env::consts::OS, - escape_markdown(host), - escape_markdown(scenario), - status, - duration.as_millis(), - if message.is_empty() { "-" } else { &message }, - ), - ); - } -} - -fn run_host_case(scenario: &str, spec: &HostSpec, test: F) -> Option> +fn run_host_case_with_outcome( + case: CaseSpec, + spec: &HostSpec, + test: F, +) -> Option> where - F: FnOnce(Fixture), + F: FnOnce(&mut Fixture) -> Observation, { + write_declaration_from_env(&case).expect("write E2E case declaration"); let started = Instant::now(); + let mut evidence = Evidence::default(); + let delivery = case.delivery; let outcome = panic::catch_unwind(AssertUnwindSafe(|| { - let Some(fixture) = launch_host(spec, scenario) else { - return false; + let mut fixture = launch_host_with_evidence(spec, &case.cell_id, &mut evidence); + let sentinel = if delivery == Delivery::Background { + Some(ForegroundSentinel::launch(&mut fixture.driver)) + } else { + None }; - test(fixture); - true + if let Some(sentinel) = &sentinel { + sentinel + .assert_background_posture(TargetWindow { + pid: fixture.pid, + native_id: fixture.wid, + }) + .expect("establish background posture before recording"); + } + fixture.driver.start_behavior_recording(); + let mut observation = if delivery == Delivery::Background { + let sentinel = sentinel.as_ref().expect("background sentinel"); + let (mut observation, passed) = sentinel + .observe_background( + TargetWindow { + pid: fixture.pid, + native_id: fixture.wid, + }, + || test(&mut fixture), + ) + .expect("observe background desktop side effects"); + observation.passed_oracles.extend(passed); + observation + } else { + test(&mut fixture) + }; + drop(sentinel); + observation.evidence = evidence.clone(); + observation })); match outcome { - Ok(true) => { - record_result(scenario, spec.name, "PASS", "", started.elapsed()); - None - } - Ok(false) => { - record_result( - scenario, - spec.name, - "SKIP", - "fixture unavailable", - started.elapsed(), - ); + Ok(observation) => { + let result = CaseResult::evaluate(case, observation, started.elapsed()); + write_result_from_env(&result).expect("write E2E case result"); + if result.test_status == cua_driver_testkit::e2e::TestStatus::Fail { + return Some(Box::new(result.message)); + } None } Err(payload) => { let message = panic_message(&payload); - record_result(scenario, spec.name, "FAIL", &message, started.elapsed()); + let result = CaseResult::evaluate( + case, + Observation::error(&message, evidence), + started.elapsed(), + ); + write_result_from_env(&result).expect("write failed E2E case result"); Some(payload) } } @@ -212,7 +221,7 @@ fn resume_first_failure(failure: Option>) { fn spawn_driver(recording_label: &str) -> Option { #[cfg(target_os = "macos")] { - return McpDriver::spawn_macos_daemon_proxy_named(recording_label); + McpDriver::spawn_macos_daemon_proxy_named(recording_label) } #[cfg(not(target_os = "macos"))] { @@ -227,38 +236,48 @@ struct Fixture { window_x: f64, window_y: f64, name: &'static str, + journal: FixtureJournal, } -fn launch_host(spec: &HostSpec, scenario: &str) -> Option { +fn evidence_for_driver(driver: &McpDriver) -> Evidence { + recording_evidence(driver.recording_dir()) +} + +fn allocate_loopback_port() -> u16 { + let listener = + std::net::TcpListener::bind(("127.0.0.1", 0)).expect("allocate an ephemeral fixture port"); + listener.local_addr().expect("read fixture port").port() +} + +fn launch_host_with_evidence(spec: &HostSpec, scenario: &str, evidence: &mut Evidence) -> Fixture { if !spec.path.exists() { - if std::env::var_os("CUA_TEST_REQUIRE_FIXTURES").is_some() { - panic!( - "{} fixture is required but was not staged at {:?}", - spec.name, spec.path - ); - } - eprintln!( - "[{}] fixture not staged at {:?}; skipping", + panic!( + "{} fixture is required but was not staged at {:?}", spec.name, spec.path ); - return None; } - let recording_label = format!("{scenario}-{}", spec.name); - let Some(mut driver) = spawn_driver(&recording_label) else { - if std::env::var_os("CUA_TEST_REQUIRE_FIXTURES").is_some() { - panic!( - "cua-driver could not be started for the required {} fixture", - spec.name - ); - } - return None; - }; + let recording_label = scenario.to_owned(); + let mut driver = spawn_driver(&recording_label).unwrap_or_else(|| { + panic!( + "cua-driver could not be started for the required {} fixture", + spec.name + ) + }); + *evidence = evidence_for_driver(&driver); + let journal = FixtureJournal::start(); let mut command = Command::new(&spec.path); command .args(&spec.args) - .stdout(Stdio::null()) - .stderr(Stdio::null()); + .env("CUA_E2E_FIXTURE_JOURNAL_URL", journal.url()) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()); + if spec.name == "electron" { + command.env( + "CUA_ELECTRON_CDP_PORT", + allocate_loopback_port().to_string(), + ); + } let before_windows = driver .call("list_windows", serde_json::json!({})) .structured()["windows"] @@ -270,7 +289,8 @@ fn launch_host(spec: &HostSpec, scenario: &str) -> Option { .collect::>() }) .unwrap_or_default(); - let child = spawn_in_job(&mut command).ok()?; + let child = spawn_in_job(&mut command) + .unwrap_or_else(|error| panic!("failed to launch {} fixture: {error}", spec.name)); let pid = child.id(); driver.reaper().push(child); @@ -296,20 +316,24 @@ fn launch_host(spec: &HostSpec, scenario: &str) -> Option { window_x: bounds["x"].as_f64().unwrap_or(0.0), window_y: bounds["y"].as_f64().unwrap_or(0.0), name: spec.name, + journal, }; let ax_deadline = Instant::now() + Duration::from_secs(10); + let mut last_tree = String::new(); while Instant::now() < ax_deadline { - if snapshot(&mut fixture) - .tree_text() - .contains("WEB_HARNESS_MARKER_v1") + let state = snapshot(&mut fixture); + last_tree.clear(); + last_tree.push_str(state.tree_text()); + if last_tree.contains("WEB_HARNESS_MARKER_v1") + && fixture.journal.contains("WEB_HARNESS_MARKER_v1") { - return Some(fixture); + return fixture; } thread::sleep(Duration::from_millis(250)); } panic!( - "{} fixture accessibility tree did not become ready", - spec.name + "{} fixture accessibility tree did not become ready; last tree:\n{}", + spec.name, last_tree ); } } @@ -366,7 +390,7 @@ fn screenshot_scale(state: &ToolResponse) -> f64 { }) .and_then(|window| window["frame"]["w"].as_f64()) .unwrap_or(0.0); - if window_width > 0.0 && width > window_width { + if window_width > 0.0 && width > 0.0 { width / window_width } else { 1.0 @@ -375,18 +399,27 @@ fn screenshot_scale(state: &ToolResponse) -> f64 { fn require_element(snapshot: &ToolResponse, id: &str) -> u64 { // Some WebKit/Chromium AX adapters preserve the DOM id as an annotation - // (`(calc-1 1)`) instead of emitting the common `id=...` form. Searching - // the visible label fallback handles adapters that omit both forms. + // instead of emitting the common `id=...` form. Searching the visible + // label fallback handles adapters that omit both forms. let visible_label = match id { - "calc-1" | "calc-2" | "calc-4" | "calc-plus" | "calc-equals" | "editor-document" - | "editor-save" | "scroll-tall" => id, + "editor-document" | "editor-save" | "scroll-tall" => id, + "border-click-target" => "Click target (left / right / double)", + "txt-input" => "type here", "keyboard-input" => "keyboard-input", "drag-source" => "Drag source", "drop-target" => "Drop target", "btn-open-child-window" => "Open child window", _ => id, }; + let platform_id = match id { + // Chromium exposes the DOM id through AX, while data-cua-id is the + // canonical identifier shared with the native harnesses. + "border-click-target" => "click-target", + _ => id, + }; element_index_by_id(snapshot.tree_text(), id) + .or_else(|| element_index_by_id(snapshot.tree_text(), platform_id)) + .or_else(|| element_index_containing(snapshot.tree_text(), id)) .or_else(|| element_index_containing(snapshot.tree_text(), visible_label)) .unwrap_or_else(|| { panic!( @@ -413,352 +446,622 @@ fn element_center(snapshot: &ToolResponse, element_index: u64) -> (f64, f64) { ) } -fn assert_tree_contains(fixture: &mut Fixture, marker: &str) { - let post = snapshot(fixture); +fn assert_fixture_contains(fixture: &Fixture, marker: &str) { + let deadline = Instant::now() + Duration::from_secs(2); + loop { + if fixture.journal.contains(marker) { + return; + } + if Instant::now() >= deadline { + panic!( + "{}: fixture journal did not reach {marker:?}: {}", + fixture.name, + fixture.journal.snapshot() + ); + } + thread::sleep(Duration::from_millis(100)); + } +} + +fn action_target_args( + fixture: &Fixture, + state: &ToolResponse, + id: &str, + addressing: &str, + delivery: &str, +) -> serde_json::Value { + let index = require_element(state, id); + let mut args = serde_json::json!({ + "pid": fixture.pid as i64, + "window_id": fixture.wid, + "delivery_mode": delivery, + }); + let object = args.as_object_mut().expect("action arguments object"); + if addressing == "ax" { + object.insert("element_index".to_owned(), serde_json::json!(index)); + } else { + let origin = window_origin(fixture, state); + let scale = screenshot_scale(state); + let (x, y) = element_center(state, index); + let local_x = (x - origin.0) * scale; + let local_y = (y - origin.1) * scale; + let width = state.structured()["screenshot_width"] + .as_f64() + .expect("PX action requires screenshot_width"); + let height = state.structured()["screenshot_height"] + .as_f64() + .expect("PX action requires screenshot_height"); + eprintln!( + "[shared-px] {} target={id} screen=({x:.1},{y:.1}) origin=({:.1},{:.1}) scale={scale:.3} local=({local_x:.1},{local_y:.1}) capture=({width:.1}x{height:.1})", + fixture.name, origin.0, origin.1 + ); + assert!( + local_x >= 0.0 && local_x < width && local_y >= 0.0 && local_y < height, + "{}: PX target {id:?} center ({local_x:.1}, {local_y:.1}) is outside the captured window ({width:.1}x{height:.1}); fix the harness layout", + fixture.name + ); + object.insert("x".to_owned(), serde_json::json!(local_x)); + object.insert("y".to_owned(), serde_json::json!(local_y)); + } + args +} + +fn run_pointer_action( + fixture: &mut Fixture, + tool: &str, + addressing: &str, + delivery: &str, + expected_marker: &str, +) -> Observation { + let pre = snapshot(fixture); + let journal_before = fixture.journal.snapshot(); + let args = action_target_args(fixture, &pre, "border-click-target", addressing, delivery); + let response = fixture.driver.call(tool, args); + if let Some(code) = background_refusal_code(&response, delivery) { + return refused_without_fixture_mutation(fixture, &journal_before, code, &response); + } + assert!( + !response.is_error(), + "{}: {tool} {addressing}/{delivery} failed: {}", + fixture.name, + response.text() + ); + assert_fixture_contains(fixture, expected_marker); + delivered_observation() +} + +fn delivered_observation() -> Observation { + Observation::delivered(vec![OracleKind::FixtureState], Evidence::default()) +} + +fn refused_without_fixture_mutation( + fixture: &Fixture, + before: &serde_json::Value, + code: RefusalCode, + response: &ToolResponse, +) -> Observation { + thread::sleep(Duration::from_millis(150)); + let after = fixture.journal.snapshot(); + assert_eq!( + &after, before, + "{}: refused action mutated fixture state: before={before}, after={after}", + fixture.name + ); + Observation::refused( + code, + vec![OracleKind::FixtureState], + response.text(), + Evidence::default(), + ) +} + +fn unverified_background_protocol_oracle( + response: &ToolResponse, + delivery: &str, +) -> Vec { + if !cfg!(target_os = "windows") || delivery != "background" { + return Vec::new(); + } + assert_eq!( + response.verified(), + Some(false), + "background dispatch without independent read-back must remain unverified: {}", + response.text() + ); + assert_ne!( + response.structured()["verify"].as_str(), + Some("confirmed"), + "background dispatch overclaimed a confirmed read-back: {}", + response.text() + ); + vec![OracleKind::Protocol] +} + +fn background_refusal_code(response: &ToolResponse, delivery: &str) -> Option { + if delivery != "background" || !response.is_error() { + return None; + } + response.structured()["code"] + .as_str() + .and_then(RefusalCode::from_driver_code) +} + +fn run_text_action(fixture: &mut Fixture, addressing: &str, delivery: &str) -> Observation { + let pre = snapshot(fixture); + let journal_before = fixture.journal.snapshot(); + let text = format!("cua-{addressing}-{delivery}"); + let mut args = action_target_args(fixture, &pre, "txt-input", addressing, delivery); + args.as_object_mut() + .expect("type_text arguments object") + .insert("text".to_owned(), serde_json::json!(text)); + let response = fixture.driver.call("type_text", args); + if let Some(code) = background_refusal_code(&response, delivery) { + return refused_without_fixture_mutation(fixture, &journal_before, code, &response); + } + assert!( + !response.is_error(), + "{}: type_text {addressing}/{delivery} failed: {}", + fixture.name, + response.text() + ); + let mut passed = vec![OracleKind::FixtureState]; + if addressing == "px" { + passed.extend(unverified_background_protocol_oracle(&response, delivery)); + } + thread::sleep(Duration::from_millis(250)); + assert_fixture_contains(fixture, &format!("mirror={text}")); + Observation::delivered(passed, Evidence::default()) +} + +fn run_press_key_action(fixture: &mut Fixture, addressing: &str, delivery: &str) -> Observation { + let pre = snapshot(fixture); + let journal_before = fixture.journal.snapshot(); + let mut press_args = action_target_args(fixture, &pre, "keyboard-input", addressing, delivery); + press_args + .as_object_mut() + .expect("press_key arguments object") + .insert("key".to_owned(), serde_json::json!("return")); + let response = fixture.driver.call("press_key", press_args); + if let Some(code) = background_refusal_code(&response, delivery) { + return refused_without_fixture_mutation(fixture, &journal_before, code, &response); + } + assert!( + !response.is_error(), + "{}: press_key {addressing}/{delivery} failed: {}", + fixture.name, + response.text() + ); + let mut passed = vec![OracleKind::FixtureState]; + passed.extend(unverified_background_protocol_oracle(&response, delivery)); + assert_fixture_contains(fixture, "key_state=enter"); + + Observation::delivered(passed, Evidence::default()) +} + +fn run_hotkey_action(fixture: &mut Fixture, addressing: &str, delivery: &str) -> Observation { + let pre = snapshot(fixture); + let journal_before = fixture.journal.snapshot(); + let mut hotkey_args = action_target_args(fixture, &pre, "keyboard-input", addressing, delivery); + hotkey_args + .as_object_mut() + .expect("hotkey arguments object") + .insert("keys".to_owned(), serde_json::json!(["ctrl", "shift", "7"])); + let response = fixture.driver.call("hotkey", hotkey_args); + if let Some(code) = background_refusal_code(&response, delivery) { + return refused_without_fixture_mutation(fixture, &journal_before, code, &response); + } + assert!( + !response.is_error(), + "{}: hotkey {addressing}/{delivery} failed: {}", + fixture.name, + response.text() + ); + assert_fixture_contains(fixture, "key_state=hotkey"); + let passed = unverified_background_protocol_oracle(&response, delivery); + Observation::delivered_with_fixture_state(passed) +} + +fn run_scroll_action(fixture: &mut Fixture, addressing: &str, delivery: &str) -> Observation { + let pre = snapshot(fixture); + let journal_before = fixture.journal.snapshot(); + let mut args = action_target_args(fixture, &pre, "scroll-tall", addressing, delivery); + let object = args.as_object_mut().expect("scroll arguments object"); + object.insert("direction".to_owned(), serde_json::json!("down")); + object.insert("by".to_owned(), serde_json::json!("page")); + object.insert("amount".to_owned(), serde_json::json!(2)); + let response = fixture.driver.call("scroll", args); + if let Some(code) = background_refusal_code(&response, delivery) { + return refused_without_fixture_mutation(fixture, &journal_before, code, &response); + } assert!( - post.tree_text().contains(marker), - "{}: application state did not reach {marker:?}: {}", + !response.is_error(), + "{}: scroll {addressing}/{delivery} failed: {}; raw={}", fixture.name, - post.tree_text() + response.text(), + response.raw ); + thread::sleep(Duration::from_millis(250)); + let offset = fixture + .journal + .text("lbl-scroll-offset") + .and_then(|text| text.split("scroll_offset=").nth(1).map(str::to_owned)) + .and_then(|tail| { + tail.split(|ch: char| !ch.is_ascii_digit()) + .next() + .map(str::to_owned) + }) + .and_then(|value| value.parse::().ok()) + .unwrap_or(0); + assert!( + offset > 0, + "{}: scroll did not advance the external oracle: {}; response={}; raw={}", + fixture.name, + fixture.journal.snapshot(), + response.text(), + response.raw + ); + delivered_observation() } -fn click_ax(fixture: &mut Fixture, id: &str) { +fn run_drag_action(fixture: &mut Fixture, delivery: &str) -> Observation { let pre = snapshot(fixture); - let element_index = require_element(&pre, id); + let journal_before = fixture.journal.snapshot(); + let source = require_element(&pre, "drag-source"); + let target = require_element(&pre, "drop-target"); + let origin = window_origin(fixture, &pre); + let scale = screenshot_scale(&pre); + let point = |index: u64| { + let (x, y) = element_center(&pre, index); + ((x - origin.0) * scale, (y - origin.1) * scale) + }; + let (from_x, from_y) = point(source); + let (to_x, to_y) = point(target); let response = fixture.driver.call( - "click", + "drag", serde_json::json!({ "pid": fixture.pid as i64, "window_id": fixture.wid, - "element_index": element_index, - "delivery_mode": "background" + "from_x": from_x, + "from_y": from_y, + "to_x": to_x, + "to_y": to_y, + "duration_ms": 400, + "steps": 20, + "delivery_mode": delivery, }), ); + if let Some(code) = background_refusal_code(&response, delivery) { + return refused_without_fixture_mutation(fixture, &journal_before, code, &response); + } assert!( !response.is_error(), - "{}: AX click {id} failed: {}", + "{}: drag PX/{delivery} failed: {}; raw={}", + fixture.name, + response.text(), + response.raw + ); + assert_fixture_contains(fixture, "drag_status=dropped"); + delivered_observation() +} + +fn run_child_window_action(fixture: &mut Fixture, addressing: &str, delivery: &str) -> Observation { + let pre = snapshot(fixture); + let journal_before = fixture.journal.snapshot(); + let args = action_target_args(fixture, &pre, "btn-open-child-window", addressing, delivery); + let response = fixture.driver.call("click", args); + if let Some(code) = background_refusal_code(&response, delivery) { + return refused_without_fixture_mutation(fixture, &journal_before, code, &response); + } + assert!( + !response.is_error(), + "{}: child-window click {addressing}/{delivery} failed: {}", fixture.name, response.text() ); + assert_fixture_contains(fixture, "child_windows=1"); + delivered_observation() } -#[test] -#[ignore] -fn shared_web_calculator_ax_route_is_state_verified() { - let mut failure = None; - for spec in host_specs() { - let result = run_host_case("calculator_ax", &spec, |mut fixture| { - click_ax(&mut fixture, "calc-1"); - click_ax(&mut fixture, "calc-2"); - click_ax(&mut fixture, "calc-plus"); - click_ax(&mut fixture, "calc-4"); - click_ax(&mut fixture, "calc-equals"); - assert_tree_contains(&mut fixture, "display=16"); - println!("✅ {} calculator AX route", fixture.name); - }); - if failure.is_none() { - failure = result; - } +fn run_editor_save_action(fixture: &mut Fixture, delivery: &str) -> Observation { + let pre = snapshot(fixture); + let journal_before = fixture.journal.snapshot(); + let text = format!("cua-editor-{delivery}"); + let mut text_args = action_target_args(fixture, &pre, "editor-document", "ax", delivery); + text_args + .as_object_mut() + .expect("editor type_text arguments object") + .insert("text".to_owned(), serde_json::json!(text)); + let response = fixture.driver.call("type_text", text_args); + if let Some(code) = background_refusal_code(&response, delivery) { + return refused_without_fixture_mutation(fixture, &journal_before, code, &response); } - resume_first_failure(failure); + assert!( + !response.is_error(), + "{}: editor type_text failed: {}", + fixture.name, + response.text() + ); + + let post_text = snapshot(fixture); + let journal_before_save = fixture.journal.snapshot(); + let save_args = action_target_args(fixture, &post_text, "editor-save", "ax", delivery); + let response = fixture.driver.call("click", save_args); + if let Some(code) = background_refusal_code(&response, delivery) { + return refused_without_fixture_mutation(fixture, &journal_before_save, code, &response); + } + assert!( + !response.is_error(), + "{}: editor save failed: {}", + fixture.name, + response.text() + ); + assert_fixture_contains(fixture, "editor_status=saved:"); + delivered_observation() } -#[test] -#[ignore] -fn shared_web_keyboard_routes_are_state_verified() { - let mut failure = None; - for spec in host_specs() { - let result = run_host_case("keyboard", &spec, |mut fixture| { - let pre = snapshot(&mut fixture); - let input = require_element(&pre, "keyboard-input"); - let (origin_x, origin_y) = window_origin(&fixture, &pre); - let scale = screenshot_scale(&pre); - let (screen_x, screen_y) = element_center(&pre, input); - let focus = fixture.driver.call( - "click", - serde_json::json!({ - "pid": fixture.pid as i64, - "window_id": fixture.wid, - "x": (screen_x - origin_x) * scale, - "y": (screen_y - origin_y) * scale, - "delivery_mode": "background" - }), - ); - assert!( - !focus.is_error(), - "{}: keyboard input focus click failed: {}", - fixture.name, - focus.text() - ); - thread::sleep(Duration::from_millis(150)); - let enter = fixture.driver.call( - "press_key", - serde_json::json!({ - "pid": fixture.pid as i64, - "window_id": fixture.wid, - "element_index": input, - "key": "return", - "delivery_mode": "background" - }), - ); - assert!( - !enter.is_error(), - "{}: return failed: {}", - fixture.name, - enter.text() - ); - assert_tree_contains(&mut fixture, "key_state=enter"); +#[cfg(target_os = "linux")] +fn linux_real_pointer_input_available() -> bool { + platform_linux::input::real_pointer_input_available() +} - let hotkey = fixture.driver.call( - "hotkey", - serde_json::json!({ - "pid": fixture.pid as i64, - "window_id": fixture.wid, - "keys": ["ctrl", "shift", "7"], - "x": (screen_x - origin_x) * scale, - "y": (screen_y - origin_y) * scale, - "delivery_mode": "background" - }), - ); - #[cfg(target_os = "windows")] - { - if hotkey.is_error() - || hotkey - .text() - .contains("Background delivery is not available") - { - println!( - "✅ {} keyboard AX route: background hotkey refused honestly", - fixture.name - ); - return; - } +#[cfg(not(target_os = "linux"))] +fn linux_real_pointer_input_available() -> bool { + false +} + +fn shared_case(spec: &HostSpec, action: &str, addressing: &str, delivery: &str) -> CaseSpec { + let targeting = match addressing { + "ax" => Targeting::Ax, + "px" => Targeting::Px, + _ => Targeting::NotApplicable, + }; + let delivery_kind = match delivery { + "background" => Delivery::Background, + "foreground" => Delivery::Foreground, + _ => Delivery::NotApplicable, + }; + let scenario = format!("{action}_{addressing}_{delivery}"); + let cell_id = format!("{}-{}-{scenario}", std::env::consts::OS, spec.name).replace('_', "-"); + let expected_refusals = if cfg!(target_os = "windows") && delivery_kind == Delivery::Background + { + match (spec.name, action, targeting) { + ("electron", "right_click" | "double_click" | "drag", _) => { + vec![RefusalCode::BackgroundOccluded] + } + ("electron", "type_text" | "press_key" | "hotkey" | "scroll" | "editor_save", _) => { + vec![RefusalCode::BackgroundUnavailable] + } + ("tauri", "hotkey", _) | ("tauri", "scroll", Targeting::Px) => { + vec![RefusalCode::BackgroundUnavailable] + } + ("tauri", "drag", Targeting::Px) => vec![RefusalCode::BackgroundOccluded], + _ => Vec::new(), + } + } else if cfg!(target_os = "macos") && delivery_kind == Delivery::Background { + match (spec.name, action, targeting) { + ("electron", "scroll", _) | (_, "drag", Targeting::Px) => { + vec![RefusalCode::BackgroundUnavailable] } - #[cfg(target_os = "linux")] + _ => Vec::new(), + } + } else if cfg!(target_os = "linux") + && spec.name == "electron" + && delivery_kind == Delivery::Background + { + // Chromium's X11 renderer drops synthetic input addressed to a fully + // occluded, unfocused toplevel. Focus-free AT-SPI button actions are + // the exception: they are externally verified by the fixture and the + // focus/z-order/leak sentinels. + if std::env::var_os("CUA_INJECT_SOCKET").is_some() { + Vec::new() + } else { + match (action, targeting) { + ("left_click" | "child_window", Targeting::Ax) => Vec::new(), + _ => vec![RefusalCode::BackgroundUnavailable], + } + } + } else if cfg!(target_os = "linux") + && spec.name == "tauri" + && delivery_kind == Delivery::Background + { + // WebKitGTK accepts AT-SPI text writes without emitting the renderer's + // user-input event, and its keyboard channel is focus-bound. The driver + // must refuse those background keyboard composites instead of reporting + // a successful write that the page never observes. + match (action, targeting) { + ("type_text", _) | ("editor_save", Targeting::Ax) | ("press_key" | "hotkey", _) => { + vec![RefusalCode::BackgroundUnavailable] + } + ("right_click" | "double_click" | "scroll", _) | ("drag", Targeting::Px) + if !linux_real_pointer_input_available() => { - if hotkey.is_error() - && hotkey.structured()["code"].as_str() == Some("background_unavailable") - { - println!( - "✅ {} keyboard AX route: background hotkey refused honestly", - fixture.name - ); - return; - } + vec![RefusalCode::BackgroundUnavailable] } - assert!( - !hotkey.is_error(), - "{}: Ctrl+Shift+7 failed: {}", - fixture.name, - hotkey.text() - ); - assert_tree_contains(&mut fixture, "key_state=hotkey"); - println!("✅ {} keyboard AX routes", fixture.name); - }); - if failure.is_none() { - failure = result; + _ => Vec::new(), + } + } else { + Vec::new() + }; + let expected_background_refusal = !expected_refusals.is_empty(); + let mut oracles = if expected_background_refusal { + vec![OracleKind::FixtureState] + } else { + vec![OracleKind::FixtureState] + }; + if delivery_kind == Delivery::Background { + oracles.extend([ + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::NoLeakedInput, + ]); + if cua_driver_testkit::e2e::DisplayServer::current() + != cua_driver_testkit::e2e::DisplayServer::Wayland + { + oracles.push(OracleKind::Cursor); + } + if !expected_background_refusal + && cfg!(target_os = "windows") + && (matches!(action, "press_key" | "hotkey") + || (action == "type_text" && targeting == Targeting::Px)) + { + oracles.push(OracleKind::Protocol); } } - resume_first_failure(failure); + let mut route = shared_web_route( + cua_driver_testkit::e2e::Platform::current(), + cua_driver_testkit::e2e::DisplayServer::current(), + action, + targeting, + delivery_kind, + ) + .unwrap_or_else(|error| panic!("{error}")); + if cfg!(target_os = "windows") + && spec.name == "electron" + && action == "left_click" + && delivery_kind == Delivery::Background + { + route = cua_driver_testkit::e2e::DriverRoute::PostMessage; + } else if cfg!(target_os = "macos") + && targeting == Targeting::Px + && delivery_kind == Delivery::Background + && matches!(action, "left_click" | "child_window") + { + route = cua_driver_testkit::e2e::DriverRoute::MacosAxAction; + } + let case = CaseSpec::delivered( + cell_id, + spec.name, + if spec.name == "electron" { + "chromium" + } else { + "platform-webview" + }, + action, + targeting, + delivery_kind, + Scope::Window, + route, + oracles, + ); + if expected_background_refusal { + case.expecting_refusal(expected_refusals) + } else { + case + } } -#[test] -#[ignore] -fn shared_web_calculator_pixel_route_is_state_verified() { - let mut failure = None; - for spec in host_specs() { - let result = run_host_case("calculator_pixel", &spec, |mut fixture| { - for id in ["calc-1", "calc-2", "calc-plus", "calc-4", "calc-equals"] { - let pre = snapshot(&mut fixture); - let origin = window_origin(&fixture, &pre); - let scale = screenshot_scale(&pre); - let index = require_element(&pre, id); - let (screen_x, screen_y) = element_center(&pre, index); - let response = fixture.driver.call( - "click", - serde_json::json!({ - "pid": fixture.pid as i64, - "window_id": fixture.wid, - "x": (screen_x - origin.0) * scale, - "y": (screen_y - origin.1) * scale, - "delivery_mode": "background" - }), - ); - assert!( - !response.is_error(), - "{}: pixel click {id} failed: {}", - fixture.name, - response.text() - ); - } - assert_tree_contains(&mut fixture, "display=16"); - println!("✅ {} calculator pixel route", fixture.name); - }); - if failure.is_none() { - failure = result; - } - } - resume_first_failure(failure); +fn cell_selected(case: &CaseSpec) -> bool { + let Ok(filter) = std::env::var("CUA_E2E_CELL_FILTER") else { + return true; + }; + let mut parts = filter + .split(',') + .map(str::trim) + .filter(|part| !part.is_empty()) + .peekable(); + parts.peek().is_none() || parts.any(|part| case.cell_id == part || case.cell_id.contains(part)) +} + +fn cell_filter_active() -> bool { + std::env::var("CUA_E2E_CELL_FILTER") + .map(|filter| filter.split(',').any(|part| !part.trim().is_empty())) + .unwrap_or(false) } #[test] #[ignore] -fn shared_web_editor_and_scroll_are_state_verified() { +fn shared_web_action_matrix_is_state_verified() { let mut failure = None; + let mut selected = 0usize; for spec in host_specs() { - let result = run_host_case("editor_scroll", &spec, |mut fixture| { - let pre = snapshot(&mut fixture); - let editor = require_element(&pre, "editor-document"); - let text = fixture.driver.call( + for (action, tool, marker) in [ + ("left_click", "click", "last_action=left_click"), + ("right_click", "right_click", "last_action=right_click"), + ("double_click", "double_click", "last_action=double_click"), + ] { + for addressing in ["ax", "px"] { + for delivery in ["background", "foreground"] { + let case = shared_case(&spec, action, addressing, delivery); + if !cell_selected(&case) { + continue; + } + selected += 1; + let result = run_host_case_with_outcome(case, &spec, |fixture| { + run_pointer_action(fixture, tool, addressing, delivery, marker) + }); + if failure.is_none() { + failure = result; + } + } + } + } + for (action, run) in [ + ( "type_text", - serde_json::json!({ - "pid": fixture.pid as i64, - "window_id": fixture.wid, - "element_index": editor, - "text": "CUA saved this note.", - "delivery_mode": "background" - }), - ); - assert!( - !text.is_error(), - "{}: editor type_text failed: {}", - fixture.name, - text.text() - ); - click_ax(&mut fixture, "editor-save"); - assert_tree_contains(&mut fixture, "editor_status=saved:"); - - let scroll = require_element(&snapshot(&mut fixture), "scroll-tall"); - let response = fixture.driver.call( + run_text_action as fn(&mut Fixture, &str, &str) -> Observation, + ), + ( + "press_key", + run_press_key_action as fn(&mut Fixture, &str, &str) -> Observation, + ), + ( + "hotkey", + run_hotkey_action as fn(&mut Fixture, &str, &str) -> Observation, + ), + ( "scroll", - serde_json::json!({ - "pid": fixture.pid as i64, - "window_id": fixture.wid, - "element_index": scroll, - "direction": "down", - "by": "page", - "amount": 4, - "delivery_mode": "background" - }), - ); - #[cfg(target_os = "windows")] - { - if response.is_error() - || response - .text() - .contains("Background delivery is not available") - { - println!( - "✅ {} editor + scroll: background scroll refused honestly", - fixture.name - ); - return; + run_scroll_action as fn(&mut Fixture, &str, &str) -> Observation, + ), + ( + "child_window", + run_child_window_action as fn(&mut Fixture, &str, &str) -> Observation, + ), + ] { + for addressing in ["ax", "px"] { + for delivery in ["background", "foreground"] { + let case = shared_case(&spec, action, addressing, delivery); + if !cell_selected(&case) { + continue; + } + selected += 1; + let result = run_host_case_with_outcome(case, &spec, |fixture| { + run(fixture, addressing, delivery) + }); + if failure.is_none() { + failure = result; + } } } - assert!( - !response.is_error(), - "{}: scroll failed: {}", - fixture.name, - response.text() - ); - let post = snapshot(&mut fixture); - let offset = post - .tree_text() - .lines() - .find_map(|line| line.split("scroll_offset=").nth(1)) - .and_then(|tail| tail.split(|ch: char| !ch.is_ascii_digit()).next()) - .and_then(|value| value.parse::().ok()) - .unwrap_or(0); - assert!( - offset > 0, - "{}: successful background scroll did not advance the external scroll oracle: {}", - fixture.name, - post.tree_text() - ); - println!("✅ {} editor + scroll state oracles", fixture.name); - }); - if failure.is_none() { - failure = result; } - } - resume_first_failure(failure); -} - -#[test] -#[ignore] -fn shared_web_child_window_and_drag_have_external_oracles() { - let mut failure = None; - for spec in host_specs() { - let result = run_host_case("child_window_drag", &spec, |mut fixture| { - let pre = snapshot(&mut fixture); - let (window_x, window_y) = window_origin(&fixture, &pre); - let scale = screenshot_scale(&pre); - let source = require_element(&pre, "drag-source"); - let frame = pre.structured()["elements"] - .as_array() - .and_then(|elements| { - elements - .iter() - .find(|element| element["element_index"].as_u64() == Some(source)) - }) - .and_then(|element| element["frame"].as_object()) - .unwrap_or_else(|| panic!("{}: drag-source has no frame", fixture.name)); - let x = (frame["x"].as_f64().unwrap_or(0.0) - window_x - + frame["w"].as_f64().unwrap_or(0.0) / 2.0) - * scale; - let y = (frame["y"].as_f64().unwrap_or(0.0) - window_y - + frame["h"].as_f64().unwrap_or(0.0) / 2.0) - * scale; - let target_index = require_element(&pre, "drop-target"); - let target = pre.structured()["elements"] - .as_array() - .and_then(|elements| { - elements - .iter() - .find(|element| element["element_index"].as_u64() == Some(target_index)) - }) - .and_then(|element| element["frame"].as_object()) - .unwrap_or_else(|| panic!("{}: drop-target has no frame", fixture.name)); - let tx = (target["x"].as_f64().unwrap_or(0.0) - window_x - + target["w"].as_f64().unwrap_or(0.0) / 2.0) - * scale; - let ty = (target["y"].as_f64().unwrap_or(0.0) - window_y - + target["h"].as_f64().unwrap_or(0.0) / 2.0) - * scale; - #[cfg(not(target_os = "macos"))] - let _ = (x, y, tx, ty); - #[cfg(target_os = "macos")] - { - let drag = fixture.driver.call( - "drag", - serde_json::json!({ - "pid": fixture.pid as i64, - "window_id": fixture.wid, - "from_x": x, - "from_y": y, - "to_x": tx, - "to_y": ty, - "duration_ms": 500, - "delivery_mode": "foreground" - }), - ); - assert!( - !drag.is_error(), - "{}: drag failed: {}", - fixture.name, - drag.text() - ); - assert_tree_contains(&mut fixture, "drag_status=dropped"); + for delivery in ["background", "foreground"] { + let case = shared_case(&spec, "drag", "px", delivery); + if !cell_selected(&case) { + continue; + } + selected += 1; + let result = run_host_case_with_outcome(case, &spec, |fixture| { + run_drag_action(fixture, delivery) + }); + if failure.is_none() { + failure = result; + } + } + for delivery in ["background", "foreground"] { + let case = shared_case(&spec, "editor_save", "ax", delivery); + if cell_selected(&case) { + selected += 1; + let result = run_host_case_with_outcome(case, &spec, |fixture| { + run_editor_save_action(fixture, delivery) + }); + if failure.is_none() { + failure = result; + } } - - click_ax(&mut fixture, "btn-open-child-window"); - assert_tree_contains(&mut fixture, "child_windows=1"); - println!("✅ {} child-window + drag state oracles", fixture.name); - }); - if failure.is_none() { - failure = result; } } + assert!( + !cell_filter_active() || selected > 0, + "CUA_E2E_CELL_FILTER matched no shared E2E cells" + ); resume_first_failure(failure); } diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_linux_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_linux_test.rs new file mode 100644 index 0000000000..a7c887f167 --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_linux_test.rs @@ -0,0 +1,277 @@ +//! Linux **desktop-scope** (vision/foreground) modality through the SAME +//! cua-driver interface as Windows/macOS: `set_config capture_scope=desktop` + +//! a window-less screen-absolute `click` (no `pid`/`window_id`/`list_windows`). +//! The Linux actuator warps the pointer and injects a real button press via the +//! XTest extension — the peer of the Windows `WindowFromPoint` + macOS +//! global-HID desktop click. (XTest delivering to the under-pointer window is +//! why the *background* paths use `XSendEvent`; for desktop scope that delivery +//! is exactly what we want.) +//! +//! Grounds the click on the GTK3 harness increment button's screen-absolute +//! `frame` (AT-SPI Component extents), asserts the counter advanced, plus the +//! window-scope rejection gate. +//! +//! Linux config is global-only (no per-session override), so `set_config` +//! capture_scope writes the on-disk default — the test resets it to `window` +//! before asserting so a failure can't leave the sandbox in desktop scope. +//! +//! #[ignore] (needs an X11/Xwayland display + AT-SPI + the GTK3 harness). Run: +//! cargo test -p cua-driver --test desktop_scope_linux_test -- --ignored --nocapture --test-threads=1 + +#![cfg(target_os = "linux")] + +use std::panic::{self, AssertUnwindSafe}; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +use cua_driver_testkit::e2e::{ + execute_case, recording_evidence, CaseSpec, Delivery, DriverRoute, Evidence, Observation, + OracleKind, Scope, Targeting, +}; +use cua_driver_testkit::{harness_app, Driver, McpDriver}; + +fn harness_exe() -> std::path::PathBuf { + std::env::var("HARNESS_GTK3_EXE") + .map(std::path::PathBuf::from) + .ok() + .filter(|p| p.exists()) + .unwrap_or_else(|| harness_app("harness-gtk3", "CuaTestHarness.Gtk3")) +} + +fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> { + let exe = harness_exe(); + if !exe.exists() { + if std::env::var_os("CUA_TEST_REQUIRE_FIXTURES").is_some() { + panic!("required GTK3 harness is missing at {exe:?}"); + } + eprintln!("[desktop-linux] GTK3 harness not built ({exe:?}) — run tests/fixtures/build/linux.sh; skipping"); + return None; + } + let launched = driver.reaper().spawn( + Command::new(&exe) + .stdout(Stdio::null()) + .stderr(Stdio::null()), + ); + if let Err(error) = launched { + if std::env::var_os("CUA_TEST_REQUIRE_FIXTURES").is_some() { + panic!("failed to launch required GTK3 harness {exe:?}: {error}"); + } + eprintln!("[desktop-linux] GTK3 harness launch failed: {error}; skipping"); + return None; + } + let deadline = Instant::now() + Duration::from_secs(14); + while Instant::now() < deadline { + let r = driver.call("list_windows", serde_json::json!({})); + if let Some(wins) = r.structured()["windows"].as_array() { + for w in wins { + if w["title"] + .as_str() + .unwrap_or("") + .contains("CuaTestHarness GTK3") + { + let pid = w["pid"].as_u64().unwrap_or(0) as u32; + let wid = w["window_id"].as_u64().unwrap_or(0); + if pid != 0 && wid != 0 { + driver.reaper().track_pid(pid); + return Some((pid, wid)); + } + } + } + } + std::thread::sleep(Duration::from_millis(400)); + } + if std::env::var_os("CUA_TEST_REQUIRE_FIXTURES").is_some() { + panic!("required GTK3 harness window never appeared"); + } + eprintln!("[desktop-linux] harness window never appeared — graphical session + AT-SPI available? skipping"); + None +} + +fn ax_snapshot(driver: &mut McpDriver, pid: u32, wid: u64) -> serde_json::Value { + driver + .call( + "get_window_state", + serde_json::json!({ "pid": pid as i64, "window_id": wid, "capture_mode": "ax" }), + ) + .structured() + .clone() +} + +/// Screen-absolute center (px) of the increment button from `elements[].frame` +/// (AT-SPI Component extents are screen-absolute). The GTK3 harness sets the +/// button's accessible NAME to `btn-increment`; match any element whose blob +/// carries that aid and has a frame, so we're robust to the exact field name. +fn increment_center(snap: &serde_json::Value) -> Option<(i64, i64)> { + let els = snap["elements"].as_array()?; + let btn = els.iter().find(|e| { + serde_json::to_string(e) + .map(|s| s.contains("btn-increment")) + .unwrap_or(false) + && e.get("frame").map(|f| f.is_object()).unwrap_or(false) + })?; + let f = &btn["frame"]; + let (x, y, w, h) = ( + f["x"].as_f64()?, + f["y"].as_f64()?, + f["w"].as_f64()?, + f["h"].as_f64()?, + ); + Some(((x + w / 2.0) as i64, (y + h / 2.0) as i64)) +} + +fn counter(snap: &serde_json::Value) -> Option { + let tree = snap["tree_markdown"].as_str()?; + let idx = tree.find("counter=")? + "counter=".len(); + let digits: String = tree[idx..] + .chars() + .take_while(|c| c.is_ascii_digit()) + .collect(); + digits.parse().ok() +} + +fn set_scope(driver: &mut McpDriver, scope: &str) { + let r = driver.call( + "set_config", + serde_json::json!({ "key": "capture_scope", "value": scope }), + ); + assert!( + !r.is_error(), + "set_config capture_scope={scope} failed: {}", + r.text() + ); +} + +fn with_desktop_scope(driver: &mut McpDriver, test: impl FnOnce(&mut McpDriver) -> R) -> R { + set_scope(driver, "desktop"); + let result = panic::catch_unwind(AssertUnwindSafe(|| test(driver))); + set_scope(driver, "window"); + match result { + Ok(value) => value, + Err(payload) => panic::resume_unwind(payload), + } +} + +// ── tests ─────────────────────────────────────────────────────────────────────── + +/// In desktop scope, a window-less screen-absolute click (no pid/window_id) +/// lands on the increment button — its counter advances. +#[test] +#[ignore] +fn desktop_scope_windowless_click_lands_on_control() { + let cell_id = "linux-gtk3-desktop-left-click-px-foreground"; + let case = CaseSpec::delivered( + cell_id, + "gtk3", + "gtk3", + "left_click", + Targeting::Px, + Delivery::Foreground, + Scope::Desktop, + if std::env::var_os("CUA_INJECT_SOCKET").is_some() { + DriverRoute::LinuxCuaCompositorInject + } else { + DriverRoute::LinuxXTest + }, + vec![OracleKind::FixtureState], + ); + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_named(cell_id).expect("start source-built Linux driver"); + *evidence = recording_evidence(driver.recording_dir()); + let (pid, wid) = launch(&mut driver).expect("required GTK3 harness did not launch"); + + // Settle for the AT-SPI tree to register the button + its extents. + let mut snap = ax_snapshot(&mut driver, pid, wid); + let mut center = increment_center(&snap); + let deadline = Instant::now() + Duration::from_secs(8); + while center.is_none() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(400)); + snap = ax_snapshot(&mut driver, pid, wid); + center = increment_center(&snap); + } + let Some((cx, cy)) = center else { + panic!("increment button frame not found in required GTK3 AT-SPI tree"); + }; + let pre = counter(&snap).unwrap_or(0); + println!("[desktop-linux] increment button screen-center=({cx},{cy}) pre-counter={pre}"); + let posture = driver.call( + "bring_to_front", + serde_json::json!({"pid": pid as i64, "window_id": wid}), + ); + assert!(!posture.is_error(), "could not foreground GTK3 fixture: {}", posture.text()); + std::thread::sleep(Duration::from_millis(300)); + driver.start_behavior_recording(); + + // Retry the window-less desktop click until the counter advances. A + // freshly-mapped harness window may not yet be raised under the pointer on + // the first click (X11 window-raise timing differs across WMs — XFCE/Openbox + // lag GNOME), so the screen-absolute XTest click can miss the first attempt. + // Re-issuing the SAME click is safe: extra landed clicks only increment the + // counter further, and `post > pre` still holds. We assert the click was + // *dispatched as desktop scope* on the first attempt, and that it eventually + // *lands* within the budget. + let (post, first_text) = with_desktop_scope(&mut driver, |driver| { + let mut post = pre; + let mut first_text = String::new(); + for attempt in 0..12 { + let clicked = driver.call("click", serde_json::json!({ "x": cx, "y": cy })); + if attempt == 0 { + first_text = clicked.text().to_string(); + assert!( + !clicked.is_error(), + "desktop-scope click errored: {}", + clicked.text() + ); + } + std::thread::sleep(Duration::from_millis(500)); + post = counter(&ax_snapshot(driver, pid, wid)).unwrap_or(pre); + if post > pre { + break; + } + } + (post, first_text) + }); + + assert!( + first_text.to_lowercase().contains("desktop scope"), + "click not reported as desktop-scope: {first_text}" + ); + assert!( + post > pre, + "counter did not advance after window-less desktop clicks: pre={pre} post={post} \ + (the harness window never became clickable at ({cx},{cy}) within the retry budget)" + ); + Observation::delivered_with_fixture_state(Vec::new()) + }); +} + +/// Negative gate: a window-less click under `capture_scope=window` is rejected. +#[test] +#[ignore] +fn window_scope_rejects_windowless_click() { + let cell_id = "linux-window-scope-gate-px-not-applicable"; + let case = CaseSpec::delivered( + cell_id, + "desktop", + "x11", + "window_scope_gate", + Targeting::Px, + Delivery::NotApplicable, + Scope::Window, + DriverRoute::Composite, + vec![OracleKind::Protocol], + ); + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_named(cell_id).expect("start source-built Linux driver"); + *evidence = recording_evidence(driver.recording_dir()); + set_scope(&mut driver, "window"); + driver.start_behavior_recording(); + let r = driver.call("click", serde_json::json!({ "x": 100, "y": 100 })); + assert!( + r.is_error() + && r.structured()["code"].as_str() == Some("desktop_scope_disabled"), + "window-scope window-less click was NOT rejected: {}", + r.text() + ); + Observation::delivered(vec![OracleKind::Protocol], Evidence::default()) + }); +} diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_macos_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_macos_test.rs similarity index 54% rename from libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_macos_test.rs rename to libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_macos_test.rs index d1cef67d7c..9cf4a123fd 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_macos_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_macos_test.rs @@ -1,6 +1,6 @@ //! macOS **desktop-scope** (vision/foreground) modality, exercised through the -//! SAME cua-driver interface as the Windows `modality_desktop_scope_test`: -//! `set_config capture_scope=desktop` + a window-less screen-absolute `click` +//! SAME cua-driver interface as the Windows `desktop_scope_windows_test`: +//! a window-less screen-absolute `click` with `scope=desktop` //! (no `pid`, no `window_id`, no `list_windows`). The macOS actuator resolves //! the frontmost on-screen window under the point (the `WindowFromPoint` peer, //! via `CGWindowList`) and clicks it through the proven window-local pixel path, @@ -14,24 +14,21 @@ //! asserts the `window`-scope gate rejects a window-less click //! (`desktop_scope_disabled`), matching the Windows contract. //! -//! `set_config` is made session-scoped (a `session` arg → `_session_id`), so it -//! is in-memory only and never writes the developer's `~/.cua-driver/config.json`. -//! //! #[ignore] (needs a real desktop session + TCC Accessibility + the AppKit //! harness). Run: -//! cargo test -p cua-driver --test modality_desktop_scope_macos_test -- --ignored --nocapture --test-threads=1 +//! cargo test -p cua-driver --test desktop_scope_macos_test -- --ignored --nocapture --test-threads=1 #![cfg(target_os = "macos")] use std::process::{Command, Stdio}; use std::time::{Duration, Instant}; +use cua_driver_testkit::e2e::{ + execute_case, recording_evidence, CaseSpec, Delivery, DriverRoute, Evidence, Observation, + OracleKind, Scope, Targeting, +}; use cua_driver_testkit::{harness_app, Driver, McpDriver}; -/// Session id so `set_config capture_scope=desktop` is session-scoped (no disk -/// write) and the `click` resolves the same scope override. -const SESSION: &str = "vf-desktop"; - fn harness_exe() -> std::path::PathBuf { std::env::var("HARNESS_APPKIT_APP") .map(std::path::PathBuf::from) @@ -46,22 +43,36 @@ fn harness_exe() -> std::path::PathBuf { fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> { let exe = harness_exe(); if !exe.exists() { + if std::env::var_os("CUA_TEST_REQUIRE_FIXTURES").is_some() { + panic!("required AppKit harness is missing at {exe:?}"); + } eprintln!("[desktop-mac] AppKit harness not built ({exe:?}) — run tests/fixtures/build/macos.sh; skipping"); return None; } - driver - .reaper() - .spawn( - Command::new(&exe) - .stdout(Stdio::null()) - .stderr(Stdio::null()), - ) - .ok()?; + let child = match cua_driver_testkit::spawn_in_job( + Command::new(&exe) + .stdout(Stdio::null()) + .stderr(Stdio::null()), + ) { + Ok(child) => child, + Err(error) => { + if std::env::var_os("CUA_TEST_REQUIRE_FIXTURES").is_some() { + panic!("failed to launch required AppKit harness {exe:?}: {error}"); + } + eprintln!("[desktop-mac] AppKit harness launch failed: {error}; skipping"); + return None; + } + }; + let launched_pid = child.id(); + driver.reaper().push(child); let deadline = Instant::now() + Duration::from_secs(14); while Instant::now() < deadline { let r = driver.call("list_windows", serde_json::json!({})); if let Some(wins) = r.structured()["windows"].as_array() { for w in wins { + if w["pid"].as_u64() != Some(launched_pid as u64) { + continue; + } if w["title"] .as_str() .unwrap_or("") @@ -70,7 +81,6 @@ fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> { let pid = w["pid"].as_u64().unwrap_or(0) as u32; let wid = w["window_id"].as_u64().unwrap_or(0); if pid != 0 && wid != 0 { - driver.reaper().track_pid(pid); return Some((pid, wid)); } } @@ -78,6 +88,9 @@ fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> { } std::thread::sleep(Duration::from_millis(400)); } + if std::env::var_os("CUA_TEST_REQUIRE_FIXTURES").is_some() { + panic!("required AppKit harness window never appeared"); + } eprintln!( "[desktop-mac] harness window never appeared — graphical session available? skipping" ); @@ -156,67 +169,68 @@ fn activate_pid(pid: u32) { #[test] #[ignore] fn desktop_scope_windowless_click_lands_on_control() { - let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else { - return; - }; - let Some((pid, wid)) = launch(&mut driver) else { - return; - }; + let cell_id = "macos-appkit-desktop-left-click-px-foreground"; + let case = CaseSpec::delivered( + cell_id, + "appkit", + "appkit", + "left_click", + Targeting::Px, + Delivery::Foreground, + Scope::Desktop, + DriverRoute::MacosCgEventHid, + vec![OracleKind::FixtureState], + ); + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_macos_daemon_proxy_named(cell_id) + .expect("start installed macOS daemon proxy"); + *evidence = recording_evidence(driver.recording_dir()); + let (pid, wid) = launch(&mut driver).expect("required AppKit harness did not launch"); - // Settle for the AppKit AX tree to register the button + its frame. - let mut snap = ax_snapshot(&mut driver, pid, wid); - let mut center = increment_center(&snap); - let deadline = Instant::now() + Duration::from_secs(8); - while center.is_none() && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(400)); - snap = ax_snapshot(&mut driver, pid, wid); - center = increment_center(&snap); - } - let Some((cx, cy)) = center else { - eprintln!("[desktop-mac] increment button frame not found (TCC Accessibility missing?) — skipping"); - return; - }; - let pre = counter(&snap).unwrap_or(0); - println!("[desktop-mac] increment button screen-center=({cx},{cy}) pre-counter={pre}"); + // Settle for the AppKit AX tree to register the button + its frame. + let mut snap = ax_snapshot(&mut driver, pid, wid); + let mut center = increment_center(&snap); + let deadline = Instant::now() + Duration::from_secs(8); + while center.is_none() && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(400)); + snap = ax_snapshot(&mut driver, pid, wid); + center = increment_center(&snap); + } + let Some((cx, cy)) = center else { + panic!("increment button frame not found in required AppKit AX tree"); + }; + let pre = counter(&snap).unwrap_or(0); + println!("[desktop-mac] increment button screen-center=({cx},{cy}) pre-counter={pre}"); - // Desktop scope clicks the frontmost window at the point — put the harness there. - activate_pid(pid); + // Desktop scope clicks the frontmost window at the point — put the harness there. + activate_pid(pid); + driver.start_behavior_recording(); - // Window-less screen-absolute click — no pid, no window_id; scope per-call. - let clicked = driver.call( - "click", - serde_json::json!({ "x": cx, "y": cy, "scope": "desktop", "session": SESSION }), - ); - assert!( - !clicked.is_error(), - "desktop-scope click errored: {}", - clicked.text() - ); - assert!( - clicked.text().to_lowercase().contains("desktop scope"), - "click not reported as desktop-scope: {}", - clicked.text() - ); - println!("[desktop-mac] {}", clicked.text()); + // Window-less screen-absolute click — no pid, no window_id; scope per-call. + let clicked = driver.call( + "click", + serde_json::json!({ "x": cx, "y": cy, "scope": "desktop" }), + ); + assert!( + !clicked.is_error(), + "desktop-scope click errored: {}", + clicked.text() + ); + assert!( + clicked.text().to_lowercase().contains("desktop scope"), + "click not reported as desktop-scope: {}", + clicked.text() + ); + println!("[desktop-mac] {}", clicked.text()); - std::thread::sleep(Duration::from_millis(600)); - let post = counter(&ax_snapshot(&mut driver, pid, wid)).unwrap_or(pre); - if post > pre { - println!("✅ desktop_scope_windowless_click_lands_on_control: counter {pre} → {post}"); - return; - } - // The desktop click lands on whatever window is *visually frontmost* at the - // point — that is the contract. On a busy desktop another window can cover - // the harness (and `activate` may not beat a floating panel), so a - // non-advance here means the harness was not frontmost at the point, NOT a - // driver fault. The click is confirmed to have resolved a real window (see - // its result text above). Skip rather than false-fail; a clean session - // asserts the landing, as the Linux peer does end-to-end. - eprintln!( - "[desktop-mac] counter did not advance ({pre}→{post}) — the harness was not the \ - frontmost window at ({cx},{cy}) on this desktop (another window covering it). \ - Skipping the landing assertion; run on a clean GUI session to assert it." - ); + std::thread::sleep(Duration::from_millis(600)); + let post = counter(&ax_snapshot(&mut driver, pid, wid)).unwrap_or(pre); + assert!( + post > pre, + "desktop click did not advance AppKit counter at ({cx},{cy}): {pre} -> {post}" + ); + Observation::delivered_with_fixture_state(Vec::new()) + }); } /// Negative gate: a window-less screen-absolute click while `capture_scope=window` @@ -225,19 +239,33 @@ fn desktop_scope_windowless_click_lands_on_control() { #[test] #[ignore] fn window_scope_rejects_windowless_click() { - let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else { - return; - }; - // Default scope is "window" — a window-less click must be rejected. - let r = driver.call( - "click", - serde_json::json!({ "x": 100, "y": 100, "scope": "window", "session": SESSION }), - ); - let txt = r.text().to_lowercase(); - assert!( - r.is_error() || txt.contains("desktop scope") || txt.contains("desktop_scope_disabled"), - "window-scope window-less click was NOT rejected: {}", - r.text() + let cell_id = "macos-window-scope-gate-px-not-applicable"; + let case = CaseSpec::delivered( + cell_id, + "desktop", + "quartz", + "window_scope_gate", + Targeting::Px, + Delivery::NotApplicable, + Scope::Window, + DriverRoute::Composite, + vec![OracleKind::Protocol], ); - println!("✅ window_scope_rejects_windowless_click: window-less click correctly gated"); + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_macos_daemon_proxy_named(cell_id) + .expect("start installed macOS daemon proxy"); + *evidence = recording_evidence(driver.recording_dir()); + // Default scope is "window" — a window-less click must be rejected. + driver.start_behavior_recording(); + let r = driver.call( + "click", + serde_json::json!({ "x": 100, "y": 100, "scope": "window" }), + ); + assert!( + r.is_error() && r.structured()["code"].as_str() == Some("desktop_scope_disabled"), + "window-scope window-less click was NOT rejected: {}", + r.text() + ); + Observation::delivered(vec![OracleKind::Protocol], Evidence::default()) + }); } diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_windows_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_windows_test.rs new file mode 100644 index 0000000000..07d9d7de9f --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver/tests/desktop_scope_windows_test.rs @@ -0,0 +1,305 @@ +//! Harness integration test for the **desktop-scope** modality (#1968 / #2019). +//! +//! Desktop-scope is cua-driver's *foreground*, vision-only, **screen-absolute** +//! loop (the "Computer-Use 1.0" mode), the complement to the default per-window +//! background model that `harness_bg_modality_test` / `e2e_windows_bg_input_test` +//! cover. This test exercises the Windows Phase-1 actuator end-to-end against a +//! real harness app: +//! +//! 1. `set_config capture_scope=desktop` → `get_desktop_state` returns a +//! full-display capture with true `screen_width/height` (no downscale). +//! 2. A **window-less** screen-absolute `click` / `scroll` (no pid/window_id) +//! lands via `WindowFromPoint` while in desktop scope. +//! 3. Negative gate: the same window-less `click` under `capture_scope=window` +//! is rejected with the structured `desktop_scope_disabled` error. +//! +//! Note: `set_config` is a *session* override — it persists for the lifetime of +//! the one MCP server we spawn here (not across separate `cua-driver call` +//! processes), which is exactly why this test drives a single long-lived server. +//! +//! All tests are `#[ignore]` (need a real desktop session). Run explicitly: +//! cargo test -p cua-driver --test desktop_scope_windows_test -- --ignored --nocapture --test-threads=1 + +#![cfg(target_os = "windows")] + +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +use cua_driver_testkit::ax; +use cua_driver_testkit::e2e::{ + execute_case, recording_evidence, CaseSpec, Delivery, DriverRoute, Evidence, Observation, + OracleKind, Scope, Targeting, +}; +use cua_driver_testkit::{harness_app, Driver, McpDriver}; + +/// WPF harness app (built by `tests/fixtures/build/windows.ps1`). Path mirrors +/// `shared/scenarios.json`'s `wpf.exe_relative_path`. +fn harness_wpf_exe() -> std::path::PathBuf { + harness_app("harness-wpf", "CuaTestHarness.Wpf.exe") +} + +/// Launch the WPF harness app and return its pid and native window id. +/// Skips (returns None) if the harness app isn't built. +fn launch_wpf(driver: &mut McpDriver) -> Option<(u32, u64)> { + let exe = harness_wpf_exe(); + if !exe.exists() { + if std::env::var_os("CUA_TEST_REQUIRE_FIXTURES").is_some() { + panic!("required WPF harness is missing at {exe:?}"); + } + eprintln!("[desktop-scope] WPF harness not built ({exe:?}) — skipping window-target tests"); + return None; + } + let launched = driver.reaper().spawn( + Command::new(&exe) + .stdout(Stdio::null()) + .stderr(Stdio::null()), + ); + if let Err(error) = launched { + if std::env::var_os("CUA_TEST_REQUIRE_FIXTURES").is_some() { + panic!("failed to launch required WPF harness {exe:?}: {error}"); + } + eprintln!("[desktop-scope] WPF harness launch failed: {error}; skipping"); + return None; + } + + let deadline = Instant::now() + Duration::from_secs(15); + while Instant::now() < deadline { + let r = driver.call("list_windows", serde_json::json!({})); + if let Some(arr) = r.structured()["windows"].as_array() { + for w in arr { + let title = w["title"].as_str().unwrap_or(""); + if !title.contains("CuaTestHarness") { + continue; + } + let pid = w["pid"].as_u64().unwrap_or(0) as u32; + let wid = w["window_id"].as_u64().unwrap_or(0); + if pid != 0 && wid != 0 { + driver.reaper().track_pid(pid); + return Some((pid, wid)); + } + } + } + std::thread::sleep(Duration::from_millis(500)); + } + if std::env::var_os("CUA_TEST_REQUIRE_FIXTURES").is_some() { + panic!("required WPF harness window never appeared"); + } + eprintln!("[desktop-scope] WPF harness window never appeared — skipping"); + None +} + +fn snapshot(driver: &mut McpDriver, pid: u32, wid: u64) -> cua_driver_testkit::ToolResponse { + driver.call( + "get_window_state", + serde_json::json!({ "pid": pid as i64, "window_id": wid, "capture_mode": "ax" }), + ) +} + +fn element_center(state: &cua_driver_testkit::ToolResponse, id: &str) -> (i64, i64) { + let index = ax::element_index_by_id(state.tree_text(), id) + .unwrap_or_else(|| panic!("missing WPF element {id:?}: {}", state.tree_text())); + let element = state.structured()["elements"] + .as_array() + .and_then(|elements| { + elements + .iter() + .find(|element| element["element_index"].as_u64() == Some(index)) + }) + .unwrap_or_else(|| panic!("WPF element {id:?} has no structured frame")); + let frame = &element["frame"]; + ( + (frame["x"].as_f64().expect("frame x") + frame["w"].as_f64().expect("frame w") / 2.0) + as i64, + (frame["y"].as_f64().expect("frame y") + frame["h"].as_f64().expect("frame h") / 2.0) + as i64, + ) +} + +fn set_scope(driver: &mut McpDriver, scope: &str) { + let r = driver.call( + "set_config", + serde_json::json!({ "key": "capture_scope", "value": scope }), + ); + assert!( + !r.is_error(), + "set_config capture_scope={scope} failed: {}", + r.text() + ); + assert_eq!( + r.structured()["capture_scope"].as_str(), + Some(scope), + "set_config did not report capture_scope={scope}: {}", + r.text() + ); +} + +fn run_desktop_fixture_case( + action: &str, + route: DriverRoute, + test: impl FnOnce(u32, u64, &mut McpDriver), +) { + let cell_id = format!("windows-wpf-desktop-{action}-px-foreground").replace('_', "-"); + let case = CaseSpec::delivered( + cell_id.clone(), + "wpf", + "wpf", + action, + Targeting::Px, + Delivery::Foreground, + Scope::Desktop, + route, + vec![OracleKind::FixtureState], + ); + execute_case(case, |evidence| { + let mut driver = + McpDriver::spawn_named(&cell_id).expect("required source-built driver did not start"); + *evidence = recording_evidence(driver.recording_dir()); + let (pid, wid) = launch_wpf(&mut driver).expect("required WPF harness did not launch"); + set_scope(&mut driver, "desktop"); + let posture = driver.call( + "bring_to_front", + serde_json::json!({"pid": pid as i64, "window_id": wid}), + ); + assert!(!posture.is_error(), "could not foreground WPF fixture: {}", posture.text()); + std::thread::sleep(Duration::from_millis(300)); + driver.start_behavior_recording(); + test(pid, wid, &mut driver); + Observation::delivered_with_fixture_state(Vec::new()) + }); +} + +// ── tests ───────────────────────────────────────────────────────────────────── + +/// `get_desktop_state` in desktop scope returns a full-display capture with +/// real screen dimensions (the Session-0 `handle is invalid` case is only the +/// service-session wall; this needs a real interactive desktop). +#[test] +#[ignore] +fn desktop_scope_capture_returns_screen_dims() { + let case = CaseSpec::delivered( + "windows-desktop-state-px-not-applicable", + "desktop", + "win32", + "get_desktop_state", + Targeting::Px, + Delivery::NotApplicable, + Scope::Desktop, + DriverRoute::WindowState, + vec![OracleKind::Pixels], + ); + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_named("windows-desktop-state-px-not-applicable") + .expect("required source-built driver did not start"); + *evidence = recording_evidence(driver.recording_dir()); + set_scope(&mut driver, "desktop"); + driver.start_behavior_recording(); + let response = driver.call("get_desktop_state", serde_json::json!({})); + assert!( + !response.is_error(), + "get_desktop_state errored: {}", + response.text() + ); + let width = response.structured()["screen_width"].as_u64().unwrap_or(0); + let height = response.structured()["screen_height"].as_u64().unwrap_or(0); + assert!( + width > 0 && height > 0, + "get_desktop_state returned no screen size" + ); + Observation::delivered(vec![OracleKind::Pixels], Evidence::default()) + }); +} + +/// In desktop scope, a window-less screen-absolute click + scroll succeed and +/// resolve a real window via WindowFromPoint (no pid/window_id supplied). +#[test] +#[ignore] +fn desktop_scope_windowless_click_lands_on_control() { + run_desktop_fixture_case( + "left_click", + DriverRoute::WindowsSendInput, + |pid, wid, driver| { + let pre = snapshot(driver, pid, wid); + let (x, y) = element_center(&pre, "border-click-target"); + let response = driver.call("click", serde_json::json!({ "x": x, "y": y })); + assert!( + !response.is_error(), + "desktop click failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(400)); + let after = snapshot(driver, pid, wid); + assert!( + after.tree_text().contains("last_action=left_click"), + "desktop click did not update the WPF click oracle: {}", + after.tree_text() + ); + }, + ); +} + +#[test] +#[ignore] +fn desktop_scope_windowless_scroll_lands_on_control() { + run_desktop_fixture_case( + "scroll", + DriverRoute::WindowsSendInput, + |pid, wid, driver| { + let pre = snapshot(driver, pid, wid); + // The fixture's outer ScrollViewer is visible while the nested + // scroll-tall region begins below a 768px CI desktop. Wheel over a + // visible child and verify the outer viewport moved by observing a + // lower AX element's fresh screen coordinate. + let (x, y) = element_center(&pre, "border-click-target"); + let (_, before_y) = element_center(&pre, "btn-increment"); + let response = driver.call( + "scroll", + serde_json::json!({ "x": x, "y": y, "direction": "down", "amount": 5 }), + ); + assert!( + !response.is_error(), + "desktop scroll failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(500)); + let after = snapshot(driver, pid, wid); + let (_, after_y) = element_center(&after, "btn-increment"); + assert!( + after_y < before_y, + "desktop scroll did not move the WPF outer viewport: before_y={before_y}, after_y={after_y}" + ); + }, + ); +} + +/// Negative gate: a window-less screen-absolute click under `capture_scope=window` +/// must be rejected (the `desktop_scope_disabled` contract), not silently retargeted. +#[test] +#[ignore] +fn window_scope_rejects_windowless_click() { + let case = CaseSpec::delivered( + "windows-window-scope-gate-px-not-applicable", + "desktop", + "win32", + "window_scope_gate", + Targeting::Px, + Delivery::NotApplicable, + Scope::Window, + DriverRoute::Composite, + vec![OracleKind::Protocol], + ); + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_named("windows-window-scope-gate-px-not-applicable") + .expect("required source-built driver did not start"); + *evidence = recording_evidence(driver.recording_dir()); + set_scope(&mut driver, "window"); + driver.start_behavior_recording(); + let response = driver.call("click", serde_json::json!({ "x": 100, "y": 100 })); + assert!( + response.is_error() + && response.structured()["code"].as_str() == Some("desktop_scope_disabled"), + "window-scope window-less click was not rejected: {}", + response.text() + ); + Observation::delivered(vec![OracleKind::Protocol], Evidence::default()) + }); +} diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/e2e_environment_preflight_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/e2e_environment_preflight_test.rs new file mode 100644 index 0000000000..4666962983 --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver/tests/e2e_environment_preflight_test.rs @@ -0,0 +1,356 @@ +//! One strict GUI/recording preflight before a canonical E2E lane runs. + +#![cfg(any(target_os = "windows", target_os = "macos", target_os = "linux"))] + +use std::any::Any; +use std::collections::HashSet; +use std::panic::{self, AssertUnwindSafe}; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant}; + +#[cfg(target_os = "linux")] +use cua_driver_testkit::e2e::DisplayServer; +use cua_driver_testkit::e2e::{write_environment_from_env, EnvironmentRecord}; +use cua_driver_testkit::{driver_binary, harness_app, spawn_in_job, Driver, McpDriver}; + +struct PreflightFixture { + path: std::path::PathBuf, + args: Vec<&'static str>, + title: &'static str, + ax_marker: &'static str, +} + +fn preflight_fixture() -> PreflightFixture { + #[cfg(target_os = "windows")] + { + PreflightFixture { + path: harness_app("harness-electron", "CuaTestHarness.Electron.exe"), + args: vec![ + "--no-sandbox", + "--disable-gpu", + "--force-renderer-accessibility", + ], + title: "CuaTestHarness Electron", + ax_marker: "WEB_HARNESS_MARKER_v1", + } + } + #[cfg(target_os = "macos")] + { + PreflightFixture { + path: harness_app( + "harness-electron", + "CuaTestHarness.Electron.app/Contents/MacOS/Electron", + ), + args: vec!["--force-renderer-accessibility"], + title: "CuaTestHarness Electron", + ax_marker: "WEB_HARNESS_MARKER_v1", + } + } + #[cfg(target_os = "linux")] + { + if std::env::var("CUA_E2E_INTERNAL_LANE").as_deref() == Ok("native") { + PreflightFixture { + path: harness_app("harness-gtk3", "CuaTestHarness.Gtk3"), + args: Vec::new(), + title: "CuaTestHarness GTK3", + ax_marker: "HARNESS_TEXT_MARKER_v1", + } + } else { + PreflightFixture { + path: harness_app("harness-electron", "CuaTestHarness.Electron"), + args: vec![ + "--no-sandbox", + "--disable-gpu", + "--force-renderer-accessibility", + ], + title: "CuaTestHarness Electron", + ax_marker: "WEB_HARNESS_MARKER_v1", + } + } + } +} + +fn spawn_driver() -> McpDriver { + #[cfg(target_os = "macos")] + { + McpDriver::spawn_macos_daemon_proxy_named("environment-preflight") + .expect("installed CuaDriver daemon is not available") + } + #[cfg(not(target_os = "macos"))] + { + McpDriver::spawn_named("environment-preflight") + .expect("source-built cua-driver could not be started") + } +} + +fn has_image(response: &cua_driver_testkit::ToolResponse) -> bool { + response.raw["result"]["content"] + .as_array() + .map(|content| { + content + .iter() + .any(|item| item["type"].as_str() == Some("image")) + }) + .unwrap_or(false) + || response.structured()["screenshot_png_base64"] + .as_str() + .map(|png| !png.is_empty()) + .unwrap_or(false) +} + +fn run_preflight() { + if let Ok(expected_sha) = std::env::var("CUA_E2E_SOURCE_SHA") { + assert!( + expected_sha.len() == 40 && expected_sha.chars().all(|ch| ch.is_ascii_hexdigit()), + "CUA_E2E_SOURCE_SHA must be a full commit SHA" + ); + let source = Command::new("git").args(["rev-parse", "HEAD"]).output(); + let actual_sha = source + .ok() + .filter(|source| source.status.success()) + .map(|source| String::from_utf8_lossy(&source.stdout).trim().to_owned()) + .or_else(|| { + let marker = std::env::var_os("CUA_E2E_SOURCE_MARKER") + .map(std::path::PathBuf::from) + .unwrap_or_else(|| std::path::PathBuf::from(".cua-e2e-source-sha")); + std::fs::read_to_string(marker) + .ok() + .map(|source| source.trim().to_owned()) + }) + .expect("neither git HEAD nor .cua-e2e-source-sha identifies the synced source"); + assert_eq!( + actual_sha.to_ascii_lowercase(), + expected_sha.to_ascii_lowercase(), + "checked-out source does not match the workflow's resolved SHA" + ); + } + + let driver_path = driver_binary(); + assert!( + driver_path.is_file(), + "source-built driver is missing at {}", + driver_path.display() + ); + let version = Command::new(&driver_path) + .arg("--version") + .output() + .expect("source-built driver --version failed"); + assert!( + version.status.success(), + "source-built driver is not runnable" + ); + let version = String::from_utf8_lossy(&version.stdout); + assert!( + version.contains(env!("CARGO_PKG_VERSION")), + "driver version mismatch: {version}" + ); + + let fixture = preflight_fixture(); + assert!( + fixture.path.exists(), + "required preflight fixture is missing at {}", + fixture.path.display() + ); + let recordings_root = std::env::var_os("CUA_E2E_RECORDINGS_ROOT") + .expect("CUA_E2E_RECORDINGS_ROOT is required for canonical E2E"); + + let mut driver = spawn_driver(); + let config = driver.call("get_config", serde_json::json!({})); + assert!( + !config.is_error(), + "connected driver get_config failed: {}", + config.text() + ); + assert_eq!( + config.structured()["version"].as_str(), + Some(env!("CARGO_PKG_VERSION")), + "connected driver version does not match the source build" + ); + let recording_dir = driver + .recording_dir() + .expect("preflight evidence directory was not prepared") + .to_path_buf(); + assert!( + recording_dir.starts_with(&recordings_root), + "preflight recording escaped the artifact root" + ); + + let before = driver.call("list_windows", serde_json::json!({})); + assert!(!before.is_error(), "list_windows failed: {}", before.text()); + let before_ids = before.structured()["windows"] + .as_array() + .map(|windows| { + windows + .iter() + .filter_map(|window| window["window_id"].as_u64()) + .collect::>() + }) + .unwrap_or_default(); + + let mut command = Command::new(&fixture.path); + command + .args(&fixture.args) + .stdout(Stdio::null()) + .stderr(Stdio::inherit()); + let mut child = spawn_in_job(&mut command).expect("preflight fixture failed to launch"); + let launched_pid = child.id() as i64; + + let deadline = Instant::now() + Duration::from_secs(20); + let (pid, window_id) = loop { + if let Some(status) = child + .try_wait() + .expect("could not inspect preflight fixture process") + { + panic!("preflight fixture exited before mapping a window: {status}"); + } + let response = driver.call("list_windows", serde_json::json!({})); + if let Some(window) = response.structured()["windows"] + .as_array() + .and_then(|windows| { + windows.iter().find(|window| { + window["window_id"] + .as_u64() + .map(|id| !before_ids.contains(&id)) + .unwrap_or(false) + && window["title"] + .as_str() + .unwrap_or("") + .contains(fixture.title) + }) + }) + { + if let Some(window_id) = window["window_id"].as_u64() { + break (window["pid"].as_i64().unwrap_or(launched_pid), window_id); + } + } + if Instant::now() >= deadline { + let _ = child.kill(); + let _ = child.wait(); + panic!("preflight fixture window did not appear"); + } + std::thread::sleep(Duration::from_millis(200)); + }; + driver.reaper().push(child); + + #[cfg(target_os = "linux")] + { + if DisplayServer::current() == DisplayServer::X11 { + let activated = driver.call( + "bring_to_front", + serde_json::json!({ "pid": pid, "window_id": window_id }), + ); + assert!( + !activated.is_error(), + "preflight fixture could not be placed on the Linux desktop: {}", + activated.text() + ); + std::thread::sleep(Duration::from_millis(300)); + } + } + + let deadline = Instant::now() + Duration::from_secs(15); + loop { + let state = driver.call( + "get_window_state", + serde_json::json!({ + "pid": pid, + "window_id": window_id, + "capture_mode": "ax" + }), + ); + if !state.is_error() && state.tree_text().contains(fixture.ax_marker) && has_image(&state) { + break; + } + assert!( + Instant::now() < deadline, + "preflight could not read both AX state and screenshot: {}", + state.text() + ); + std::thread::sleep(Duration::from_millis(200)); + } + + driver.start_behavior_recording(); + + drop(driver); + let video = recording_dir.join("recording.mp4"); + assert!( + std::fs::metadata(&video) + .map(|metadata| metadata.len() > 0) + .unwrap_or(false), + "preflight video is missing or empty at {}", + video.display() + ); + assert!( + !recording_dir.join("recording-error.txt").exists(), + "preflight recording reported an error" + ); + let probe = Command::new("ffprobe") + .args([ + "-v", + "error", + "-show_entries", + "format=duration", + "-of", + "default=noprint_wrappers=1:nokey=1", + ]) + .arg(&video) + .status() + .expect("ffprobe is required for canonical E2E"); + assert!(probe.success(), "ffprobe rejected the preflight video"); + + let frame = recording_dir.join("preflight-frame.png"); + let extracted = Command::new("ffmpeg") + .args(["-y", "-sseof", "-0.2", "-i"]) + .arg(&video) + .args(["-frames:v", "1"]) + .arg(&frame) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .expect("ffmpeg is required for canonical E2E"); + assert!( + extracted.success(), + "could not extract preflight video frame" + ); + let frame = image::open(&frame) + .expect("preflight video frame is not a readable image") + .to_rgb8(); + let non_dark_pixels = frame + .pixels() + .filter(|pixel| pixel.0.iter().copied().max().unwrap_or(0) > 30) + .count(); + assert!( + non_dark_pixels * 1_000 >= frame.pixels().len(), + "preflight video is effectively blank: {non_dark_pixels}/{} non-dark pixels", + frame.pixels().len() + ); +} + +fn panic_message(payload: &Box) -> String { + payload + .downcast_ref::() + .cloned() + .or_else(|| { + payload + .downcast_ref::<&str>() + .map(|message| (*message).to_owned()) + }) + .unwrap_or_else(|| "preflight panicked without a string payload".to_owned()) +} + +#[test] +#[ignore] +fn canonical_e2e_environment_is_ready() { + let started = Instant::now(); + let outcome = panic::catch_unwind(AssertUnwindSafe(run_preflight)); + match outcome { + Ok(()) => write_environment_from_env(&EnvironmentRecord::ready(started.elapsed())) + .expect("write environment record"), + Err(payload) => { + let message = panic_message(&payload); + write_environment_from_env(&EnvironmentRecord::error(started.elapsed(), message)) + .expect("write failed environment record"); + panic::resume_unwind(payload); + } + } +} diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/guard_ux_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/guard_ux_test.rs deleted file mode 100644 index c2264a9590..0000000000 --- a/libs/cua-driver/rust/crates/cua-driver/tests/guard_ux_test.rs +++ /dev/null @@ -1,730 +0,0 @@ -//! UX-guard integration tests for Windows. -//! -//! These cover the UX-guard scenarios previously exercised by the legacy -//! Python suite: -//! - background focus preservation -//! - new-window click delivery -//! - visible app launch -//! - background menu shortcuts -//! -//! Invariant under test (the "UX guard"): -//! The agent must be able to click, type, and launch apps in background -//! windows WITHOUT stealing focus from the user's foreground window. -//! -//! Background target: the repo-local Electron harness staged at -//! test-apps/harness-electron/CuaTestHarness.Electron.exe. Notepad is used -//! only as a secondary fallback check. -//! -//! Background-action tests launch the target first, then foreground -//! focus-monitor-win (the "user's foreground window") before measuring focus -//! loss. The launch_app test starts the monitor first because launch behavior is -//! the action under test. -//! -//! Run in sandbox via: -//! ..\tests\runners\windows-sandbox\run-tests-in-sandbox.ps1 guard_ux - -#![cfg(target_os = "windows")] - -use std::collections::HashSet; -use std::path::PathBuf; -use std::process::{Child, Command, Stdio}; -use std::time::{Duration, Instant}; - -use cua_driver_testkit::{ax, driver_binary, spawn_in_job, workspace_root, Driver, McpDriver}; - -// ── focus-monitor + test-app fixtures ──────────────────────────────────────── - -fn gui_required() -> bool { - std::env::var("CUA_REQUIRE_GUI") - .ok() - .map(|v| matches!(v.to_ascii_lowercase().as_str(), "1" | "true" | "yes" | "on")) - .unwrap_or(false) -} - -fn skip_desktop(context: &str, reason: String) -> bool { - let msg = format!("{context}: {reason}; skipping GUI UX guard"); - if gui_required() { - panic!("{msg}"); - } - eprintln!("{msg}"); - false -} - -fn require_seedable_desktop(context: &str) -> bool { - let state = platform_windows::diagnostics::desktop_state(); - if state.session_id == Some(0) { - return skip_desktop( - context, - format!( - "running in Windows Session 0 ({}) - re-run from RDP/console/scheduled task in user session", - state.summary() - ), - ); - } - if !state.has_process_window_station { - return skip_desktop( - context, - format!("no attached process window station ({})", state.summary()), - ); - } - if !state.input_desktop_is_default() { - return skip_desktop( - context, - format!( - "input desktop is not the user Default desktop ({})", - state.summary() - ), - ); - } - true -} - -fn require_focus_monitor_foreground(context: &str, expected_hwnd: u64) -> bool { - let deadline = std::time::Instant::now() + Duration::from_secs(2); - loop { - let state = platform_windows::diagnostics::desktop_state(); - if state.foreground_hwnd == Some(expected_hwnd as usize) { - return true; - } - if std::time::Instant::now() >= deadline { - return skip_desktop( - context, - format!( - "focus monitor did not become foreground (expected HWND 0x{expected_hwnd:x}; {})", - state.summary() - ), - ); - } - std::thread::sleep(Duration::from_millis(100)); - } -} - -fn focus_monitor_path() -> PathBuf { - workspace_root().join("target/debug/focus-monitor-win.exe") -} - -fn test_app_path() -> PathBuf { - // In sandbox, sandbox-runner.ps1 copies the exe to %TEMP% to avoid the - // ShellExecuteW zone-security dialog that blocks on mapped-folder exes. - if let Ok(p) = std::env::var("HARNESS_ELECTRON_EXE") { - let pb = PathBuf::from(p); - if pb.exists() { - return pb; - } - } - workspace_root().join("test-apps/harness-electron/CuaTestHarness.Electron.exe") -} - -/// Launch the Electron harness in the background (tied to the driver's reaper) -/// and return its pid. Returns None if the binary doesn't exist or the app -/// fails to start. -fn launch_test_app(driver: &mut McpDriver) -> Option { - let exe = test_app_path(); - if !exe.exists() { - eprintln!("CuaTestHarness.Electron not found at {exe:?} - skipping"); - return None; - } - let child = spawn_in_job( - Command::new(&exe) - .stdout(Stdio::null()) - .stderr(Stdio::null()), - ) - .ok()?; - let pid = child.id(); - driver.reaper().push(child); - // Cold-start in sandbox can take a few seconds. - std::thread::sleep(Duration::from_secs(3)); - Some(pid) -} - -fn launch_driver_and_test_app() -> Option<(McpDriver, u32, u64)> { - let Some(mut driver) = McpDriver::spawn() else { - return None; - }; - - let Some(app_pid) = launch_test_app(&mut driver) else { - eprintln!("test app not available — skipping"); - return None; - }; - let Some((window_pid, app_wid)) = find_window_for_pid(&mut driver, app_pid as i64) else { - eprintln!("test app window not found — skipping"); - return None; - }; - - // Electron may create the visible window in a Chromium child process. - // Actions must use the PID that owns the HWND, not the launcher parent. - Some((driver, window_pid, app_wid)) -} - -fn kill_process_tree_by_image(image: &str) { - Command::new("taskkill") - .args(["/F", "/T", "/IM", image]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .ok(); -} - -struct KillProcessTreeOnDrop(&'static str); - -impl Drop for KillProcessTreeOnDrop { - fn drop(&mut self) { - kill_process_tree_by_image(self.0); - } -} - -fn loss_file() -> PathBuf { - std::env::temp_dir().join("focus_monitor_losses.txt") -} -fn key_loss_file() -> PathBuf { - std::env::temp_dir().join("focus_monitor_key_losses.txt") -} - -fn read_losses(path: &std::path::Path) -> u32 { - std::fs::read_to_string(path) - .ok() - .and_then(|s| s.trim().parse().ok()) - .unwrap_or(0) -} - -fn focus_pid_file() -> PathBuf { - std::env::temp_dir().join("focus_monitor_pid.txt") -} -fn focus_hwnd_file() -> PathBuf { - std::env::temp_dir().join("focus_monitor_hwnd.txt") -} - -/// Launch focus-monitor-win and return (process, hwnd, pid). -/// Reads FOCUS_PID and FOCUS_HWND from temp files written by the monitor -/// (avoids blocking on the stdout pipe if the sandbox redirects I/O). -fn launch_focus_monitor() -> Option<(Child, u64, u32)> { - if !require_seedable_desktop("focus-monitor-win") { - return None; - } - - let exe = focus_monitor_path(); - if !exe.exists() { - eprintln!("focus-monitor-win.exe not built at {exe:?} — skipping"); - return None; - } - // Reset all sentinel files so stale values are not mistaken for new ones. - let _ = std::fs::write(loss_file(), "0"); - let _ = std::fs::write(key_loss_file(), "0"); - let _ = std::fs::remove_file(focus_pid_file()); - let _ = std::fs::remove_file(focus_hwnd_file()); - - let mut child = spawn_in_job( - Command::new(&exe) - .stdout(Stdio::null()) - .stderr(Stdio::null()), - ) - .expect("spawn focus-monitor-win"); - - // Poll temp files until both PID and HWND are written (max 15s). - let deadline = std::time::Instant::now() + Duration::from_secs(15); - let (mut pid_val, mut hwnd_val) = (0u32, 0u64); - loop { - if std::time::Instant::now() > deadline { - panic!("focus-monitor-win did not write PID/HWND temp files within 15s"); - } - if pid_val == 0 { - pid_val = std::fs::read_to_string(focus_pid_file()) - .ok() - .and_then(|s| s.trim().parse().ok()) - .unwrap_or(0); - } - if hwnd_val == 0 { - hwnd_val = std::fs::read_to_string(focus_hwnd_file()) - .ok() - .and_then(|s| s.trim().parse().ok()) - .unwrap_or(0); - } - if pid_val != 0 && hwnd_val != 0 { - break; - } - std::thread::sleep(Duration::from_millis(100)); - } - if !require_focus_monitor_foreground("focus-monitor-win", hwnd_val) { - child.kill().ok(); - return None; - } - Some((child, hwnd_val, pid_val)) -} - -/// Find the first on-screen window belonging to the given pid. -fn find_window_for_pid(driver: &mut McpDriver, pid: i64) -> Option<(u32, u64)> { - let resp = driver.call( - "list_windows", - serde_json::json!({"pid": pid, "on_screen_only": true}), - ); - resp.structured()["windows"] - .as_array()? - .iter() - .find_map(|w| Some((w["pid"].as_u64()? as u32, w["window_id"].as_u64()?))) -} - -fn window_ids(driver: &mut McpDriver) -> HashSet { - let resp = driver.call("list_windows", serde_json::json!({})); - resp.structured()["windows"] - .as_array() - .map(|a| a.iter().filter_map(|w| w["window_id"].as_u64()).collect()) - .unwrap_or_default() -} - -fn wait_for_new_window(driver: &mut McpDriver, before: &HashSet) -> bool { - let deadline = Instant::now() + Duration::from_secs(3); - while Instant::now() < deadline { - let after = window_ids(driver); - if after.iter().any(|id| !before.contains(id)) { - return true; - } - std::thread::sleep(Duration::from_millis(100)); - } - false -} - -// ── UX guard assertion ──────────────────────────────────────────────────────── - -/// Assert act_losses stayed at `max_allowed` (usually 0) since `before`. -fn assert_ux_guard(before: u32, max_allowed: u32, context: &str) { - let after = read_losses(&loss_file()); - let delta = after.saturating_sub(before); - assert!( - delta <= max_allowed, - "UX guard violated: act_losses went from {before} to {after} \ - (delta={delta}, max_allowed={max_allowed}) during: {context}" - ); -} - -// ───────────────────────────────────────────────────────────────────────────── -// Test 1: background click + type do not steal focus -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn test_background_click_and_type_no_focus_steal() { - //! Background focus-preservation coverage. - //! - //! 1. Launch the Electron harness. - //! 2. Foreground FocusMonitorWin (simulates the user's active window). - //! 3. Click inside the app and type text via cua-driver. - //! 4. Assert act_losses on FocusMonitorWin stayed at 0. - - if !driver_binary().exists() { - eprintln!("Binary not found — skipping"); - return; - } - - let Some((mut driver, app_pid, app_wid)) = launch_driver_and_test_app() else { - return; - }; - - let Some((mut fm_proc, _fm_hwnd, _fm_pid)) = launch_focus_monitor() else { - return; - }; - let losses_before = read_losses(&loss_file()); - - // Click inside the app (background, via PostMessage). - let r = driver.call( - "click", - serde_json::json!({"pid": app_pid, "window_id": app_wid, "x": 200.0, "y": 200.0}), - ); - assert!( - r.raw["error"].is_null(), - "Protocol error from click: {:?}", - r.raw - ); - - // Type text into the app (background, via PostMessage). - let r = driver.call( - "type_text", - serde_json::json!({"pid": app_pid, "window_id": app_wid, "text": "ux-guard-test"}), - ); - assert!( - r.raw["error"].is_null(), - "Protocol error from type_text: {:?}", - r.raw - ); - assert_eq!( - r.verified(), - Some(false), - "background type_text without an element read-back must not report confirmed success: {}", - r.text() - ); - assert_ne!( - r.structured()["verify"].as_str(), - Some("confirmed"), - "background type_text reported a confirmed read-back on an unreadable path: {}", - r.text() - ); - - // Plain background key dispatch is likewise an unverified PostMessage send, - // not a confirmed keypress. - let r = driver.call( - "press_key", - serde_json::json!({"pid": app_pid, "window_id": app_wid, "key": "F24"}), - ); - assert!( - r.raw["error"].is_null(), - "Protocol error from press_key: {:?}", - r.raw - ); - assert_eq!( - r.verified(), - Some(false), - "background press_key must not report confirmed success: {}", - r.text() - ); - - // ux_guard: FocusMonitorWin must not have lost activation. - assert_ux_guard( - losses_before, - 0, - "background click + type_text into CuaTestHarness.Electron", - ); - - fm_proc.kill().ok(); -} - -// ───────────────────────────────────────────────────────────────────────────── -// Test 2: launch_app minimized mode does not steal focus -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn test_launch_app_minimized_no_focus_steal() { - //! Visible app-launch coverage. - //! - //! launch_app with start_minimized=true is the strict Windows background - //! launch mode: the app starts without displacing FocusMonitorWin. - - if !driver_binary().exists() { - return; - } - - let exe = test_app_path(); - if !exe.exists() { - eprintln!("test app not available — skipping"); - return; - } - let _cleanup = KillProcessTreeOnDrop("CuaTestHarness.Electron.exe"); - - let Some((mut fm_proc, _fm_hwnd, _fm_pid)) = launch_focus_monitor() else { - return; - }; - let losses_before = read_losses(&loss_file()); - - let Some(mut driver) = McpDriver::spawn() else { - fm_proc.kill().ok(); - return; - }; - - // Launch the test app via cua-driver launch_app in strict background mode. - let path_str = exe.to_string_lossy().into_owned(); - let r = driver.call( - "launch_app", - serde_json::json!({"path": path_str, "start_minimized": true}), - ); - if r.is_error() { - eprintln!("launch_app failed — skipping: {:?}", r.raw); - fm_proc.kill().ok(); - return; - } - - // Wait for the app window to appear (Electron startup ~2-3s). - let mut app_pid: Option = None; - for _ in 0..20 { - std::thread::sleep(Duration::from_millis(500)); - let r2 = driver.call("list_apps", serde_json::json!({})); - if let Some(procs) = r2.structured()["processes"].as_array() { - if let Some(p) = procs.iter().find(|p| { - p["name"] - .as_str() - .map(|n| { - let n = n.to_lowercase(); - n.contains("cuatestharness.electron") || n.contains("electron") - }) - .unwrap_or(false) - }) { - app_pid = p["pid"].as_i64(); - break; - } - } - } - if app_pid.is_none() { - eprintln!("CuaTestHarness.Electron not found in process list after launch_app - skipping"); - fm_proc.kill().ok(); - return; - } - - // ux_guard: FocusMonitorWin must not have lost activation. - assert_ux_guard( - losses_before, - 0, - "launch_app CuaTestHarness.Electron with start_minimized=true", - ); - - // Kill the launched app by exe name. - kill_process_tree_by_image("CuaTestHarness.Electron.exe"); - fm_proc.kill().ok(); -} - -// ───────────────────────────────────────────────────────────────────────────── -// Test 3: background hotkey does not steal focus -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn test_background_hotkey_no_focus_steal() { - //! Background menu-shortcut coverage. - //! - //! Send Ctrl+A to a background Electron harness window. - //! FocusMonitorWin must never lose activation. - - if !driver_binary().exists() { - return; - } - - let Some((mut driver, app_pid, app_wid)) = launch_driver_and_test_app() else { - return; - }; - - let Some((mut fm_proc, _fm_hwnd, _fm_pid)) = launch_focus_monitor() else { - return; - }; - let losses_before = read_losses(&loss_file()); - - // Send Ctrl+A hotkey to background app (PostMessage, no focus steal). - let r = driver.call( - "hotkey", - serde_json::json!({"pid": app_pid, "window_id": app_wid, "keys": ["ctrl", "a"]}), - ); - assert!( - r.raw["error"].is_null(), - "Protocol error from hotkey: {:?}", - r.raw - ); - - // ux_guard: FocusMonitorWin must not have lost activation. - assert_ux_guard( - losses_before, - 0, - "background hotkey ctrl+a to CuaTestHarness.Electron", - ); - - fm_proc.kill().ok(); -} - -// ───────────────────────────────────────────────────────────────────────────── -// Test 4: background click that opens a new window (e.g. File→New dialog) -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn test_background_click_opens_new_window_focus_preserved() { - //! New-window click-delivery coverage. - //! - //! 1. FocusMonitorWin is foreground. - //! 2. Click the CuaTestHarness.Electron child-window button. - //! 3. FocusMonitorWin must remain active throughout (UX guard). - //! - //! Verifies that a background click can cause a child window to appear - //! without activating either the original target or the new window. - - if !driver_binary().exists() { - return; - } - - let Some((mut driver, app_pid, app_wid)) = launch_driver_and_test_app() else { - return; - }; - let snap = driver.call( - "get_window_state", - serde_json::json!({"pid": app_pid, "window_id": app_wid, "capture_mode": "ax"}), - ); - let Some(open_idx) = ax::element_index_containing(snap.text(), "Open child window") else { - eprintln!("child-window button not found in test app — skipping"); - return; - }; - - let Some((mut fm_proc, _fm_hwnd, _fm_pid)) = launch_focus_monitor() else { - return; - }; - let windows_before = window_ids(&mut driver); - let losses_before = read_losses(&loss_file()); - - // Click the explicit child-window button. - let r = driver.call( - "click", - serde_json::json!({"pid": app_pid, "window_id": app_wid, "element_index": open_idx}), - ); - assert!( - r.raw["error"].is_null(), - "Protocol error from click: {:?}", - r.raw - ); - - assert!( - wait_for_new_window(&mut driver, &windows_before), - "background click did not open a new harness window" - ); - - // ux_guard: FocusMonitorWin must not have lost activation. - assert_ux_guard( - losses_before, - 0, - "background click in CuaTestHarness.Electron (may open new window)", - ); - - // Verify FocusMonitorWin is still alive. - assert!( - fm_proc.try_wait().expect("try_wait").is_none(), - "FocusMonitorWin crashed during the test" - ); - - fm_proc.kill().ok(); -} - -// ───────────────────────────────────────────────────────────────────────────── -// Test 5: screenshot of background window doesn't steal focus -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn test_background_screenshot_no_focus_steal() { - //! PrintWindow captures a background window without activating it. - - if !driver_binary().exists() { - return; - } - - let Some((mut driver, _app_pid, app_wid)) = launch_driver_and_test_app() else { - return; - }; - - let Some((mut fm_proc, _fm_hwnd, _fm_pid)) = launch_focus_monitor() else { - return; - }; - let losses_before = read_losses(&loss_file()); - - // Screenshot via PrintWindow — must not activate the window. - let r = driver.call("screenshot", serde_json::json!({"window_id": app_wid})); - assert!( - r.raw["error"].is_null(), - "Protocol error from screenshot: {:?}", - r.raw - ); - - // ux_guard - assert_ux_guard( - losses_before, - 0, - "screenshot of background CuaTestHarness.Electron", - ); - - fm_proc.kill().ok(); -} - -// ───────────────────────────────────────────────────────────────────────────── -// Test 6: agent cursor is visually present on screen after move_cursor -// ───────────────────────────────────────────────────────────────────────────── - -#[test] -fn test_agent_cursor_visible_on_screen() { - //! Computer-vision check: after move_cursor the overlay must be visible. - //! - //! Steps: - //! 1. Enable the agent cursor and move it to a known screen position. - //! 2. Wait for the glide animation to settle (default 750ms). - //! 3. Capture the screen at the cursor position using screenshot_display_bytes - //! (BitBlt from display DC — captures layered/overlay windows). - //! 4. Decode the PNG and sample a 40×40 px patch centred on the cursor. - //! 5. Assert the patch contains cursor-like pixels (bright or saturated). - - if !driver_binary().exists() { - eprintln!("Binary not found — skipping"); - return; - } - if !require_seedable_desktop("agent cursor visibility") { - return; - } - - let Some(mut driver) = McpDriver::spawn() else { - return; - }; - - // Safe centre-ish position on primary monitor. - let cx = 640.0_f64; - let cy = 400.0_f64; - let cursor_id = "guard-ux-cursor"; - - // Enable cursor overlay and glide to target. - let r = driver.call( - "set_agent_cursor_enabled", - serde_json::json!({"enabled": true, "cursor_id": cursor_id}), - ); - assert!( - r.raw["error"].is_null(), - "set_agent_cursor_enabled failed: {:?}", - r.raw - ); - - let r = driver.call( - "set_agent_cursor_motion", - serde_json::json!({ - "cursor_id": cursor_id, - "glide_duration_ms": 100, - "idle_hide_ms": 0 - }), - ); - assert!( - r.raw["error"].is_null(), - "set_agent_cursor_motion failed: {:?}", - r.raw - ); - - let r = driver.call( - "move_cursor", - serde_json::json!({"x": cx, "y": cy, "cursor_id": cursor_id}), - ); - assert!(r.raw["error"].is_null(), "move_cursor failed: {:?}", r.raw); - - // Wait for the fixed 100ms glide plus a few render frames. - std::thread::sleep(Duration::from_millis(350)); - - // Capture the screen directly (includes layered windows like the overlay). - let png_bytes = platform_windows::capture::screenshot_display_bytes() - .expect("screenshot_display_bytes failed"); - - drop(driver); - - // Decode PNG. - let img = image::load_from_memory(&png_bytes).expect("decode PNG"); - let rgba = img.to_rgba8(); - let (iw, ih) = rgba.dimensions(); - - // Sample 40×40 patch centred on (cx, cy). - let half = 20u32; - let x0 = (cx as u32).saturating_sub(half).min(iw.saturating_sub(1)); - let x1 = (cx as u32 + half).min(iw); - let y0 = (cy as u32).saturating_sub(half).min(ih.saturating_sub(1)); - let y1 = (cy as u32 + half).min(ih); - - let mut colourful_pixels = 0u32; - for py in y0..y1 { - for px in x0..x1 { - let [r, g, b, a] = rgba.get_pixel(px, py).0; - if a < 10 { - continue; - } - let brightness = r as u32 + g as u32 + b as u32; - let saturation = r.max(g).max(b) as u32 - r.min(g).min(b) as u32; - // Accept bright-white stroke pixels OR coloured gradient pixels. - if brightness > 60 && (saturation > 30 || brightness > 600) { - colourful_pixels += 1; - } - } - } - - assert!( - colourful_pixels >= 5, - "Agent cursor not visible at ({cx},{cy}): only {colourful_pixels} qualifying pixels \ - in 40×40 patch (x={x0}..{x1}, y={y0}..{y1}, image={iw}x{ih}). \ - Overlay may not be rendering or is positioned off-screen." - ); -} diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs index 65711e784f..5fe217f093 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_appkit_test.rs @@ -19,10 +19,8 @@ //! //! Tests are `#[ignore]` so they don't run in plain `cargo test`. //! -//! **TCC caveat:** on a fresh Mac, the cua-driver process needs -//! Accessibility permission for AX queries to return non-empty trees. -//! These tests print a TCC hint and exit cleanly (PASS-by-skip) when the -//! AX tree is empty rather than misreporting as a test failure. +//! The macOS lane preflight verifies the installed daemon identity and TCC +//! grants before these tests run. Missing fixtures or AX trees fail here too. #![cfg(target_os = "macos")] @@ -30,7 +28,13 @@ use std::path::PathBuf; use std::process::{Child, Command, Stdio}; use std::time::Duration; -use cua_driver_testkit::ax::{element_index_by_id, has_id, looks_empty}; +use cua_driver_testkit::ax::{element_index_by_id, element_index_containing, has_id, looks_empty}; +use cua_driver_testkit::e2e::{ + execute_case, native_background_case, native_foreground_case, native_readonly_case, + recording_evidence, DriverRoute, Evidence, Observation, OracleKind, RefusalCode, Targeting, +}; +use cua_driver_testkit::observer::TargetWindow; +use cua_driver_testkit::sentinel::run_with_background_oracles; use cua_driver_testkit::{Driver, McpDriver, ToolResponse}; // ── paths ──────────────────────────────────────────────────────────────────── @@ -57,15 +61,12 @@ struct Harness { } impl Harness { - fn launch() -> Option { + fn launch() -> Self { let exe = harness_exe(); - if !exe.exists() { - eprintln!( - "harness exe not found at {exe:?} \ - — run libs/cua-driver/tests/fixtures/build/macos.sh first" - ); - return None; - } + assert!( + exe.exists(), + "required AppKit harness is missing at {exe:?}; run the fixture build" + ); // Launch the binary directly (not via `open`) so we control the pid // and can kill it cleanly on Drop. The app still installs an AppKit // window via NSApp.run(). @@ -73,11 +74,11 @@ impl Harness { .stdout(Stdio::null()) .stderr(Stdio::null()) .spawn() - .ok()?; + .unwrap_or_else(|error| panic!("launch AppKit harness {exe:?}: {error}")); let pid = app.id(); // Settle for window creation + activation. std::thread::sleep(Duration::from_millis(800)); - Some(Self { _app: app, pid }) + Self { _app: app, pid } } } @@ -102,74 +103,148 @@ fn snapshot_elements(driver: &mut McpDriver, pid: u32, window_id: u64) -> ToolRe ) } -// ── tests ──────────────────────────────────────────────────────────────────── +fn element_pixel_frame(snapshot: &ToolResponse, identifier: &str) -> (f64, f64, f64, f64) { + let index = element_index_by_id(snapshot.tree_text(), identifier) + .unwrap_or_else(|| panic!("{identifier} element_index not found")); + let elements = snapshot.structured()["elements"] + .as_array() + .expect("AppKit structured elements"); + let element = elements + .iter() + .find(|element| element["element_index"].as_u64() == Some(index)) + .unwrap_or_else(|| panic!("{identifier} element frame not found")); + let window = elements + .iter() + .find(|element| element["role"].as_str() == Some("AXWindow")) + .expect("AppKit window frame"); + let scale = snapshot.structured()["screenshot_width"] + .as_f64() + .unwrap_or(1.0) + / window["frame"]["w"].as_f64().unwrap_or(1.0).max(1.0); + ( + (element["frame"]["x"].as_f64().unwrap_or(0.0) + - window["frame"]["x"].as_f64().unwrap_or(0.0)) + * scale, + (element["frame"]["y"].as_f64().unwrap_or(0.0) + - window["frame"]["y"].as_f64().unwrap_or(0.0)) + * scale, + element["frame"]["w"].as_f64().unwrap_or(0.0) * scale, + element["frame"]["h"].as_f64().unwrap_or(0.0) * scale, + ) +} -#[test] -#[ignore] -fn harness_appkit_smoke() { - let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else { - return; - }; - let harness = match Harness::launch() { - Some(h) => h, - None => { - eprintln!("harness not built — skipping"); - return; +fn run_case( + case: cua_driver_testkit::e2e::CaseSpec, + test: impl FnOnce(u32, u64, &mut McpDriver) -> Observation, +) { + let cell_id = case.cell_id.clone(); + let delivery = case.delivery; + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_macos_daemon_proxy_named(&cell_id) + .expect("start installed macOS daemon proxy"); + *evidence = recording_evidence(driver.recording_dir()); + let harness = Harness::launch(); + let (wid, _) = driver + .find_window(harness.pid as i64, "CuaTestHarness AppKit") + .expect("AppKit main window not found"); + if delivery != cua_driver_testkit::e2e::Delivery::Background { + driver.start_behavior_recording(); } - }; - println!("harness pid={}", harness.pid); + test(harness.pid, wid, &mut driver) + }); +} - let (wid, title) = driver - .find_window(harness.pid as i64, "CuaTestHarness AppKit") - .expect("main window not found via list_windows"); - println!("main window: id={wid} title={title:?}"); +fn run_background_case( + action: &str, + route: DriverRoute, + test: impl FnOnce(u32, u64, &mut McpDriver), +) { + run_background_case_targeting(action, Targeting::Ax, route, test); +} - let snap = snapshot_elements(&mut driver, harness.pid, wid); +fn run_background_case_targeting( + action: &str, + targeting: Targeting, + route: DriverRoute, + test: impl FnOnce(u32, u64, &mut McpDriver), +) { + run_case( + native_background_case("appkit", action, targeting, route), + |pid, wid, driver| { + let (_, passed) = run_with_background_oracles( + driver, + TargetWindow { + pid, + native_id: wid, + }, + |driver| test(pid, wid, driver), + ) + .unwrap_or_else(|error| panic!("background desktop contract failed: {error}")); + Observation::delivered_with_fixture_state(passed) + }, + ); +} - if looks_empty(snap.tree_text()) { - eprintln!( - "AX tree empty — likely TCC Accessibility not granted to the test runner. \ - Skipping element-assertion phase. To enable: System Settings → Privacy & \ - Security → Accessibility → add the binary running `cargo test`." - ); - return; - } +// ── tests ──────────────────────────────────────────────────────────────────── - let text = snap.tree_text(); - println!("snapshot:\n{text}"); - - // AppKit AX quirk (mirrors the WPF behavior documented in - // harness_wpf_test.rs::harness_wpf_smoke): NSTextField in label mode - // and other AXStaticText leaves do NOT propagate - // setAccessibilityIdentifier into the AX tree's identifier slot, so - // we don't assert on ids for labels. We assert on text-presence for - // those, and on AX ids only for actionable controls (Buttons, - // TextFields). - for aid in [ - "wnd-main", // NSWindow - "btn-increment", - "btn-reset", // NSButton - "txt-input", // editable NSTextField - "menu-test-item", // NSMenuItem (Mac-specific) - "btn-exit", - ] { - assert!( - has_id(snap.tree_text(), aid), - "missing AX identifier {aid} in AppKit snapshot" - ); - } +#[test] +#[ignore] +fn harness_appkit_smoke() { + run_case( + native_readonly_case( + "appkit", + "ax_tree", + Targeting::Ax, + DriverRoute::AxRead, + vec![OracleKind::AxState], + ), + |pid, wid, driver| { + let snap = snapshot_elements(driver, pid, wid); + + assert!( + !looks_empty(snap.tree_text()), + "required AppKit AX tree is empty" + ); - // text_body marker carried by the visible string of the NSTextField - assert!( - text.contains("HARNESS_TEXT_MARKER_v1"), - "text_body marker not in AppKit snapshot" - ); - // The two label-mode NSTextFields under click_target render as - // AXStaticText nodes — assert on their starting text instead of ids. - assert!(text.contains("clicks=0"), "click_count label missing"); - assert!( - text.contains("last_action=none"), - "last_action label missing" + let text = snap.tree_text(); + println!("snapshot:\n{text}"); + + // AppKit AX quirk (mirrors the WPF behavior documented in + // harness_wpf_test.rs::harness_wpf_smoke): NSTextField in label mode + // and other AXStaticText leaves do NOT propagate + // setAccessibilityIdentifier into the AX tree's identifier slot, so + // we don't assert on ids for labels. We assert on text-presence for + // those, and on AX ids only for actionable controls (Buttons, + // TextFields). + for aid in [ + "wnd-main", // NSWindow + "btn-increment", + "btn-reset", // NSButton + "txt-input", // editable NSTextField + "menu-test-item", // NSMenuItem (Mac-specific) + "btn-exit", + ] { + assert!( + has_id(snap.tree_text(), aid), + "missing AX identifier {aid} in AppKit snapshot" + ); + } + + // text_body marker carried by the visible string of the NSTextField + assert!( + text.contains("HARNESS_TEXT_MARKER_v1"), + "text_body marker not in AppKit snapshot" + ); + // The two label-mode NSTextFields under click_target render as + // AXStaticText nodes — assert on their starting text instead of ids. + assert!(text.contains("counter=0"), "counter label missing"); + assert!(text.contains("clicks=0"), "click_count label missing"); + assert!( + text.contains("last_action=none"), + "last_action label missing" + ); + Observation::delivered(vec![OracleKind::AxState], Evidence::default()) + }, ); } @@ -179,47 +254,40 @@ fn harness_appkit_smoke() { #[test] #[ignore] fn harness_appkit_text_input() { - let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else { - return; - }; - let harness = match Harness::launch() { - Some(h) => h, - None => { - eprintln!("harness not built — skipping"); - return; - } - }; - - let (wid, _) = driver - .find_window(harness.pid as i64, "CuaTestHarness AppKit") - .expect("main window not found"); - let snap_pre = snapshot_elements(&mut driver, harness.pid, wid); - if looks_empty(snap_pre.tree_text()) { - eprintln!("AX empty — TCC not granted; skipping"); - return; - } - let idx = element_index_by_id(snap_pre.tree_text(), "txt-input") - .expect("txt-input element_index not found"); - - // set_value via AX is the deterministic background path; type_text would - // also work but races with cursor focus on cold-launched windows. - let resp = driver.call( + run_background_case( "set_value", - serde_json::json!({ - "pid": harness.pid as i64, - "window_id": wid, - "element_index": idx, - "value": "hello-cua" - }), - ); - println!("set_value resp: {}", resp.text()); - - std::thread::sleep(Duration::from_millis(250)); - let snap_post = snapshot_elements(&mut driver, harness.pid, wid); - let post_text = snap_post.tree_text().to_owned(); - assert!( - post_text.contains("hello-cua"), - "text_input value did not propagate to mirror; snapshot:\n{post_text}" + DriverRoute::MacosAxValue, + |pid, wid, driver| { + let snap_pre = snapshot_elements(driver, pid, wid); + assert!( + !looks_empty(snap_pre.tree_text()), + "required AppKit AX tree is empty" + ); + let idx = element_index_by_id(snap_pre.tree_text(), "txt-input") + .expect("txt-input element_index not found"); + + // set_value via AX is the deterministic background path; type_text would + // also work but races with cursor focus on cold-launched windows. + let resp = driver.call( + "set_value", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "element_index": idx, + "value": "hello-cua" + }), + ); + assert!(!resp.is_error(), "AppKit set_value failed: {}", resp.text()); + println!("set_value resp: {}", resp.text()); + + std::thread::sleep(Duration::from_millis(250)); + let snap_post = snapshot_elements(driver, pid, wid); + let post_text = snap_post.tree_text().to_owned(); + assert!( + post_text.contains("hello-cua"), + "text_input value did not propagate to mirror; snapshot:\n{post_text}" + ); + }, ); } @@ -228,150 +296,125 @@ fn harness_appkit_text_input() { /// dispatch chain reaches a backgrounded Cocoa text input. #[test] #[ignore] -fn harness_appkit_type_text_keystroke() { - let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else { - return; - }; - let harness = match Harness::launch() { - Some(h) => h, - None => { - eprintln!("harness not built — skipping"); - return; - } - }; - - let (wid, _) = driver - .find_window(harness.pid as i64, "CuaTestHarness AppKit") - .expect("main window not found"); - let snap_pre = snapshot_elements(&mut driver, harness.pid, wid); - if looks_empty(snap_pre.tree_text()) { - eprintln!("AX empty — TCC not granted; skipping"); - return; - } - let idx: u64 = if let Some(i) = element_index_by_id(snap_pre.tree_text(), "txt-input") { - i - } else { - eprintln!("txt-input not found; skipping"); - return; - }; - - // Focus the field first so the keystrokes land in it. AX press on - // a text field has the side effect of giving it keyboard focus. - let _ = driver.call( - "click", - serde_json::json!({ - "pid": harness.pid as i64, "window_id": wid, - "element_index": idx, "action": "press" - }), - ); - std::thread::sleep(Duration::from_millis(150)); - - // CGEvent-based type_text against the focused field (does NOT use - // set_value — exercises the keystroke synthesis chain). - let resp = driver.call( +fn harness_appkit_type_text_background() { + run_background_case( "type_text", - serde_json::json!({ - "pid": harness.pid as i64, "window_id": wid, - "text": "kbd-cua" - }), - ); - println!("type_text resp: {}", resp.text()); - std::thread::sleep(Duration::from_millis(250)); - - let snap_post = snapshot_elements(&mut driver, harness.pid, wid); - let post = snap_post.tree_text().to_owned(); - assert!( - post.contains("kbd-cua"), - "type_text keystroke did not land in the text field; snapshot:\n{post}" + DriverRoute::MacosAxValue, + |pid, wid, driver| { + let snap_pre = snapshot_elements(driver, pid, wid); + assert!( + !looks_empty(snap_pre.tree_text()), + "required AppKit AX tree is empty" + ); + let idx = element_index_by_id(snap_pre.tree_text(), "txt-input") + .expect("txt-input element_index not found"); + + // Address the field through type_text itself. AXTextField does not + // advertise AXPress, so a preparatory click would test an invalid + // action and fail before the keyboard/value delivery path runs. + let resp = driver.call( + "type_text", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "text": "kbd-cua", "delivery_mode": "background" + }), + ); + assert!(!resp.is_error(), "AppKit type_text failed: {}", resp.text()); + println!("type_text resp: {}", resp.text()); + std::thread::sleep(Duration::from_millis(250)); + + let snap_post = snapshot_elements(driver, pid, wid); + let post = snap_post.tree_text().to_owned(); + assert!( + post.contains("kbd-cua"), + "type_text keystroke did not land in the text field; snapshot:\n{post}" + ); + }, ); } -/// scroll: scroll the NSScrollView downward, verify the offset label -/// changes (it mirrors the clip view's documentVisibleRect.origin.y). -/// -/// **Status:** EXPECTED-FAIL today (see notes below). The `scroll` tool -/// itself works at the API level — `libs/cua-driver/tests/fixtures/smoke/macos.sh` confirms it -/// PASSes against the same harness window. What this test would -/// verify additionally is that the scroll event actually moved the -/// scroll view's bounds (state-change observation, not just API -/// success). -/// -/// Why it's expected-fail: on macOS, `CGEvent.scroll` requires the -/// cursor position to lie inside the target NSScrollView for the event -/// to be routed to it (Cocoa scroll-routing is cursor-anchored). The -/// `move_cursor` tool we expose is overlay-only — it doesn't move the -/// OS hardware cursor on macOS. So state-change tests for scroll need -/// either (a) an OS-cursor warp (intentionally not exposed) or (b) a -/// different scroll dispatch primitive (NSEvent.otherEvent -/// keyDown(.swipeUp) on focused window, or an AXScrollAreaScrollTo -/// action). Both are open implementation work; tracking in the journal's -/// "Open items" section. #[test] #[ignore] -#[should_panic(expected = "scroll offset label did not advance from 0")] -fn harness_appkit_scroll_expected_fail() { - let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else { - return; - }; - let harness = match Harness::launch() { - Some(h) => h, - None => { - eprintln!("harness not built — skipping"); - return; - } - }; - - let (wid, _) = driver - .find_window(harness.pid as i64, "CuaTestHarness AppKit") - .expect("main window not found"); - let snap_pre = snapshot_elements(&mut driver, harness.pid, wid); - if looks_empty(snap_pre.tree_text()) { - eprintln!("AX empty — TCC not granted; skipping"); - return; - } - // Pre-condition: offset label should be at "0" (the controller - // initial state). The label text appears as an AXStaticText leaf - // immediately after the AXTextArea body in the rendered tree. - let pre = snap_pre.tree_text().to_owned(); - let pre_has_zero_offset = pre.lines().any(|l| l.trim() == "- AXStaticText = \"0\""); - assert!( - pre_has_zero_offset, - "scroll offset label not at 0 pre-scroll" +fn harness_appkit_scroll_foreground() { + run_case( + native_foreground_case( + "appkit", + "scroll", + Targeting::Ax, + DriverRoute::MacosAxAction, + ), + |pid, wid, driver| { + let pre = snapshot_elements(driver, pid, wid); + assert!(pre.tree_text().contains("scroll_offset=0")); + let index = element_index_by_id(pre.tree_text(), "scroll-tall") + .or_else(|| element_index_containing(pre.tree_text(), "SCROLL_TOP_MARKER_v1")) + .unwrap_or_else(|| { + panic!("scroll-tall element_index not found:\n{}", pre.tree_text()) + }); + let response = driver.call( + "scroll", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "element_index": index, + "direction": "down", + "amount": 5, + "delivery_mode": "foreground" + }), + ); + assert!( + !response.is_error(), + "AppKit foreground scroll failed: {}; raw={}", + response.text(), + response.raw + ); + std::thread::sleep(Duration::from_millis(300)); + let post = snapshot_elements(driver, pid, wid); + assert!( + !post.tree_text().contains("scroll_offset=0"), + "AppKit foreground scroll did not move the NSScrollView; response={}; raw={}", + response.text(), + response.raw + ); + Observation::delivered_with_fixture_state(Vec::new()) + }, ); +} - // Scroll the scroll view down a few ticks. The scroll tool takes - // window-local pixel coords; pick a point inside the scroller - // (the scroll target sits roughly mid-window). - let resp = driver.call( - "scroll", - serde_json::json!({ - "pid": harness.pid as i64, "window_id": wid, - "x": 180, "y": 450, // inside the scroll view - "direction": "down", - "amount": 5 - }), - ); - println!("scroll resp: {}", resp.text()); - std::thread::sleep(Duration::from_millis(250)); - - let snap_post = snapshot_elements(&mut driver, harness.pid, wid); - let post = snap_post.tree_text().to_owned(); - // After scroll, the offset label should no longer be "0" — any - // positive integer indicates the bounds-change notification fired - // and the label updated. We don't pin a specific value (scroll - // wheel pixel delta varies by macOS version + accessibility setting). - let still_zero = post.lines().any(|l| l.trim() == "- AXStaticText = \"0\""); - let unchanged_count = post.matches("- AXStaticText = \"0\"").count(); - let pre_count = pre.matches("- AXStaticText = \"0\"").count(); - // Counter label is also "0" so the bare presence isn't a signal — - // instead check the COUNT decreased by 1 (only the offset label - // moved off zero, not the counter). - assert!( - unchanged_count < pre_count || !still_zero, - "scroll offset label did not advance from 0; pre: {} \"0\" leaves; post: {} \"0\" leaves", - pre_count, - unchanged_count - ); +#[test] +#[ignore] +fn harness_appkit_scroll_background() { + run_background_case("scroll", DriverRoute::MacosAxAction, |pid, wid, driver| { + let pre = snapshot_elements(driver, pid, wid); + assert!(pre.tree_text().contains("scroll_offset=0")); + let index = element_index_by_id(pre.tree_text(), "scroll-tall") + .or_else(|| element_index_containing(pre.tree_text(), "SCROLL_TOP_MARKER_v1")) + .unwrap_or_else(|| panic!("scroll-tall element_index not found:\n{}", pre.tree_text())); + let response = driver.call( + "scroll", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "element_index": index, + "direction": "down", + "amount": 5, + "delivery_mode": "background" + }), + ); + assert!( + !response.is_error(), + "AppKit background scroll failed: {}; raw={}", + response.text(), + response.raw + ); + std::thread::sleep(Duration::from_millis(200)); + assert!( + !snapshot_elements(driver, pid, wid) + .tree_text() + .contains("scroll_offset=0"), + "AppKit background AX scroll did not move the NSScrollView" + ); + }); } /// counter: click the increment button via element_index, verify the @@ -379,52 +422,352 @@ fn harness_appkit_scroll_expected_fail() { #[test] #[ignore] fn harness_appkit_counter() { - let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else { - return; - }; - let harness = match Harness::launch() { - Some(h) => h, - None => { - eprintln!("harness not built — skipping"); - return; - } - }; - - let (wid, _) = driver - .find_window(harness.pid as i64, "CuaTestHarness AppKit") - .expect("main window not found"); - let snap_pre = snapshot_elements(&mut driver, harness.pid, wid); - if looks_empty(snap_pre.tree_text()) { - eprintln!("AX empty — TCC not granted; skipping"); - return; - } - let pre_text = snap_pre.tree_text().to_owned(); - assert!( - pre_text.contains("\"0\""), - "counter not 0 pre-click; snapshot:\n{pre_text}" + run_background_case( + "left_click", + DriverRoute::MacosAxAction, + |pid, wid, driver| { + let snap_pre = snapshot_elements(driver, pid, wid); + assert!( + !looks_empty(snap_pre.tree_text()), + "required AppKit AX tree is empty" + ); + let pre_text = snap_pre.tree_text().to_owned(); + assert!( + pre_text.contains("counter=0"), + "counter not 0 pre-click; snapshot:\n{pre_text}" + ); + + let idx = element_index_by_id(snap_pre.tree_text(), "btn-increment") + .expect("btn-increment element_index not found"); + + let click_resp = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "element_index": idx, + "action": "press", + "delivery_mode": "background" + }), + ); + assert!( + !click_resp.is_error(), + "AppKit counter click failed: {}", + click_resp.text() + ); + println!("click resp: {}", click_resp.text()); + + // Let the AppKit run-loop process the press and refresh the label. + std::thread::sleep(Duration::from_millis(200)); + + let snap_post = snapshot_elements(driver, pid, wid); + let post_text = snap_post.tree_text().to_owned(); + assert!( + post_text.contains("counter=1"), + "counter did not advance to 1 after press; post snapshot:\n{post_text}" + ); + }, ); +} - let idx = element_index_by_id(snap_pre.tree_text(), "btn-increment") - .expect("btn-increment element_index not found"); +/// Resolve the native AppKit button from a screenshot-space PX target, then +/// deliver through the background-safe AX hit-test bridge while another app +/// remains fully foreground. +#[test] +#[ignore] +fn harness_appkit_counter_px_background() { + run_background_case_targeting( + "left_click", + Targeting::Px, + DriverRoute::MacosAxAction, + |pid, wid, driver| { + let pre = snapshot_elements(driver, pid, wid); + let (x, y, width, height) = element_pixel_frame(&pre, "btn-increment"); + let response = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "x": x + width / 2.0, + "y": y + height / 2.0, + "delivery_mode": "background" + }), + ); + assert!( + !response.is_error(), + "AppKit PX background click failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(200)); + assert!( + snapshot_elements(driver, pid, wid) + .tree_text() + .contains("counter=1"), + "AppKit PX background click did not advance counter" + ); + }, + ); +} - let click_resp = driver.call( - "click", - serde_json::json!({ - "pid": harness.pid as i64, - "window_id": wid, - "element_index": idx, - "action": "press" - }), +#[test] +#[ignore] +fn harness_appkit_right_click_px_foreground() { + run_case( + native_foreground_case( + "appkit", + "right_click", + Targeting::Px, + DriverRoute::MacosCgEventHid, + ), + |pid, wid, driver| { + let pre = snapshot_elements(driver, pid, wid); + let (x, y, width, height) = element_pixel_frame(&pre, "btn-clicktarget"); + let response = driver.call( + "right_click", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "x": x + width / 2.0, + "y": y + height / 2.0, + "delivery_mode": "foreground" + }), + ); + assert!( + !response.is_error(), + "AppKit right click failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(250)); + assert!( + snapshot_elements(driver, pid, wid) + .tree_text() + .contains("last_action=right_click"), + "AppKit right-click handler did not fire" + ); + Observation::delivered_with_fixture_state(Vec::new()) + }, ); - println!("click resp: {}", click_resp.text()); +} - // Let the AppKit run-loop process the press and refresh the label. - std::thread::sleep(Duration::from_millis(200)); +#[test] +#[ignore] +fn harness_appkit_right_click_px_background() { + run_background_case_targeting( + "right_click", + Targeting::Px, + DriverRoute::MacosCgEventPid, + |pid, wid, driver| { + let pre = snapshot_elements(driver, pid, wid); + let (x, y, width, height) = element_pixel_frame(&pre, "btn-clicktarget"); + let response = driver.call( + "right_click", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "x": x + width / 2.0, + "y": y + height / 2.0, + "delivery_mode": "background" + }), + ); + assert!( + !response.is_error(), + "AppKit right click failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(250)); + assert!( + snapshot_elements(driver, pid, wid) + .tree_text() + .contains("last_action=right_click"), + "AppKit background right-click handler did not fire" + ); + }, + ); +} - let snap_post = snapshot_elements(&mut driver, harness.pid, wid); - let post_text = snap_post.tree_text().to_owned(); - assert!( - post_text.contains("\"1\""), - "counter did not advance to 1 after press; post snapshot:\n{post_text}" +#[test] +#[ignore] +fn harness_appkit_double_click_px_foreground() { + run_case( + native_foreground_case( + "appkit", + "double_click", + Targeting::Px, + DriverRoute::MacosCgEventHid, + ), + |pid, wid, driver| { + let pre = snapshot_elements(driver, pid, wid); + let (x, y, width, height) = element_pixel_frame(&pre, "btn-clicktarget"); + let response = driver.call( + "double_click", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "x": x + width / 2.0, + "y": y + height / 2.0, + "delivery_mode": "foreground" + }), + ); + assert!( + !response.is_error(), + "AppKit double click failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(250)); + assert!( + snapshot_elements(driver, pid, wid) + .tree_text() + .contains("last_action=double_click"), + "AppKit double-click handler did not fire" + ); + Observation::delivered_with_fixture_state(Vec::new()) + }, + ); +} + +#[test] +#[ignore] +fn harness_appkit_double_click_px_background() { + run_background_case_targeting( + "double_click", + Targeting::Px, + DriverRoute::MacosCgEventPid, + |pid, wid, driver| { + let pre = snapshot_elements(driver, pid, wid); + let (x, y, width, height) = element_pixel_frame(&pre, "btn-clicktarget"); + let response = driver.call( + "double_click", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "x": x + width / 2.0, + "y": y + height / 2.0, + "delivery_mode": "background" + }), + ); + assert!( + !response.is_error(), + "AppKit double click failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(250)); + assert!( + snapshot_elements(driver, pid, wid) + .tree_text() + .contains("last_action=double_click"), + "AppKit background double-click handler did not fire" + ); + }, + ); +} + +#[test] +#[ignore] +fn harness_appkit_slider_drag_px_foreground() { + run_case( + native_foreground_case( + "appkit", + "slider_drag", + Targeting::Px, + DriverRoute::MacosCgEventHid, + ), + |pid, wid, driver| { + let pre = snapshot_elements(driver, pid, wid); + assert!(pre.tree_text().contains("slider_value=0")); + let (x, y, width, height) = element_pixel_frame(&pre, "sld-value"); + let response = driver.call( + "drag", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "from_x": x + width * 0.05, + "from_y": y + height / 2.0, + "to_x": x + width * 0.90, + "to_y": y + height / 2.0, + "duration_ms": 500, + "steps": 30, + "delivery_mode": "foreground" + }), + ); + assert!( + !response.is_error(), + "AppKit slider drag failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(300)); + assert!( + !snapshot_elements(driver, pid, wid) + .tree_text() + .contains("slider_value=0"), + "AppKit foreground drag did not move the slider" + ); + Observation::delivered_with_fixture_state(Vec::new()) + }, ); } + +#[test] +#[ignore] +fn harness_appkit_slider_drag_px_background() { + let case = native_background_case( + "appkit", + "slider_drag", + Targeting::Px, + DriverRoute::MacosCgEventPid, + ) + .expecting_refusal(vec![RefusalCode::BackgroundUnavailable]); + run_case(case, |pid, wid, driver| { + let pre = snapshot_elements(driver, pid, wid); + assert!(pre.tree_text().contains("slider_value=0")); + let (x, y, width, height) = element_pixel_frame(&pre, "sld-value"); + let (response, mut passed) = run_with_background_oracles( + driver, + TargetWindow { + pid, + native_id: wid, + }, + |driver| { + driver.call( + "drag", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "from_x": x + width * 0.05, + "from_y": y + height / 2.0, + "to_x": x + width * 0.90, + "to_y": y + height / 2.0, + "duration_ms": 500, + "steps": 30, + "delivery_mode": "background" + }), + ) + }, + ) + .unwrap_or_else(|error| panic!("background desktop contract failed: {error}")); + assert!( + response.is_error(), + "AppKit background drag unexpectedly reported delivery: {}", + response.text() + ); + assert_eq!( + response.structured()["code"].as_str(), + Some("background_unavailable"), + "AppKit background drag returned the wrong refusal: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(200)); + assert!( + snapshot_elements(driver, pid, wid) + .tree_text() + .contains("slider_value=0"), + "refused AppKit background drag changed the slider" + ); + passed.push(OracleKind::FixtureState); + Observation::refused( + RefusalCode::BackgroundUnavailable, + passed, + response.text(), + Evidence::default(), + ) + }); +} diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk3_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk3_test.rs index 9be13a143c..54ae8ba0db 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk3_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_gtk3_test.rs @@ -1,255 +1,1090 @@ -//! Integration test against the CuaTestHarness.Gtk3 (PyGObject / GTK3) app — -//! the Linux peer of `harness_appkit_test` / `harness_swiftui_test` (macOS) and -//! `harness_wpf_test` / `harness_winui3_test` (Windows). Brings Linux to parity -//! with the other platforms' controlled-harness coverage (alongside the real-app -//! Nix GUI scenarios). +//! Evidence-bearing native GTK3 harness catalog for Linux. //! -//! ## The Linux id contract: NAME, not id -//! cua-driver's Linux `get_window_state` renders each element as -//! `[idx] ""` from AT-SPI — there is NO `id=` field like Windows -//! (UIA AutomationId) or macOS (AX identifier). So the harness sets each -//! actionable control's AT-SPI **accessible name** to the scenario aid -//! (`btn-increment`, …), and this test matches on name via -//! `ax::element_index_containing` (substring), not `ax::has_id`. +//! GTK exposes accessibility through ATK -> AT-SPI. The Linux snapshot has no +//! AutomationId/AXIdentifier field, so actionable controls publish their +//! scenario id as their accessible name and this test addresses them by name. //! -//! Requires a real Linux desktop with AT-SPI running + a display (Xwayland is -//! fine — the launcher forces `GDK_BACKEND=x11`), GTK3 + PyGObject, and the app -//! built via `tests/fixtures/build/linux.sh`. `#[ignore]`; run explicitly: +//! Requires a Linux desktop, AT-SPI, GTK3, PyGObject, and the fixture built by +//! `tests/fixtures/build/linux.sh`. The canonical lane runs this target with: //! cargo test -p cua-driver --test harness_gtk3_test -- --ignored --nocapture --test-threads=1 #![cfg(target_os = "linux")] use std::process::{Command, Stdio}; -use std::time::Duration; +use std::time::{Duration, Instant}; -use cua_driver_testkit::{ax, harness_app, Driver, McpDriver}; +use cua_driver_testkit::e2e::{ + execute_case, native_background_case, native_foreground_case, native_readonly_case, + recording_evidence, CaseSpec, Delivery, DisplayServer, DriverRoute, Evidence, Observation, + OracleKind, RefusalCode, Targeting, +}; +use cua_driver_testkit::observer::TargetWindow; +use cua_driver_testkit::sentinel::run_with_background_oracles; +use cua_driver_testkit::{ax, harness_app, Driver, McpDriver, ToolResponse}; fn harness_exe() -> std::path::PathBuf { - if let Ok(p) = std::env::var("HARNESS_GTK3_EXE") { - let pb = std::path::PathBuf::from(p); - if pb.exists() { - return pb; + if let Ok(path) = std::env::var("HARNESS_GTK3_EXE") { + let path = std::path::PathBuf::from(path); + if path.exists() { + return path; } } harness_app("harness-gtk3", "CuaTestHarness.Gtk3") } -/// Launch the GTK3 harness, tying its lifetime to the driver's reaper, and -/// return (pid, main window_id). Returns None (skip) if the app isn't built. -fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> { +fn launch(driver: &mut McpDriver) -> (u32, u64) { let exe = harness_exe(); - if !exe.exists() { - eprintln!("GTK3 harness not built at {exe:?} — run tests/fixtures/build/linux.sh"); - return None; - } + assert!(exe.exists(), "required GTK3 harness is missing: {exe:?}"); driver .reaper() .spawn( Command::new(&exe) - .stdout(Stdio::null()) - .stderr(Stdio::null()), + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()), ) - .ok()?; - // GTK app cold-start + window map + AT-SPI registration. - std::thread::sleep(Duration::from_millis(1500)); - - // Resolve the harness window by title. find_window needs the pid; the - // launcher execs python3 in-place so the spawned pid IS the GTK process. - // We don't have that pid directly here (reaper owns the child), so discover - // by title across list_windows. - let deadline = std::time::Instant::now() + Duration::from_secs(12); - while std::time::Instant::now() < deadline { - let r = driver.call("list_windows", serde_json::json!({})); - if let Some(wins) = r.structured()["windows"].as_array() { - for w in wins { - if w["title"] + .unwrap_or_else(|error| panic!("launch GTK3 harness {exe:?}: {error}")); + + let deadline = Instant::now() + Duration::from_secs(12); + while Instant::now() < deadline { + let response = driver.call("list_windows", serde_json::json!({})); + if let Some(windows) = response.structured()["windows"].as_array() { + for window in windows { + if window["title"] .as_str() .unwrap_or("") .contains("CuaTestHarness GTK3") { - let pid = w["pid"].as_u64().unwrap_or(0) as u32; - let wid = w["window_id"].as_u64().unwrap_or(0); - if pid != 0 && wid != 0 { + let pid = window["pid"].as_u64().unwrap_or(0) as u32; + let window_id = window["window_id"].as_u64().unwrap_or(0); + if pid != 0 && window_id != 0 { driver.reaper().track_pid(pid); - return Some((pid, wid)); + std::thread::sleep(Duration::from_millis(500)); + return (pid, window_id); } } } } std::thread::sleep(Duration::from_millis(400)); } - eprintln!("GTK3 harness window never appeared — is a graphical session + AT-SPI available?"); - None + panic!("required GTK3 harness window never appeared"); } -fn snapshot(driver: &mut McpDriver, pid: u32, wid: u64) -> String { - driver - .call( - "get_window_state", - serde_json::json!({ "pid": pid as i64, "window_id": wid }), - ) - .text() - .to_string() +fn snapshot(driver: &mut McpDriver, pid: u32, window_id: u64) -> ToolResponse { + driver.call( + "get_window_state", + serde_json::json!({ "pid": pid as i64, "window_id": window_id }), + ) } -fn ax_empty(text: &str) -> bool { - ax::looks_empty(text) || text.contains("D-Bus") || text.contains("AT-SPI") +fn element_index(state: &ToolResponse, name: &str) -> u64 { + ax::element_index_containing(state.tree_text(), name).unwrap_or_else(|| { + panic!( + "{name:?} not found in GTK3 AT-SPI tree:\n{}", + state.tree_text() + ) + }) } -#[test] -#[ignore] -fn harness_gtk3_smoke() { - let Some(mut driver) = McpDriver::spawn() else { - return; - }; - let Some((pid, wid)) = launch(&mut driver) else { - return; - }; - println!("gtk3 harness pid={pid} wid={wid}"); - - let text = snapshot(&mut driver, pid, wid); - if ax_empty(&text) { - eprintln!("AT-SPI tree empty — accessibility bus unavailable; skipping element assertions"); - return; - } - println!("snapshot:\n{text}"); - - // Actionable controls expose their aid as the AT-SPI accessible name. - for name in [ - "btn-increment", - "btn-reset", - "txt-input", - "btn-open-popover", - "btn-exit", - ] { - assert!( - text.contains(name), - "missing control named {name:?} in GTK3 AT-SPI tree" - ); - } - // Marker label + counter label carry their text as the accessible name. - assert!( - text.contains("HARNESS_TEXT_MARKER_v1"), - "text_body marker not in tree" - ); +fn element_rect( + driver: &mut McpDriver, + pid: u32, + window_id: u64, + state: &ToolResponse, + name: &str, +) -> (f64, f64, f64, f64) { + let index = element_index(state, name); + let elements = state.structured()["elements"] + .as_array() + .expect("PX targeting requires structured GTK3 elements"); + let target = elements + .iter() + .find(|element| element["element_index"].as_u64() == Some(index)) + .and_then(|element| element["frame"].as_object()) + .unwrap_or_else(|| panic!("GTK3 element [{index}] {name:?} has no frame")); + let windows = driver.call("list_windows", serde_json::json!({ "pid": pid as i64 })); + let window = windows.structured()["windows"] + .as_array() + .and_then(|windows| { + windows + .iter() + .find(|window| window["window_id"].as_u64() == Some(window_id)) + }) + .unwrap_or_else(|| panic!("GTK3 window {window_id} disappeared before PX targeting")); + + let window_width = window["width"].as_f64().unwrap_or(0.0); + let window_height = window["height"].as_f64().unwrap_or(0.0); + let screenshot_width = state.structured()["screenshot_width"] + .as_f64() + .expect("PX targeting requires screenshot_width"); + let screenshot_height = state.structured()["screenshot_height"] + .as_f64() + .expect("PX targeting requires screenshot_height"); assert!( - text.contains("counter=0"), - "initial counter label not in tree" + window_width > 0.0 && window_height > 0.0, + "GTK3 top-level frame has invalid geometry: {window:?}" ); + let scale_x = screenshot_width / window_width; + let scale_y = screenshot_height / window_height; + let x = (target["x"].as_f64().unwrap_or(0.0) - window["x"].as_f64().unwrap_or(0.0)) * scale_x; + let y = (target["y"].as_f64().unwrap_or(0.0) - window["y"].as_f64().unwrap_or(0.0)) * scale_y; + let width = target["w"].as_f64().unwrap_or(0.0) * scale_x; + let height = target["h"].as_f64().unwrap_or(0.0) * scale_y; assert!( - text.contains("popover_open=False"), - "initial popover state not in tree" + width > 0.0 + && height > 0.0 + && x + width / 2.0 >= 0.0 + && y + height / 2.0 >= 0.0 + && x + width / 2.0 < screenshot_width + && y + height / 2.0 < screenshot_height, + "GTK3 PX target {name:?} is outside the capture: rect=({x},{y},{width},{height}) capture=({screenshot_width},{screenshot_height})" ); + (x, y, width, height) +} - println!("✅ harness_gtk3_smoke: all scenarios present in AT-SPI tree"); +fn wait_for_state(driver: &mut McpDriver, pid: u32, window_id: u64, expected: &str) -> String { + let deadline = Instant::now() + Duration::from_secs(4); + let mut last = String::new(); + while Instant::now() < deadline { + last = snapshot(driver, pid, window_id).tree_text().to_owned(); + if last.contains(expected) { + return last; + } + std::thread::sleep(Duration::from_millis(200)); + } + let relevant = last + .lines() + .filter(|line| line.contains('=') || line.contains("MARKER")) + .collect::>() + .join(" / "); + panic!("fixture state never contained {expected:?}; final state: {relevant}"); } -#[test] -#[ignore] -fn harness_gtk3_counter_click() { - let Some(mut driver) = McpDriver::spawn() else { - return; - }; - let Some((pid, wid)) = launch(&mut driver) else { - return; - }; +fn state_number(text: &str, key: &str) -> Option { + let start = text.find(key)? + key.len(); + let digits = text[start..] + .chars() + .take_while(char::is_ascii_digit) + .collect::(); + digits.parse().ok() +} - let pre = snapshot(&mut driver, pid, wid); - if ax_empty(&pre) { - eprintln!("AT-SPI empty — skipping"); - return; +fn wait_for_positive_state(driver: &mut McpDriver, pid: u32, window_id: u64, key: &str) -> String { + let deadline = Instant::now() + Duration::from_secs(4); + let mut last = String::new(); + while Instant::now() < deadline { + last = snapshot(driver, pid, window_id).tree_text().to_owned(); + if state_number(&last, key).is_some_and(|value| value > 0) { + return last; + } + std::thread::sleep(Duration::from_millis(200)); } - let idx = match ax::element_index_containing(&pre, "btn-increment") { - Some(i) => i, - None => { - eprintln!("btn-increment not found in tree — skipping"); + panic!("fixture state {key:?} never became positive; final tree:\n{last}"); +} + +fn assert_popover_marker(driver: &mut McpDriver, pid: u32, main_window_id: u64) { + let deadline = Instant::now() + Duration::from_secs(4); + while Instant::now() < deadline { + if snapshot(driver, pid, main_window_id) + .tree_text() + .contains("POPOVER_MARKER_v1") + { return; } - }; - let click = driver.call( - "click", - serde_json::json!({ "pid": pid as i64, "window_id": wid, "element_index": idx }), - ); - println!("click [{idx}] btn-increment: {}", click.text()); - std::thread::sleep(Duration::from_millis(400)); + let windows = driver.call("list_windows", serde_json::json!({})); + if let Some(windows) = windows.structured()["windows"].as_array() { + for window in windows { + let window_id = window["window_id"].as_u64().unwrap_or(0); + if window["pid"].as_u64() == Some(pid as u64) + && window_id != 0 + && window_id != main_window_id + && snapshot(driver, pid, window_id) + .tree_text() + .contains("POPOVER_MARKER_v1") + { + return; + } + } + } + std::thread::sleep(Duration::from_millis(200)); + } + panic!("GTK3 popover opened but POPOVER_MARKER_v1 was not exposed"); +} + +fn run_case(case: CaseSpec, test: impl FnOnce(u32, u64, &mut McpDriver) -> Observation) { + let cell_id = case.cell_id.clone(); + let delivery = case.delivery; + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_named(&cell_id).expect("start source-built Linux driver"); + *evidence = recording_evidence(driver.recording_dir()); + let (pid, window_id) = launch(&mut driver); + if delivery != Delivery::Background { + driver.start_behavior_recording(); + } + test(pid, window_id, &mut driver) + }); +} + +#[derive(Clone, Copy, Debug)] +enum Operation { + AxClick { + target: &'static str, + expected: &'static str, + }, + AxTypeText { + target: &'static str, + text: &'static str, + expected: &'static str, + }, + AxSetValue { + target: &'static str, + value: &'static str, + expected: &'static str, + }, + PxClick { + target: &'static str, + button: &'static str, + count: u8, + expected: &'static str, + }, + PxTypeText { + target: &'static str, + text: &'static str, + expected: &'static str, + }, + PressKey { + key: &'static str, + expected: &'static str, + }, + Hotkey { + expected: &'static str, + }, + Scroll { + target: &'static str, + pixel: bool, + state_key: &'static str, + }, + Drag { + target: &'static str, + state_key: &'static str, + }, + Popover { + target: &'static str, + expected: &'static str, + }, +} + +#[derive(Clone, Copy, Debug)] +struct CatalogRow { + action: &'static str, + targeting: Targeting, + delivery: Delivery, + route: DriverRoute, + operation: Operation, +} - let post = snapshot(&mut driver, pid, wid); +fn assert_refusal_without_mutation( + response: &ToolResponse, + before: &str, + pid: u32, + window_id: u64, + driver: &mut McpDriver, +) { assert!( - post.contains("counter=1"), - "counter did not advance after clicking btn-increment. counter lines: {}", - post.lines() - .filter(|l| l.contains("counter=")) - .collect::>() - .join(" / ") + response.is_error(), + "expected background_unavailable refusal" + ); + assert_eq!( + response.structured()["code"].as_str(), + Some("background_unavailable"), + "unexpected refusal: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(200)); + assert_eq!( + snapshot(driver, pid, window_id).tree_text(), + before, + "refused GTK3 action mutated fixture state" ); - println!("✅ harness_gtk3_counter_click: counter advanced via AT-SPI click"); } -#[test] -#[ignore] -fn harness_gtk3_popover() { - let Some(mut driver) = McpDriver::spawn() else { - return; +fn invoke_operation( + row: CatalogRow, + pid: u32, + window_id: u64, + driver: &mut McpDriver, + expect_refusal: bool, +) -> bool { + let mode = match row.delivery { + Delivery::Background => "background", + Delivery::Foreground => "foreground", + Delivery::NotApplicable => unreachable!("catalog operations require delivery"), }; - let Some((pid, wid)) = launch(&mut driver) else { - return; - }; - - let pre = snapshot(&mut driver, pid, wid); - if ax_empty(&pre) { - eprintln!("AT-SPI empty — skipping"); - return; - } + let pre = snapshot(driver, pid, window_id); assert!( - pre.contains("popover_open=False"), - "popover state not initially closed" + !ax::looks_empty(pre.tree_text()), + "required GTK3 AT-SPI tree is empty" ); - let idx = match ax::element_index_containing(&pre, "btn-open-popover") { - Some(i) => i, - None => { - eprintln!("btn-open-popover not found — skipping"); - return; + let (response, expected) = match row.operation { + Operation::AxClick { target, expected } => { + let index = element_index(&pre, target); + ( + driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": window_id, + "element_index": index, "delivery_mode": mode + }), + ), + expected, + ) + } + Operation::AxTypeText { + target, + text, + expected, + } => { + let index = element_index(&pre, target); + ( + driver.call( + "type_text", + serde_json::json!({ + "pid": pid as i64, "window_id": window_id, + "element_index": index, "text": text, "delivery_mode": mode + }), + ), + expected, + ) + } + Operation::AxSetValue { + target, + value, + expected, + } => { + let index = element_index(&pre, target); + ( + driver.call( + "set_value", + serde_json::json!({ + "pid": pid as i64, "window_id": window_id, + "element_index": index, "value": value + }), + ), + expected, + ) + } + Operation::PxClick { + target, + button, + count, + expected, + } => { + let (x, y, width, height) = element_rect(driver, pid, window_id, &pre, target); + let tool = if count == 2 { + "double_click" + } else if button == "right" { + "right_click" + } else { + "click" + }; + let mut args = serde_json::json!({ + "pid": pid as i64, "window_id": window_id, + "x": x + width / 2.0, "y": y + height / 2.0, + "delivery_mode": mode + }); + if tool == "click" { + args["button"] = serde_json::json!(button); + args["count"] = serde_json::json!(count); + } + (driver.call(tool, args), expected) + } + Operation::PxTypeText { + target, + text, + expected, + } => { + let (x, y, width, height) = element_rect(driver, pid, window_id, &pre, target); + ( + driver.call( + "type_text", + serde_json::json!({ + "pid": pid as i64, "window_id": window_id, + "x": x + width / 2.0, "y": y + height / 2.0, + "text": text, "delivery_mode": mode + }), + ), + expected, + ) + } + Operation::PressKey { key, expected } => ( + driver.call( + "press_key", + serde_json::json!({ + "pid": pid as i64, "window_id": window_id, + "key": key, "delivery_mode": mode + }), + ), + expected, + ), + Operation::Hotkey { expected } => ( + driver.call( + "hotkey", + serde_json::json!({ + "pid": pid as i64, "window_id": window_id, + "keys": ["ctrl", "shift", "k"], "delivery_mode": mode + }), + ), + expected, + ), + Operation::Scroll { + target, + pixel, + state_key, + } => { + let mut args = serde_json::json!({ + "pid": pid as i64, "window_id": window_id, + "direction": "down", "amount": 6, "delivery_mode": mode + }); + if pixel { + let (x, y, width, height) = element_rect(driver, pid, window_id, &pre, target); + args["x"] = serde_json::json!(x + width / 2.0); + args["y"] = serde_json::json!(y + height / 2.0); + } else { + args["element_index"] = serde_json::json!(element_index(&pre, target)); + } + let response = driver.call("scroll", args); + if expect_refusal { + assert_refusal_without_mutation(&response, pre.tree_text(), pid, window_id, driver); + return true; + } + assert!( + !response.is_error(), + "GTK3 scroll failed: {}", + response.text() + ); + wait_for_positive_state(driver, pid, window_id, state_key); + return false; + } + Operation::Drag { target, state_key } => { + let (x, y, width, height) = element_rect(driver, pid, window_id, &pre, target); + let response = driver.call( + "drag", + serde_json::json!({ + "pid": pid as i64, "window_id": window_id, + "from_x": x + width * 0.05, "from_y": y + height / 2.0, + "to_x": x + width * 0.90, "to_y": y + height / 2.0, + "duration_ms": 500, "steps": 30, "delivery_mode": mode + }), + ); + if expect_refusal { + assert_refusal_without_mutation(&response, pre.tree_text(), pid, window_id, driver); + return true; + } + assert!( + !response.is_error(), + "GTK3 drag failed: {}; raw={}", + response.text(), + response.raw + ); + wait_for_positive_state(driver, pid, window_id, state_key); + return false; + } + Operation::Popover { target, expected } => { + let index = element_index(&pre, target); + let response = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": window_id, + "element_index": index, "delivery_mode": mode + }), + ); + assert!( + !response.is_error(), + "GTK3 popover click failed: {}", + response.text() + ); + wait_for_state(driver, pid, window_id, expected); + assert_popover_marker(driver, pid, window_id); + return false; } }; - let _ = driver.call( - "click", - serde_json::json!({ "pid": pid as i64, "window_id": wid, "element_index": idx }), - ); - std::thread::sleep(Duration::from_millis(500)); - // GtkPopover may surface as a separate top-level; check the main window - // first, then any new window of this pid. - let post = snapshot(&mut driver, pid, wid); + if expect_refusal { + assert_refusal_without_mutation(&response, pre.tree_text(), pid, window_id, driver); + return true; + } + assert!( - post.contains("popover_open=True"), - "popover state did not flip open after click. Popover lines: {}", - post.lines() - .filter(|l| l.contains("popover_open=")) - .collect::>() - .join(" / ") + !response.is_error(), + "GTK3 {} {:?}/{:?} failed: {}", + row.action, + row.targeting, + row.delivery, + response.text() ); - let mut found = post.contains("POPOVER_MARKER_v1"); - if !found { - let r = driver.call("list_windows", serde_json::json!({})); - if let Some(wins) = r.structured()["windows"].as_array() { - for w in wins { - if w["pid"].as_u64() == Some(pid as u64) { - if let Some(other) = w["window_id"].as_u64() { - if other != wid - && snapshot(&mut driver, pid, other).contains("POPOVER_MARKER_v1") - { - found = true; - break; - } - } - } + wait_for_state(driver, pid, window_id, expected); + false +} + +fn row_expects_refusal(row: CatalogRow) -> bool { + if row.delivery != Delivery::Background { + return false; + } + let inject_mode = std::env::var_os("CUA_INJECT_SOCKET").is_some(); + if !inject_mode + && matches!( + row.operation, + Operation::PressKey { .. } | Operation::Hotkey { .. } + ) + { + return true; + } + let focus_bound_pointer = matches!( + row.operation, + Operation::PxClick { + button: "right", + .. + } | Operation::PxClick { count: 2, .. } + | Operation::Scroll { pixel: true, .. } + | Operation::Drag { .. } + ); + if DisplayServer::current() == DisplayServer::X11 + && (focus_bound_pointer || matches!(row.operation, Operation::PxTypeText { .. })) + && !platform_linux::input::real_pointer_input_available() + { + return true; + } + DisplayServer::current() == DisplayServer::Wayland + && !inject_mode + && (focus_bound_pointer || matches!(row.operation, Operation::PxTypeText { .. })) +} + +fn run_catalog_row(row: CatalogRow) { + let expect_refusal = row_expects_refusal(row); + let route = if DisplayServer::current() == DisplayServer::Wayland + && (row.targeting == Targeting::Px + || matches!( + row.operation, + Operation::PressKey { .. } | Operation::Hotkey { .. } + )) { + if std::env::var_os("CUA_INJECT_SOCKET").is_some() { + DriverRoute::LinuxCuaCompositorInject + } else { + DriverRoute::LinuxWaylandVirtualPointer + } + } else { + row.route + }; + let case = match row.delivery { + Delivery::Background => native_background_case("gtk3", row.action, row.targeting, route), + Delivery::Foreground => native_foreground_case("gtk3", row.action, row.targeting, route), + Delivery::NotApplicable => unreachable!("catalog operations require delivery"), + }; + let case = if expect_refusal { + case.expecting_refusal(vec![RefusalCode::BackgroundUnavailable]) + } else { + case + }; + run_case(case, |pid, window_id, driver| match row.delivery { + Delivery::Background => { + let mut refused = false; + let (_, passed) = run_with_background_oracles( + driver, + TargetWindow { + pid, + native_id: window_id, + }, + |driver| { + refused = invoke_operation(row, pid, window_id, driver, expect_refusal); + }, + ) + .unwrap_or_else(|error| panic!("background desktop contract failed: {error}")); + if refused { + let mut passed = passed; + passed.push(OracleKind::FixtureState); + Observation::refused( + RefusalCode::BackgroundUnavailable, + passed, + "GTK3 focus-bound background input was refused", + Evidence::default(), + ) + } else { + Observation::delivered_with_fixture_state(passed) } } - } - assert!( - found, - "popover body marker POPOVER_MARKER_v1 not found after open" + Delivery::Foreground => { + assert!(!invoke_operation(row, pid, window_id, driver, false)); + Observation::delivered_with_fixture_state(Vec::new()) + } + Delivery::NotApplicable => unreachable!(), + }); +} + +#[test] +#[ignore] +fn harness_gtk3_ax_tree() { + run_case( + native_readonly_case( + "gtk3", + "ax_tree", + Targeting::Ax, + DriverRoute::AxRead, + vec![OracleKind::AxState], + ), + |pid, window_id, driver| { + let state = snapshot(driver, pid, window_id); + let text = state.tree_text(); + assert!(!ax::looks_empty(text), "required GTK3 AT-SPI tree is empty"); + for name in [ + "btn-increment", + "txt-input", + "btn-clicktarget", + "sld-value", + "chk-agree", + "scroll-tall", + "btn-open-popover", + "btn-exit", + ] { + assert!(text.contains(name), "missing GTK3 control named {name:?}"); + } + for state in [ + "HARNESS_TEXT_MARKER_v1", + "counter=0", + "mirror=", + "last_action=none", + "last_key=none", + "last_hotkey=none", + "slider_value=0", + "agreed=False", + "scroll_offset=0", + "popover_open=False", + ] { + assert!(text.contains(state), "missing GTK3 fixture state {state:?}"); + } + Observation::delivered(vec![OracleKind::AxState], Evidence::default()) + }, ); - println!("✅ harness_gtk3_popover: popover body enumerated after open"); } + +macro_rules! catalog_test { + ($name:ident, $row:expr) => { + #[test] + #[ignore] + fn $name() { + run_catalog_row($row); + } + }; +} + +catalog_test!( + harness_gtk3_left_click_ax_background, + CatalogRow { + action: "left_click", + targeting: Targeting::Ax, + delivery: Delivery::Background, + route: DriverRoute::LinuxAtSpiAction, + operation: Operation::AxClick { + target: "btn-increment", + expected: "counter=1" + }, + } +); +catalog_test!( + harness_gtk3_left_click_ax_foreground, + CatalogRow { + action: "left_click", + targeting: Targeting::Ax, + delivery: Delivery::Foreground, + route: DriverRoute::LinuxAtSpiAction, + operation: Operation::AxClick { + target: "btn-increment", + expected: "counter=1" + }, + } +); +catalog_test!( + harness_gtk3_left_click_px_background, + CatalogRow { + action: "left_click", + targeting: Targeting::Px, + delivery: Delivery::Background, + route: DriverRoute::LinuxXSendEvent, + operation: Operation::PxClick { + target: "btn-clicktarget", + button: "left", + count: 1, + expected: "last_action=click clicks=1" + }, + } +); +catalog_test!( + harness_gtk3_left_click_px_foreground, + CatalogRow { + action: "left_click", + targeting: Targeting::Px, + delivery: Delivery::Foreground, + route: DriverRoute::LinuxXTest, + operation: Operation::PxClick { + target: "btn-clicktarget", + button: "left", + count: 1, + expected: "last_action=click clicks=1" + }, + } +); +catalog_test!( + harness_gtk3_right_click_px_background, + CatalogRow { + action: "right_click", + targeting: Targeting::Px, + delivery: Delivery::Background, + route: DriverRoute::LinuxXSendEvent, + operation: Operation::PxClick { + target: "btn-clicktarget", + button: "right", + count: 1, + expected: "last_action=right_click clicks=0" + }, + } +); +catalog_test!( + harness_gtk3_right_click_px_foreground, + CatalogRow { + action: "right_click", + targeting: Targeting::Px, + delivery: Delivery::Foreground, + route: DriverRoute::LinuxXTest, + operation: Operation::PxClick { + target: "btn-clicktarget", + button: "right", + count: 1, + expected: "last_action=right_click clicks=0" + }, + } +); +catalog_test!( + harness_gtk3_double_click_px_background, + CatalogRow { + action: "double_click", + targeting: Targeting::Px, + delivery: Delivery::Background, + route: DriverRoute::LinuxXSendEvent, + operation: Operation::PxClick { + target: "btn-clicktarget", + button: "left", + count: 2, + expected: "last_action=double_click clicks=2" + }, + } +); +catalog_test!( + harness_gtk3_double_click_px_foreground, + CatalogRow { + action: "double_click", + targeting: Targeting::Px, + delivery: Delivery::Foreground, + route: DriverRoute::LinuxXTest, + operation: Operation::PxClick { + target: "btn-clicktarget", + button: "left", + count: 2, + expected: "last_action=double_click clicks=2" + }, + } +); +catalog_test!( + harness_gtk3_type_text_ax_background, + CatalogRow { + action: "type_text", + targeting: Targeting::Ax, + delivery: Delivery::Background, + route: DriverRoute::LinuxAtSpiValue, + operation: Operation::AxTypeText { + target: "txt-input", + text: "ax-typed", + expected: "mirror=ax-typed" + }, + } +); +catalog_test!( + harness_gtk3_type_text_ax_foreground, + CatalogRow { + action: "type_text", + targeting: Targeting::Ax, + delivery: Delivery::Foreground, + route: DriverRoute::LinuxAtSpiValue, + operation: Operation::AxTypeText { + target: "txt-input", + text: "ax-typed", + expected: "mirror=ax-typed" + }, + } +); +catalog_test!( + harness_gtk3_type_text_px_background, + CatalogRow { + action: "type_text", + targeting: Targeting::Px, + delivery: Delivery::Background, + route: DriverRoute::LinuxXSendEvent, + operation: Operation::PxTypeText { + target: "txt-input", + text: "px-typed", + expected: "mirror=px-typed" + }, + } +); +catalog_test!( + harness_gtk3_type_text_px_foreground, + CatalogRow { + action: "type_text", + targeting: Targeting::Px, + delivery: Delivery::Foreground, + route: DriverRoute::LinuxXTest, + operation: Operation::PxTypeText { + target: "txt-input", + text: "px-typed", + expected: "mirror=px-typed" + }, + } +); +catalog_test!( + harness_gtk3_set_value_ax_background, + CatalogRow { + action: "set_value", + targeting: Targeting::Ax, + delivery: Delivery::Background, + route: DriverRoute::LinuxAtSpiValue, + operation: Operation::AxSetValue { + target: "txt-input", + value: "set-value", + expected: "mirror=set-value" + }, + } +); +catalog_test!( + harness_gtk3_set_value_ax_foreground, + CatalogRow { + action: "set_value", + targeting: Targeting::Ax, + delivery: Delivery::Foreground, + route: DriverRoute::LinuxAtSpiValue, + operation: Operation::AxSetValue { + target: "txt-input", + value: "set-value", + expected: "mirror=set-value" + }, + } +); +catalog_test!( + harness_gtk3_check_ax_background, + CatalogRow { + action: "checkbox_toggle", + targeting: Targeting::Ax, + delivery: Delivery::Background, + route: DriverRoute::LinuxAtSpiAction, + operation: Operation::AxClick { + target: "chk-agree", + expected: "agreed=True" + }, + } +); +catalog_test!( + harness_gtk3_check_ax_foreground, + CatalogRow { + action: "checkbox_toggle", + targeting: Targeting::Ax, + delivery: Delivery::Foreground, + route: DriverRoute::LinuxAtSpiAction, + operation: Operation::AxClick { + target: "chk-agree", + expected: "agreed=True" + }, + } +); +catalog_test!( + harness_gtk3_slider_set_value_ax_background, + CatalogRow { + action: "slider_set_value", + targeting: Targeting::Ax, + delivery: Delivery::Background, + route: DriverRoute::LinuxAtSpiValue, + operation: Operation::AxSetValue { + target: "sld-value", + value: "64", + expected: "slider_value=64" + }, + } +); +catalog_test!( + harness_gtk3_slider_set_value_ax_foreground, + CatalogRow { + action: "slider_set_value", + targeting: Targeting::Ax, + delivery: Delivery::Foreground, + route: DriverRoute::LinuxAtSpiValue, + operation: Operation::AxSetValue { + target: "sld-value", + value: "64", + expected: "slider_value=64" + }, + } +); +catalog_test!( + harness_gtk3_press_key_px_background, + CatalogRow { + action: "press_key", + targeting: Targeting::Px, + delivery: Delivery::Background, + route: DriverRoute::LinuxXSendEvent, + operation: Operation::PressKey { + key: "f5", + expected: "last_key=f5 key_presses=1" + }, + } +); +catalog_test!( + harness_gtk3_press_key_px_foreground, + CatalogRow { + action: "press_key", + targeting: Targeting::Px, + delivery: Delivery::Foreground, + route: DriverRoute::LinuxXTest, + operation: Operation::PressKey { + key: "f5", + expected: "last_key=f5 key_presses=1" + }, + } +); +catalog_test!( + harness_gtk3_hotkey_px_background, + CatalogRow { + action: "hotkey", + targeting: Targeting::Px, + delivery: Delivery::Background, + route: DriverRoute::LinuxXSendEvent, + operation: Operation::Hotkey { + expected: "last_hotkey=ctrl+shift+k hotkeys=1" + }, + } +); +catalog_test!( + harness_gtk3_hotkey_px_foreground, + CatalogRow { + action: "hotkey", + targeting: Targeting::Px, + delivery: Delivery::Foreground, + route: DriverRoute::LinuxXTest, + operation: Operation::Hotkey { + expected: "last_hotkey=ctrl+shift+k hotkeys=1" + }, + } +); +catalog_test!( + harness_gtk3_scroll_ax_background, + CatalogRow { + action: "scroll", + targeting: Targeting::Ax, + delivery: Delivery::Background, + route: DriverRoute::LinuxAtSpiAction, + operation: Operation::Scroll { + target: "scroll-tall-vertical", + pixel: false, + state_key: "scroll_offset=" + }, + } +); +catalog_test!( + harness_gtk3_scroll_ax_foreground, + CatalogRow { + action: "scroll", + targeting: Targeting::Ax, + delivery: Delivery::Foreground, + route: DriverRoute::LinuxAtSpiAction, + operation: Operation::Scroll { + target: "scroll-tall-vertical", + pixel: false, + state_key: "scroll_offset=" + }, + } +); +catalog_test!( + harness_gtk3_scroll_px_background, + CatalogRow { + action: "scroll", + targeting: Targeting::Px, + delivery: Delivery::Background, + route: DriverRoute::LinuxXSendEvent, + operation: Operation::Scroll { + target: "scroll-tall-viewport", + pixel: true, + state_key: "scroll_offset=" + }, + } +); +catalog_test!( + harness_gtk3_scroll_px_foreground, + CatalogRow { + action: "scroll", + targeting: Targeting::Px, + delivery: Delivery::Foreground, + route: DriverRoute::LinuxXTest, + operation: Operation::Scroll { + target: "scroll-tall-viewport", + pixel: true, + state_key: "scroll_offset=" + }, + } +); +catalog_test!( + harness_gtk3_drag_px_background, + CatalogRow { + action: "drag", + targeting: Targeting::Px, + delivery: Delivery::Background, + route: DriverRoute::LinuxXSendEvent, + operation: Operation::Drag { + target: "sld-value", + state_key: "slider_value=" + }, + } +); +catalog_test!( + harness_gtk3_drag_px_foreground, + CatalogRow { + action: "drag", + targeting: Targeting::Px, + delivery: Delivery::Foreground, + route: DriverRoute::LinuxXTest, + operation: Operation::Drag { + target: "sld-value", + state_key: "slider_value=" + }, + } +); +catalog_test!( + harness_gtk3_child_window_ax_background, + CatalogRow { + action: "child_window", + targeting: Targeting::Ax, + delivery: Delivery::Background, + route: DriverRoute::LinuxAtSpiAction, + operation: Operation::Popover { + target: "btn-open-popover", + expected: "popover_open=True" + }, + } +); +catalog_test!( + harness_gtk3_child_window_ax_foreground, + CatalogRow { + action: "child_window", + targeting: Targeting::Ax, + delivery: Delivery::Foreground, + route: DriverRoute::LinuxAtSpiAction, + operation: Operation::Popover { + target: "btn-open-popover", + expected: "popover_open=True" + }, + } +); diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_swiftui_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_swiftui_test.rs index 27435a4f47..5df102bdcd 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_swiftui_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_swiftui_test.rs @@ -21,6 +21,12 @@ use std::process::{Child, Command, Stdio}; use std::time::Duration; use cua_driver_testkit::ax::{element_index_by_id, has_id, looks_empty}; +use cua_driver_testkit::e2e::{ + execute_case, native_background_case, native_foreground_case, native_readonly_case, + recording_evidence, DriverRoute, Evidence, Observation, OracleKind, Targeting, +}; +use cua_driver_testkit::observer::TargetWindow; +use cua_driver_testkit::sentinel::run_with_background_oracles; use cua_driver_testkit::{harness_app, Driver, McpDriver, ToolResponse}; fn harness_exe() -> PathBuf { @@ -42,20 +48,17 @@ struct Harness { } impl Harness { - fn launch() -> Option { + fn launch() -> Self { let exe = harness_exe(); - if !exe.exists() { - eprintln!("harness exe not found at {exe:?}"); - return None; - } + assert!(exe.exists(), "required SwiftUI harness is missing: {exe:?}"); let app = Command::new(&exe) .stdout(Stdio::null()) .stderr(Stdio::null()) .spawn() - .ok()?; + .unwrap_or_else(|error| panic!("launch SwiftUI harness {exe:?}: {error}")); let pid = app.id(); std::thread::sleep(Duration::from_millis(900)); - Some(Self { _app: app, pid }) + Self { _app: app, pid } } } @@ -78,134 +81,252 @@ fn snapshot_elements(driver: &mut McpDriver, pid: u32, window_id: u64) -> ToolRe ) } +fn run_case( + case: cua_driver_testkit::e2e::CaseSpec, + test: impl FnOnce(u32, u64, &mut McpDriver) -> Observation, +) { + let cell_id = case.cell_id.clone(); + let delivery = case.delivery; + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_macos_daemon_proxy_named(&cell_id) + .expect("start installed macOS daemon proxy"); + *evidence = recording_evidence(driver.recording_dir()); + let harness = Harness::launch(); + let (wid, _) = driver + .find_window(harness.pid as i64, "CuaTestHarness SwiftUI") + .expect("SwiftUI main window not found"); + if delivery != cua_driver_testkit::e2e::Delivery::Background { + driver.start_behavior_recording(); + } + test(harness.pid, wid, &mut driver) + }); +} + +fn run_foreground_case(action: &str, test: impl FnOnce(u32, u64, &mut McpDriver)) { + run_case( + native_foreground_case("swiftui", action, Targeting::Ax, DriverRoute::MacosAxAction), + |pid, wid, driver| { + test(pid, wid, driver); + Observation::delivered_with_fixture_state(Vec::new()) + }, + ); +} + +fn run_background_case( + action: &str, + route: DriverRoute, + test: impl FnOnce(u32, u64, &mut McpDriver), +) { + run_case( + native_background_case("swiftui", action, Targeting::Ax, route), + |pid, wid, driver| { + let (_, passed) = run_with_background_oracles( + driver, + TargetWindow { + pid, + native_id: wid, + }, + |driver| test(pid, wid, driver), + ) + .unwrap_or_else(|error| panic!("background desktop contract failed: {error}")); + Observation::delivered_with_fixture_state(passed) + }, + ); +} + #[test] #[ignore] fn harness_swiftui_smoke() { - let harness = match Harness::launch() { - Some(h) => h, - None => { - eprintln!("harness not built — skipping"); - return; - } - }; - println!("harness pid={}", harness.pid); - - let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else { - return; - }; - - let (wid, title) = driver - .find_window(harness.pid as i64, "CuaTestHarness SwiftUI") - .expect("main window not found"); - println!("main window: id={wid} title={title:?}"); - - let snap = snapshot_elements(&mut driver, harness.pid, wid); - if looks_empty(snap.tree_text()) { - eprintln!("AX empty — TCC not granted; skipping element-assertions"); - return; - } - let text = snap.tree_text(); - println!("snapshot:\n{text}"); - - // SwiftUI Text views render as AXStaticText leaves and don't propagate - // accessibilityIdentifier into the AX tree's identifier slot (same - // quirk as AppKit's NSTextField label mode + WPF's TextBlock). Assert - // on text content for labels, AX-id only for actionable controls. - for aid in [ - "btn-increment", - "btn-reset", - "txt-input", - "btn-open-popover", - "btn-exit", - ] { - assert!( - has_id(snap.tree_text(), aid), - "missing AX identifier {aid} in SwiftUI snapshot" - ); - } + run_case( + native_readonly_case( + "swiftui", + "ax_tree", + Targeting::Ax, + DriverRoute::AxRead, + vec![OracleKind::AxState], + ), + |pid, wid, driver| { + let snap = snapshot_elements(driver, pid, wid); + assert!( + !looks_empty(snap.tree_text()), + "required SwiftUI AX tree is empty" + ); + let text = snap.tree_text(); + println!("snapshot:\n{text}"); + + // SwiftUI Text views render as AXStaticText leaves and don't propagate + // accessibilityIdentifier into the AX tree's identifier slot (same + // quirk as AppKit's NSTextField label mode + WPF's TextBlock). Assert + // on text content for labels, AX-id only for actionable controls. + for aid in [ + "btn-increment", + "btn-reset", + "txt-input", + "btn-open-popover", + "btn-exit", + ] { + assert!( + has_id(snap.tree_text(), aid), + "missing AX identifier {aid} in SwiftUI snapshot" + ); + } - assert!( - text.contains("HARNESS_TEXT_MARKER_v1"), - "text_body marker not in SwiftUI snapshot" + assert!( + text.contains("HARNESS_TEXT_MARKER_v1"), + "text_body marker not in SwiftUI snapshot" + ); + Observation::delivered(vec![OracleKind::AxState], Evidence::default()) + }, ); } -/// popover: click the popover trigger, verify the popover body text appears -/// in the AX tree after the open. SwiftUI's analogue of WinUI3 CommandBarFlyout. #[test] #[ignore] -fn harness_swiftui_popover() { - let harness = match Harness::launch() { - Some(h) => h, - None => { - eprintln!("harness not built — skipping"); - return; - } - }; - - let Some(mut driver) = McpDriver::spawn_macos_daemon_proxy() else { - return; - }; - - let (wid, _) = driver - .find_window(harness.pid as i64, "CuaTestHarness SwiftUI") - .expect("main window not found"); - let snap_pre = snapshot_elements(&mut driver, harness.pid, wid); - if looks_empty(snap_pre.tree_text()) { - eprintln!("AX empty — TCC not granted; skipping"); - return; - } - // Verify popover body is NOT yet in the tree. - let pre_text = snap_pre.tree_text().to_owned(); - assert!( - !pre_text.contains("POPOVER_MARKER_v1"), - "popover body unexpectedly present BEFORE open" +fn harness_swiftui_counter_background() { + run_background_case( + "left_click", + DriverRoute::MacosAxAction, + |pid, wid, driver| { + let pre = snapshot_elements(driver, pid, wid); + assert!(pre.tree_text().contains("counter=0")); + let index = element_index_by_id(pre.tree_text(), "btn-increment") + .expect("btn-increment element_index not found"); + let response = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "element_index": index, + "action": "press", + "delivery_mode": "background" + }), + ); + assert!( + !response.is_error(), + "SwiftUI counter click failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(200)); + assert!( + snapshot_elements(driver, pid, wid) + .tree_text() + .contains("counter=1"), + "SwiftUI background AX click did not advance counter" + ); + }, ); +} - let trigger_idx: u64 = - if let Some(i) = element_index_by_id(snap_pre.tree_text(), "btn-open-popover") { - i - } else { - eprintln!("popover trigger not found, skipping"); - return; - }; - let click = driver.call( - "click", - serde_json::json!({ - "pid": harness.pid as i64, - "window_id": wid, - "element_index": trigger_idx, - "action": "press" - }), +#[test] +#[ignore] +fn harness_swiftui_set_value_background() { + run_background_case( + "set_value", + DriverRoute::MacosAxValue, + |pid, wid, driver| { + let pre = snapshot_elements(driver, pid, wid); + let index = element_index_by_id(pre.tree_text(), "txt-input") + .expect("txt-input element_index not found"); + let response = driver.call( + "set_value", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "element_index": index, + "value": "swiftui-cua" + }), + ); + assert!( + !response.is_error(), + "SwiftUI set_value failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(200)); + assert!( + snapshot_elements(driver, pid, wid) + .tree_text() + .contains("swiftui-cua"), + "SwiftUI background AX value did not reach the field" + ); + }, ); - println!("popover trigger click: {}", click.text()); - - std::thread::sleep(Duration::from_millis(300)); - - // Popovers may live in a separate AXWindow on macOS — try the main - // window first, then list_windows for additional candidates. - let snap_post = snapshot_elements(&mut driver, harness.pid, wid); - let mut found_marker = snap_post.tree_text().contains("POPOVER_MARKER_v1"); - if !found_marker { - // Walk any new windows for the same pid. - let resp = driver.call( - "list_windows", - serde_json::json!({ "pid": harness.pid as i64 }), +} + +/// Popover activation: click the trigger and verify fixture-owned state changes. +/// Transient-window AX discovery is observed separately so it cannot hide a +/// correctly delivered action. +#[test] +#[ignore] +fn harness_swiftui_popover_foreground() { + run_foreground_case("popover_open", |pid, wid, driver| { + let snap_pre = snapshot_elements(driver, pid, wid); + assert!( + !looks_empty(snap_pre.tree_text()), + "required SwiftUI AX tree is empty" ); - if let Some(wins) = resp.structured()["windows"].as_array() { - for w in wins { - if let Some(other_wid) = w["window_id"].as_u64() { - if other_wid == wid { - continue; - } - let s = snapshot_elements(&mut driver, harness.pid, other_wid); - if s.tree_text().contains("POPOVER_MARKER_v1") { - found_marker = true; - break; + // Verify popover body is NOT yet in the tree. + let pre_text = snap_pre.tree_text().to_owned(); + assert!( + !pre_text.contains("POPOVER_MARKER_v1"), + "popover body unexpectedly present BEFORE open" + ); + assert!( + pre_text.contains("popover_open=false"), + "popover state was not false before open" + ); + + let trigger_idx = element_index_by_id(snap_pre.tree_text(), "btn-open-popover") + .expect("popover trigger not found"); + let click = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "element_index": trigger_idx, + "action": "press", + "delivery_mode": "foreground" + }), + ); + assert!( + !click.is_error(), + "SwiftUI popover click failed: {}", + click.text() + ); + println!("popover trigger click: {}", click.text()); + + // First prove the button action reached SwiftUI's state independently + // of whether AX can enumerate the transient panel. + let deadline = std::time::Instant::now() + Duration::from_secs(3); + let mut state_open = false; + let mut found_marker = false; + while !state_open && std::time::Instant::now() < deadline { + let owner = snapshot_elements(driver, pid, wid); + state_open = owner.tree_text().contains("popover_open=true"); + found_marker = owner.tree_text().contains("POPOVER_MARKER_v1"); + let resp = driver.call("list_windows", serde_json::json!({ "pid": pid as i64 })); + if let Some(wins) = resp.structured()["windows"].as_array() { + for w in wins { + if let Some(other_wid) = w["window_id"].as_u64() { + if other_wid == wid { + continue; + } + let s = snapshot_elements(driver, pid, other_wid); + if s.tree_text().contains("POPOVER_MARKER_v1") { + found_marker = true; + break; + } } } } + if !state_open { + std::thread::sleep(Duration::from_millis(100)); + } } - } - assert!(found_marker, "popover body marker not found after open"); + assert!(state_open, "popover trigger did not change fixture state"); + if !found_marker { + eprintln!( + "SwiftUI popover opened, but its transient panel remains absent from targeted AX enumeration" + ); + } + }); } diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs index 89b1bfc1b7..3e85411c51 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_web_test.rs @@ -6,116 +6,380 @@ //! Run via: //! cargo test --test harness_web_test -- --ignored --nocapture //! -//! ## Known cua-driver gaps these tests document -//! -//! - **CDP `/json` HTTP read uses `read_to_end`** — `mcp-server/src/cdp.rs` -//! sends `Connection: close` and then calls `stream.read_to_end()`, but -//! Chromium's CDP HTTP server ignores `Connection: close` and keeps the -//! socket alive, so `read_to_end` hangs until the 10 s discovery timeout. -//! Confirmed against Electron 31 on port 9223 (verified manually via -//! curl: instant 200, JSON body present). Fix: parse `Content-Length` -//! and `read_exact` that many bytes, or honour `Transfer-Encoding: -//! chunked`. Tracked in this test as a structural assertion (window -//! discoverable) rather than a behavioural one (page tool round-trip). -//! -//! - **WebView2 `--remote-debugging-port` ignored** — passing -//! `AdditionalBrowserArguments = "--remote-debugging-port=9222"` via -//! `CoreWebView2EnvironmentOptions` does not open a CDP listener on the -//! WebView2 helper processes. WebView2 may be filtering the flag. -//! Tracked here as a TODO for the harness rather than a cua-driver -//! issue (since this is a WebView2 configuration concern). +//! The page-tool tests cover CDP discovery and a DOM round-trip together. +//! WebView2 can expose its listener before its first page target is ready, so +//! the driver must tolerate a briefly empty `/json` response. #![cfg(target_os = "windows")] +use std::cell::Cell; +use std::io::Read; use std::path::PathBuf; use std::process::{Command, Stdio}; -use std::time::Duration; +use std::time::{Duration, Instant}; -use cua_driver_testkit::{harness_app, spawn_in_job, Driver, McpDriver}; +use cua_driver_testkit::e2e::{ + execute_case, native_background_case, recording_evidence, DriverRoute, Observation, Targeting, +}; +use cua_driver_testkit::observer::TargetWindow; +use cua_driver_testkit::sentinel::run_with_background_oracles; +use cua_driver_testkit::{harness_app, spawn_in_job, Driver, FixtureJournal, McpDriver, ToolResponse}; // ── workspace paths ────────────────────────────────────────────────────────── fn webview_exe() -> PathBuf { if let Ok(p) = std::env::var("HARNESS_WEBVIEW_EXE") { let pb = PathBuf::from(p); - if pb.exists() { return pb; } + if pb.exists() { + return pb; + } } harness_app("harness-webview", "CuaTestHarness.WebView.exe") } fn electron_exe() -> PathBuf { if let Ok(p) = std::env::var("HARNESS_ELECTRON_EXE") { let pb = PathBuf::from(p); - if pb.exists() { return pb; } + if pb.exists() { + return pb; + } } harness_app("harness-electron", "CuaTestHarness.Electron.exe") } // ── shared session helper ──────────────────────────────────────────────────── -/// Wait (up to ~5s) for `port` to become free. These web tests use FIXED CDP -/// ports (9222/9223) and a process-global `CUA_DRIVER_CDP_PORT`, so they must -/// run serially (`--test-threads=1`). A previous test's host can still be -/// releasing its port when the next launches; reusing it before then makes the -/// daemon discover the OLD host's page (`pages[0]`), so the click lands on a -/// stale window and the counter check fails. This guard closes that teardown -/// overlap — belt-and-braces on top of serial execution. -fn wait_port_free(port: u16) { - for _ in 0..50 { - if std::net::TcpStream::connect(("127.0.0.1", port)).is_err() { - return; +fn allocate_loopback_port() -> u16 { + let listener = + std::net::TcpListener::bind(("127.0.0.1", 0)).expect("allocate an ephemeral CDP port"); + listener.local_addr().expect("read CDP port").port() +} + +fn wait_for_page_text( + driver: &mut McpDriver, + pid: i64, + wid: u64, + javascript: &str, + expected: &str, +) -> String { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + let response = driver.call( + "page", + serde_json::json!({ + "pid": pid, + "window_id": wid, + "action": "execute_javascript", + "javascript": javascript, + }), + ); + let text = response.text().to_owned(); + if text.contains(expected) { + return text; } + assert!( + std::time::Instant::now() < deadline, + "page state did not reach {expected:?}: {text:?}" + ); std::thread::sleep(Duration::from_millis(100)); } - eprintln!("warning: CDP port {port} still bound after 5s — prior host may not have released it"); } /// Launch the harness exe + a cua-driver child with `CUA_DRIVER_CDP_PORT` /// pointing at the harness's CDP endpoint. Polls list_windows until the /// host's window appears. -fn run_with_session(label: &str, host_exe: PathBuf, title_substr: &str, cdp_port: u16, f: F) +fn run_web_case(toolkit: &str, action: &str, host_exe: PathBuf, title_substr: &str, f: F) where F: FnOnce(i64, u64, &mut McpDriver), { - if !host_exe.exists() { - eprintln!("{label} host exe not found at {host_exe:?} — run tests/fixtures/build/windows.ps1"); - return; + let case = native_background_case(toolkit, action, Targeting::Page, DriverRoute::Cdp); + run_web_case_with_preparation( + case, + toolkit, + host_exe, + title_substr, + |_, _, _, _| {}, + |pid, wid, driver, _| f(pid, wid, driver), + ); +} + +fn run_web_case_with_preparation( + case: cua_driver_testkit::e2e::CaseSpec, + toolkit: &str, + host_exe: PathBuf, + title_substr: &str, + prepare: P, + f: F, +) where + P: FnOnce(i64, u64, &mut McpDriver, &FixtureJournal), + F: FnOnce(i64, u64, &mut McpDriver, &FixtureJournal), +{ + let cell_id = case.cell_id.clone(); + execute_case(case, |evidence| { + assert!( + host_exe.exists(), + "required {toolkit} host is missing at {host_exe:?}" + ); + let cdp_port = allocate_loopback_port(); + let cdp_port_string = cdp_port.to_string(); + let journal = FixtureJournal::start(); + let mut driver = McpDriver::spawn_named_with_env( + &cell_id, + &[("CUA_DRIVER_CDP_PORT", cdp_port_string.as_str())], + ) + .expect("required source-built Windows driver did not start"); + *evidence = recording_evidence(driver.recording_dir()); + + let env_var = if toolkit == "webview2" { + "CUA_WEBVIEW_CDP_PORT" + } else { + "CUA_ELECTRON_CDP_PORT" + }; + let mut cmd = Command::new(&host_exe); + cmd.env(env_var, &cdp_port_string) + .env("CUA_E2E_FIXTURE_JOURNAL_URL", journal.url()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()); + let mut app = spawn_in_job(&mut cmd).expect("spawn web harness"); + let pid = app.id() as i64; + // WebView2's first CoreWebView2Environment creation can exceed 12s on a + // cold hosted runner. Keep polling the externally visible ready title; + // process exit and the final deadline still fail closed. + let deadline = std::time::Instant::now() + Duration::from_secs(30); + let mut observed_titles = Vec::new(); + let (wid, _) = 'ready: loop { + if let Some(status) = app.try_wait().expect("poll web harness process") { + let mut stderr = String::new(); + if let Some(mut pipe) = app.stderr.take() { + let _ = pipe.read_to_string(&mut stderr); + } + panic!( + "{toolkit} fixture exited before readiness with {status}: {}", + stderr.trim() + ); + } + let response = driver.call("list_windows", serde_json::json!({ "pid": pid })); + observed_titles.clear(); + if let Some(windows) = response.structured()["windows"].as_array() { + for window in windows { + let title = window["title"].as_str().unwrap_or(""); + observed_titles.push(title.to_owned()); + if title.contains(title_substr) { + if let Some(wid) = window["window_id"].as_u64() { + break 'ready (wid, title.to_owned()); + } + } + } + } + if std::time::Instant::now() >= deadline { + let _ = app.kill(); + let _ = app.wait(); + let mut stderr = String::new(); + if let Some(mut pipe) = app.stderr.take() { + let _ = pipe.read_to_string(&mut stderr); + } + panic!( + "{toolkit} window with title containing {title_substr:?} did not become ready; \ + observed titles={observed_titles:?}; stderr={:?}", + stderr.trim() + ); + } + std::thread::sleep(Duration::from_millis(100)); + }; + driver.reaper().push(app); + let journal_deadline = Instant::now() + Duration::from_secs(5); + while !journal.contains("WEB_HARNESS_MARKER_v1") { + assert!( + Instant::now() < journal_deadline, + "{toolkit} fixture journal did not become ready: {}", + journal.snapshot() + ); + std::thread::sleep(Duration::from_millis(50)); + } + prepare(pid, wid, &mut driver, &journal); + let (_, passed) = run_with_background_oracles( + &mut driver, + TargetWindow { + pid: pid as u32, + native_id: wid, + }, + |driver| f(pid, wid, driver, &journal), + ) + .unwrap_or_else(|error| panic!("background desktop contract failed: {error}")); + Observation::delivered_with_fixture_state(passed) + }); +} + +fn snapshot(driver: &mut McpDriver, pid: i64, wid: u64) -> ToolResponse { + driver.call( + "get_window_state", + serde_json::json!({ + "pid": pid, + "window_id": wid, + "capture_mode": "ax" + }), + ) +} + +fn window_bounds(driver: &mut McpDriver, pid: i64, wid: u64) -> (f64, f64, f64, f64) { + let response = driver.call("list_windows", serde_json::json!({ "pid": pid })); + let window = response.structured()["windows"] + .as_array() + .and_then(|windows| { + windows + .iter() + .find(|window| window["window_id"].as_u64() == Some(wid)) + }) + .unwrap_or_else(|| { + panic!( + "WebView2 window {wid} is missing from list_windows: {}", + response.text() + ) + }); + let bounds = &window["bounds"]; + ( + bounds["x"].as_f64().expect("WebView2 bounds need x"), + bounds["y"].as_f64().expect("WebView2 bounds need y"), + bounds["width"] + .as_f64() + .expect("WebView2 bounds need width"), + bounds["height"] + .as_f64() + .expect("WebView2 bounds need height"), + ) +} + +fn pixel_from_screen( + state: &ToolResponse, + screen_x: f64, + screen_y: f64, + window: (f64, f64, f64, f64), +) -> (f64, f64) { + let (window_x, window_y, window_w, window_h) = window; + assert!( + window_w > 0.0 && window_h > 0.0, + "WebView2 window needs positive geometry: {window:?}" + ); + let screenshot_w = state.structured()["screenshot_width"] + .as_f64() + .expect("PX targeting requires screenshot_width"); + let screenshot_h = state.structured()["screenshot_height"] + .as_f64() + .expect("PX targeting requires screenshot_height"); + let scale_x = screenshot_w / window_w; + let scale_y = screenshot_h / window_h; + let x = (screen_x - window_x) * scale_x; + let y = (screen_y - window_y) * scale_y; + assert!( + x >= 0.0 && x < screenshot_w && y >= 0.0 && y < screenshot_h, + "WebView2 PX target center ({x:.1}, {y:.1}) is outside the capture ({screenshot_w:.1}x{screenshot_h:.1})" + ); + (x, y) +} + +fn wait_for_journal_text(journal: &FixtureJournal, id: &str, expected: &str) { + let deadline = Instant::now() + Duration::from_secs(3); + loop { + if journal.text(id).as_deref() == Some(expected) { + return; + } + assert!( + Instant::now() < deadline, + "WebView2 fixture journal {id:?} did not reach {expected:?}: {}", + journal.snapshot() + ); + std::thread::sleep(Duration::from_millis(50)); } - // A prior test's host may still hold this fixed CDP port — wait for it to - // free so the daemon doesn't discover the stale host's page. - wait_port_free(cdp_port); - // Set the CDP port the daemon should probe; the spawned cua-driver child - // inherits it from this process's environment. - std::env::set_var("CUA_DRIVER_CDP_PORT", cdp_port.to_string()); - let Some(mut driver) = McpDriver::spawn() else { return }; - - // Set the CDP port the host should use so the daemon can find it. - let env_var = if label == "webview" { "CUA_WEBVIEW_CDP_PORT" } else { "CUA_ELECTRON_CDP_PORT" }; - let mut cmd = Command::new(&host_exe); - cmd.env(env_var, cdp_port.to_string()) - .stdout(Stdio::null()) - .stderr(Stdio::null()); - let app = spawn_in_job(&mut cmd).expect("spawn host"); - let pid = app.id() as i64; - driver.reaper().push(app); - println!("{label} pid={pid} cdp_port={cdp_port}"); - std::thread::sleep(Duration::from_secs(2)); // small cold-start for runtime spin-up - - let (wid, _title) = driver - .find_window(pid, title_substr) - .unwrap_or_else(|| panic!("{label} window with title containing {title_substr:?} not found")); - - f(pid, wid, &mut driver); } -// ── WebView2 structural + page tool ───────────────────────────────────────── +// ── WebView2 page tool ────────────────────────────────────────────────────── #[test] #[ignore] -fn harness_webview_window_discoverable() { - run_with_session("webview", webview_exe(), "CuaTestHarness WebView", 9222, - |pid, wid, _driver| { - println!("✅ harness_webview_window_discoverable: pid={pid} wid={wid}"); - }); +fn harness_webview_left_click_px_background() { + let case = native_background_case( + "webview2", + "left_click", + Targeting::Px, + DriverRoute::UiaInvoke, + ); + let point = Cell::new(None); + run_web_case_with_preparation( + case, + "webview2", + webview_exe(), + "CuaTestHarness WebView [ready", + |pid, wid, driver, journal| { + wait_for_journal_text(journal, "lbl-counter", "counter=0"); + let bounds = window_bounds(driver, pid, wid); + let ready_state = snapshot(driver, pid, wid); + let dom_probe = driver.call( + "page", + serde_json::json!({ + "pid": pid, + "window_id": wid, + "action": "click_element", + "selector": "#btn-increment" + }), + ); + assert!( + !dom_probe.is_error(), + "WebView2 DOM geometry probe failed: {}", + dom_probe.text() + ); + let screen_x = dom_probe.structured()["screen_x"] + .as_f64() + .expect("WebView2 DOM probe needs screen_x"); + let screen_y = dom_probe.structured()["screen_y"] + .as_f64() + .expect("WebView2 DOM probe needs screen_y"); + wait_for_journal_text(journal, "lbl-counter", "counter=1"); + let (x, y) = pixel_from_screen(&ready_state, screen_x, screen_y, bounds); + let geometry_probe = driver.call( + "click", + serde_json::json!({ + "pid": pid, + "window_id": wid, + "x": x, + "y": y, + "delivery_mode": "foreground" + }), + ); + assert!( + !geometry_probe.is_error(), + "WebView2 foreground PX geometry probe failed: {}", + geometry_probe.text() + ); + wait_for_journal_text(journal, "lbl-counter", "counter=2"); + point.set(Some((x, y))); + }, + |pid, wid, driver, journal| { + let (x, y) = point + .get() + .expect("foreground geometry probe did not set a PX target"); + let click = driver.call( + "click", + serde_json::json!({ + "pid": pid, + "window_id": wid, + "x": x, + "y": y, + "delivery_mode": "background" + }), + ); + assert!( + !click.is_error(), + "WebView2 PX background click failed: {}", + click.text() + ); + assert_eq!( + click.structured()["path"].as_str(), + Some("ax"), + "WebView2 PX background click used an unexpected driver route: {}", + click.text() + ); + wait_for_journal_text(journal, "lbl-counter", "counter=3"); + }, + ); } #[test] @@ -125,43 +389,42 @@ fn harness_webview_page_tool() { // CoreWebView2EnvironmentOptions.AdditionalBrowserArguments. // Combined with the `/json` Content-Length fix in mcp-server/src/cdp.rs, // the page tool now reaches WebView2's DOM via CDP just like Electron. - run_with_session("webview", webview_exe(), "CuaTestHarness WebView", 9222, + run_web_case( + "webview2", + "page_roundtrip", + webview_exe(), + "CuaTestHarness WebView [ready", |pid, wid, driver| { - - let marker = driver.call("page", serde_json::json!({ + let marker = driver.call("page", serde_json::json!({ "pid": pid, "window_id": wid, "action": "execute_javascript", "javascript": "document.querySelector('[data-cua-id=\"page-marker\"]').textContent" })).text().to_string(); - assert!(marker.contains("WEB_HARNESS_MARKER_v1"), - "WebView2 CDP execute_javascript marker fetch: {marker:?}"); + assert!( + marker.contains("WEB_HARNESS_MARKER_v1"), + "WebView2 CDP execute_javascript marker fetch: {marker:?}" + ); - // click_element via DOM selector + counter readback. - let _ = driver.call("page", serde_json::json!({ - "pid": pid, "window_id": wid, "action": "click_element", - "selector": "#btn-increment" - })); - std::thread::sleep(Duration::from_millis(500)); - - let post = driver.call("page", serde_json::json!({ - "pid": pid, "window_id": wid, "action": "execute_javascript", - "javascript": "document.getElementById('lbl-counter').textContent" - })).text().to_string(); - assert!(post.contains("counter=1"), - "WebView2 counter didn't advance via page.click_element: {post:?}"); - println!("✅ harness_webview_page_tool: CDP+execute_javascript+click_element green"); - }); + // click_element via DOM selector + counter readback. + let _ = driver.call( + "page", + serde_json::json!({ + "pid": pid, "window_id": wid, "action": "click_element", + "selector": "#btn-increment" + }), + ); + wait_for_page_text( + driver, + pid, + wid, + "document.getElementById('lbl-counter').textContent", + "counter=1", + ); + println!("✅ harness_webview_page_tool: CDP+execute_javascript+click_element green"); + }, + ); } -// ── Electron structural + page tool ────────────────────────────────────────── - -#[test] -#[ignore] -fn harness_electron_window_discoverable() { - run_with_session("electron", electron_exe(), "CuaTestHarness Electron", 9223, - |pid, wid, _driver| { - println!("✅ harness_electron_window_discoverable: pid={pid} wid={wid}"); - }); -} +// ── Electron page tool ─────────────────────────────────────────────────────── #[test] #[ignore] @@ -169,34 +432,47 @@ fn harness_electron_page_tool() { // Regression guard for the CDP /json discovery fix (parse // Content-Length / Transfer-Encoding instead of read_to_end). // cua-driver's page tool now reaches Electron's CDP successfully. - run_with_session("electron", electron_exe(), "CuaTestHarness Electron", 9223, + run_web_case( + "electron", + "page_execute", + electron_exe(), + "CuaTestHarness Electron", |pid, wid, driver| { - - // 1. execute_javascript via CDP. - let marker = driver.call("page", serde_json::json!({ + // 1. execute_javascript via CDP. + let marker = driver.call("page", serde_json::json!({ "pid": pid, "window_id": wid, "action": "execute_javascript", "javascript": "document.querySelector('[data-cua-id=\"page-marker\"]').textContent" })).text().to_string(); - assert!(marker.contains("WEB_HARNESS_MARKER_v1"), - "Electron CDP execute_javascript marker fetch: {marker:?}"); - - // 2. Increment counter via direct execute_javascript (the - // click_element path has a separate probe-JSON-parsing gap - // documented below — track separately). - let _ = driver.call("page", serde_json::json!({ - "pid": pid, "window_id": wid, "action": "execute_javascript", - "javascript": "document.getElementById('btn-increment').click()" - })); - std::thread::sleep(Duration::from_millis(300)); + assert!( + marker.contains("WEB_HARNESS_MARKER_v1"), + "Electron CDP execute_javascript marker fetch: {marker:?}" + ); - let post = driver.call("page", serde_json::json!({ - "pid": pid, "window_id": wid, "action": "execute_javascript", - "javascript": "document.getElementById('lbl-counter').textContent" - })).text().to_string(); - assert!(post.contains("counter=1"), - "Electron counter did not advance via execute_javascript: {post:?}"); - println!("✅ harness_electron_page_tool: CDP+execute_javascript green"); - }); + // 2. Increment counter via direct execute_javascript (the + // click_element path has a separate probe-JSON-parsing gap + // documented below — track separately). + let click = driver.call( + "page", + serde_json::json!({ + "pid": pid, "window_id": wid, "action": "execute_javascript", + "javascript": "document.getElementById('btn-increment').click()" + }), + ); + assert!( + !click.is_error(), + "Electron execute_javascript click failed: {}", + click.text() + ); + wait_for_page_text( + driver, + pid, + wid, + "document.getElementById('lbl-counter').textContent", + "counter=1", + ); + println!("✅ harness_electron_page_tool: CDP+execute_javascript green"); + }, + ); } /// Regression guard for the page.click_element double-encode fix. @@ -211,29 +487,41 @@ fn harness_electron_page_tool() { #[test] #[ignore] fn harness_electron_click_element() { - run_with_session("electron", electron_exe(), "CuaTestHarness Electron", 9223, + run_web_case( + "electron", + "click_element_probe", + electron_exe(), + "CuaTestHarness Electron", |pid, wid, driver| { - let resp = driver.call("page", serde_json::json!({ - "pid": pid, "window_id": wid, "action": "click_element", - "selector": "#btn-increment" - })); - // Prefer the tool text; fall back to a JSON-RPC error message. - let text = if resp.text().is_empty() { - resp.raw["error"]["message"].as_str().unwrap_or("").to_string() - } else { - resp.text().to_string() - }; - assert!(!text.contains("probe JSON missing") && !text.contains("required field"), - "click_element probe parse regressed: {text:?}"); - std::thread::sleep(Duration::from_millis(400)); - - // Verify the click actually fired in the DOM. - let post = driver.call("page", serde_json::json!({ - "pid": pid, "window_id": wid, "action": "execute_javascript", - "javascript": "document.getElementById('lbl-counter').textContent" - })).text().to_string(); - assert!(post.contains("counter=1"), - "Counter didn't advance after page.click_element: {post:?}"); - println!("✅ harness_electron_click_element: probe parsed, click fired, counter=1"); - }); + let resp = driver.call( + "page", + serde_json::json!({ + "pid": pid, "window_id": wid, "action": "click_element", + "selector": "#btn-increment" + }), + ); + // Prefer the tool text; fall back to a JSON-RPC error message. + let text = if resp.text().is_empty() { + resp.raw["error"]["message"] + .as_str() + .unwrap_or("") + .to_string() + } else { + resp.text().to_string() + }; + assert!( + !text.contains("probe JSON missing") && !text.contains("required field"), + "click_element probe parse regressed: {text:?}" + ); + // Verify the click actually fired in the DOM. + wait_for_page_text( + driver, + pid, + wid, + "document.getElementById('lbl-counter').textContent", + "counter=1", + ); + println!("✅ harness_electron_click_element: probe parsed, click fired, counter=1"); + }, + ); } diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs index 2731fb9dc7..827a7de0c9 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_winui3_test.rs @@ -22,9 +22,15 @@ use std::path::PathBuf; use std::process::{Command, Stdio}; -use std::time::Duration; +use std::time::{Duration, Instant}; use cua_driver_testkit::ax::element_index_by_id; +use cua_driver_testkit::e2e::{ + execute_case, native_background_case, native_readonly_case, recording_evidence, DriverRoute, + Evidence, Observation, OracleKind, Targeting, +}; +use cua_driver_testkit::observer::TargetWindow; +use cua_driver_testkit::sentinel::run_with_background_oracles; use cua_driver_testkit::{harness_app, spawn_in_job, Driver, McpDriver}; /// Resolve the WinUI3 harness exe — the `HARNESS_WINUI3_EXE` override wins (if it @@ -47,132 +53,198 @@ fn launch_winui3(driver: &mut McpDriver) -> Option { eprintln!("WinUI3 harness exe not found at {exe:?} — run tests/fixtures/build/windows.ps1"); return None; } - let child = spawn_in_job(Command::new(&exe).stdout(Stdio::null()).stderr(Stdio::null())).ok()?; + let child = spawn_in_job( + Command::new(&exe) + .stdout(Stdio::null()) + .stderr(Stdio::null()), + ) + .ok()?; let pid = child.id(); driver.reaper().push(child); - // Short fixed cold-start settle (window creation + foreground - // establishment after spawn). `find_window`'s polling handles the - // variable tail (WinUI3 first-run cold-start under sandbox load). - std::thread::sleep(Duration::from_millis(1500)); Some(pid) } -#[test] -#[ignore] -fn harness_winui3_smoke() { - let Some(mut driver) = McpDriver::spawn() else { return }; - let Some(pid) = launch_winui3(&mut driver) else { return }; - println!("WinUI3 harness pid={pid}"); - - let (wid, _) = driver - .find_window(pid as i64, "CuaTestHarness WinUI3") - .expect("WinUI3 main window not found"); - - let snap = driver.call( - "get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode":"ax"}), - ); - let text = snap.text(); - - // Button-class controls surface AutomationIds in the UIA tree. - for aid in [ - "btn-increment", "btn-reset", - "btn-open-flyout", - "btn-open-popup", - "btn-exit", - ] { - assert!(text.contains(&format!("id={aid}")), - "missing AutomationId {aid} in WinUI3 UIA snapshot"); +fn wait_for_winui3_ready(driver: &mut McpDriver, pid: u32, window_id: u64) { + let deadline = Instant::now() + Duration::from_secs(20); + loop { + let state = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": window_id}), + ); + let last_state = state.text(); + if !state.is_error() + && last_state.contains("HARNESS_TEXT_MARKER_v1") + && last_state.contains("id=chk-agreed") + { + return; + } + assert!( + Instant::now() < deadline, + "WinUI3 UIA tree did not become ready: {last_state}" + ); + std::thread::sleep(Duration::from_millis(100)); } +} - // TextBlock content (no AutomationId surfaces) — assert markers. - assert!(text.contains("HARNESS_TEXT_MARKER_v1"), "WinUI3 text_body marker not in snapshot"); - assert!(text.contains("counter=0"), "WinUI3 initial counter label not in snapshot"); +fn run_case( + case: cua_driver_testkit::e2e::CaseSpec, + test: impl FnOnce(u32, u64, &mut McpDriver) -> Observation, +) { + let cell_id = case.cell_id.clone(); + let delivery = case.delivery; + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_named(&cell_id) + .expect("required source-built Windows driver did not start"); + *evidence = recording_evidence(driver.recording_dir()); + let pid = launch_winui3(&mut driver).expect("required WinUI3 harness did not launch"); + let (wid, _) = driver + .find_window(pid as i64, "CuaTestHarness WinUI3") + .expect("WinUI3 main window not found"); + wait_for_winui3_ready(&mut driver, pid, wid); + if delivery != cua_driver_testkit::e2e::Delivery::Background { + driver.start_behavior_recording(); + } + test(pid, wid, &mut driver) + }); +} - println!("✅ harness_winui3_smoke: all expected scenarios present in UIA tree"); +fn run_background_case( + action: &str, + route: DriverRoute, + test: impl FnOnce(u32, u64, &mut McpDriver), +) { + run_case( + native_background_case("winui3", action, Targeting::Ax, route), + |pid, wid, driver| { + let (_, passed) = run_with_background_oracles( + driver, + TargetWindow { + pid, + native_id: wid, + }, + |driver| test(pid, wid, driver), + ) + .unwrap_or_else(|error| panic!("background desktop contract failed: {error}")); + Observation::delivered_with_fixture_state(passed) + }, + ); } #[test] #[ignore] -fn harness_winui3_type_text() { - let Some(mut driver) = McpDriver::spawn() else { return }; - let Some(pid) = launch_winui3(&mut driver) else { return }; - - let (wid, _) = driver - .find_window(pid as i64, "CuaTestHarness WinUI3") - .expect("WinUI3 main window"); - - let snap = driver.call( - "get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode":"ax"}), +fn harness_winui3_smoke() { + run_case( + native_readonly_case( + "winui3", + "ax_tree", + Targeting::Ax, + DriverRoute::AxRead, + vec![OracleKind::AxState], + ), + |pid, wid, driver| { + println!("WinUI3 harness pid={pid}"); + let snap = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode":"ax"}), + ); + assert!( + !snap.is_error(), + "WinUI3 AX snapshot failed: {}", + snap.text() + ); + let text = snap.text(); + for aid in [ + "btn-increment", + "btn-reset", + "btn-open-flyout", + "btn-open-popup", + "btn-exit", + ] { + assert!( + text.contains(&format!("id={aid}")), + "missing AutomationId {aid} in WinUI3 UIA snapshot" + ); + } + assert!( + text.contains("HARNESS_TEXT_MARKER_v1"), + "WinUI3 text_body marker not in snapshot" + ); + assert!( + text.contains("counter=0"), + "WinUI3 initial counter label not in snapshot" + ); + Observation::delivered(vec![OracleKind::AxState], Evidence::default()) + }, ); - let idx = element_index_by_id(snap.text(), "txt-input").expect("txt-input not in WinUI3 snapshot"); - - // WinUI3 is a XAML host — type_text requires element_index + window_id - // (routes through UIA ValuePattern.SetValue, see Windows backend docs). - let resp = driver.call("type_text", serde_json::json!({ - "pid": pid as i64, - "window_id": wid, - "element_index": idx, - "text": "winui3-typed" - })); - println!("type_text (WinUI3): {}", resp.text()); - std::thread::sleep(Duration::from_millis(500)); +} - let post = driver.call( - "get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode":"ax"}), - ); - assert!(post.text().contains("mirror=winui3-typed"), - "WinUI3 TextBox mirror did not advance. Snapshot excerpt: {}", - post.text().chars().take(600).collect::()); - println!("✅ harness_winui3_type_text: WinUI3 TextBox mirror reflects 'winui3-typed'"); +#[test] +#[ignore] +fn harness_winui3_type_text() { + run_background_case("type_text", DriverRoute::UiaValue, |pid, wid, driver| { + let snap = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode":"ax"}), + ); + let idx = element_index_by_id(snap.text(), "txt-input") + .expect("txt-input not in WinUI3 snapshot"); + let resp = driver.call( + "type_text", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "text": "winui3-typed", "delivery_mode": "background" + }), + ); + assert!(!resp.is_error(), "WinUI3 type_text failed: {}", resp.text()); + std::thread::sleep(Duration::from_millis(500)); + let post = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode":"ax"}), + ); + assert!( + post.text().contains("mirror=winui3-typed"), + "WinUI3 TextBox mirror did not advance. Snapshot excerpt: {}", + post.text().chars().take(600).collect::() + ); + }); } #[test] #[ignore] fn harness_winui3_xaml_popup_open() { - let Some(mut driver) = McpDriver::spawn() else { return }; - let Some(pid) = launch_winui3(&mut driver) else { return }; - - let (wid, _) = driver - .find_window(pid as i64, "CuaTestHarness WinUI3") - .expect("WinUI3 main window"); - - let snap = driver.call( - "get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode":"ax"}), + run_background_case( + "xaml_popup_open", + DriverRoute::UiaExpandCollapse, + |pid, wid, driver| { + let snap = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode":"ax"}), + ); + let idx = element_index_by_id(snap.text(), "btn-open-popup").expect("btn-open-popup"); + let response = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "background" + }), + ); + assert!( + !response.is_error(), + "open popup failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(500)); + let post = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode":"ax"}), + ); + assert!( + post.text().contains("XAML_POPUP_MARKER_v1"), + "XAML popup body did not appear in tree after click. Excerpt: {}", + post.text().chars().take(600).collect::() + ); + }, ); - let idx = element_index_by_id(snap.text(), "btn-open-popup").expect("btn-open-popup"); - - let _ = driver.call("click", serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx - })); - std::thread::sleep(Duration::from_millis(500)); - - let post = driver.call( - "get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode":"ax"}), - ); - let text = post.text(); - assert!(text.contains("XAML_POPUP_MARKER_v1"), - "XAML popup body did not appear in tree after click. Excerpt: {}", - text.chars().take(600).collect::()); - println!("✅ harness_winui3_xaml_popup_open: popup body visible in UIA tree"); -} - -// ── Session helper for the additional control tests ────────────────────────── - -fn winui3_with_session(f: F) -where - F: FnOnce(u32, u64, &mut McpDriver), -{ - let Some(mut driver) = McpDriver::spawn() else { return }; - let Some(pid) = launch_winui3(&mut driver) else { return }; - let (wid, _) = driver - .find_window(pid as i64, "CuaTestHarness WinUI3") - .expect("WinUI3 main window"); - f(pid, wid, &mut driver); } /// Regression guard for the click → TogglePattern dispatch fix. @@ -182,40 +254,78 @@ where #[test] #[ignore] fn harness_winui3_checkbox_toggle() { - winui3_with_session(|pid, wid, driver| { - let snap = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); - let idx = element_index_by_id(snap.text(), "chk-agreed").expect("chk-agreed"); - let _ = driver.call("click", serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx - })); - std::thread::sleep(Duration::from_millis(400)); - let post = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); - assert!(post.text().contains("agreed=True"), - "WinUI3 CheckBox didn't toggle: TogglePattern dispatch may have regressed."); - println!("✅ harness_winui3_checkbox_toggle: agreed=True via UIA Toggle"); - }); + run_background_case( + "checkbox_toggle", + DriverRoute::UiaToggle, + |pid, wid, driver| { + let snap = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}), + ); + let idx = element_index_by_id(snap.text(), "chk-agreed").expect("chk-agreed"); + let response = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "background" + }), + ); + assert!( + !response.is_error(), + "checkbox toggle failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(400)); + let post = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}), + ); + assert!( + post.text().contains("agreed=True"), + "WinUI3 CheckBox didn't toggle: TogglePattern dispatch may have regressed." + ); + println!("✅ harness_winui3_checkbox_toggle: agreed=True via UIA Toggle"); + }, + ); } /// Regression guard for SelectionItem.Select dispatch on RadioButton. #[test] #[ignore] fn harness_winui3_radio_select() { - winui3_with_session(|pid, wid, driver| { - let snap = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); - let idx = element_index_by_id(snap.text(), "rdo-high").expect("rdo-high"); - let _ = driver.call("click", serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx - })); - std::thread::sleep(Duration::from_millis(400)); - let post = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); - assert!(post.text().contains("prio=High"), - "WinUI3 radio didn't select High via SelectionItem.Select."); - println!("✅ harness_winui3_radio_select: prio=High via UIA SelectionItem"); - }); + run_background_case( + "radio_select", + DriverRoute::UiaSelection, + |pid, wid, driver| { + let snap = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}), + ); + let idx = element_index_by_id(snap.text(), "rdo-high").expect("rdo-high"); + let response = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "background" + }), + ); + assert!( + !response.is_error(), + "radio select failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(400)); + let post = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}), + ); + assert!( + post.text().contains("prio=High"), + "WinUI3 radio didn't select High via SelectionItem.Select." + ); + println!("✅ harness_winui3_radio_select: prio=High via UIA SelectionItem"); + }, + ); } /// Documents cua-driver gap: WinUI3 Slider implements @@ -233,28 +343,45 @@ fn harness_winui3_radio_select() { #[test] #[ignore] fn harness_winui3_slider_set_value() { - winui3_with_session(|pid, wid, driver| { - let snap = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); - let idx = element_index_by_id(snap.text(), "sld-value") + run_background_case( + "slider_set_value", + DriverRoute::UiaRangeValue, + |pid, wid, driver| { + let snap = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}), + ); + let idx = element_index_by_id(snap.text(), "sld-value") .expect("sld-value should now be in the UIA flat tree after RangeValuePattern detection fix"); - let resp = driver.call("set_value", serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx, - "value": "42" - })); - println!("set_value sld-value=42: {}", resp.text()); - std::thread::sleep(Duration::from_millis(400)); - let post = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); - let text = post.text(); - let advanced = text.lines().any(|l| - l.contains("slider_value=") && !l.contains("slider_value=0\"")); - assert!(advanced, - "WinUI3 Slider didn't move via RangeValuePattern.SetValue. Lines: {}", - text.lines().filter(|l| l.contains("slider_value")) - .collect::>().join(" / ")); - println!("✅ harness_winui3_slider_set_value: value moved via UIA RangeValuePattern.SetValue"); - }); + let resp = driver.call( + "set_value", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "value": "42" + }), + ); + println!("set_value sld-value=42: {}", resp.text()); + assert!(!resp.is_error(), "slider set_value failed: {}", resp.text()); + std::thread::sleep(Duration::from_millis(400)); + let post = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}), + ); + let text = post.text(); + let advanced = text + .lines() + .any(|l| l.contains("slider_value=") && !l.contains("slider_value=0\"")); + assert!( + advanced, + "WinUI3 Slider didn't move via RangeValuePattern.SetValue. Lines: {}", + text.lines() + .filter(|l| l.contains("slider_value")) + .collect::>() + .join(" / ") + ); + println!("✅ harness_winui3_slider_set_value: value moved via UIA RangeValuePattern.SetValue"); + }, + ); } /// Regression guard for ExpandCollapse.Expand + SelectionItem.Select on @@ -262,29 +389,51 @@ fn harness_winui3_slider_set_value() { #[test] #[ignore] fn harness_winui3_combo_select() { - winui3_with_session(|pid, wid, driver| { - let snap = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); - let combo_idx = element_index_by_id(snap.text(), "cbo-color").expect("cbo-color"); - // Expand the dropdown via ExpandCollapse. - let _ = driver.call("click", serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": combo_idx - })); - std::thread::sleep(Duration::from_millis(400)); - // Re-snapshot — items materialize after expand. - let snap2 = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); - let item_idx = element_index_by_id(snap2.text(), "cbo-item-orange") - .expect("cbo-item-orange after expand"); - // Select the item via SelectionItem.Select. - let _ = driver.call("click", serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": item_idx - })); - std::thread::sleep(Duration::from_millis(400)); - let post = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); - assert!(post.text().contains("color=orange"), - "WinUI3 combo didn't switch to orange via ExpandCollapse + SelectionItem.Select."); - println!("✅ harness_winui3_combo_select: color=orange via UIA Expand + Select"); - }); + run_background_case( + "combo_select", + DriverRoute::Composite, + |pid, wid, driver| { + let snap = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}), + ); + let combo_idx = element_index_by_id(snap.text(), "cbo-color").expect("cbo-color"); + // Expand the dropdown via ExpandCollapse. + let expand = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": combo_idx, + "delivery_mode": "background" + }), + ); + assert!(!expand.is_error(), "combo expand failed: {}", expand.text()); + std::thread::sleep(Duration::from_millis(400)); + // Re-snapshot — items materialize after expand. + let snap2 = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}), + ); + let item_idx = element_index_by_id(snap2.text(), "cbo-item-orange") + .expect("cbo-item-orange after expand"); + // Select the item via SelectionItem.Select. + let select = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": item_idx, + "delivery_mode": "background" + }), + ); + assert!(!select.is_error(), "combo select failed: {}", select.text()); + std::thread::sleep(Duration::from_millis(400)); + let post = driver.call( + "get_window_state", + serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}), + ); + assert!( + post.text().contains("color=orange"), + "WinUI3 combo didn't switch to orange via ExpandCollapse + SelectionItem.Select." + ); + println!("✅ harness_winui3_combo_select: color=orange via UIA Expand + Select"); + }, + ); } diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs index ede8d31e8b..97987e4910 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/harness_wpf_test.rs @@ -39,8 +39,15 @@ use std::path::PathBuf; use std::process::{Command, Stdio}; -use std::time::Duration; - +use std::time::{Duration, Instant}; + +use cua_driver_testkit::e2e::{ + execute_case, native_background_case, native_foreground_case, native_readonly_case, + recording_evidence, CaseSpec, Delivery, DriverRoute, Evidence, Observation, OracleKind, + RefusalCode, Scope, Targeting, +}; +use cua_driver_testkit::observer::TargetWindow; +use cua_driver_testkit::sentinel::{run_with_background_oracles, ForegroundSentinel}; use cua_driver_testkit::{ax, harness_app, spawn_in_job, Driver, McpDriver, ToolResponse}; // ── harness launcher ───────────────────────────────────────────────────────── @@ -64,17 +71,24 @@ fn harness_exe() -> PathBuf { /// hasn't been fully reaped and there are briefly two CuaTestHarness.Wpf /// windows on the desktop. fn launch_harness(driver: &mut McpDriver) -> Option { + launch_harness_with_state_file(driver, None) +} + +fn launch_harness_with_state_file( + driver: &mut McpDriver, + state_path: Option<&std::path::Path>, +) -> Option { let exe = harness_exe(); if !exe.exists() { eprintln!("harness exe not found at {exe:?} — run tests/fixtures/build/windows.ps1 first"); return None; } - let app = spawn_in_job( - Command::new(&exe) - .stdout(Stdio::null()) - .stderr(Stdio::null()), - ) - .ok()?; + let mut command = Command::new(&exe); + command.stdout(Stdio::null()).stderr(Stdio::null()); + if let Some(path) = state_path { + command.env("CUA_E2E_FIXTURE_STATE_PATH", path); + } + let app = spawn_in_job(&mut command).ok()?; let pid = app.id(); driver.reaper().push(app); // Short fixed settle for cold-start (window-creation + initial @@ -120,66 +134,136 @@ fn snapshot_lines_containing(text: &str, needles: &[&str]) -> String { } } -// ── tests ──────────────────────────────────────────────────────────────────── +fn fixture_state_line<'a>(text: &'a str, marker: &str) -> &'a str { + text.lines() + .find(|line| line.contains(marker)) + .unwrap_or_else(|| panic!("fixture state marker {marker:?} missing from snapshot")) +} -#[test] -#[ignore] -fn harness_wpf_smoke() { - let Some(mut driver) = McpDriver::spawn() else { - return; - }; - let Some(pid) = launch_harness(&mut driver) else { - return; - }; - println!("harness pid={}", pid); - - let (wid, title) = driver - .find_window(pid as i64, "CuaTestHarness WPF") - .expect("main window not found via list_windows"); - println!("main window: id={} title={:?}", wid, title); - - let snap = snapshot(&mut driver, pid, wid); - let text = snap.text(); - - // Buttons appear with explicit id= tags in the UIA markdown. - for aid in [ - "btn-increment", - "btn-reset", - "btn-open-msgbox", - "btn-save", - "btn-cancel", // regression guard for #1696 - "btn-open-owned", - "btn-open-layered", - "btn-exit", - ] { - assert!( - ax::has_id(text, aid), - "missing AutomationId {aid} in WPF UIA snapshot" - ); - } +fn window_bounds(driver: &mut McpDriver, pid: u32, wid: u64) -> (f64, f64, f64, f64) { + let response = driver.call("list_windows", serde_json::json!({ "pid": pid as i64 })); + let window = response.structured()["windows"] + .as_array() + .and_then(|windows| { + windows + .iter() + .find(|window| window["window_id"].as_u64() == Some(wid)) + }) + .unwrap_or_else(|| { + panic!( + "WPF window {wid} is missing from list_windows: {}", + response.text() + ) + }); + let bounds = &window["bounds"]; + ( + bounds["x"].as_f64().expect("WPF window bounds need x"), + bounds["y"].as_f64().expect("WPF window bounds need y"), + bounds["width"] + .as_f64() + .expect("WPF window bounds need width"), + bounds["height"] + .as_f64() + .expect("WPF window bounds need height"), + ) +} - // TextBlocks are reported as bare Text nodes (no UIA Invoke/Value pattern, - // no AutomationId in the rendered tree). Assert on their content instead. +fn pixel_center(state: &ToolResponse, target_id: &str, window: (f64, f64, f64, f64)) -> (f64, f64) { + let target_index = ax::element_index_by_id(state.text(), target_id) + .unwrap_or_else(|| panic!("missing PX target {target_id:?}: {}", state.text())); + let elements = state.structured()["elements"] + .as_array() + .expect("PX targeting requires structured elements"); + let target = elements + .iter() + .find(|element| element["element_index"].as_u64() == Some(target_index)) + .and_then(|element| element["frame"].as_object()) + .unwrap_or_else(|| panic!("element [{target_index}] has no structured frame")); + let target_w = target["w"].as_f64().unwrap_or(0.0); + let target_h = target["h"].as_f64().unwrap_or(0.0); + let (window_x, window_y, window_w, window_h) = window; assert!( - text.contains("HARNESS_TEXT_MARKER_v1"), - "text_body marker not in snapshot" + target_w > 0.0 && target_h > 0.0 && window_w > 0.0 && window_h > 0.0, + "WPF PX target and window need positive geometry: target={target:?}, window={window:?}" ); + let screenshot_w = state.structured()["screenshot_width"] + .as_f64() + .expect("PX targeting requires screenshot_width"); + let screenshot_h = state.structured()["screenshot_height"] + .as_f64() + .expect("PX targeting requires screenshot_height"); + let scale_x = screenshot_w / window_w; + let scale_y = screenshot_h / window_h; + let x = (target["x"].as_f64().unwrap_or(0.0) + target_w / 2.0 - window_x) * scale_x; + let y = (target["y"].as_f64().unwrap_or(0.0) + target_h / 2.0 - window_y) * scale_y; assert!( - text.contains("counter=0"), - "initial counter label not in snapshot" - ); - assert!( - text.contains("accel_fired=0"), - "initial accel label not in snapshot" + x >= 0.0 && x < screenshot_w && y >= 0.0 && y < screenshot_h, + "WPF PX target center ({x:.1}, {y:.1}) is outside the capture ({screenshot_w:.1}x{screenshot_h:.1})" ); + (x, y) +} - // HwndHost child should surface the native Win32 BUTTON as a UIA Button. - assert!( - text.contains("\"Native Win32 Child\""), - "native HWND child button not in snapshot" - ); +fn wait_for_fixture_file_text(path: &std::path::Path, id: &str, expected: &str) { + let deadline = Instant::now() + Duration::from_secs(3); + loop { + if let Ok(body) = std::fs::read(path) { + if let Ok(state) = serde_json::from_slice::(&body) { + if state[id]["text"].as_str() == Some(expected) { + return; + } + } + } + assert!( + Instant::now() < deadline, + "WPF fixture state {id:?} did not reach {expected:?}: {}", + std::fs::read_to_string(path).unwrap_or_else(|_| "".to_owned()) + ); + std::thread::sleep(Duration::from_millis(50)); + } +} - println!("✅ harness_wpf_smoke: all expected scenarios present in UIA tree"); +// ── tests ──────────────────────────────────────────────────────────────────── + +#[test] +#[ignore] +fn harness_wpf_smoke() { + run_case( + native_readonly_case( + "wpf", + "ax_tree", + Targeting::Ax, + DriverRoute::AxRead, + vec![OracleKind::AxState], + ), + |pid, wid, driver| { + let snap = snapshot(driver, pid, wid); + assert!(!snap.is_error(), "WPF AX snapshot failed: {}", snap.text()); + let text = snap.text(); + for aid in [ + "btn-increment", + "btn-reset", + "btn-open-msgbox", + "btn-save", + "btn-cancel", + "btn-open-owned", + "btn-open-layered", + "btn-exit", + ] { + assert!( + ax::has_id(text, aid), + "missing AutomationId {aid} in WPF UIA snapshot" + ); + } + for marker in ["HARNESS_TEXT_MARKER_v1", "counter=0", "accel_fired=0"] { + assert!(text.contains(marker), "missing WPF AX marker {marker}"); + } + assert!( + text.contains("\"Native Win32 Child\""), + "native HWND child button not in snapshot" + ); + Observation::delivered(vec![OracleKind::AxState], Evidence::default()) + }, + ); } // ── shared driver session helper ───────────────────────────────────────────── @@ -188,11 +272,11 @@ fn harness_wpf_smoke() { /// everything down (the harness app is reaped with the driver via the Job /// Object). Returns whatever the closure returns. The closure receives the /// harness pid, a pre-resolved main window_id, and the driver. -fn with_session(f: F) -> Option +fn with_named_session(label: &str, f: F) -> Option where F: FnOnce(u32, u64, &mut McpDriver) -> R, { - let mut driver = McpDriver::spawn()?; + let mut driver = McpDriver::spawn_named(label)?; let pid = launch_harness(&mut driver)?; let (wid, _) = driver .find_window(pid as i64, "CuaTestHarness WPF") @@ -200,142 +284,343 @@ where Some(f(pid, wid, &mut driver)) } -#[test] -#[ignore] -fn harness_wpf_counter_invoke() { - let Some(mut driver) = McpDriver::spawn() else { - return; - }; - let Some(pid) = launch_harness(&mut driver) else { - return; - }; +fn run_case(case: CaseSpec, test: impl FnOnce(u32, u64, &mut McpDriver) -> Observation) { + let cell_id = case.cell_id.clone(); + let delivery = case.delivery; + execute_case(case, |evidence| { + with_named_session(&cell_id, |pid, wid, driver| { + *evidence = recording_evidence(driver.recording_dir()); + if delivery == Delivery::NotApplicable { + driver.start_behavior_recording(); + } + test(pid, wid, driver) + }) + .expect("required WPF session did not start") + }); +} - let (wid, _) = driver - .find_window(pid as i64, "CuaTestHarness WPF") - .expect("main window"); - // Pre-snapshot so element_cache has indices we can address. - let pre = snapshot(&mut driver, pid, wid); - let idx = ax::element_index_by_id(pre.text(), "btn-increment") - .expect("btn-increment not in pre-snapshot"); +fn run_foreground_case( + action: &str, + targeting: Targeting, + route: DriverRoute, + extra_oracles: Vec, + test: impl FnOnce(u32, u64, &mut McpDriver) -> Vec, +) { + let mut case = native_foreground_case("wpf", action, targeting, route); + case.oracles.extend(extra_oracles); + case.oracles.sort(); + case.oracles.dedup(); + run_case(case, |pid, wid, driver| { + let mut passed = test(pid, wid, driver); + passed.push(OracleKind::FixtureState); + Observation::delivered(passed, Evidence::default()) + }); +} - let click = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, - "window_id": wid, - "element_index": idx - }), +fn run_background_case( + action: &str, + route: DriverRoute, + test: impl FnOnce(u32, u64, &mut McpDriver), +) { + run_case( + native_background_case("wpf", action, Targeting::Ax, route), + |pid, wid, driver| { + let (_, passed) = run_with_background_oracles( + driver, + TargetWindow { + pid, + native_id: wid, + }, + |driver| test(pid, wid, driver), + ) + .unwrap_or_else(|error| panic!("background desktop contract failed: {error}")); + Observation::delivered_with_fixture_state(passed) + }, ); - println!("click [{idx}] btn-increment: {}", click.text()); +} - std::thread::sleep(Duration::from_millis(300)); +fn observe_background( + driver: &mut McpDriver, + pid: u32, + wid: u64, + action: impl FnOnce(&mut McpDriver) -> R, +) -> (R, Vec) { + let sentinel = ForegroundSentinel::launch(driver); + sentinel + .assert_background_posture(TargetWindow { + pid, + native_id: wid, + }) + .expect("establish WPF background posture before recording"); + driver.start_behavior_recording(); + let (result, passed) = sentinel + .observe_background( + TargetWindow { + pid, + native_id: wid, + }, + || action(driver), + ) + .unwrap_or_else(|error| panic!("background desktop contract failed: {error}")); + for required in [ + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::Cursor, + OracleKind::NoLeakedInput, + ] { + assert!( + passed.contains(&required), + "background observer omitted required {required:?} oracle" + ); + } + (result, passed) +} - let post = snapshot(&mut driver, pid, wid); - let text = post.text(); - assert!( - text.contains("counter=1"), - "counter label did not advance after click — snapshot text: {}", - text.chars().take(400).collect::() - ); - println!("✅ harness_wpf_counter_invoke: counter advanced to 1"); +fn background_case(action: &str, route: DriverRoute) -> CaseSpec { + CaseSpec::delivered( + format!("windows-wpf-{action}-ax-background").replace('_', "-"), + "wpf", + "wpf", + action, + Targeting::Ax, + Delivery::Background, + Scope::Window, + route, + vec![ + OracleKind::FixtureState, + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::Cursor, + OracleKind::NoLeakedInput, + ], + ) +} + +fn delivered_with_fixture_state(mut passed: Vec) -> Observation { + passed.push(OracleKind::FixtureState); + passed.sort(); + passed.dedup(); + Observation::delivered(passed, Evidence::default()) } #[test] #[ignore] -fn harness_wpf_type_text() { - with_session(|pid, wid, driver| { - let snap = snapshot(driver, pid, wid); - let idx = - ax::element_index_by_id(snap.text(), "txt-input").expect("txt-input not in snapshot"); - - // WPF's TextBox needs *keyboard focus* for WM_CHAR delivery — and - // PostMessage(WM_LBUTTONDOWN) doesn't reliably transfer keyboard - // focus (WPF's input system treats posted events differently from - // real ones). Use dispatch:"foreground" → SendInput synthesizes - // an OS-level click that WPF treats identically to a user mouse, - // landing actual keyboard focus on the TextBox. - let _ = driver.call( - "bring_to_front", - serde_json::json!({ - "pid": pid as i64, "window_id": wid - }), - ); - std::thread::sleep(Duration::from_millis(300)); +fn harness_wpf_counter_invoke() { + execute_case( + background_case("left_click", DriverRoute::UiaInvoke), + |evidence| { + let mut driver = McpDriver::spawn_named("windows-wpf-left-click-ax-background") + .expect("required source-built driver did not start"); + *evidence = recording_evidence(driver.recording_dir()); + let pid = launch_harness(&mut driver).expect("required WPF harness did not launch"); + let (wid, _) = driver + .find_window(pid as i64, "CuaTestHarness WPF") + .expect("main window"); + let pre = snapshot(&mut driver, pid, wid); + let idx = ax::element_index_by_id(pre.text(), "btn-increment") + .expect("btn-increment not in pre-snapshot"); + let (click, passed) = observe_background(&mut driver, pid, wid, |driver| { + driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "background" + }), + ) + }); + assert!(!click.is_error(), "counter click failed: {}", click.text()); + std::thread::sleep(Duration::from_millis(300)); + let post = snapshot(&mut driver, pid, wid); + assert!( + post.text().contains("counter=1"), + "counter label did not advance after click: {}", + post.text().chars().take(400).collect::() + ); + delivered_with_fixture_state(passed) + }, + ); +} - let _ = driver.call( +#[test] +#[ignore] +fn harness_wpf_left_click_px_background() { + let case = native_background_case("wpf", "left_click", Targeting::Px, DriverRoute::UiaInvoke); + execute_case(case, |evidence| { + let mut driver = McpDriver::spawn_named("windows-wpf-left-click-px-background") + .expect("required source-built driver did not start"); + *evidence = recording_evidence(driver.recording_dir()); + let state_dir = tempfile::tempdir().expect("create WPF fixture state directory"); + let state_path = state_dir.path().join("state.json"); + let pid = launch_harness_with_state_file(&mut driver, Some(&state_path)) + .expect("required WPF harness did not launch"); + let (wid, _) = driver + .find_window(pid as i64, "CuaTestHarness WPF") + .expect("main window"); + wait_for_fixture_file_text(&state_path, "lbl-click-count", "clicks=0"); + + let bounds = window_bounds(&mut driver, pid, wid); + let ready_state = snapshot(&mut driver, pid, wid); + let (x, y) = pixel_center(&ready_state, "border-click-target", bounds); + let geometry_probe = driver.call( "click", serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx, + "pid": pid as i64, + "window_id": wid, + "x": x, + "y": y, "delivery_mode": "foreground" }), ); - std::thread::sleep(Duration::from_millis(400)); - - // SendInput's restore_foreground_polling_best_effort may yank - // foreground back from the harness window between click and - // type_text. Re-assert foreground so PostMessage WM_CHAR finds - // the TextBox with keyboard focus. - let _ = driver.call( - "bring_to_front", - serde_json::json!({ - "pid": pid as i64, "window_id": wid - }), + assert!( + !geometry_probe.is_error(), + "WPF foreground PX geometry probe failed: {}", + geometry_probe.text() ); - std::thread::sleep(Duration::from_millis(300)); + wait_for_fixture_file_text(&state_path, "lbl-click-count", "clicks=1"); - let resp = driver.call( - "type_text", - serde_json::json!({ - "pid": pid as i64, - "text": "harness-typed", - "delivery_mode": "foreground" - }), - ); - println!("type_text: {}", resp.text()); - std::thread::sleep(Duration::from_millis(700)); - - let post = snapshot(driver, pid, wid); - let text = post.text(); - let mirror_lines: Vec<&str> = text - .lines() - .filter(|l| l.contains("mirror=") || l.contains("txt-input")) - .collect(); + let (click, passed) = observe_background(&mut driver, pid, wid, |driver| { + driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "x": x, + "y": y, + "delivery_mode": "background" + }), + ) + }); assert!( - text.contains("mirror=harness-typed"), - "TextBox mirror did not reflect typed text. Mirror/input lines: {:?}", - mirror_lines + !click.is_error(), + "WPF PX background click failed: {}", + click.text() + ); + assert_eq!( + click.structured()["path"].as_str(), + Some("ax"), + "WPF PX background click used an unexpected driver route: {}", + click.text() ); - println!("✅ harness_wpf_type_text: TextBox mirror advanced to 'harness-typed'"); + wait_for_fixture_file_text(&state_path, "lbl-click-count", "clicks=2"); + wait_for_fixture_file_text(&state_path, "lbl-last-action", "last_action=left_click"); + delivered_with_fixture_state(passed) }); } #[test] #[ignore] -fn harness_wpf_set_value() { - // Companion to harness_wpf_type_text: exercises the UIA ValuePattern - // write path via the `set_value` tool. No focus needed — purely UIA. - with_session(|pid, wid, driver| { - let snap = snapshot(driver, pid, wid); - let idx = - ax::element_index_by_id(snap.text(), "txt-input").expect("txt-input not in snapshot"); - let _ = driver.call( - "set_value", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx, - "value": "via-uia-setvalue" - }), - ); - std::thread::sleep(Duration::from_millis(400)); +fn harness_wpf_type_text() { + run_foreground_case( + "type_text", + Targeting::Ax, + DriverRoute::WindowsSendInput, + Vec::new(), + |pid, wid, driver| { + let snap = snapshot(driver, pid, wid); + let idx = ax::element_index_by_id(snap.text(), "txt-input") + .expect("txt-input not in snapshot"); + + // WPF's TextBox needs *keyboard focus* for WM_CHAR delivery — and + // PostMessage(WM_LBUTTONDOWN) doesn't reliably transfer keyboard + // focus (WPF's input system treats posted events differently from + // real ones). Use dispatch:"foreground" → SendInput synthesizes + // an OS-level click that WPF treats identically to a user mouse, + // landing actual keyboard focus on the TextBox. + let _ = driver.call( + "bring_to_front", + serde_json::json!({ + "pid": pid as i64, "window_id": wid + }), + ); + std::thread::sleep(Duration::from_millis(300)); + driver.start_behavior_recording(); - let post = snapshot(driver, pid, wid); - let text = post.text(); - assert!( - text.contains("mirror=via-uia-setvalue"), - "set_value did not update TextBox. Excerpt: {}", - text.chars().take(500).collect::() - ); - println!("✅ harness_wpf_set_value: ValuePattern.SetValue wrote to TextBox"); - }); + let _ = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "foreground" + }), + ); + std::thread::sleep(Duration::from_millis(400)); + + // SendInput's restore_foreground_polling_best_effort may yank + // foreground back from the harness window between click and + // type_text. Re-assert foreground so PostMessage WM_CHAR finds + // the TextBox with keyboard focus. + let _ = driver.call( + "bring_to_front", + serde_json::json!({ + "pid": pid as i64, "window_id": wid + }), + ); + std::thread::sleep(Duration::from_millis(300)); + + let resp = driver.call( + "type_text", + serde_json::json!({ + "pid": pid as i64, + "text": "harness-typed", + "delivery_mode": "foreground" + }), + ); + println!("type_text: {}", resp.text()); + std::thread::sleep(Duration::from_millis(700)); + + let post = snapshot(driver, pid, wid); + let text = post.text(); + let mirror_lines: Vec<&str> = text + .lines() + .filter(|l| l.contains("mirror=") || l.contains("txt-input")) + .collect(); + assert!( + text.contains("mirror=harness-typed"), + "TextBox mirror did not reflect typed text. Mirror/input lines: {:?}", + mirror_lines + ); + println!("✅ harness_wpf_type_text: TextBox mirror advanced to 'harness-typed'"); + Vec::new() + }, + ); +} + +#[test] +#[ignore] +fn harness_wpf_set_value() { + execute_case( + background_case("set_value", DriverRoute::UiaValue), + |evidence| { + with_named_session("windows-wpf-set-value-ax-background", |pid, wid, driver| { + *evidence = recording_evidence(driver.recording_dir()); + let snap = snapshot(driver, pid, wid); + let idx = ax::element_index_by_id(snap.text(), "txt-input") + .expect("txt-input not in snapshot"); + let (response, passed) = observe_background(driver, pid, wid, |driver| { + driver.call( + "set_value", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "value": "via-uia-setvalue" + }), + ) + }); + assert!( + !response.is_error(), + "set_value failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(400)); + let post = snapshot(driver, pid, wid); + assert!( + post.text().contains("mirror=via-uia-setvalue"), + "set_value did not update TextBox: {}", + post.text().chars().take(500).collect::() + ); + delivered_with_fixture_state(passed) + }) + .expect("required WPF session did not start") + }, + ); } // In test-batch mode (many harnesses launched/killed in sequence) the WPF @@ -353,374 +638,453 @@ fn focus_harness(driver: &mut McpDriver, pid: u32, wid: u64) { }), ); std::thread::sleep(Duration::from_millis(300)); + driver.start_behavior_recording(); } #[test] #[ignore] fn harness_wpf_right_click() { - with_session(|pid, wid, driver| { - focus_harness(driver, pid, wid); - let snap = snapshot(driver, pid, wid); - let idx = ax::element_index_by_id(snap.text(), "border-click-target") - .expect("border-click-target not in snapshot"); - // Same dispatch:foreground rationale as type_text — PostMessage - // WM_RBUTTONDOWN doesn't always reach WPF's MouseRightButtonDown - // routed-event chain (intermittent in batch runs). - let resp = driver.call( - "right_click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx, - "delivery_mode": "foreground" - }), - ); - println!("right_click: {}", resp.text()); - std::thread::sleep(Duration::from_millis(400)); - - let post = snapshot(driver, pid, wid); - let text = post.text(); - let action_lines: Vec<&str> = text - .lines() - .filter(|l| l.contains("last_action=") || l.contains("clicks=")) - .collect(); - assert!( - text.contains("last_action=right_click"), - "right_click handler did not fire. Action/click lines: {:?}", - action_lines - ); - println!("✅ harness_wpf_right_click: last_action=right_click"); - }); + run_foreground_case( + "right_click", + Targeting::Ax, + DriverRoute::WindowsSendInput, + Vec::new(), + |pid, wid, driver| { + focus_harness(driver, pid, wid); + let snap = snapshot(driver, pid, wid); + let idx = ax::element_index_by_id(snap.text(), "border-click-target") + .expect("border-click-target not in snapshot"); + // Same dispatch:foreground rationale as type_text — PostMessage + // WM_RBUTTONDOWN doesn't always reach WPF's MouseRightButtonDown + // routed-event chain (intermittent in batch runs). + let resp = driver.call( + "right_click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "foreground" + }), + ); + println!("right_click: {}", resp.text()); + std::thread::sleep(Duration::from_millis(400)); + + let post = snapshot(driver, pid, wid); + let text = post.text(); + let action_lines: Vec<&str> = text + .lines() + .filter(|l| l.contains("last_action=") || l.contains("clicks=")) + .collect(); + assert!( + text.contains("last_action=right_click"), + "right_click handler did not fire. Action/click lines: {:?}", + action_lines + ); + println!("✅ harness_wpf_right_click: last_action=right_click"); + Vec::new() + }, + ); } #[test] #[ignore] fn harness_wpf_double_click() { - with_session(|pid, wid, driver| { - focus_harness(driver, pid, wid); - let snap = snapshot(driver, pid, wid); - let idx = ax::element_index_by_id(snap.text(), "border-click-target") - .expect("border-click-target not in snapshot"); - // dispatch:foreground for the same reason as right_click — - // PostMessage WM_LBUTTONDOWN ×2 doesn't always reach WPF's - // MouseDoubleClick / ClickCount=2 path under test-batch load. - let resp = driver.call( - "double_click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx, - "delivery_mode": "foreground" - }), - ); - println!("double_click: {}", resp.text()); - std::thread::sleep(Duration::from_millis(400)); - - let post = snapshot(driver, pid, wid); - let text = post.text(); - let action_lines: Vec<&str> = text - .lines() - .filter(|l| l.contains("last_action=") || l.contains("clicks=")) - .collect(); - assert!( - text.contains("last_action=double_click"), - "double_click handler did not register a 2nd click. \ + run_foreground_case( + "double_click", + Targeting::Ax, + DriverRoute::WindowsSendInput, + Vec::new(), + |pid, wid, driver| { + focus_harness(driver, pid, wid); + let snap = snapshot(driver, pid, wid); + let idx = ax::element_index_by_id(snap.text(), "border-click-target") + .expect("border-click-target not in snapshot"); + // dispatch:foreground for the same reason as right_click — + // PostMessage WM_LBUTTONDOWN ×2 doesn't always reach WPF's + // MouseDoubleClick / ClickCount=2 path under test-batch load. + let resp = driver.call( + "double_click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "foreground" + }), + ); + println!("double_click: {}", resp.text()); + std::thread::sleep(Duration::from_millis(400)); + + let post = snapshot(driver, pid, wid); + let text = post.text(); + let action_lines: Vec<&str> = text + .lines() + .filter(|l| l.contains("last_action=") || l.contains("clicks=")) + .collect(); + assert!( + text.contains("last_action=double_click"), + "double_click handler did not register a 2nd click. \ Action/click lines: {:?}", - action_lines - ); - println!("✅ harness_wpf_double_click: last_action=double_click"); - }); + action_lines + ); + println!("✅ harness_wpf_double_click: last_action=double_click"); + Vec::new() + }, + ); } #[test] #[ignore] fn harness_wpf_press_key_accelerator() { - // F5 binding rather than the Ctrl+Shift+H one: cua-driver's hotkey - // PostMessage path doesn't update OS modifier-key state (GetKeyState - // returns "not pressed" for VK_CONTROL), so WPF's KeyBinding with - // Modifiers=Control+Shift never matches. The UIA-worker SendInput - // path would handle modifiers but requires the cua-driver-uia.exe - // helper that isn't in our test config. F5 has no modifier and works - // on the PostMessage path. - with_session(|pid, wid, driver| { - let resp = driver.call( - "press_key", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "key": "f5" - }), - ); - println!("press_key f5: {}", resp.text()); - std::thread::sleep(Duration::from_millis(400)); - - let post = snapshot(driver, pid, wid); - let text = post.text(); - assert!( - text.contains("accel_fired=1"), - "F5 KeyBinding did not fire. Snapshot excerpt: {}", - text.chars().take(500).collect::() - ); - println!("✅ harness_wpf_press_key_accelerator: accel_fired=1 (F5 via PostMessage)"); - }); + // WPF's InputManager ignores posted key messages while another native + // window owns foreground, even for an unmodified F5 binding. The driver + // must refuse before posting instead of returning an unverifiable success. + execute_case( + background_case("keyboard", DriverRoute::PostMessage) + .expecting_refusal(vec![RefusalCode::BackgroundUnavailable]), + |evidence| { + with_named_session("windows-wpf-keyboard-ax-background", |pid, wid, driver| { + *evidence = recording_evidence(driver.recording_dir()); + let (response, mut passed) = observe_background(driver, pid, wid, |driver| { + driver.call( + "press_key", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "key": "f5", + "delivery_mode": "background" + }), + ) + }); + assert!( + response.is_error(), + "WPF background press_key unexpectedly reported delivery: {}", + response.text() + ); + let code = response.structured()["code"] + .as_str() + .and_then(RefusalCode::from_driver_code) + .expect("WPF background key refusal needs a structured code"); + let post = snapshot(driver, pid, wid); + assert!( + post.text().contains("accel_fired=0"), + "refused WPF key mutated fixture state: {}", + post.text().chars().take(500).collect::() + ); + passed.push(OracleKind::FixtureState); + Observation::refused(code, passed, response.text(), Evidence::default()) + }) + .expect("required WPF session did not start") + }, + ); } #[test] #[ignore] fn harness_wpf_scroll() { - with_session(|pid, wid, driver| { - // Pre-snapshot to populate the cache + read initial offset. - let pre = snapshot(driver, pid, wid); - let pre_text = pre.text(); - assert!( - pre_text.contains("scroll_offset=0"), - "expected initial scroll_offset=0, got: {}", - pre_text - .lines() - .filter(|l| l.contains("scroll_offset")) - .collect::>() - .join(" / ") - ); - - // Click into the ScrollViewer so it gets focus / its descendants - // become the WM_VSCROLL target. - let idx = ax::element_index_by_id(pre.text(), "scroll-tall") - .expect("scroll-tall not in snapshot"); - let _ = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, - "window_id": wid, - "element_index": idx, - "delivery_mode": "foreground" - }), - ); - std::thread::sleep(Duration::from_millis(200)); - - // Scroll down 5 lines. Keep this on the default background rung: the - // WPF harness translates the driver's WM_VSCROLL messages into the - // ScrollViewer movement we assert below. - let resp = driver.call( - "scroll", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, - "direction": "down", "by": "line", "amount": 5, - }), - ); - println!("scroll down: {}", resp.text()); - std::thread::sleep(Duration::from_millis(400)); - - let post = snapshot(driver, pid, wid); - let text = post.text(); - let advanced = text - .lines() - .any(|l| l.contains("scroll_offset=") && !l.contains("scroll_offset=0\"")); - assert!( - advanced, - "scroll offset did not advance after WM_VSCROLL. Lines: {}", - text.lines() - .filter(|l| l.contains("scroll_offset")) - .collect::>() - .join(" / ") - ); - println!("✅ harness_wpf_scroll: scroll_offset advanced past 0"); - }); + execute_case( + background_case("scroll", DriverRoute::UiaScroll), + |evidence| { + with_named_session("windows-wpf-scroll-ax-background", |pid, wid, driver| { + *evidence = recording_evidence(driver.recording_dir()); + // Pre-snapshot to populate the cache + read initial offset. + let pre = snapshot(driver, pid, wid); + let pre_text = pre.text(); + assert!( + pre_text.contains("scroll_offset=0"), + "expected initial scroll_offset=0, got: {}", + pre_text + .lines() + .filter(|l| l.contains("scroll_offset")) + .collect::>() + .join(" / ") + ); + + // Click into the ScrollViewer so it gets focus / its descendants + // become the WM_VSCROLL target. + let idx = ax::element_index_by_id(pre.text(), "scroll-tall") + .expect("scroll-tall not in snapshot"); + let _ = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, + "window_id": wid, + "element_index": idx, + "delivery_mode": "foreground" + }), + ); + std::thread::sleep(Duration::from_millis(200)); + + // Scroll down 5 lines. Keep this on the default background rung: the + // WPF harness translates the driver's WM_VSCROLL messages into the + // ScrollViewer movement we assert below. + let (resp, passed) = observe_background(driver, pid, wid, |driver| { + driver.call( + "scroll", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, + "direction": "down", "by": "line", "amount": 5, + "delivery_mode": "background" + }), + ) + }); + assert!(!resp.is_error(), "scroll failed: {}", resp.text()); + println!("scroll down: {}", resp.text()); + std::thread::sleep(Duration::from_millis(400)); + + let post = snapshot(driver, pid, wid); + let text = post.text(); + let advanced = text + .lines() + .any(|l| l.contains("scroll_offset=") && !l.contains("scroll_offset=0\"")); + assert!( + advanced, + "scroll offset did not advance after WM_VSCROLL. Lines: {}", + text.lines() + .filter(|l| l.contains("scroll_offset")) + .collect::>() + .join(" / ") + ); + delivered_with_fixture_state(passed) + }) + .expect("required WPF session did not start") + }, + ); } #[test] #[ignore] fn harness_wpf_modal_messagebox() { - with_session(|pid, wid, driver| { - let snap = snapshot(driver, pid, wid); - let idx = ax::element_index_by_id(snap.text(), "btn-open-msgbox") - .expect("btn-open-msgbox not in snapshot"); - let _ = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx - }), - ); - std::thread::sleep(Duration::from_millis(600)); + run_foreground_case( + "modal_messagebox", + Targeting::Ax, + DriverRoute::WindowsSendInput, + vec![OracleKind::AxState], + |pid, wid, driver| { + focus_harness(driver, pid, wid); + let snap = snapshot(driver, pid, wid); + let idx = ax::element_index_by_id(snap.text(), "btn-open-msgbox") + .expect("btn-open-msgbox not in snapshot"); + let open = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "foreground" + }), + ); + assert!(!open.is_error(), "open message box failed: {}", open.text()); + std::thread::sleep(Duration::from_millis(600)); - // List windows — the modal MessageBox should be a new top-level window - // owned by the same pid. - let resp = driver.call( - "list_windows", - serde_json::json!({ - "pid": pid as i64 - }), - ); - let windows = resp.structured()["windows"] - .as_array() - .expect("windows array"); - let modal = windows - .iter() - .find(|w| { - w["title"] - .as_str() - .map(|t| t.contains("Harness MessageBox")) - .unwrap_or(false) - }) - .expect("Harness MessageBox modal window not found"); - let modal_wid = modal["window_id"].as_u64().unwrap(); - println!("modal window_id={}", modal_wid); + // List windows — the modal MessageBox should be a new top-level window + // owned by the same pid. + let resp = driver.call( + "list_windows", + serde_json::json!({ + "pid": pid as i64 + }), + ); + let windows = resp.structured()["windows"] + .as_array() + .expect("windows array"); + let modal = windows + .iter() + .find(|w| { + w["title"] + .as_str() + .map(|t| t.contains("Harness MessageBox")) + .unwrap_or(false) + }) + .expect("Harness MessageBox modal window not found"); + let modal_wid = modal["window_id"].as_u64().unwrap(); + println!("modal window_id={}", modal_wid); - // Walk the modal's UIA tree — expect OK and Cancel buttons. - let modal_snap = driver.call( - "get_window_state", - serde_json::json!({ - "pid": pid as i64, "window_id": modal_wid, "capture_mode": "ax" - }), - ); - let modal_text = modal_snap.text(); - assert!( - modal_text.contains("\"OK\""), - "MessageBox UIA tree missing OK button. Tree: {}", - modal_text.chars().take(800).collect::() - ); - assert!( - modal_text.contains("\"Cancel\""), - "MessageBox UIA tree missing Cancel button" - ); + // Walk the modal's UIA tree — expect OK and Cancel buttons. + let modal_snap = driver.call( + "get_window_state", + serde_json::json!({ + "pid": pid as i64, "window_id": modal_wid, "capture_mode": "ax" + }), + ); + let modal_text = modal_snap.text(); + assert!( + modal_text.contains("\"OK\""), + "MessageBox UIA tree missing OK button. Tree: {}", + modal_text.chars().take(800).collect::() + ); + assert!( + modal_text.contains("\"Cancel\""), + "MessageBox UIA tree missing Cancel button" + ); - // Dismiss by clicking Cancel in the modal. - let cancel_idx = modal_text - .lines() - .find(|l| l.contains("\"Cancel\"") && l.contains('[')) - .and_then(|l| { - let s = l.find('[')? + 1; - let e = l[s..].find(']')? + s; - l[s..e].trim().parse::().ok() - }) - .expect("Cancel button element_index not parseable"); - let _ = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": modal_wid, "element_index": cancel_idx - }), - ); - std::thread::sleep(Duration::from_millis(400)); - println!("✅ harness_wpf_modal_messagebox: opened + parsed + dismissed"); - }); + // Dismiss by clicking Cancel in the modal. + let cancel_idx = modal_text + .lines() + .find(|l| l.contains("\"Cancel\"") && l.contains('[')) + .and_then(|l| { + let s = l.find('[')? + 1; + let e = l[s..].find(']')? + s; + l[s..e].trim().parse::().ok() + }) + .expect("Cancel button element_index not parseable"); + let dismiss = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": modal_wid, "element_index": cancel_idx, + "delivery_mode": "foreground" + }), + ); + assert!( + !dismiss.is_error(), + "dismiss message box failed: {}", + dismiss.text() + ); + std::thread::sleep(Duration::from_millis(400)); + println!("✅ harness_wpf_modal_messagebox: opened + parsed + dismissed"); + vec![OracleKind::AxState] + }, + ); } #[test] #[ignore] fn harness_wpf_owned_popup() { - with_session(|pid, wid, driver| { - let snap = snapshot(driver, pid, wid); - let idx = ax::element_index_by_id(snap.text(), "btn-open-owned") - .expect("btn-open-owned not in snapshot"); - let _ = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx - }), - ); - std::thread::sleep(Duration::from_millis(500)); + run_foreground_case( + "owned_popup", + Targeting::Ax, + DriverRoute::WindowsSendInput, + vec![OracleKind::AxState], + |pid, wid, driver| { + focus_harness(driver, pid, wid); + let snap = snapshot(driver, pid, wid); + let idx = ax::element_index_by_id(snap.text(), "btn-open-owned") + .expect("btn-open-owned not in snapshot"); + let open = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "foreground" + }), + ); + assert!(!open.is_error(), "open owned popup failed: {}", open.text()); + std::thread::sleep(Duration::from_millis(500)); - let resp = driver.call( - "list_windows", - serde_json::json!({ - "pid": pid as i64 - }), - ); - let windows = resp.structured()["windows"].as_array().unwrap(); - let owned = windows - .iter() - .find(|w| { - w["title"] - .as_str() - .map(|t| t.contains("Harness Owned Popup")) - .unwrap_or(false) - }) - .expect("Harness Owned Popup window not found in list_windows"); - let owned_wid = owned["window_id"].as_u64().unwrap(); + let resp = driver.call( + "list_windows", + serde_json::json!({ + "pid": pid as i64 + }), + ); + let windows = resp.structured()["windows"].as_array().unwrap(); + let owned = windows + .iter() + .find(|w| { + w["title"] + .as_str() + .map(|t| t.contains("Harness Owned Popup")) + .unwrap_or(false) + }) + .expect("Harness Owned Popup window not found in list_windows"); + let owned_wid = owned["window_id"].as_u64().unwrap(); - let owned_snap = driver.call( - "get_window_state", - serde_json::json!({ - "pid": pid as i64, "window_id": owned_wid, "capture_mode": "ax" - }), - ); - let owned_text = owned_snap.text(); - assert!( - owned_text.contains("OWNED_POPUP_MARKER_v1"), - "owned popup body marker missing. Tree: {}", - owned_text.chars().take(600).collect::() - ); - println!("✅ harness_wpf_owned_popup: opened + parsed"); - }); + let owned_snap = driver.call( + "get_window_state", + serde_json::json!({ + "pid": pid as i64, "window_id": owned_wid, "capture_mode": "ax" + }), + ); + let owned_text = owned_snap.text(); + assert!( + owned_text.contains("OWNED_POPUP_MARKER_v1"), + "owned popup body marker missing. Tree: {}", + owned_text.chars().take(600).collect::() + ); + println!("✅ harness_wpf_owned_popup: opened + parsed"); + vec![OracleKind::AxState] + }, + ); } #[test] #[ignore] fn harness_wpf_layered_popup_capture() { - with_session(|pid, wid, driver| { - let snap = snapshot(driver, pid, wid); - let idx = ax::element_index_by_id(snap.text(), "btn-open-layered") - .expect("btn-open-layered not in snapshot"); - let _ = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx - }), - ); - std::thread::sleep(Duration::from_millis(600)); + run_foreground_case( + "layered_popup_capture", + Targeting::Ax, + DriverRoute::Composite, + vec![OracleKind::Pixels], + |pid, wid, driver| { + focus_harness(driver, pid, wid); + let snap = snapshot(driver, pid, wid); + let idx = ax::element_index_by_id(snap.text(), "btn-open-layered") + .expect("btn-open-layered not in snapshot"); + let open = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "foreground" + }), + ); + assert!( + !open.is_error(), + "open layered popup failed: {}", + open.text() + ); + std::thread::sleep(Duration::from_millis(600)); - let resp = driver.call( - "list_windows", - serde_json::json!({ - "pid": pid as i64 - }), - ); - let windows = resp.structured()["windows"].as_array().unwrap(); - let layered = windows - .iter() - .find(|w| { - w["title"] - .as_str() - .map(|t| t.contains("Harness Layered Popup")) - .unwrap_or(false) - }) - .expect("Harness Layered Popup window not found"); - let layered_wid = layered["window_id"].as_u64().unwrap(); - - // Capture-only path — assert the screenshot is not all-black, which - // is the failure mode for PrintWindow against layered windows - // without the WGC fallback. - let cap = driver.call( - "get_window_state", - serde_json::json!({ - "pid": pid as i64, "window_id": layered_wid, "capture_mode": "vision" - }), - ); - let img_b64 = cap.raw["result"]["content"] - .as_array() - .and_then(|arr| { - arr.iter().find_map(|c| { - if c["type"] == "image" { - c["data"].as_str() - } else { - None - } + let resp = driver.call( + "list_windows", + serde_json::json!({ + "pid": pid as i64 + }), + ); + let windows = resp.structured()["windows"].as_array().unwrap(); + let layered = windows + .iter() + .find(|w| { + w["title"] + .as_str() + .map(|t| t.contains("Harness Layered Popup")) + .unwrap_or(false) }) - }) - .expect("layered window capture returned no image"); - // Decode the PNG and look for any non-black pixel. - let bytes = base64::Engine::decode(&base64::engine::general_purpose::STANDARD, img_b64) - .expect("base64"); - let img = image::load_from_memory(&bytes).expect("png decode"); - let rgb = img.to_rgb8(); - let any_color = rgb - .pixels() - .any(|p| p.0[0] > 12 || p.0[1] > 12 || p.0[2] > 12); - assert!( + .expect("Harness Layered Popup window not found"); + let layered_wid = layered["window_id"].as_u64().unwrap(); + + // Capture-only path — assert the screenshot is not all-black, which + // is the failure mode for PrintWindow against layered windows + // without the WGC fallback. + let cap = driver.call( + "get_window_state", + serde_json::json!({ + "pid": pid as i64, "window_id": layered_wid, "capture_mode": "vision" + }), + ); + let img_b64 = cap.raw["result"]["content"] + .as_array() + .and_then(|arr| { + arr.iter().find_map(|c| { + if c["type"] == "image" { + c["data"].as_str() + } else { + None + } + }) + }) + .expect("layered window capture returned no image"); + // Decode the PNG and look for any non-black pixel. + let bytes = base64::Engine::decode(&base64::engine::general_purpose::STANDARD, img_b64) + .expect("base64"); + let img = image::load_from_memory(&bytes).expect("png decode"); + let rgb = img.to_rgb8(); + let any_color = rgb + .pixels() + .any(|p| p.0[0] > 12 || p.0[1] > 12 || p.0[2] > 12); + assert!( any_color, "layered window capture is all-black ({}x{}). PrintWindow likely needs WGC fallback.", rgb.width(), rgb.height() ); - println!( - "✅ harness_wpf_layered_popup_capture: capture has non-black pixels ({}x{})", - rgb.width(), - rgb.height() - ); - }); + println!( + "✅ harness_wpf_layered_popup_capture: capture has non-black pixels ({}x{})", + rgb.width(), + rgb.height() + ); + vec![OracleKind::Pixels] + }, + ); } // ── slider / checkable / combo / list / menu coverage ──────────────────────── @@ -740,52 +1104,112 @@ fn harness_wpf_slider_drag() { // bring_to_front first to make the harness foreground (via // AttachThreadInput), then SendInput's own SetForegroundWindow is a // no-op success. - with_session(|pid, wid, driver| { - focus_harness(driver, pid, wid); - let pre = snapshot(driver, pid, wid); - assert!( - pre.text().contains("slider_value=0"), - "initial slider_value=0 missing" - ); + run_foreground_case( + "slider_drag", + Targeting::Px, + DriverRoute::WindowsSendInput, + Vec::new(), + |pid, wid, driver| { + focus_harness(driver, pid, wid); + let pre = snapshot(driver, pid, wid); + assert!( + pre.text().contains("slider_value=0"), + "initial slider_value=0 missing" + ); - let resp = driver.call( - "drag", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, - // Window-local coords along the slider TRACK. The track row sits at - // window-local y≈304 (verified on the VM: y=275 landed ~29px above - // it, on empty GroupBox space, so the thumb never moved); the thumb - // rests at the left (x≈44) at value=0. Dragging left→right advances - // the value. (TODO: derive these from the `sld-value` element frame - // in get_window_state for DPI/placement independence.) - "from_x": 44.0, "from_y": 304.0, - "to_x": 330.0, "to_y": 304.0, - "duration_ms": 700, "steps": 40, - "delivery_mode": "foreground" - }), - ); - let msg = resp.text(); - println!("drag slider (foreground): {msg}"); - assert!( - msg.starts_with("✅"), - "drag tool returned non-success: {msg}" - ); - std::thread::sleep(Duration::from_millis(500)); + let resp = driver.call( + "drag", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, + // Window-local coords along the slider TRACK. The track row sits at + // window-local y≈304 (verified on the VM: y=275 landed ~29px above + // it, on empty GroupBox space, so the thumb never moved); the thumb + // rests at the left (x≈44) at value=0. Dragging left→right advances + // the value. (TODO: derive these from the `sld-value` element frame + // in get_window_state for DPI/placement independence.) + "from_x": 44.0, "from_y": 304.0, + "to_x": 330.0, "to_y": 304.0, + "duration_ms": 700, "steps": 40, + "delivery_mode": "foreground" + }), + ); + let msg = resp.text(); + println!("drag slider (foreground): {msg}"); + assert!( + msg.starts_with("✅"), + "drag tool returned non-success: {msg}" + ); + std::thread::sleep(Duration::from_millis(500)); + + let post = snapshot(driver, pid, wid); + let text = post.text(); + let advanced = text + .lines() + .any(|l| l.contains("slider_value=") && !l.contains("slider_value=0\"")); + assert!( + advanced, + "Slider value did not advance via SendInput drag. Lines: {}", + text.lines() + .filter(|l| l.contains("slider_value")) + .collect::>() + .join(" / ") + ); + println!("✅ harness_wpf_slider_drag: thumb tracked via SendInput drag"); + Vec::new() + }, + ); +} - let post = snapshot(driver, pid, wid); - let text = post.text(); - let advanced = text - .lines() - .any(|l| l.contains("slider_value=") && !l.contains("slider_value=0\"")); +#[test] +#[ignore] +fn harness_wpf_slider_drag_background_refusal() { + let case = native_background_case( + "wpf", + "slider_drag", + Targeting::Px, + DriverRoute::WindowsTargetedInjection, + ) + .expecting_refusal(vec![RefusalCode::BackgroundUnavailable]); + run_case(case, |pid, wid, driver| { + let before = snapshot(driver, pid, wid); + let before_value = fixture_state_line(before.text(), "slider_value="); + let (response, mut passed) = observe_background(driver, pid, wid, |driver| { + driver.call( + "drag", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, + "from_x": 44.0, "from_y": 304.0, + "to_x": 330.0, "to_y": 304.0, + "duration_ms": 700, "steps": 40, + "delivery_mode": "background" + }), + ) + }); assert!( - advanced, - "Slider value did not advance via SendInput drag. Lines: {}", - text.lines() - .filter(|l| l.contains("slider_value")) - .collect::>() - .join(" / ") + response.is_error(), + "WPF background drag unexpectedly reported delivery: {}", + response.text() + ); + assert_eq!( + response.structured()["code"].as_str(), + Some("background_unavailable"), + "WPF background drag returned the wrong refusal: {}", + response.text() ); - println!("✅ harness_wpf_slider_drag: thumb tracked via SendInput drag"); + std::thread::sleep(Duration::from_millis(200)); + let after = snapshot(driver, pid, wid); + assert_eq!( + fixture_state_line(after.text(), "slider_value="), + before_value, + "refused WPF background drag changed the slider value" + ); + passed.push(OracleKind::FixtureState); + Observation::refused( + RefusalCode::BackgroundUnavailable, + passed, + response.text(), + Evidence::default(), + ) }); } @@ -795,228 +1219,294 @@ fn harness_wpf_slider_increase_large() { // Companion to slider_drag — exercises UIA Invoke on the Slider's // internal IncreaseLarge "page-up" button. Doesn't depend on screen // coords, so it's the more robust slider integration test. - with_session(|pid, wid, driver| { - focus_harness(driver, pid, wid); - let snap = snapshot(driver, pid, wid); - let idx = ax::element_index_by_id(snap.text(), "IncreaseLarge") - .expect("slider IncreaseLarge button not in snapshot"); - for i in 0..3 { - let resp = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx - }), + run_background_case( + "slider_increase_large", + DriverRoute::UiaInvoke, + |pid, wid, driver| { + let snap = snapshot(driver, pid, wid); + let idx = ax::element_index_by_id(snap.text(), "IncreaseLarge") + .expect("slider IncreaseLarge button not in snapshot"); + for i in 0..3 { + let resp = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx + }), + ); + println!("invoke IncreaseLarge #{i}: {}", resp.text()); + assert!( + !resp.is_error(), + "IncreaseLarge invoke failed: {}", + resp.text() + ); + std::thread::sleep(Duration::from_millis(150)); + } + std::thread::sleep(Duration::from_millis(300)); + let post = snapshot(driver, pid, wid); + let text = post.text(); + let advanced = text + .lines() + .any(|l| l.contains("slider_value=") && !l.contains("slider_value=0\"")); + assert!( + advanced, + "slider IncreaseLarge invokes did not advance value. Lines: {}", + text.lines() + .filter(|l| l.contains("slider_value")) + .collect::>() + .join(" / ") ); - println!("invoke IncreaseLarge #{i}: {}", resp.text()); - std::thread::sleep(Duration::from_millis(150)); - } - std::thread::sleep(Duration::from_millis(300)); - let post = snapshot(driver, pid, wid); - let text = post.text(); - let advanced = text - .lines() - .any(|l| l.contains("slider_value=") && !l.contains("slider_value=0\"")); - assert!( - advanced, - "slider IncreaseLarge invokes did not advance value. Lines: {}", - text.lines() - .filter(|l| l.contains("slider_value")) - .collect::>() - .join(" / ") - ); - println!("✅ harness_wpf_slider_increase_large: advanced via UIA Invoke"); - }); + println!("✅ harness_wpf_slider_increase_large: advanced via UIA Invoke"); + }, + ); } #[test] #[ignore] fn harness_wpf_checkbox_toggle() { - with_session(|pid, wid, driver| { - focus_harness(driver, pid, wid); - let snap = snapshot(driver, pid, wid); - let idx = ax::element_index_by_id(snap.text(), "chk-agreed").expect("chk-agreed missing"); - // CheckBox exposes UIA TogglePattern (actions=[toggle]), not Invoke. - // cua-driver's click tool tries UIA Invoke first; for elements that - // don't support it the PostMessage fallback path runs. Use - // dispatch:"foreground" to land a SendInput click that WPF - // recognises as a real user click and processes through Toggle. - let resp = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx, - "delivery_mode": "foreground" - }), - ); - println!("click chk-agreed: {}", resp.text()); - std::thread::sleep(Duration::from_millis(400)); - let post = snapshot(driver, pid, wid); - assert!( - post.text().contains("agreed=True"), - "checkbox didn't toggle: {}", - post.text() - .lines() - .filter(|l| l.contains("agreed=")) - .collect::>() - .join(" / ") - ); - println!("✅ harness_wpf_checkbox_toggle: agreed=True"); - }); + run_foreground_case( + "checkbox_toggle", + Targeting::Ax, + DriverRoute::WindowsSendInput, + Vec::new(), + |pid, wid, driver| { + focus_harness(driver, pid, wid); + let snap = snapshot(driver, pid, wid); + let idx = + ax::element_index_by_id(snap.text(), "chk-agreed").expect("chk-agreed missing"); + // CheckBox exposes UIA TogglePattern (actions=[toggle]), not Invoke. + // cua-driver's click tool tries UIA Invoke first; for elements that + // don't support it the PostMessage fallback path runs. Use + // dispatch:"foreground" to land a SendInput click that WPF + // recognises as a real user click and processes through Toggle. + let resp = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "foreground" + }), + ); + println!("click chk-agreed: {}", resp.text()); + std::thread::sleep(Duration::from_millis(400)); + let post = snapshot(driver, pid, wid); + assert!( + post.text().contains("agreed=True"), + "checkbox didn't toggle: {}", + post.text() + .lines() + .filter(|l| l.contains("agreed=")) + .collect::>() + .join(" / ") + ); + println!("✅ harness_wpf_checkbox_toggle: agreed=True"); + Vec::new() + }, + ); } #[test] #[ignore] fn harness_wpf_radio_select() { - with_session(|pid, wid, driver| { - focus_harness(driver, pid, wid); - let snap = snapshot(driver, pid, wid); - let idx = ax::element_index_by_id(snap.text(), "rdo-high").expect("rdo-high missing"); - // RadioButton exposes SelectionItem pattern (actions=[select]). - // Same dispatch:foreground rationale as the checkbox test. - let _ = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx, - "delivery_mode": "foreground" - }), - ); - std::thread::sleep(Duration::from_millis(400)); - let post = snapshot(driver, pid, wid); - assert!( - post.text().contains("prio=High"), - "radio didn't switch to High" - ); - println!("✅ harness_wpf_radio_select: prio=High"); - }); + run_foreground_case( + "radio_select", + Targeting::Ax, + DriverRoute::WindowsSendInput, + Vec::new(), + |pid, wid, driver| { + focus_harness(driver, pid, wid); + let snap = snapshot(driver, pid, wid); + let idx = ax::element_index_by_id(snap.text(), "rdo-high").expect("rdo-high missing"); + // RadioButton exposes SelectionItem pattern (actions=[select]). + // Same dispatch:foreground rationale as the checkbox test. + let response = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "foreground" + }), + ); + assert!( + !response.is_error(), + "radio select failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(400)); + let post = snapshot(driver, pid, wid); + assert!( + post.text().contains("prio=High"), + "radio didn't switch to High" + ); + println!("✅ harness_wpf_radio_select: prio=High"); + Vec::new() + }, + ); } #[test] #[ignore] fn harness_wpf_combo_select() { - with_session(|pid, wid, driver| { - focus_harness(driver, pid, wid); - let snap = snapshot(driver, pid, wid); - let combo_idx = - ax::element_index_by_id(snap.text(), "cbo-color").expect("cbo-color missing"); - // WPF ComboBox UIA peer surfaces ExpandCollapsePattern (actions=[expand]) - // but not ValuePattern — set_value at the parent is a no-op. Standard - // recipe: invoke the combo to expand the dropdown, re-snapshot so the - // item AIDs land in the element cache, then click the target item. - let _ = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": combo_idx, - "delivery_mode": "foreground" - }), - ); - std::thread::sleep(Duration::from_millis(500)); - - let snap2 = snapshot(driver, pid, wid); - let item_idx = ax::element_index_by_id(snap2.text(), "cbo-item-orange") - .expect("cbo-item-orange missing after expand"); - let _ = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": item_idx, - "delivery_mode": "foreground" - }), - ); - std::thread::sleep(Duration::from_millis(500)); + run_background_case( + "combo_select", + DriverRoute::Composite, + |pid, wid, driver| { + let snap = snapshot(driver, pid, wid); + let combo_idx = + ax::element_index_by_id(snap.text(), "cbo-color").expect("cbo-color missing"); + // WPF ComboBox UIA peer surfaces ExpandCollapsePattern (actions=[expand]) + // but not ValuePattern — set_value at the parent is a no-op. Standard + // recipe: invoke the combo to expand the dropdown, re-snapshot so the + // item AIDs land in the element cache, then click the target item. + let expand = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": combo_idx, + "action": "expand", "delivery_mode": "background" + }), + ); + assert!(!expand.is_error(), "combo expand failed: {}", expand.text()); + std::thread::sleep(Duration::from_millis(500)); - let post = snapshot(driver, pid, wid); - assert!( - post.text().contains("color=orange"), - "combo didn't switch to orange: {}", - post.text() - .lines() - .filter(|l| l.contains("color=")) - .collect::>() - .join(" / ") - ); - println!("✅ harness_wpf_combo_select: color=orange"); - }); + let snap2 = snapshot(driver, pid, wid); + let item_idx = ax::element_index_by_id(snap2.text(), "cbo-item-orange") + .expect("cbo-item-orange missing after expand"); + let select = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": item_idx, + "delivery_mode": "background" + }), + ); + assert!(!select.is_error(), "combo select failed: {}", select.text()); + std::thread::sleep(Duration::from_millis(500)); + + let post = snapshot(driver, pid, wid); + assert!( + post.text().contains("color=orange"), + "combo didn't switch to orange: {}", + post.text() + .lines() + .filter(|l| l.contains("color=")) + .collect::>() + .join(" / ") + ); + println!("✅ harness_wpf_combo_select: color=orange"); + }, + ); } #[test] #[ignore] fn harness_wpf_listbox_select() { - with_session(|pid, wid, driver| { - focus_harness(driver, pid, wid); - let snap = snapshot(driver, pid, wid); - let idx = ax::element_index_by_id(snap.text(), "lst-cherry").expect("lst-cherry missing"); - let _ = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx, - "delivery_mode": "foreground" - }), - ); - std::thread::sleep(Duration::from_millis(400)); - let post = snapshot(driver, pid, wid); - assert!( - post.text().contains("selected=cherry"), - "list didn't select cherry: {}", - post.text() - .lines() - .filter(|l| l.contains("selected=")) - .collect::>() - .join(" / ") - ); - println!("✅ harness_wpf_listbox_select: selected=cherry"); - }); + run_foreground_case( + "listbox_select", + Targeting::Ax, + DriverRoute::WindowsSendInput, + Vec::new(), + |pid, wid, driver| { + focus_harness(driver, pid, wid); + let snap = snapshot(driver, pid, wid); + let idx = + ax::element_index_by_id(snap.text(), "lst-cherry").expect("lst-cherry missing"); + let response = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": idx, + "delivery_mode": "foreground" + }), + ); + assert!( + !response.is_error(), + "listbox select failed: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(400)); + let post = snapshot(driver, pid, wid); + assert!( + post.text().contains("selected=cherry"), + "list didn't select cherry: {}", + post.text() + .lines() + .filter(|l| l.contains("selected=")) + .collect::>() + .join(" / ") + ); + println!("✅ harness_wpf_listbox_select: selected=cherry"); + Vec::new() + }, + ); } #[test] #[ignore] fn harness_wpf_menu_invoke() { - with_session(|pid, wid, driver| { - focus_harness(driver, pid, wid); - // Expand File menu first (UIA expand pattern on MenuItem) - let snap = snapshot(driver, pid, wid); - let file_idx = ax::element_index_by_id(snap.text(), "menu-file") - .or_else(|| ax::element_index_containing(snap.text(), "File")) - .unwrap_or_else(|| { - panic!( - "menu-file missing. Menu-related snapshot lines: {}", - snapshot_lines_containing(snap.text(), &["menu", "file", "new", "open"]) - ) - }); - let _ = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": file_idx, - }), - ); - std::thread::sleep(Duration::from_millis(400)); - - // Re-snapshot so menu-file-new is in the cache (it materialized - // when the menu expanded). - let snap2 = snapshot(driver, pid, wid); - let new_idx = ax::element_index_by_id(snap2.text(), "menu-file-new") - .or_else(|| ax::element_index_containing(snap2.text(), "New")) - .unwrap_or_else(|| { - panic!( - "menu-file-new missing after expand. Menu-related snapshot lines: {}", - snapshot_lines_containing(snap2.text(), &["menu", "file", "new", "open"]) - ) - }); - let _ = driver.call( - "click", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": new_idx, - }), - ); - std::thread::sleep(Duration::from_millis(500)); - - let post = snapshot(driver, pid, wid); - assert!( - post.text().contains("menu_action=file_new"), - "File>New didn't invoke: {}", - post.text() - .lines() - .filter(|l| l.contains("menu_action=")) - .collect::>() - .join(" / ") - ); - println!("✅ harness_wpf_menu_invoke: menu_action=file_new"); - }); + run_foreground_case( + "menu_invoke", + Targeting::Ax, + DriverRoute::UiaExpandCollapse, + Vec::new(), + |pid, wid, driver| { + focus_harness(driver, pid, wid); + // Expand File menu first (UIA expand pattern on MenuItem) + let snap = snapshot(driver, pid, wid); + let file_idx = ax::element_index_by_id(snap.text(), "menu-file") + .or_else(|| ax::element_index_containing(snap.text(), "File")) + .unwrap_or_else(|| { + panic!( + "menu-file missing. Menu-related snapshot lines: {}", + snapshot_lines_containing(snap.text(), &["menu", "file", "new", "open"]) + ) + }); + let expand = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": file_idx, + "action": "expand", + "delivery_mode": "foreground" + }), + ); + assert!( + !expand.is_error(), + "File menu expand failed: {}", + expand.text() + ); + std::thread::sleep(Duration::from_millis(400)); + + // Re-snapshot so menu-file-new is in the cache (it materialized + // when the menu expanded). + let snap2 = snapshot(driver, pid, wid); + let new_idx = ax::element_index_by_id(snap2.text(), "menu-file-new") + .or_else(|| ax::element_index_containing(snap2.text(), "New")) + .unwrap_or_else(|| { + panic!( + "menu-file-new missing after expand. Menu-related snapshot lines: {}", + snapshot_lines_containing(snap2.text(), &["menu", "file", "new", "open"]) + ) + }); + let invoke = driver.call( + "click", + serde_json::json!({ + "pid": pid as i64, "window_id": wid, "element_index": new_idx, + "delivery_mode": "foreground" + }), + ); + assert!( + !invoke.is_error(), + "File > New invoke failed: {}", + invoke.text() + ); + std::thread::sleep(Duration::from_millis(500)); + + let post = snapshot(driver, pid, wid); + assert!( + post.text().contains("menu_action=file_new"), + "File>New didn't invoke: {}", + post.text() + .lines() + .filter(|l| l.contains("menu_action=")) + .collect::>() + .join(" / ") + ); + println!("✅ harness_wpf_menu_invoke: menu_action=file_new"); + Vec::new() + }, + ); } diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_launch_focus_macos_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/installed_app_launch_macos_test.rs similarity index 97% rename from libs/cua-driver/rust/crates/cua-driver/tests/modality_launch_focus_macos_test.rs rename to libs/cua-driver/rust/crates/cua-driver/tests/installed_app_launch_macos_test.rs index 7fee4f4deb..d5cd265a10 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/modality_launch_focus_macos_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/installed_app_launch_macos_test.rs @@ -5,7 +5,7 @@ //! they exercise external apps and a live user desktop, so run them explicitly. //! //! Run: -//! cargo test -p cua-driver --test modality_launch_focus_macos_test -- --ignored --nocapture --test-threads=1 +//! cargo test -p cua-driver --test installed_app_launch_macos_test -- --ignored --nocapture --test-threads=1 #![cfg(target_os = "macos")] diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/installed_app_textedit_macos_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/installed_app_textedit_macos_test.rs new file mode 100644 index 0000000000..86c26d87a8 --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver/tests/installed_app_textedit_macos_test.rs @@ -0,0 +1,52 @@ +//! Optional TextEdit background-delivery integration check for macOS. +//! +//! Covers the `{path, verified}` structured outcome on a real Cocoa app. +//! +//! The schema contract lives in `protocol_schema_test.rs`. This optional +//! installed-app check needs a focus-sensitive app and GUI session. + +#![cfg(target_os = "macos")] + +// ── End-to-end ladder behavior (interactive; needs a GUI session) ──────────── + +/// On a NATIVE Cocoa field (TextEdit), `delivery_mode:"background"` lands via the +/// AX value-write and the driver confirms it: `path:"ax", verified:true`. This is +/// the driver-verifiable happy path — no foreground needed, no screenshot needed. +#[test] +#[ignore] +fn background_type_on_native_cocoa_is_ax_verified() { + use cua_driver_testkit::{Driver, McpDriver}; + let Some(mut driver) = McpDriver::spawn() else { return }; + + // Launch TextEdit and open a blank document. + let launch = driver.call("launch_app", serde_json::json!({ "bundle_id": "com.apple.TextEdit" })); + if launch.is_error() { + eprintln!("[dispatch] could not launch TextEdit — skipping"); + return; + } + let pid = launch.structured()["pid"].as_i64().expect("pid"); + let windows = launch.structured()["windows"].as_array().cloned().unwrap_or_default(); + let Some(wid) = windows.first().and_then(|w| w["window_id"].as_u64()) else { + eprintln!("[dispatch] TextEdit opened no window — skipping"); + return; + }; + + // Find the AXTextArea. + let state = driver.call("get_window_state", + serde_json::json!({ "pid": pid, "window_id": wid, "capture_mode": "ax" })); + let el = state.structured()["elements"].as_array().and_then(|els| { + els.iter().find(|e| e["role"] == "AXTextArea").and_then(|e| e["element_index"].as_u64()) + }); + let Some(el) = el else { + eprintln!("[dispatch] no AXTextArea in TextEdit (AX permission?) — skipping"); + return; + }; + + let typed = driver.call("type_text", serde_json::json!({ + "pid": pid, "window_id": wid, "element_index": el, + "text": "ladder", "delivery_mode": "background" + })); + assert!(!typed.is_error(), "type_text errored: {}", typed.text()); + assert_eq!(typed.path(), Some("ax"), "native Cocoa field should land via AX: {}", typed.text()); + assert_eq!(typed.verified(), Some(true), "AX write should read back as verified: {}", typed.text()); +} diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/launch_windows_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/launch_windows_test.rs new file mode 100644 index 0000000000..8720bbd8ac --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver/tests/launch_windows_test.rs @@ -0,0 +1,242 @@ +//! Windows launch behavior against the repo-local Electron harness. + +#![cfg(target_os = "windows")] + +use std::collections::HashSet; +use std::process::{Command, Stdio}; +use std::time::Duration; + +use cua_driver_testkit::e2e::{ + execute_case, recording_evidence, CaseSpec, Delivery, DriverRoute, Evidence, Observation, + OracleKind, RefusalCode, Scope, Targeting, +}; +use cua_driver_testkit::sentinel::ForegroundSentinel; +use cua_driver_testkit::{harness_app, spawn_in_job, Driver, McpDriver}; +use windows::core::BOOL; +use windows::Win32::Foundation::{HWND, LPARAM, TRUE}; +use windows::Win32::UI::WindowsAndMessaging::{ + EnumWindows, GetWindowTextLengthW, GetWindowTextW, GetWindowThreadProcessId, IsIconic, + ShowWindow, SW_MINIMIZE, +}; + +#[test] +#[ignore] +fn minimized_window_is_listed_and_bring_to_front_restores_it() { + let executable = harness_app("harness-electron", "CuaTestHarness.Electron.exe"); + assert!( + executable.exists(), + "required Electron launch harness is missing: {}", + executable.display() + ); + let mut driver = McpDriver::spawn_named("windows-electron-minimized-window-restore") + .expect("required source-built driver did not start"); + let mut command = Command::new(&executable); + command.stdout(Stdio::null()).stderr(Stdio::null()); + let app = spawn_in_job(&mut command).expect("Electron harness did not start"); + let pid = app.id(); + driver.reaper().push(app); + + let (window_id, _) = wait_for_window(pid); + let hwnd = HWND(window_id as *mut _); + unsafe { + let _ = ShowWindow(hwnd, SW_MINIMIZE); + } + wait_until(Duration::from_secs(2), || { + unsafe { IsIconic(hwnd) }.as_bool() + }); + + let listed = driver.call("list_windows", serde_json::json!({"pid": pid as i64})); + let window = listed.structured()["windows"] + .as_array() + .and_then(|windows| { + windows + .iter() + .find(|window| window["window_id"].as_u64() == Some(window_id)) + }) + .expect("minimized target window missing from list_windows"); + assert_eq!(window["minimized"].as_bool(), Some(true)); + assert_eq!(window["is_on_screen"].as_bool(), Some(false)); + + let on_screen = driver.call( + "list_windows", + serde_json::json!({"pid": pid as i64, "on_screen_only": true}), + ); + assert!( + on_screen.structured()["windows"] + .as_array() + .is_some_and(|windows| windows.iter().all(|window| { + window["window_id"].as_u64() != Some(window_id) + })), + "on_screen_only retained minimized target" + ); + + let restored = driver.call( + "bring_to_front", + serde_json::json!({"pid": pid as i64, "window_id": window_id}), + ); + assert!(!restored.is_error(), "restore failed: {}", restored.text()); + assert_eq!(restored.structured()["restored"].as_bool(), Some(true)); + wait_until(Duration::from_secs(2), || { + !unsafe { IsIconic(hwnd) }.as_bool() + }); + + let listed = driver.call("list_windows", serde_json::json!({"pid": pid as i64})); + let window = listed.structured()["windows"] + .as_array() + .and_then(|windows| { + windows + .iter() + .find(|window| window["window_id"].as_u64() == Some(window_id)) + }) + .expect("restored target window missing from list_windows"); + assert_eq!(window["minimized"].as_bool(), Some(false)); + assert_eq!(window["is_on_screen"].as_bool(), Some(true)); +} + +#[test] +#[ignore] +fn launch_app_minimized_preserves_foreground() { + let case = CaseSpec::delivered( + "windows-electron-launch-app-background", + "electron", + "chromium", + "launch_app", + Targeting::NotApplicable, + Delivery::Background, + Scope::Window, + DriverRoute::WindowsShellExecute, + vec![ + OracleKind::FixtureState, + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::Cursor, + OracleKind::NoLeakedInput, + ], + ) + .expecting_refusal(vec![RefusalCode::BackgroundUnavailable]); + execute_case(case, |evidence| { + let executable = harness_app("harness-electron", "CuaTestHarness.Electron.exe"); + assert!( + executable.exists(), + "required Electron launch harness is missing: {}", + executable.display() + ); + let mut driver = McpDriver::spawn_named("windows-electron-launch-app-background") + .expect("required source-built driver did not start"); + *evidence = recording_evidence(driver.recording_dir()); + let sentinel = ForegroundSentinel::launch(&mut driver); + let before = window_ids(); + driver.start_behavior_recording(); + + let (response, mut passed) = sentinel + .observe_desktop(|| { + driver.call( + "launch_app", + serde_json::json!({ + "path": executable.to_string_lossy(), + "start_minimized": true + }), + ) + }) + .unwrap_or_else(|error| panic!("minimized launch disturbed the desktop: {error}")); + assert_required_background_oracles(&passed); + assert!(response.is_error(), "minimized launch unexpectedly proceeded"); + assert_eq!( + response.structured()["code"].as_str(), + Some("background_unavailable"), + "minimized launch returned the wrong refusal: {}", + response.text() + ); + std::thread::sleep(Duration::from_millis(500)); + assert!( + window_ids().is_subset(&before), + "refused minimized launch created a new desktop window" + ); + passed.push(OracleKind::FixtureState); + Observation::refused( + RefusalCode::BackgroundUnavailable, + passed, + response.text(), + Evidence::default(), + ) + }); +} + +fn native_windows() -> Vec<(u32, u64, String)> { + unsafe extern "system" fn callback(hwnd: HWND, lparam: LPARAM) -> BOOL { + let windows = &mut *(lparam.0 as *mut Vec<(u32, u64, String)>); + let title_len = GetWindowTextLengthW(hwnd); + if title_len > 0 { + let mut title = vec![0u16; title_len as usize + 1]; + let copied = GetWindowTextW(hwnd, &mut title); + if copied > 0 { + let mut pid = 0u32; + GetWindowThreadProcessId(hwnd, Some(&mut pid)); + windows.push(( + pid, + hwnd.0 as u64, + String::from_utf16_lossy(&title[..copied as usize]), + )); + } + } + TRUE + } + + let mut windows = Vec::new(); + unsafe { + let _ = EnumWindows( + Some(callback), + LPARAM(&mut windows as *mut Vec<(u32, u64, String)> as isize), + ); + } + windows +} + +fn wait_for_window(pid: u32) -> (u64, String) { + let deadline = std::time::Instant::now() + Duration::from_secs(10); + loop { + if let Some((_, window_id, title)) = native_windows() + .into_iter() + .find(|(window_pid, _, _)| *window_pid == pid) + { + return (window_id, title); + } + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for pid {pid} to create a window" + ); + std::thread::sleep(Duration::from_millis(100)); + } +} + +fn wait_until(timeout: Duration, predicate: impl Fn() -> bool) { + let deadline = std::time::Instant::now() + timeout; + while !predicate() { + assert!( + std::time::Instant::now() < deadline, + "condition did not become true within {timeout:?}" + ); + std::thread::sleep(Duration::from_millis(25)); + } +} + +fn window_ids() -> HashSet { + native_windows() + .into_iter() + .map(|(_, window_id, _)| window_id) + .collect() +} + +fn assert_required_background_oracles(passed: &[OracleKind]) { + for required in [ + OracleKind::Focus, + OracleKind::ZOrder, + OracleKind::Cursor, + OracleKind::NoLeakedInput, + ] { + assert!( + passed.contains(&required), + "minimized launch omitted required {required:?} oracle" + ); + } +} diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_background_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/modality_background_test.rs deleted file mode 100644 index da1f125b40..0000000000 --- a/libs/cua-driver/rust/crates/cua-driver/tests/modality_background_test.rs +++ /dev/null @@ -1,398 +0,0 @@ -//! Background-modality + capture-mode tests for the CuaTestHarness.Wpf -//! harness. -//! -//! These verify the **core cua-driver promise**: background automation -//! must not steal foreground from the user's active window. The harness -//! window is at z+0 when shown (whatever WPF gives it), the -//! focus-monitor-win sentinel is then activated to z+0 (displacing the -//! harness to z+1), and cua-driver actions targeting the harness must -//! NOT make the harness regain foreground. -//! -//! Sentinel: `focus-monitor-win` (already part of the workspace; built by -//! `cargo build`). It writes `focus_monitor_losses.txt` to %TEMP% — the -//! count of times its window lost activation. We snapshot before / after -//! each action and assert delta == 0. -//! -//! Also covers **capture_mode ax** (UIA-only, no screenshot — the default) and -//! **capture_mode vision** (screenshot-only, no UIA tree). (`som` is a -//! deprecated alias for `ax`.) - -#![cfg(target_os = "windows")] - -use std::path::PathBuf; -use std::process::{Command, Stdio}; -use std::time::Duration; - -use cua_driver_testkit::ax::element_index_by_id; -use cua_driver_testkit::{harness_app, workspace_root, Driver, McpDriver}; - -// ── paths ──────────────────────────────────────────────────────────────────── - -fn focus_monitor_binary() -> PathBuf { workspace_root().join("target/debug/focus-monitor-win.exe") } -fn harness_wpf_exe() -> PathBuf { - if let Ok(p) = std::env::var("HARNESS_WPF_EXE") { - let pb = PathBuf::from(p); - if pb.exists() { return pb; } - } - harness_app("harness-wpf", "CuaTestHarness.Wpf.exe") -} - -fn loss_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_losses.txt") } -fn gain_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_gains.txt") } -fn key_loss_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_key_losses.txt") } -fn key_gain_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_key_gains.txt") } -fn focus_pid_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_pid.txt") } -fn focus_hwnd_file() -> PathBuf { std::env::temp_dir().join("focus_monitor_hwnd.txt") } - -fn read_count(p: &std::path::Path) -> u32 { - // Surface read / parse errors instead of swallowing them as 0 — a silent - // 0 would mask actual focus-steal regressions by making delta computation - // look "no change" when really the sentinel file is unreadable. - let raw = std::fs::read_to_string(p) - .unwrap_or_else(|e| panic!("failed reading sentinel counter {p:?}: {e}")); - raw.trim() - .parse::() - .unwrap_or_else(|e| panic!("failed parsing sentinel counter {p:?} as u32: {e}")) -} - -// ── shared fixture ─────────────────────────────────────────────────────────── - -/// Launch sequence: -/// 0. cua-driver MCP server (headless stdio; its reaper owns the harness + -/// sentinel below, so any early-return or panic reaps the whole tree). -/// 1. WPF harness (becomes foreground briefly on its own activation). -/// 2. focus-monitor-win sentinel — its OnLoad SetForegroundWindow displaces -/// the harness; sentinel is now z+0, harness z+1. -/// 3. Reset losses.txt to 0 (sentinel may have logged its own startup activate) -/// and resolve the harness window (pid + window_id). -fn setup() -> Option<(McpDriver, u32, u64)> { - let fm_bin = focus_monitor_binary(); - if !fm_bin.exists() { - eprintln!("focus-monitor-win.exe not built — skipping"); return None; - } - let h_exe = harness_wpf_exe(); - if !h_exe.exists() { - eprintln!("harness WPF exe not built — skipping"); return None; - } - - // Reset sentinel files so we start from a known baseline. - let _ = std::fs::write(loss_file(), "0"); - let _ = std::fs::write(gain_file(), "0"); - let _ = std::fs::write(key_loss_file(), "0"); - let _ = std::fs::write(key_gain_file(), "0"); - let _ = std::fs::remove_file(focus_pid_file()); - let _ = std::fs::remove_file(focus_hwnd_file()); - - // Spawn the cua-driver MCP server (skips if the binary isn't built). - let mut driver = McpDriver::spawn()?; - - // 1. WPF harness, launched through the reaper so it can't outlive the test. - driver - .reaper() - .spawn(Command::new(&h_exe).stdout(Stdio::null()).stderr(Stdio::null())) - .ok()?; - std::thread::sleep(Duration::from_secs(1)); - - // 2. Launch sentinel. focus-monitor-win activates its own window which - // displaces the harness. - if driver - .reaper() - .spawn(Command::new(&fm_bin).stdout(Stdio::null()).stderr(Stdio::null())) - .is_err() - { - eprintln!("focus-monitor-win spawn failed"); - return None; - } - - // Wait for sentinel to publish its pid+hwnd files (so we know it's up and - // has claimed foreground). On timeout we return None; dropping `driver` - // reaps the harness + sentinel so they don't poison subsequent tests. - let deadline = std::time::Instant::now() + Duration::from_secs(10); - loop { - let pid_ok = std::fs::read_to_string(focus_pid_file()).ok() - .and_then(|s| s.trim().parse::().ok()).unwrap_or(0) != 0; - let hwnd_ok = std::fs::read_to_string(focus_hwnd_file()).ok() - .and_then(|s| s.trim().parse::().ok()).unwrap_or(0) != 0; - if pid_ok && hwnd_ok { break; } - if std::time::Instant::now() > deadline { - eprintln!("focus-monitor sentinel never published pid/hwnd files"); - return None; - } - std::thread::sleep(Duration::from_millis(100)); - } - std::thread::sleep(Duration::from_millis(400)); // sentinel-active settle - - // Reset losses again now that sentinel has settled. Any prior loss - // counts (e.g. from harness coming up after the sentinel did) shouldn't - // count against the test. - let _ = std::fs::write(loss_file(), "0"); - let _ = std::fs::write(gain_file(), "0"); - let _ = std::fs::write(key_loss_file(), "0"); - let _ = std::fs::write(key_gain_file(), "0"); - - // Resolve the harness window (pid + window_id) by title. The WPF harness is - // its own window process, so list_windows surfaces it directly. - let resp = driver.call("list_windows", serde_json::json!({})); - let (harness_pid, harness_wid) = resp.structured()["windows"].as_array() - .and_then(|a| a.iter().find_map(|w| { - let title = w["title"].as_str().unwrap_or(""); - if title.contains("CuaTestHarness WPF") { - Some((w["pid"].as_u64()? as u32, w["window_id"].as_u64()?)) - } else { - None - } - })) - .expect("harness window not found"); - - Some((driver, harness_pid, harness_wid)) -} - -/// Strict mode: action must not generate ANY sentinel-loss event. -fn assert_no_focus_steal(label: &str, f: F) -where F: FnOnce() { - let before_act = read_count(&loss_file()); - let before_key = read_count(&key_loss_file()); - f(); - std::thread::sleep(Duration::from_millis(400)); - let after_act = read_count(&loss_file()); - let after_key = read_count(&key_loss_file()); - let d_act = after_act.saturating_sub(before_act); - let d_key = after_key.saturating_sub(before_key); - assert_eq!(d_act, 0, - "{label}: sentinel act_losses went {before_act} -> {after_act} (delta={d_act}). \ - cua-driver's background action stole foreground from the user."); - assert_eq!(d_key, 0, - "{label}: sentinel key_losses went {before_key} -> {after_key} (delta={d_key})."); - println!("✅ {label}: no focus steal (act_losses=0, key_losses=0)"); -} - -/// Relaxed mode: read GetForegroundWindow() after the action and assert -/// it's still the sentinel HWND. Tolerates a transient blip during the -/// action (which UIA Invoke against WPF Buttons creates unavoidably — -/// the target's handler calls Focus() before cua-driver gets control). -/// The cua-driver fg_bypass restores foreground after, and this check -/// asserts that restoration actually worked. -// Documented oracle kept for the foreground-restore scenarios; not currently -// wired to a live test (preserved verbatim from before the testkit migration). -#[allow(dead_code)] -fn assert_foreground_restored(label: &str, f: F) -where F: FnOnce() { - let sentinel_hwnd: u64 = std::fs::read_to_string(focus_hwnd_file()) - .ok().and_then(|s| s.trim().parse().ok()).unwrap_or(0); - assert!(sentinel_hwnd != 0, "{label}: sentinel hwnd unknown (file missing)"); - - let before_loss = read_count(&loss_file()); - f(); - std::thread::sleep(Duration::from_millis(500)); - let after_loss = read_count(&loss_file()); - let d_loss = after_loss.saturating_sub(before_loss); - - // Read current foreground via Win32 directly. - let now_fg: u64 = unsafe { - let h = windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow(); - h.0 as u64 - }; - assert_eq!(now_fg, sentinel_hwnd, - "{label}: GetForegroundWindow={now_fg:#x} but sentinel hwnd={sentinel_hwnd:#x}. \ - Foreground was NOT restored to the user's window after the action."); - if d_loss == 0 { - println!("✅ {label}: no focus blip at all (losses=0)"); - } else { - println!("✅ {label}: foreground restored after {d_loss} blip(s) — sentinel HWND={sentinel_hwnd:#x} matches GetForegroundWindow"); - } -} - -// ── BACKGROUND MODALITY: cua-driver must keep harness at z+1 ──────────────── - -#[test] -#[ignore] -fn bg_modality_get_window_state_no_focus_steal() { - let (mut driver, pid, wid) = match setup() { Some(x) => x, None => return }; - // Exercise the screenshot-bearing path (`vision`) — screen capture is the - // focus-sensitive operation; `ax` no longer grabs a frame at all. - assert_no_focus_steal("get_window_state(vision)", || { - let _ = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "vision"})); - }); -} - -/// Documents the cua-driver focus-steal gap for UIA Invoke on a WPF -/// Button. Root cause: WPF's ButtonBase.OnClick handler synchronously -/// calls `UIElement.Focus()` which routes through `SetForegroundWindow` -/// and is NOT gated by the EnableWindow(false) bypass used for UWP -/// hosts. The daemon CAN'T restore foreground reliably either, because -/// non-UIAccess processes are subject to the foreground-lock. -/// -/// **Mitigation**: route UIA activations through `cua-driver-uia.exe` -/// (UIAccess-manifested worker). With UIAccess, the worker can both -/// suppress the self-foreground and restore the user's foreground if -/// it leaked through. -/// -/// This test ASSERTS the gap currently exists. When cua-driver gains -/// the UIAccess worker path, flip this assertion to `delta == 0` and -/// rename to `..._no_focus_steal`. -#[test] -#[ignore] -// Name intentionally documents a KNOWN focus-steal (see doc comment above). -#[allow(non_snake_case)] -fn bg_modality_uia_invoke_click_DOCUMENTED_steals_focus() { - let (mut driver, pid, wid) = match setup() { Some(x) => x, None => return }; - let _ = driver.call("set_agent_cursor_enabled", - serde_json::json!({"enabled": false})); - std::thread::sleep(Duration::from_millis(200)); - - let snap = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); - let idx = element_index_by_id(snap.text(), "btn-increment").expect("btn-increment"); - - let before = read_count(&loss_file()); - let _ = driver.call("click", - serde_json::json!({"pid": pid as i64, "window_id": wid, "element_index": idx})); - std::thread::sleep(Duration::from_millis(500)); - let after = read_count(&loss_file()); - let delta = after.saturating_sub(before); - assert!(delta >= 1, - "Expected the documented focus-steal gap (delta>=1). Got delta={delta}. \ - If this now passes, cua-driver has fixed UIA Invoke focus-steal — \ - flip this assertion to delta==0 and update the docstring."); - println!("⚠️ bg_modality_uia_invoke_click: gap confirmed (delta={delta}). \ - Mitigation = route UIA via cua-driver-uia.exe (UIAccess worker)."); -} - -/// Companion gap: UIA ValuePattern.SetValue on WPF TextBox. -/// Same root cause and mitigation path as -/// bg_modality_uia_invoke_click_DOCUMENTED_steals_focus. -#[test] -#[ignore] -// Name intentionally documents a KNOWN focus-steal (see doc comment above). -#[allow(non_snake_case)] -fn bg_modality_set_value_DOCUMENTED_steals_focus() { - let (mut driver, pid, wid) = match setup() { Some(x) => x, None => return }; - let _ = driver.call("set_agent_cursor_enabled", - serde_json::json!({"enabled": false})); - std::thread::sleep(Duration::from_millis(200)); - - let snap = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"})); - let idx = element_index_by_id(snap.text(), "txt-input").expect("txt-input"); - - let before = read_count(&loss_file()); - let _ = driver.call("set_value", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, "element_index": idx, - "value": "via-uia-no-focus-steal" - })); - std::thread::sleep(Duration::from_millis(500)); - let after = read_count(&loss_file()); - let delta = after.saturating_sub(before); - assert!(delta >= 1, - "Expected the documented SetValue focus-steal gap. delta={delta}."); - println!("⚠️ bg_modality_set_value: gap confirmed (delta={delta})"); -} - -#[test] -#[ignore] -fn bg_modality_press_key_no_focus_steal() { - let (mut driver, pid, wid) = match setup() { Some(x) => x, None => return }; - // F5 fires the harness accelerator (KeyBinding) via PostMessage - // WM_KEYDOWN — background path, no foreground swap. - assert_no_focus_steal("press_key(f5, PostMessage)", || { - let _ = driver.call("press_key", - serde_json::json!({"pid": pid as i64, "window_id": wid, "key": "f5"})); - }); -} - -#[test] -#[ignore] -fn bg_modality_scroll_no_focus_steal() { - let (mut driver, pid, wid) = match setup() { Some(x) => x, None => return }; - assert_no_focus_steal("scroll(down, PostMessage WM_VSCROLL)", || { - let _ = driver.call("scroll", - serde_json::json!({ - "pid": pid as i64, "window_id": wid, - "direction": "down", "by": "line", "amount": 3 - })); - }); -} - -// ── PERCEPTION: returns both tree + screenshot, with an opt-out ───────────── - -#[test] -#[ignore] -fn default_returns_tree_and_image() { - let (mut driver, pid, wid) = match setup() { Some(x) => x, None => return }; - // Default (no capture_mode): BOTH the tree AND a screenshot. - let resp = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid})); - assert!(resp.text().contains("id=btn-increment"), - "default tree missing btn-increment AID"); - let has_image = resp.raw["result"]["content"].as_array() - .map(|a| a.iter().any(|c| c["type"].as_str() == Some("image"))) - .unwrap_or(false); - assert!(has_image, "default get_window_state must return a screenshot alongside the tree"); - println!("✅ default_returns_tree_and_image: tree + image both present"); - - assert_no_focus_steal("get_window_state(default)", || { - let _ = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid})); - }); -} - -#[test] -#[ignore] -fn include_screenshot_false_returns_tree_only() { - let (mut driver, pid, wid) = match setup() { Some(x) => x, None => return }; - // The perf opt-out: tree present, NO image (the cheap re-index path). - let resp = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "include_screenshot": false})); - assert!(resp.text().contains("id=btn-increment"), - "tree-only snapshot missing btn-increment AID"); - let has_image = resp.raw["result"]["content"].as_array() - .map(|a| a.iter().any(|c| c["type"].as_str() == Some("image"))) - .unwrap_or(false); - assert!(!has_image, - "include_screenshot:false should NOT return image content (got one anyway)"); - println!("✅ include_screenshot_false_returns_tree_only: tree present, no image"); -} - -#[test] -#[ignore] -fn deprecated_capture_mode_is_ignored() { - // The legacy `capture_mode:"vision"` used to suppress the tree. It is now - // ignored — both halves must still come back. - let (mut driver, pid, wid) = match setup() { Some(x) => x, None => return }; - let resp = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "vision"})); - assert!(resp.text().contains("id=btn-increment"), - "capture_mode=vision must NOT suppress the tree (it is deprecated/ignored)"); - let has_image = resp.raw["result"]["content"].as_array() - .map(|a| a.iter().any(|c| c["type"].as_str() == Some("image"))) - .unwrap_or(false); - assert!(has_image, "capture_mode=vision must NOT suppress the screenshot either"); - println!("✅ deprecated_capture_mode_is_ignored: both tree and image returned"); -} - -#[test] -#[ignore] -fn ground_invoke_reground_roundtrip() { - // Ground (both tree + image) → find element → element ax action (Invoke) → - // re-ground and confirm the post-action state. Mirrors how an agent works. - let (mut driver, pid, wid) = match setup() { Some(x) => x, None => return }; - let snap = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid})); - let idx = element_index_by_id(snap.text(), "btn-increment").expect("btn-increment"); - - let _ = driver.call("click", - serde_json::json!({"pid": pid as i64, "window_id": wid, "element_index": idx})); - std::thread::sleep(Duration::from_millis(300)); - - let snap2 = driver.call("get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid})); - let has_image = snap2.raw["result"]["content"].as_array() - .map(|a| a.iter().any(|c| c["type"].as_str() == Some("image"))) - .unwrap_or(false); - assert!(has_image, "re-ground snapshot didn't return an image"); - assert!(snap2.text().contains("counter=1"), - "counter didn't advance after UIA Invoke"); - println!("✅ ground_invoke_reground_roundtrip: ground→invoke→reground (tree+image) green"); -} diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_linux_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_linux_test.rs deleted file mode 100644 index ec2c43cc39..0000000000 --- a/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_linux_test.rs +++ /dev/null @@ -1,186 +0,0 @@ -//! Linux **desktop-scope** (vision/foreground) modality through the SAME -//! cua-driver interface as Windows/macOS: `set_config capture_scope=desktop` + -//! a window-less screen-absolute `click` (no `pid`/`window_id`/`list_windows`). -//! The Linux actuator warps the pointer and injects a real button press via the -//! XTest extension — the peer of the Windows `WindowFromPoint` + macOS -//! global-HID desktop click. (XTest delivering to the under-pointer window is -//! why the *background* paths use `XSendEvent`; for desktop scope that delivery -//! is exactly what we want.) -//! -//! Grounds the click on the GTK3 harness increment button's screen-absolute -//! `frame` (AT-SPI Component extents), asserts the counter advanced, plus the -//! window-scope rejection gate. -//! -//! Linux config is global-only (no per-session override), so `set_config` -//! capture_scope writes the on-disk default — the test resets it to `window` -//! before asserting so a failure can't leave the sandbox in desktop scope. -//! -//! #[ignore] (needs an X11/Xwayland display + AT-SPI + the GTK3 harness). Run: -//! cargo test -p cua-driver --test modality_desktop_scope_linux_test -- --ignored --nocapture --test-threads=1 - -#![cfg(target_os = "linux")] - -use std::process::{Command, Stdio}; -use std::time::{Duration, Instant}; - -use cua_driver_testkit::{harness_app, Driver, McpDriver}; - -fn harness_exe() -> std::path::PathBuf { - std::env::var("HARNESS_GTK3_EXE") - .map(std::path::PathBuf::from) - .ok() - .filter(|p| p.exists()) - .unwrap_or_else(|| harness_app("harness-gtk3", "CuaTestHarness.Gtk3")) -} - -fn launch(driver: &mut McpDriver) -> Option<(u32, u64)> { - let exe = harness_exe(); - if !exe.exists() { - eprintln!("[desktop-linux] GTK3 harness not built ({exe:?}) — run tests/fixtures/build/linux.sh; skipping"); - return None; - } - driver - .reaper() - .spawn(Command::new(&exe).stdout(Stdio::null()).stderr(Stdio::null())) - .ok()?; - let deadline = Instant::now() + Duration::from_secs(14); - while Instant::now() < deadline { - let r = driver.call("list_windows", serde_json::json!({})); - if let Some(wins) = r.structured()["windows"].as_array() { - for w in wins { - if w["title"].as_str().unwrap_or("").contains("CuaTestHarness GTK3") { - let pid = w["pid"].as_u64().unwrap_or(0) as u32; - let wid = w["window_id"].as_u64().unwrap_or(0); - if pid != 0 && wid != 0 { - driver.reaper().track_pid(pid); - return Some((pid, wid)); - } - } - } - } - std::thread::sleep(Duration::from_millis(400)); - } - eprintln!("[desktop-linux] harness window never appeared — graphical session + AT-SPI available? skipping"); - None -} - -fn ax_snapshot(driver: &mut McpDriver, pid: u32, wid: u64) -> serde_json::Value { - driver - .call( - "get_window_state", - serde_json::json!({ "pid": pid as i64, "window_id": wid, "capture_mode": "ax" }), - ) - .structured() - .clone() -} - -/// Screen-absolute center (px) of the increment button from `elements[].frame` -/// (AT-SPI Component extents are screen-absolute). The GTK3 harness sets the -/// button's accessible NAME to `btn-increment`; match any element whose blob -/// carries that aid and has a frame, so we're robust to the exact field name. -fn increment_center(snap: &serde_json::Value) -> Option<(i64, i64)> { - let els = snap["elements"].as_array()?; - let btn = els.iter().find(|e| { - serde_json::to_string(e).map(|s| s.contains("btn-increment")).unwrap_or(false) - && e.get("frame").map(|f| f.is_object()).unwrap_or(false) - })?; - let f = &btn["frame"]; - let (x, y, w, h) = (f["x"].as_f64()?, f["y"].as_f64()?, f["w"].as_f64()?, f["h"].as_f64()?); - Some(((x + w / 2.0) as i64, (y + h / 2.0) as i64)) -} - -fn counter(snap: &serde_json::Value) -> Option { - let tree = snap["tree_markdown"].as_str()?; - let idx = tree.find("counter=")? + "counter=".len(); - let digits: String = tree[idx..].chars().take_while(|c| c.is_ascii_digit()).collect(); - digits.parse().ok() -} - -fn set_scope(driver: &mut McpDriver, scope: &str) { - let r = driver.call( - "set_config", - serde_json::json!({ "key": "capture_scope", "value": scope }), - ); - assert!(!r.is_error(), "set_config capture_scope={scope} failed: {}", r.text()); -} - -// ── tests ─────────────────────────────────────────────────────────────────────── - -/// In desktop scope, a window-less screen-absolute click (no pid/window_id) -/// lands on the increment button — its counter advances. -#[test] -#[ignore] -fn desktop_scope_windowless_click_lands_on_control() { - let Some(mut driver) = McpDriver::spawn() else { return }; - let Some((pid, wid)) = launch(&mut driver) else { return }; - - // Settle for the AT-SPI tree to register the button + its extents. - let mut snap = ax_snapshot(&mut driver, pid, wid); - let mut center = increment_center(&snap); - let deadline = Instant::now() + Duration::from_secs(8); - while center.is_none() && Instant::now() < deadline { - std::thread::sleep(Duration::from_millis(400)); - snap = ax_snapshot(&mut driver, pid, wid); - center = increment_center(&snap); - } - let Some((cx, cy)) = center else { - eprintln!("[desktop-linux] increment button frame not found (AT-SPI extents missing?) — skipping"); - return; - }; - let pre = counter(&snap).unwrap_or(0); - println!("[desktop-linux] increment button screen-center=({cx},{cy}) pre-counter={pre}"); - - set_scope(&mut driver, "desktop"); - - // Retry the window-less desktop click until the counter advances. A - // freshly-mapped harness window may not yet be raised under the pointer on - // the first click (X11 window-raise timing differs across WMs — XFCE/Openbox - // lag GNOME), so the screen-absolute XTest click can miss the first attempt. - // Re-issuing the SAME click is safe: extra landed clicks only increment the - // counter further, and `post > pre` still holds. We assert the click was - // *dispatched as desktop scope* on the first attempt, and that it eventually - // *lands* within the budget. - let mut post = pre; - let mut first_text = String::new(); - for attempt in 0..12 { - let clicked = driver.call("click", serde_json::json!({ "x": cx, "y": cy })); - if attempt == 0 { - first_text = clicked.text().to_string(); - assert!(!clicked.is_error(), "desktop-scope click errored: {}", clicked.text()); - } - std::thread::sleep(Duration::from_millis(500)); - post = counter(&ax_snapshot(&mut driver, pid, wid)).unwrap_or(pre); - if post > pre { - break; - } - } - // Reset scope so a later failure can't leave the box in desktop scope. - set_scope(&mut driver, "window"); - - assert!( - first_text.to_lowercase().contains("desktop scope"), - "click not reported as desktop-scope: {first_text}" - ); - assert!( - post > pre, - "counter did not advance after window-less desktop clicks: pre={pre} post={post} \ - (the harness window never became clickable at ({cx},{cy}) within the retry budget)" - ); - println!("✅ desktop_scope_windowless_click_lands_on_control: counter {pre} → {post}"); -} - -/// Negative gate: a window-less click under `capture_scope=window` is rejected. -#[test] -#[ignore] -fn window_scope_rejects_windowless_click() { - let Some(mut driver) = McpDriver::spawn() else { return }; - set_scope(&mut driver, "window"); - let r = driver.call("click", serde_json::json!({ "x": 100, "y": 100 })); - let txt = r.text().to_lowercase(); - assert!( - r.is_error() || txt.contains("desktop scope") || txt.contains("desktop_scope_disabled"), - "window-scope window-less click was NOT rejected: {}", - r.text() - ); - println!("✅ window_scope_rejects_windowless_click: window-less click correctly gated"); -} diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_test.rs deleted file mode 100644 index 1ca79a9989..0000000000 --- a/libs/cua-driver/rust/crates/cua-driver/tests/modality_desktop_scope_test.rs +++ /dev/null @@ -1,152 +0,0 @@ -//! Harness integration test for the **desktop-scope** modality (#1968 / #2019). -//! -//! Desktop-scope is cua-driver's *foreground*, vision-only, **screen-absolute** -//! loop (the "Computer-Use 1.0" mode), the complement to the default per-window -//! background model that `harness_bg_modality_test` / `e2e_windows_bg_input_test` -//! cover. This test exercises the Windows Phase-1 actuator end-to-end against a -//! real harness app: -//! -//! 1. `set_config capture_scope=desktop` → `get_desktop_state` returns a -//! full-display capture with true `screen_width/height` (no downscale). -//! 2. A **window-less** screen-absolute `click` / `scroll` (no pid/window_id) -//! lands via `WindowFromPoint` while in desktop scope. -//! 3. Negative gate: the same window-less `click` under `capture_scope=window` -//! is rejected with the structured `desktop_scope_disabled` error. -//! -//! Note: `set_config` is a *session* override — it persists for the lifetime of -//! the one MCP server we spawn here (not across separate `cua-driver call` -//! processes), which is exactly why this test drives a single long-lived server. -//! -//! All tests are `#[ignore]` (need a real desktop session). Run explicitly: -//! cargo test -p cua-driver --test harness_desktop_scope_test -- --ignored --nocapture --test-threads=1 - -#![cfg(target_os = "windows")] - -use std::process::{Command, Stdio}; -use std::time::{Duration, Instant}; - -use cua_driver_testkit::{harness_app, Driver, McpDriver}; - -/// WPF harness app (built by `tests/fixtures/build/windows.ps1`). Path mirrors -/// `shared/scenarios.json`'s `wpf.exe_relative_path`. -fn harness_wpf_exe() -> std::path::PathBuf { - harness_app("harness-wpf", "CuaTestHarness.Wpf.exe") -} - -/// Launch the WPF harness app and return (pid, window center in screen px). -/// Skips (returns None) if the harness app isn't built. -fn launch_wpf_and_center(driver: &mut McpDriver) -> Option<(u32, i32, i32)> { - let exe = harness_wpf_exe(); - if !exe.exists() { - eprintln!("[desktop-scope] WPF harness not built ({exe:?}) — skipping window-target tests"); - return None; - } - driver - .reaper() - .spawn(Command::new(&exe).stdout(Stdio::null()).stderr(Stdio::null())) - .ok()?; - - let deadline = Instant::now() + Duration::from_secs(15); - while Instant::now() < deadline { - let r = driver.call("list_windows", serde_json::json!({})); - if let Some(arr) = r.structured()["windows"].as_array() { - for w in arr { - let title = w["title"].as_str().unwrap_or(""); - if !title.contains("CuaTestHarness") { - continue; - } - let pid = w["pid"].as_u64().unwrap_or(0) as u32; - // #2018: bounds is nested {x,y,width,height} on Windows. - let b = &w["bounds"]; - let (x, y, ww, h) = ( - b["x"].as_i64().unwrap_or(0) as i32, - b["y"].as_i64().unwrap_or(0) as i32, - b["width"].as_i64().unwrap_or(0) as i32, - b["height"].as_i64().unwrap_or(0) as i32, - ); - if pid != 0 && ww > 0 && h > 0 { - driver.reaper().track_pid(pid); - return Some((pid, x + ww / 2, y + h / 2)); - } - } - } - std::thread::sleep(Duration::from_millis(500)); - } - eprintln!("[desktop-scope] WPF harness window never appeared — skipping"); - None -} - -fn set_scope(driver: &mut McpDriver, scope: &str) { - let r = driver.call( - "set_config", - serde_json::json!({ "key": "capture_scope", "value": scope }), - ); - assert!(!r.is_error(), "set_config capture_scope={scope} failed: {}", r.text()); - assert_eq!( - r.structured()["capture_scope"].as_str(), - Some(scope), - "set_config did not report capture_scope={scope}: {}", - r.text() - ); -} - -// ── tests ───────────────────────────────────────────────────────────────────── - -/// `get_desktop_state` in desktop scope returns a full-display capture with -/// real screen dimensions (the Session-0 `handle is invalid` case is only the -/// service-session wall; this needs a real interactive desktop). -#[test] -#[ignore] -fn desktop_scope_capture_returns_screen_dims() { - let Some(mut driver) = McpDriver::spawn() else { return }; - set_scope(&mut driver, "desktop"); - let r = driver.call("get_desktop_state", serde_json::json!({})); - assert!(!r.is_error(), "get_desktop_state errored: {}", r.text()); - let sw = r.structured()["screen_width"].as_u64().unwrap_or(0); - let sh = r.structured()["screen_height"].as_u64().unwrap_or(0); - assert!(sw > 0 && sh > 0, "get_desktop_state returned no/zero screen size: {}", r.text()); - eprintln!("[desktop-scope] get_desktop_state OK — screen {sw}x{sh}"); -} - -/// In desktop scope, a window-less screen-absolute click + scroll succeed and -/// resolve a real window via WindowFromPoint (no pid/window_id supplied). -#[test] -#[ignore] -fn desktop_scope_windowless_click_and_scroll_land() { - let Some(mut driver) = McpDriver::spawn() else { return }; - set_scope(&mut driver, "desktop"); - let Some((_pid, cx, cy)) = launch_wpf_and_center(&mut driver) else { return }; - - let clicked = driver.call("click", serde_json::json!({ "x": cx, "y": cy })); - assert!(!clicked.is_error(), "desktop-scope click errored: {}", clicked.text()); - let ct = clicked.text().to_lowercase(); - assert!(ct.contains("desktop scope"), "click not reported as desktop-scope: {}", clicked.text()); - assert!(ct.contains("hwnd"), "click did not resolve a window via WindowFromPoint: {}", clicked.text()); - - let scrolled = driver.call( - "scroll", - serde_json::json!({ "x": cx, "y": cy, "direction": "down" }), - ); - assert!(!scrolled.is_error(), "desktop-scope scroll errored: {}", scrolled.text()); - assert!( - scrolled.text().to_lowercase().contains("desktop scope"), - "scroll not reported as desktop-scope: {}", - scrolled.text() - ); -} - -/// Negative gate: a window-less screen-absolute click under `capture_scope=window` -/// must be rejected (the `desktop_scope_disabled` contract), not silently retargeted. -#[test] -#[ignore] -fn window_scope_rejects_windowless_click() { - let Some(mut driver) = McpDriver::spawn() else { return }; - set_scope(&mut driver, "window"); - let r = driver.call("click", serde_json::json!({ "x": 100, "y": 100 })); - let txt = r.text().to_lowercase(); - assert!( - r.is_error() || txt.contains("desktop scope") || txt.contains("desktop_scope_disabled"), - "window-scope window-less click was NOT rejected: {}", - r.text() - ); -} diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_dispatch_linux_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/modality_dispatch_linux_test.rs deleted file mode 100644 index 26d71cf6fd..0000000000 --- a/libs/cua-driver/rust/crates/cua-driver/tests/modality_dispatch_linux_test.rs +++ /dev/null @@ -1,61 +0,0 @@ -//! Modality axis: the Linux dispatch ladder (delivery_mode parity). -//! -//! Asserts the contract the Linux parity work added (X11 background/foreground -//! + bring_to_front). Uses the binary's own `describe` (which computes each -//! ToolDef schema locally) rather than a daemon round-trip, so it reflects the -//! freshly-built binary and runs in CI without a display or daemon. - -#![cfg(target_os = "linux")] - -use std::path::PathBuf; -use std::process::Command; - -fn driver_bin() -> PathBuf { - // tests run with CARGO_MANIFEST_DIR = crates/cua-driver; the workspace - // target/debug is two levels up. - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../target/debug/cua-driver") -} - -/// `cua-driver describe ` → the tool's advertised inputSchema as text. -/// Returns None (test skips) when the binary isn't built. -fn describe(tool: &str) -> Option { - let bin = driver_bin(); - if !bin.exists() { - eprintln!("[dispatch-linux] {bin:?} not built — skipping"); - return None; - } - let out = Command::new(&bin).arg("describe").arg(tool).output().ok()?; - Some(String::from_utf8_lossy(&out.stdout).into_owned()) -} - -/// Every Linux input tool advertises `delivery_mode` with the two-mode enum — -/// the per-call background/foreground rung selector (parity with macOS/Windows). -#[test] -fn linux_input_tools_advertise_delivery_mode() { - for tool in ["click", "type_text", "press_key", "hotkey", "double_click", "right_click", "scroll"] { - let Some(schema) = describe(tool) else { return }; - assert!( - schema.contains("delivery_mode"), - "{tool} schema is missing delivery_mode:\n{schema}" - ); - assert!( - schema.contains("foreground") && schema.contains("background"), - "{tool} delivery_mode enum should include background + foreground:\n{schema}" - ); - } -} - -/// `bring_to_front` is a real EWMH activation now (not the old Windows-only -/// stub) — its description must reflect the X11 _NET_ACTIVE_WINDOW path. -#[test] -fn linux_bring_to_front_is_real_not_windows_only_stub() { - let Some(schema) = describe("bring_to_front") else { return }; - assert!( - !schema.contains("Windows-only"), - "bring_to_front should be a real Linux EWMH activation now, not a Windows-only stub:\n{schema}" - ); - assert!( - schema.contains("_NET_ACTIVE_WINDOW") || schema.to_lowercase().contains("activate"), - "bring_to_front description should mention the activation mechanism:\n{schema}" - ); -} diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_dispatch_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/modality_dispatch_test.rs deleted file mode 100644 index e9332c5cfa..0000000000 --- a/libs/cua-driver/rust/crates/cua-driver/tests/modality_dispatch_test.rs +++ /dev/null @@ -1,120 +0,0 @@ -//! Modality axis: the best-effort-background dispatch ladder (macOS). -//! -//! Covers the contract introduced by the modality-ladder work: -//! - `delivery_mode` (background | foreground) on the input tools, -//! - `scope` (window | desktop) as a per-call param on `click`, -//! - the `{path, verified}` structured outcome, -//! - `bring_to_front` as a real macOS activation (no longer a Windows-only stub). -//! -//! The schema assertions are deterministic and run in CI (they only inspect -//! `tools/list`). The end-to-end ladder behavior is `#[ignore]` — it needs a -//! real focus-sensitive app + a GUI session, so it runs interactively / on the -//! VMs, asserting per-rung outcomes via the testkit `path()` / `verified()` -//! accessors instead of scraping result text. - -#![cfg(target_os = "macos")] - -use cua_driver_testkit::RawDriver; - -/// `delivery_mode` (type_text + click) and `scope` (click) are advertised with -/// their enums in `tools/list`. Consumers branch on these being present. -#[test] -fn dispatch_and_scope_schema_advertised() { - let Some(mut d) = RawDriver::spawn() else { return }; - d.send(&serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}})); - d.recv(); - d.send(&serde_json::json!({"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}})); - let resp = d.recv(); - let tools = resp["result"]["tools"].as_array().expect("tools array"); - - let props = |name: &str| { - tools.iter().find(|t| t["name"] == name) - .unwrap_or_else(|| panic!("{name} not in tools/list")) - ["inputSchema"]["properties"].clone() - }; - - let enum_values = |v: &serde_json::Value| -> Vec { - v["enum"].as_array() - .map(|a| a.iter().filter_map(|x| x.as_str().map(str::to_owned)).collect()) - .unwrap_or_default() - }; - - for tool in ["type_text", "click"] { - let dm = props(tool)["delivery_mode"].clone(); - let en = enum_values(&dm); - assert!( - en.iter().any(|s| s == "background") && en.iter().any(|s| s == "foreground"), - "{tool}.delivery_mode enum should be [background, foreground], got {dm:?}" - ); - } - - let scope = props("click")["scope"].clone(); - let sen = enum_values(&scope); - assert!( - sen.iter().any(|s| s == "window") && sen.iter().any(|s| s == "desktop"), - "click.scope enum should be [window, desktop], got {scope:?}" - ); -} - -/// `bring_to_front` is a real macOS activation now (the user's change), not the -/// old "Windows-only" error stub — its description must reflect that. -#[test] -fn bring_to_front_is_macos_native_not_windows_only() { - let Some(mut d) = RawDriver::spawn() else { return }; - d.send(&serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}})); - d.recv(); - d.send(&serde_json::json!({"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}})); - let resp = d.recv(); - let tools = resp["result"]["tools"].as_array().expect("tools array"); - let btf = tools.iter().find(|t| t["name"] == "bring_to_front") - .expect("bring_to_front not in tools/list"); - let desc = btf["description"].as_str().unwrap_or(""); - assert!( - !desc.contains("Windows-only"), - "bring_to_front should be a real macOS activation now, not Windows-only: {desc}" - ); -} - -// ── End-to-end ladder behavior (interactive; needs a GUI session) ──────────── - -/// On a NATIVE Cocoa field (TextEdit), `delivery_mode:"background"` lands via the -/// AX value-write and the driver confirms it: `path:"ax", verified:true`. This is -/// the driver-verifiable happy path — no foreground needed, no screenshot needed. -#[test] -#[ignore] -fn background_type_on_native_cocoa_is_ax_verified() { - use cua_driver_testkit::{Driver, McpDriver}; - let Some(mut driver) = McpDriver::spawn() else { return }; - - // Launch TextEdit and open a blank document. - let launch = driver.call("launch_app", serde_json::json!({ "bundle_id": "com.apple.TextEdit" })); - if launch.is_error() { - eprintln!("[dispatch] could not launch TextEdit — skipping"); - return; - } - let pid = launch.structured()["pid"].as_i64().expect("pid"); - let windows = launch.structured()["windows"].as_array().cloned().unwrap_or_default(); - let Some(wid) = windows.first().and_then(|w| w["window_id"].as_u64()) else { - eprintln!("[dispatch] TextEdit opened no window — skipping"); - return; - }; - - // Find the AXTextArea. - let state = driver.call("get_window_state", - serde_json::json!({ "pid": pid, "window_id": wid, "capture_mode": "ax" })); - let el = state.structured()["elements"].as_array().and_then(|els| { - els.iter().find(|e| e["role"] == "AXTextArea").and_then(|e| e["element_index"].as_u64()) - }); - let Some(el) = el else { - eprintln!("[dispatch] no AXTextArea in TextEdit (AX permission?) — skipping"); - return; - }; - - let typed = driver.call("type_text", serde_json::json!({ - "pid": pid, "window_id": wid, "element_index": el, - "text": "ladder", "delivery_mode": "background" - })); - assert!(!typed.is_error(), "type_text errored: {}", typed.text()); - assert_eq!(typed.path(), Some("ax"), "native Cocoa field should land via AX: {}", typed.text()); - assert_eq!(typed.verified(), Some(true), "AX write should read back as verified: {}", typed.text()); -} diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_focus_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/modality_focus_test.rs deleted file mode 100644 index c6d6eda124..0000000000 --- a/libs/cua-driver/rust/crates/cua-driver/tests/modality_focus_test.rs +++ /dev/null @@ -1,172 +0,0 @@ -//! Integration test: verify that background automation does NOT steal focus. -//! -//! Test strategy: -//! 1. Open a "focus check window" (Terminal) and confirm it is focused. -//! 2. Start a cua-driver MCP server (via the shared `McpDriver` testkit). -//! 3. Send automation actions (click, type_text) to Calculator (a different app). -//! 4. After each action, verify the Terminal window is STILL the active window. -//! -//! Requires: macOS, Calculator (com.apple.calculator), Accessibility permission, -//! and a built `cua-driver` (`cargo build` first). -//! -//! Run with: cargo test --test focus_check_test focus_not_stolen -- --ignored --nocapture - -#[cfg(target_os = "macos")] -mod macos_focus_tests { - use cua_driver_testkit::{Driver, McpDriver}; - use std::process::Command; - use std::thread; - use std::time::Duration; - - /// Get the bundle ID of the current frontmost app via osascript. - fn frontmost_bundle_id() -> String { - let out = Command::new("osascript") - .arg("-e") - .arg(r#"tell application "System Events" to bundle identifier of first application process whose frontmost is true"#) - .output() - .expect("osascript"); - String::from_utf8_lossy(&out.stdout).trim().to_owned() - } - - /// Bring Terminal to the front (our "focus check window"). - fn focus_terminal() { - let _ = Command::new("osascript") - .arg("-e") - .arg(r#"tell application "Terminal" to activate"#) - .output(); - thread::sleep(Duration::from_millis(300)); - } - - /// Open Calculator in background. - fn open_calculator_background() { - let _ = Command::new("open") - .args(["-g", "-b", "com.apple.calculator"]) - .output(); - thread::sleep(Duration::from_secs(1)); - } - - fn find_calculator_pid(driver: &mut McpDriver) -> Option { - let resp = driver.call("list_apps", serde_json::json!({})); - // Use structuredContent.apps array (preferred over text parsing). - if let Some(apps) = resp.structured()["apps"].as_array() { - for app in apps { - let bundle = app["bundle_id"].as_str().unwrap_or(""); - let name = app["name"].as_str().unwrap_or(""); - if bundle.contains("calculator") || name.eq_ignore_ascii_case("calculator") { - if let Some(pid) = app["pid"].as_i64() { - return Some(pid as i32); - } - } - } - } - // Fallback: parse text content for older binary versions. - for line in resp.text().lines() { - if line.contains("com.apple.calculator") || line.contains("Calculator") { - if let Some(pid_str) = line.split("(pid ").nth(1).and_then(|s| s.split(')').next()) { - return pid_str.trim().parse().ok(); - } - } - } - None - } - - #[test] - #[ignore] // Run explicitly: cargo test --test focus_check_test focus_not_stolen -- --ignored --nocapture - fn focus_not_stolen_during_calculator_click() { - // Setup: open Calculator in background, bring Terminal to front. - open_calculator_background(); - focus_terminal(); - - let initial_focus = frontmost_bundle_id(); - println!("Initial focus: {}", initial_focus); - - // Precondition: the decoy (Terminal) must REALLY be the frontmost, - // non-harness window before we touch the background app. Without this - // guard the test passes vacuously: if `focus_terminal()` silently - // failed (Terminal not installed / activation blocked) and Calculator - // were frontmost instead, every "focus unchanged" assertion below would - // still hold — even though the harness owns the foreground, which is - // exactly the regression these tests exist to catch. Mirrors the - // sentinel-is-up precondition the Windows background tests get for free - // from the focus-monitor-win pid/hwnd files. - assert!( - !initial_focus.is_empty() - && !initial_focus.eq_ignore_ascii_case("com.apple.calculator"), - "decoy precondition failed: frontmost app is {initial_focus:?}, expected a \ - non-harness control window (Terminal) to hold the foreground before the \ - background action. Calculator (the harness) must NOT be frontmost, otherwise \ - the no-focus-steal assertions below are vacuous." - ); - - // Start the MCP driver (skips if the binary isn't built). - let Some(mut driver) = McpDriver::spawn() else { return }; - - // Find Calculator. - let calc_pid = match find_calculator_pid(&mut driver) { - Some(p) => p, - None => { - eprintln!("Calculator not found in running apps"); - return; - } - }; - println!("Calculator pid: {}", calc_pid); - - // Get Calculator's window. - let resp = driver.call("list_windows", serde_json::json!({ "pid": calc_pid })); - let windows = resp.structured()["windows"].as_array().expect("windows array"); - if windows.is_empty() { - eprintln!("No windows for Calculator"); - return; - } - let window_id = windows[0]["window_id"].as_u64().unwrap() as u32; - println!("Calculator window_id: {}", window_id); - - // Walk AX tree. - let resp = driver.call( - "get_window_state", - serde_json::json!({ "pid": calc_pid, "window_id": window_id, "capture_mode": "ax" }), - ); - println!( - "get_window_state: {}", - resp.text().chars().take(200).collect::() - ); - - // Verify focus hasn't been stolen yet. - let focus_after_get = frontmost_bundle_id(); - println!("Focus after get_window_state: {}", focus_after_get); - assert_eq!( - focus_after_get, initial_focus, - "get_window_state STOLE FOCUS! Was: {}, now: {}", initial_focus, focus_after_get - ); - - // Click element_index 1 (a Calculator button supporting AXPress; [0] is - // the AXWindow itself, which only raises). - let resp = driver.call( - "click", - serde_json::json!({ "pid": calc_pid, "window_id": window_id, "element_index": 1 }), - ); - println!("click result: {}", resp.text()); - thread::sleep(Duration::from_millis(200)); - - // CRITICAL: verify focus was not stolen. - let focus_after_click = frontmost_bundle_id(); - println!("Focus after click: {}", focus_after_click); - assert_eq!( - focus_after_click, initial_focus, - "click() STOLE FOCUS! Expected {} to remain focused, got {}", initial_focus, focus_after_click - ); - - // type_text also must not steal focus. - let resp = driver.call("type_text", serde_json::json!({ "pid": calc_pid, "text": "5" })); - println!("type_text result: {}", resp.text()); - thread::sleep(Duration::from_millis(200)); - let focus_after_type = frontmost_bundle_id(); - println!("Focus after type_text: {}", focus_after_type); - assert_eq!( - focus_after_type, initial_focus, - "type_text() STOLE FOCUS! Expected {} to remain focused, got {}", initial_focus, focus_after_type - ); - - println!("✅ Focus was NOT stolen by click or type_text. Background automation confirmed."); - } -} diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/modality_input_e2e_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/modality_input_e2e_test.rs deleted file mode 100644 index 006bde69a1..0000000000 --- a/libs/cua-driver/rust/crates/cua-driver/tests/modality_input_e2e_test.rs +++ /dev/null @@ -1,579 +0,0 @@ -//! End-to-end Windows test for the **unified background input interface**: -//! a caller targets an app and plays click/key actions WITHOUT knowing the -//! app's internals (Electron/Chromium, Tauri/WebView2, classic Win32) and -//! WITHOUT the target window ever being raised to the foreground. -//! -//! Verified two ways per action: -//! 1. The tool succeeds in the DEFAULT dispatch mode — no -//! `background_unavailable` error, no `dispatch:"foreground"` needed. -//! 2. The `focus-monitor-win` sentinel records ZERO foreground losses across -//! the action == the target window was not z-raised over the user's -//! window (same oracle as `harness_bg_modality_test`). -//! -//! ## Process hygiene -//! The driver, target app, and sentinel are all owned by the shared testkit -//! `ChildReaper` (via `McpDriver`), which on Windows assigns them to a -//! kill-on-close Job Object — the OS reaps the whole tree when the test process -//! exits for ANY reason (normal, panic, SIGKILL), so no orphaned windows or -//! held ports. Broker-spawned window pids (packaged apps / Electron) are -//! tree-killed via `reaper().track_pid`. -//! -//! Targets (repo-local harness apps staged by `tests/fixtures/build/windows.ps1`): -//! - Electron (Chromium content) — `CuaTestHarness.Electron.exe`. -//! - Tauri (WebView2 content) — `CuaTestHarness.Tauri.exe`. -//! - Win32 baseline — `notepad.exe`. -//! Override with `HARNESS_ELECTRON_EXE` / `HARNESS_TAURI_EXE` -//! (or legacy `CUA_ELECTRON_EXE` / `CUA_TAURI_EXE`). -//! -//! All tests are `#[ignore]` (GUI, real desktop session). Run explicitly, -//! serially (GUI focus oracle, so never in parallel): -//! cargo test -p cua-driver --test e2e_windows_bg_input_test -- --ignored --nocapture --test-threads=1 - -#![cfg(target_os = "windows")] - -use std::collections::HashSet; -use std::fs; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::time::{Duration, Instant}; - -use cua_driver_testkit::{ax, workspace_root, Driver, McpDriver}; - -// ── paths ──────────────────────────────────────────────────────────────────── - -fn focus_monitor_binary() -> PathBuf { - workspace_root().join("target/debug/focus-monitor-win.exe") -} - -/// Best-effort kill of any prior instance of `exe` by basename, so a leftover -/// from an earlier (e.g. force-killed) run can't hold the app's fixed HTTP port. -fn kill_prior_by_name(exe: &Path) { - if let Some(name) = exe.file_name().and_then(|n| n.to_str()) { - let _ = Command::new("taskkill") - .args(["/F", "/T", "/IM", name]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - } -} - -fn resolve_harness(primary_env: &str, legacy_env: &str, rel_path: &str) -> Option { - for env_var in [primary_env, legacy_env] { - if let Ok(p) = std::env::var(env_var) { - let pb = PathBuf::from(p); - if pb.exists() { - return Some(pb); - } - } - } - let pb = workspace_root().join(rel_path); - pb.exists().then_some(pb) -} -fn electron_exe() -> Option { - resolve_harness( - "HARNESS_ELECTRON_EXE", - "CUA_ELECTRON_EXE", - "test-apps/harness-electron/CuaTestHarness.Electron.exe", - ) -} -fn tauri_exe() -> Option { - resolve_harness( - "HARNESS_TAURI_EXE", - "CUA_TAURI_EXE", - "test-apps/harness-tauri/CuaTestHarness.Tauri.exe", - ) -} - -fn loss_file() -> PathBuf { - std::env::temp_dir().join("focus_monitor_losses.txt") -} -fn key_loss_file() -> PathBuf { - std::env::temp_dir().join("focus_monitor_key_losses.txt") -} -fn focus_pid_file() -> PathBuf { - std::env::temp_dir().join("focus_monitor_pid.txt") -} -fn focus_hwnd_file() -> PathBuf { - std::env::temp_dir().join("focus_monitor_hwnd.txt") -} -fn read_count(p: &Path) -> u32 { - fs::read_to_string(p) - .ok() - .and_then(|s| s.trim().parse::().ok()) - .unwrap_or(0) -} -fn read_focus_hwnd() -> u64 { - fs::read_to_string(focus_hwnd_file()) - .ok() - .and_then(|s| s.trim().parse::().ok()) - .unwrap_or(0) -} - -fn window_ids(driver: &mut McpDriver) -> HashSet { - let r = driver.call("list_windows", serde_json::json!({})); - r.structured()["windows"] - .as_array() - .map(|a| a.iter().filter_map(|w| w["window_id"].as_u64()).collect()) - .unwrap_or_default() -} - -fn gui_required() -> bool { - std::env::var("CUA_REQUIRE_GUI") - .ok() - .map(|v| matches!(v.to_ascii_lowercase().as_str(), "1" | "true" | "yes" | "on")) - .unwrap_or(false) -} - -fn skip_desktop(context: &str, reason: String) -> bool { - let msg = format!("{context}: {reason}; skipping GUI modality e2e"); - if gui_required() { - panic!("{msg}"); - } - eprintln!("{msg}"); - false -} - -fn require_seedable_desktop(context: &str) -> bool { - let state = platform_windows::diagnostics::desktop_state(); - if state.session_id == Some(0) { - return skip_desktop( - context, - format!( - "running in Windows Session 0 ({}) - re-run from RDP/console/scheduled task in user session", - state.summary() - ), - ); - } - if !state.has_process_window_station { - return skip_desktop( - context, - format!("no attached process window station ({})", state.summary()), - ); - } - if !state.input_desktop_is_default() { - return skip_desktop( - context, - format!( - "input desktop is not the user Default desktop ({})", - state.summary() - ), - ); - } - true -} - -fn require_focus_monitor_foreground(context: &str, expected_hwnd: u64) -> bool { - let deadline = Instant::now() + Duration::from_secs(2); - loop { - let state = platform_windows::diagnostics::desktop_state(); - if state.foreground_hwnd == Some(expected_hwnd as usize) { - return true; - } - if Instant::now() >= deadline { - return skip_desktop( - context, - format!( - "focus monitor did not become foreground (expected HWND 0x{expected_hwnd:x}; {})", - state.summary() - ), - ); - } - std::thread::sleep(Duration::from_millis(100)); - } -} - -// ── fixture ─────────────────────────────────────────────────────────────────── - -struct E2eFixture { - driver: McpDriver, // owns + reaps target+sentinel+driver on drop - pid: u32, - wid: u64, -} - -/// Launch order: driver (no window) → snapshot windows → app → discover the -/// app's NEW window (works for multi-process apps like Electron whose window -/// belongs to a child pid) → sentinel (grabs foreground, pushing the app to -/// the background) → reset counters. -fn setup(target_exe: &Path, _title_hint: &str) -> Option { - if !require_seedable_desktop("modality_input_e2e_test") { - return None; - } - - let fm_bin = focus_monitor_binary(); - if !fm_bin.exists() { - eprintln!("[e2e] focus-monitor-win.exe not built — skipping"); - return None; - } - if target_exe.is_absolute() && !target_exe.exists() { - eprintln!("[e2e] target {target_exe:?} missing — skipping"); - return None; - } - - // Defensive: clear any leftover instance holding the app's fixed port. - kill_prior_by_name(target_exe); - let _ = fs::write(loss_file(), "0"); - let _ = fs::write(key_loss_file(), "0"); - let _ = fs::remove_file(focus_pid_file()); - let _ = fs::remove_file(focus_hwnd_file()); - - // 1. Driver (daemon, no visible window) — spawned + initialized by the testkit. - let mut driver = McpDriver::spawn()?; - - // 2. Snapshot existing windows, then launch the target app into the reaper. - let before = window_ids(&mut driver); - if driver - .reaper() - .spawn( - Command::new(target_exe) - .stdout(Stdio::null()) - .stderr(Stdio::null()), - ) - .is_err() - { - eprintln!("[e2e] target spawn failed"); // driver drops → kills itself - return None; - } - - // 3. Discover the app's NEW window (its pid may be a child of the launched - // process — Electron/WebView2 are multi-process). - let deadline = Instant::now() + Duration::from_secs(15); - let mut found: Option<(u32, u64)> = None; - while Instant::now() < deadline { - let r = driver.call("list_windows", serde_json::json!({})); - if let Some(arr) = r.structured()["windows"].as_array() { - for w in arr { - let Some(wid) = w["window_id"].as_u64() else { - continue; - }; - let pid = w["pid"].as_u64().unwrap_or(0) as u32; - let title = w["title"].as_str().unwrap_or(""); - if !before.contains(&wid) && !title.is_empty() && pid != 0 { - found = Some((pid, wid)); - break; - } - } - } - if found.is_some() { - break; - } - std::thread::sleep(Duration::from_millis(500)); - } - let (pid, wid) = match found { - Some(p) => p, - None => { - eprintln!("[e2e] app window never appeared — skipping"); - return None; - } - }; - // The window's process is often broker-spawned (packaged apps, Electron), - // i.e. NOT our direct child. track_pid job-assigns it (hard-kill safety) and - // tree-kills it on teardown so it can't orphan. - driver.reaper().track_pid(pid); - - // 4. Sentinel grabs foreground; app drops to z+1 (the background target). - if driver - .reaper() - .spawn( - Command::new(&fm_bin) - .stdout(Stdio::null()) - .stderr(Stdio::null()), - ) - .is_err() - { - eprintln!("[e2e] sentinel spawn failed — skipping"); - return None; - } - let sdeadline = Instant::now() + Duration::from_secs(10); - loop { - let ok = read_count(&focus_pid_file()) != 0 && read_focus_hwnd() != 0; - if ok { - break; - } - if Instant::now() > sdeadline { - eprintln!("[e2e] sentinel never published — skipping"); - return None; - } - std::thread::sleep(Duration::from_millis(100)); - } - std::thread::sleep(Duration::from_millis(400)); - if !require_focus_monitor_foreground("modality_input_e2e_test sentinel", read_focus_hwnd()) { - return None; - } - let _ = fs::write(loss_file(), "0"); - let _ = fs::write(key_loss_file(), "0"); - - Some(E2eFixture { driver, pid, wid }) -} - -// ── DOM registration oracle (trycua test apps serve an event log on 6769) ───── - -const APP_API: &str = "http://127.0.0.1:6769"; - -fn http_reset() { - let _ = Command::new("curl.exe") - .args(["-s", "-m", "3", "-X", "POST", &format!("{APP_API}/reset")]) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); -} -/// Returns the `/events` body, or None if the app isn't serving the API. -fn http_events() -> Option { - let out = Command::new("curl.exe") - .args(["-s", "-m", "3", &format!("{APP_API}/events")]) - .output() - .ok()?; - if !out.status.success() { - return None; - } - let body = String::from_utf8_lossy(&out.stdout).into_owned(); - if body.trim_start().starts_with('[') { - Some(body) - } else { - None - } -} -/// True if the event log recorded at least one DOM event since the last reset. -fn registered_since_reset() -> Option { - http_events().map(|b| b.trim() != "[]" && !b.trim().is_empty()) -} - -fn foreground_pid() -> u32 { - use windows::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, GetWindowThreadProcessId}; - unsafe { - let h = GetForegroundWindow(); - let mut pid = 0u32; - GetWindowThreadProcessId(h, Some(&mut pid)); - pid - } -} - -/// The reliable "no z-raise / no foreground steal" oracle, independent of the -/// machine's foreground-lock setting: after acting on a BACKGROUND target, the -/// target's process must never become the foreground window. Polls for ~1.2s -/// to also catch async self-reactivation (Chromium). -fn assert_target_stays_background(label: &str, target_pid: u32, action: F) { - let user_pid = read_count(&focus_pid_file()); - let fg_before = foreground_pid(); - if fg_before == target_pid { - eprintln!("[e2e] WARN {label}: target pid {target_pid} was already foreground before the action (setup couldn't background it)"); - } - action(); - let deadline = Instant::now() + Duration::from_millis(1200); - let mut stole = false; - let mut last_fg = fg_before; - while Instant::now() < deadline { - last_fg = foreground_pid(); - if last_fg == target_pid { - stole = true; - break; - } - std::thread::sleep(Duration::from_millis(80)); - } - assert!( - !stole, - "{label}: target pid {target_pid} BECAME the foreground window — z-raise / foreground steal. \ - (expected user pid {user_pid}, fg_before={fg_before})" - ); - println!("✅ {label}: target pid {target_pid} stayed background (foreground pid={last_fg}, user={user_pid})"); -} - -/// Shared body: default-mode left click into a webview target. -fn webview_click_case(label: &str, exe: PathBuf) { - let mut fx = match setup(&exe, "") { - Some(f) => f, - None => return, - }; - let (pid, wid) = (fx.pid, fx.wid); - let snap = fx.driver.call( - "get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}), - ); - let elem = ax::element_index_containing(snap.text(), "button") - .or_else(|| ax::element_index_containing(snap.text(), "click")); - - http_reset(); - let mut delivered = String::new(); - let mut errored = false; - let mut needs_foreground = false; - assert_target_stays_background(label, pid, || { - let args = match elem { - Some(idx) => { - serde_json::json!({"pid": pid as i64, "window_id": wid, "element_index": idx}) - } - None => serde_json::json!({"pid": pid as i64, "window_id": wid, "x": 200, "y": 200}), - }; - let last = fx.driver.call("click", args); - errored = last.is_error(); - needs_foreground = last.text().contains("background_unavailable"); - delivered = last.text().to_string(); - }); - assert!(!errored, "{label}: default-mode click errored: {delivered}"); - assert!( - !needs_foreground, - "{label}: click should not need dispatch:foreground, got {delivered:?}" - ); - // Delivery is confirmed by the driver's own ✅ result (UIA Invoke fires the - // element's default action / DOM `click`). The /events pointer-log is info - // only — it does NOT capture accessibility-driven UIA Invoke, so a `[]` - // there is expected for the UIA path. The hard invariant is no-foreground-steal. - std::thread::sleep(Duration::from_millis(300)); - let reg = registered_since_reset() - .map(|b| b.to_string()) - .unwrap_or_else(|| "n/a".into()); - println!("{label}: delivered={delivered:?} pointer-events-logged={reg}"); -} - -// ── tests ───────────────────────────────────────────────────────────────────── - -#[test] -#[ignore] -fn e2e_electron_background_click_no_z_raise() { - let Some(exe) = electron_exe() else { - eprintln!("[e2e] no electron app — skipping"); - return; - }; - webview_click_case("electron click (default dispatch)", exe); -} - -#[test] -#[ignore] -fn e2e_tauri_background_click_no_z_raise() { - let Some(exe) = tauri_exe() else { - eprintln!("[e2e] no tauri app — skipping"); - return; - }; - webview_click_case("tauri click (default dispatch)", exe); -} - -#[test] -#[ignore] -fn e2e_win32_notepad_background_click_no_z_raise() { - let mut fx = match setup(Path::new(r"C:\Windows\System32\notepad.exe"), "") { - Some(f) => f, - None => return, - }; - let (pid, wid) = (fx.pid, fx.wid); - let mut last_text = String::new(); - let mut errored = false; - assert_target_stays_background("notepad click (default dispatch)", pid, || { - let r = fx.driver.call( - "click", - serde_json::json!({"pid": pid as i64, "window_id": wid, "x": 120, "y": 120}), - ); - errored = r.is_error(); - last_text = r.text().to_string(); - }); - assert!(!errored, "notepad click errored: {last_text}"); - println!("notepad click result: {last_text:?}"); -} - -/// Electron right-click (pen-barrel injection): no raise. Pen→right promotion -/// is app-dependent, so landing is best-effort; the hard invariant is no -/// z-raise + no background_unavailable. -#[test] -#[ignore] -fn e2e_electron_background_right_click_no_z_raise() { - let Some(exe) = electron_exe() else { - eprintln!("[e2e] no electron app — skipping"); - return; - }; - let mut fx = match setup(&exe, "") { - Some(f) => f, - None => return, - }; - let (pid, wid) = (fx.pid, fx.wid); - let mut last_text = String::new(); - assert_target_stays_background("electron right-click (default dispatch)", pid, || { - let r = fx.driver.call( - "click", - serde_json::json!({"pid": pid as i64, "window_id": wid, "x": 200, "y": 200, "button": "right"}), - ); - last_text = r.text().to_string(); - }); - println!("electron right-click result: {last_text:?}"); -} - -/// Electron TEXT typing: focus a field, then type — must register the text in -/// the DOM AND never steal foreground / move the cursor. -#[test] -#[ignore] -fn e2e_electron_background_type_text_no_z_raise() { - let Some(exe) = electron_exe() else { - eprintln!("[e2e] no electron app — skipping"); - return; - }; - let mut fx = match setup(&exe, "") { - Some(f) => f, - None => return, - }; - let (pid, wid) = (fx.pid, fx.wid); - - // Focus a text field if the page exposes one (so the chars have a sink). - let snap = fx.driver.call( - "get_window_state", - serde_json::json!({"pid": pid as i64, "window_id": wid, "capture_mode": "ax"}), - ); - let field = ax::element_index_containing(snap.text(), "edit") - .or_else(|| ax::element_index_containing(snap.text(), "text")) - .or_else(|| ax::element_index_containing(snap.text(), "input")); - if let Some(idx) = field { - let _ = fx.driver.call( - "click", - serde_json::json!({"pid": pid as i64, "window_id": wid, "element_index": idx}), - ); - std::thread::sleep(Duration::from_millis(200)); - } - - http_reset(); - let mut last_text = String::new(); - let mut errored = false; - assert_target_stays_background("electron type_text (default dispatch)", pid, || { - let r = fx.driver.call( - "type_text", - serde_json::json!({"pid": pid as i64, "window_id": wid, "text": "cuatest"}), - ); - errored = r.is_error(); - last_text = r.text().to_string(); - }); - assert!(!errored, "type_text errored: {last_text}"); - std::thread::sleep(Duration::from_millis(300)); - if let Some(reg) = registered_since_reset() { - println!("electron type_text: registered={reg} result={last_text:?}"); - } else { - println!("electron type_text result: {last_text:?}"); - } -} - -/// Electron key-combo (Ctrl+A): background Chromium key-combos are currently -/// outside the safe delivery capability. The contract is an explicit refusal, -/// with no z-raise or silent partial chord. -#[test] -#[ignore] -fn e2e_electron_background_keycombo_no_z_raise() { - let Some(exe) = electron_exe() else { - eprintln!("[e2e] no electron app — skipping"); - return; - }; - let mut fx = match setup(&exe, "") { - Some(f) => f, - None => return, - }; - let (pid, wid) = (fx.pid, fx.wid); - let mut last_text = String::new(); - let mut errored = false; - assert_target_stays_background("electron Ctrl+A (default dispatch)", pid, || { - let r = fx.driver.call( - "press_key", - serde_json::json!({"pid": pid as i64, "window_id": wid, "key": "a", "modifiers": ["control"]}), - ); - errored = r.is_error(); - last_text = r.text().to_string(); - }); - // W2: refuse honestly instead of claiming delivery through a route that - // cannot preserve the background/no-focus contract for Chromium. - assert!( - !errored && last_text.contains("Background delivery is not available"), - "Ctrl+A must return a structured background capability refusal, got: {last_text:?}" - ); - println!("electron Ctrl+A refused honestly: {last_text:?}"); -} diff --git a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs index 84c506b597..7cbde076ff 100644 --- a/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs +++ b/libs/cua-driver/rust/crates/cua-driver/tests/protocol_schema_test.rs @@ -2,10 +2,10 @@ //! //! These never invoke a tool — they only inspect the advertised inputSchemas: //! that `type_text_chars` is hidden, the `list_windows.on_screen_only` knob, the -//! `set_agent_cursor_motion` Bezier knobs, and the `set_config.capture_mode` -//! enum. +//! `set_agent_cursor_motion` Bezier knobs, delivery and scope enums, and the +//! `set_config.capture_mode` enum. -#![cfg(any(target_os = "macos", target_os = "windows"))] +#![cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))] use cua_driver_testkit::RawDriver; @@ -16,7 +16,7 @@ fn tools_list_schema_shape() { //! no `type_text_chars` either — the old Windows mirror asserted it was //! exposed, but that had never been run). The advertised schemas must still //! carry their expected knobs. - let Some(mut d) = RawDriver::spawn() else { return; }; + let mut d = RawDriver::spawn().expect("spawn source-built driver for schema test"); d.send(&serde_json::json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{}})); d.recv(); @@ -25,12 +25,60 @@ fn tools_list_schema_shape() { let list_resp = d.recv(); let tools = list_resp["result"]["tools"].as_array().expect("tools array"); + let properties = |name: &str| { + &tools + .iter() + .find(|tool| tool["name"] == name) + .unwrap_or_else(|| panic!("{name} not found in tools/list"))["inputSchema"] + ["properties"] + }; + let enum_contains = |schema: &serde_json::Value, expected: &str| { + schema["enum"] + .as_array() + .map(|values| values.iter().any(|value| value.as_str() == Some(expected))) + .unwrap_or(false) + }; + // Deprecated alias is hidden from tools/list (accepted at invoke time only). assert!( tools.iter().all(|t| t["name"] != "type_text_chars"), "type_text_chars should be hidden from tools/list" ); + for tool in [ + "click", + "double_click", + "right_click", + "type_text", + "press_key", + "hotkey", + "scroll", + ] { + let delivery = &properties(tool)["delivery_mode"]; + assert!( + enum_contains(delivery, "background") && enum_contains(delivery, "foreground"), + "{tool}.delivery_mode should advertise background and foreground: {delivery:?}" + ); + } + // macOS selects desktop coordinates per click. Linux and Windows retain + // the process-level capture_scope gate used by their desktop-state tools. + #[cfg(target_os = "macos")] + { + let scope = &properties("click")["scope"]; + assert!( + enum_contains(scope, "window") && enum_contains(scope, "desktop"), + "click.scope should advertise window and desktop: {scope:?}" + ); + } + #[cfg(any(target_os = "linux", target_os = "windows"))] + { + let capture_scope = &properties("set_config")["capture_scope"]; + assert!( + enum_contains(capture_scope, "window") && enum_contains(capture_scope, "desktop"), + "set_config.capture_scope should advertise window and desktop: {capture_scope:?}" + ); + } + // list_windows schema has on_screen_only. let lw = tools.iter().find(|t| t["name"] == "list_windows") .expect("list_windows not found in tools/list"); @@ -64,3 +112,42 @@ fn tools_list_schema_shape() { "set_config capture_mode should have enum: {:?}", sc["inputSchema"]["properties"]); } } + +#[test] +#[cfg(target_os = "linux")] +fn linux_cursor_motion_knobs_are_applied() { + let mut driver = RawDriver::spawn().expect("spawn source-built Linux driver"); + driver.send(&serde_json::json!({ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": {} + })); + driver.recv(); + + driver.send(&serde_json::json!({ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": { + "name": "set_agent_cursor_motion", + "arguments": { + "session": "schema-linux", + "arc_size": 0.4, + "spring": 0.85, + "glide_duration_ms": 500, + "dwell_after_click_ms": 200, + "idle_hide_ms": 5000 + } + } + })); + let response = driver.recv(); + assert!( + !response["result"]["isError"].as_bool().unwrap_or(false), + "Linux cursor motion update failed: {response:?}" + ); + let structured = &response["result"]["structuredContent"]; + assert_eq!(structured["arc_size"].as_f64(), Some(0.4)); + assert_eq!(structured["glide_duration_ms"].as_f64(), Some(500.0)); + assert_eq!(structured["idle_hide_ms"].as_f64(), Some(5000.0)); +} diff --git a/libs/cua-driver/rust/crates/focus-monitor-win/Cargo.toml b/libs/cua-driver/rust/crates/focus-monitor-win/Cargo.toml deleted file mode 100644 index fbe73e08ef..0000000000 --- a/libs/cua-driver/rust/crates/focus-monitor-win/Cargo.toml +++ /dev/null @@ -1,18 +0,0 @@ -[package] -name = "focus-monitor-win" -version.workspace = true -edition.workspace = true -# Windows-only binary used by integration tests to verify the UX guard: -# clicking/typing into background windows must not steal focus from this window. - -[[bin]] -name = "focus-monitor-win" -path = "src/main.rs" - -[target.'cfg(target_os = "windows")'.dependencies] -windows = { version = "0.58", features = [ - "Win32_Foundation", - "Win32_UI_WindowsAndMessaging", - "Win32_System_Threading", - "Win32_Graphics_Gdi", -] } diff --git a/libs/cua-driver/rust/crates/focus-monitor-win/src/main.rs b/libs/cua-driver/rust/crates/focus-monitor-win/src/main.rs deleted file mode 100644 index bc1deb68c5..0000000000 --- a/libs/cua-driver/rust/crates/focus-monitor-win/src/main.rs +++ /dev/null @@ -1,184 +0,0 @@ -/// focus-monitor-win — Windows equivalent of macOS FocusMonitorApp. -/// -/// Creates a visible Win32 window and tracks three kinds of focus loss: -/// -/// 1. WM_ACTIVATE (wParam==WA_INACTIVE): the window loses activation. -/// Written to %TEMP%\focus_monitor_losses.txt -/// -/// 2. WM_KILLFOCUS: the window loses keyboard focus. -/// Written to %TEMP%\focus_monitor_key_losses.txt -/// -/// Prints FOCUS_PID= on stdout at startup so the test harness can -/// discover the process, then prints FOCUS_HWND= so tests -/// can target it with cua-driver tools. -/// -/// Exits cleanly on WM_DESTROY. - -#[cfg(not(target_os = "windows"))] -fn main() { - eprintln!("focus-monitor-win is Windows-only."); - std::process::exit(1); -} - -#[cfg(target_os = "windows")] -mod win { - use std::ffi::OsStr; - use std::os::windows::ffi::OsStrExt; - use std::sync::atomic::{AtomicU32, Ordering}; - use windows::Win32::Foundation::*; - use windows::Win32::Graphics::Gdi::*; - use windows::Win32::System::Threading::GetCurrentProcessId; - use windows::Win32::UI::WindowsAndMessaging::*; - - // ── global loss counters ───────────────────────────────────────────────── - static ACTIVATE_LOSSES: AtomicU32 = AtomicU32::new(0); - static ACTIVATE_GAINS: AtomicU32 = AtomicU32::new(0); - static KEY_LOSSES: AtomicU32 = AtomicU32::new(0); - static KEY_GAINS: AtomicU32 = AtomicU32::new(0); - - fn loss_file() -> std::path::PathBuf { - loss_path("focus_monitor_losses.txt") - } - fn gain_file() -> std::path::PathBuf { - loss_path("focus_monitor_gains.txt") - } - fn key_loss_file() -> std::path::PathBuf { - loss_path("focus_monitor_key_losses.txt") - } - fn key_gain_file() -> std::path::PathBuf { - loss_path("focus_monitor_key_gains.txt") - } - - fn loss_path(name: &str) -> std::path::PathBuf { - let mut p = std::env::temp_dir(); - p.push(name); - p - } - - fn write_count(path: &std::path::Path, n: u32) { - let _ = std::fs::write(path, n.to_string()); - } - - fn wide(s: &str) -> Vec { - OsStr::new(s) - .encode_wide() - .chain(std::iter::once(0)) - .collect() - } - - unsafe extern "system" fn wnd_proc( - hwnd: HWND, - msg: u32, - wparam: WPARAM, - lparam: LPARAM, - ) -> LRESULT { - match msg { - WM_ACTIVATE => { - // WA_INACTIVE == 0 in the low word of wParam; WA_ACTIVE == 1, WA_CLICKACTIVE == 2 - if (wparam.0 & 0xFFFF) == 0 { - let n = ACTIVATE_LOSSES.fetch_add(1, Ordering::SeqCst) + 1; - write_count(&loss_file(), n); - } else { - let n = ACTIVATE_GAINS.fetch_add(1, Ordering::SeqCst) + 1; - write_count(&gain_file(), n); - } - let _ = InvalidateRect(hwnd, None, true); - } - WM_KILLFOCUS => { - let n = KEY_LOSSES.fetch_add(1, Ordering::SeqCst) + 1; - write_count(&key_loss_file(), n); - let _ = InvalidateRect(hwnd, None, true); - } - WM_SETFOCUS => { - let n = KEY_GAINS.fetch_add(1, Ordering::SeqCst) + 1; - write_count(&key_gain_file(), n); - let _ = InvalidateRect(hwnd, None, true); - } - WM_PAINT => { - let mut ps = PAINTSTRUCT::default(); - let hdc = BeginPaint(hwnd, &mut ps); - let act_l = ACTIVATE_LOSSES.load(Ordering::SeqCst); - let act_g = ACTIVATE_GAINS.load(Ordering::SeqCst); - let key_l = KEY_LOSSES.load(Ordering::SeqCst); - let key_g = KEY_GAINS.load(Ordering::SeqCst); - let text = wide(&format!( - "act: {act_l}L / {act_g}G key: {key_l}L / {key_g}G (should stay net 0)" - )); - let _ = TextOutW(hdc, 10, 10, &text); - let _ = EndPaint(hwnd, &ps); - } - WM_DESTROY => { - PostQuitMessage(0); - } - _ => return DefWindowProcW(hwnd, msg, wparam, lparam), - } - LRESULT(0) - } - - pub fn run() { - unsafe { - let class_name = wide("FocusMonitorWin"); - - let wc = WNDCLASSW { - lpfnWndProc: Some(wnd_proc), - hInstance: HINSTANCE(std::ptr::null_mut()), - lpszClassName: windows::core::PCWSTR(class_name.as_ptr()), - hbrBackground: HBRUSH(COLOR_WINDOW.0 as *mut _), - ..Default::default() - }; - RegisterClassW(&wc); - - let title = wide("Focus Monitor (cua-driver UX guard)"); - let hwnd = CreateWindowExW( - WINDOW_EX_STYLE(0), - windows::core::PCWSTR(class_name.as_ptr()), - windows::core::PCWSTR(title.as_ptr()), - WS_OVERLAPPEDWINDOW, - 100, - 100, - 600, - 140, - None, - None, - HINSTANCE(std::ptr::null_mut()), - None, - ) - .expect("CreateWindowExW failed"); - - let _ = ShowWindow(hwnd, SW_SHOWNORMAL); - let _ = UpdateWindow(hwnd).ok(); - - // Write initial zeros so tests can read even before any event. - write_count(&loss_file(), 0); - write_count(&gain_file(), 0); - write_count(&key_loss_file(), 0); - write_count(&key_gain_file(), 0); - - // Signal the test harness via temp files (avoids pipe-blocking issues - // when stdout is captured by the test runner in sandbox environments). - let pid = GetCurrentProcessId(); - let hwnd_val = hwnd.0 as usize; - let pid_file = std::env::temp_dir().join("focus_monitor_pid.txt"); - let hwnd_file = std::env::temp_dir().join("focus_monitor_hwnd.txt"); - let _ = std::fs::write(&pid_file, pid.to_string()); - let _ = std::fs::write(&hwnd_file, hwnd_val.to_string()); - // Also print to stdout as a secondary signal. - println!("FOCUS_PID={pid}"); - println!("FOCUS_HWND={hwnd_val}"); - use std::io::Write; - std::io::stdout().flush().ok(); - - // Message loop. - let mut msg = MSG::default(); - while GetMessageW(&mut msg, None, 0, 0).as_bool() { - let _ = TranslateMessage(&msg); - DispatchMessageW(&msg); - } - } - } -} - -#[cfg(target_os = "windows")] -fn main() { - win::run(); -} diff --git a/libs/cua-driver/rust/crates/platform-linux/Cargo.toml b/libs/cua-driver/rust/crates/platform-linux/Cargo.toml index f8fbedf8e4..d9fc5e5bae 100644 --- a/libs/cua-driver/rust/crates/platform-linux/Cargo.toml +++ b/libs/cua-driver/rust/crates/platform-linux/Cargo.toml @@ -51,10 +51,8 @@ wayland-protocols = { version = "0.32", features = ["client", "staging"] } # (org.freedesktop.portal.RemoteDesktop.ConnectToEIS). `tokio` shares the # driver's async runtime. # -# `screencast` + `remote_desktop` ashpd features pull pipewire-rs (libspa-sys), -# which needs `libpipewire-0.3 >= 0.3.40` headers at build time — newer than -# Debian bullseye ships. They're moved behind the `portal-libei` feature so -# the cross-platform CD (Debian 11, GLIBC_2.31 floor) can keep building. +# ashpd's RemoteDesktop feature is pure Rust. Only our ScreenCast frame reader +# pulls pipewire-rs/libspa, whose headers are newer than Debian bullseye. ashpd = { version = "0.13", default-features = false, features = ["tokio", "screenshot"] } # zbus is pulled transitively via ashpd + atspi but declared directly so # the portal-probe path in wayland::portal_screenshot can talk to the @@ -68,11 +66,11 @@ url = "2" # frames from the node id returned by `open_pipe_wire_remote`. Pinned to # 0.8 (last well-documented release; 0.10 had docs.rs build failures in # mid-2026). Needs libpipewire-0.3 headers at build time — gated behind -# the `portal-libei` feature. +# the `portal-capture` feature. pipewire = { version = "0.8", optional = true } # Companion to pipewire 0.8 — needed to construct the SPA_PARAM_EnumFormat # pod (SPA_VIDEO_FORMAT_BGRx at announced width/height) we hand to -# `Stream::connect`. Same `portal-libei` gating as `pipewire`. +# `Stream::connect`. Same `portal-capture` gating as `pipewire`. libspa = { version = "0.8", optional = true } # Pure-Rust libei/libeis bindings for the GNOME/KDE input path (handshake # → seat → device → pointer_absolute/button/scroll/keyboard/text). API is @@ -107,26 +105,27 @@ dirs = "5" [features] default = [] linux-example = [] -# Opt-in stack that wires the portal ScreenCast per-window capture path -# (`wayland::portal_screencast`) and the libei input path -# (`wayland::libei`). Pulls in `pipewire`, `libspa` (need -# `libpipewire-0.3 >= 0.3.40` headers — newer than Debian bullseye) and -# `reis` (pure-Rust libei) plus the ashpd RemoteDesktop+ScreenCast -# features. Disabled in the cross-platform release CD (debian:11 -# container, GLIBC_2.31 floor) and enabled in the Nix build that -# already has the modern PipeWire + libei from nixpkgs. The wlroots -# screencopy + virtual-pointer paths and the X11 fallback work either -# way — only GNOME/KDE-portal-specific tiers go dark when this is off. -portal-libei = [ - "ashpd/screencast", +# GNOME/KDE input is pure Rust plus libxkbcommon, which is available in the +# Debian 11 release container. Ashpd's RemoteDesktop API exposes ScreenCast +# stream metadata, so its lightweight screencast module is required here too. +# The PipeWire frame reader remains isolated in portal-capture below. +portal-input = [ "ashpd/remote_desktop", - "dep:pipewire", - "dep:libspa", + "ashpd/screencast", "dep:reis", "dep:calloop", "dep:enumflags2", "dep:xkbcommon", ] +# PipeWire-backed per-window ScreenCast capture keeps its newer system-header +# requirement and remains enabled by Nix/modern desktop builds. +portal-capture = [ + "ashpd/screencast", + "dep:pipewire", + "dep:libspa", +] +# Backward-compatible full desktop feature used by the Nix package. +portal-libei = ["portal-input", "portal-capture"] [dev-dependencies] # Used by the `screenshot_cascade` example to surface the @@ -135,11 +134,11 @@ portal-libei = [ tracing-subscriber = { workspace = true } # Cargo example that exercises the libei input path directly. Lives under -# the `portal-libei` feature gate so a stock `cargo build --examples` +# the `portal-input` feature gate so a stock `cargo build --examples` # doesn't try to compile against the (now-optional) reis/xkbcommon deps. [[example]] name = "libei_input" -required-features = ["portal-libei"] +required-features = ["portal-input"] [[example]] name = "screenshot_cascade" diff --git a/libs/cua-driver/rust/crates/platform-linux/src/a11y.rs b/libs/cua-driver/rust/crates/platform-linux/src/a11y.rs index 5a96af06d9..53532de47a 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/a11y.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/a11y.rs @@ -12,14 +12,12 @@ //! to the driver. Electron apps shipped as AppImages behave identically; they //! embed the same Chromium. //! -//! A real screen reader (Orca) turns the tree on simply by writing that status -//! property when it starts. We do the same once, at daemon startup. Because the -//! status object lives on the session-scoped accessibility-bus launcher rather -//! than in our process, the flag is session-wide, outlives us, and takes effect -//! retroactively on apps that are already running — no relaunch and no -//! per-application command-line flag. GTK and Qt gate their own AT-SPI bridges -//! on the companion `IsEnabled` property, so setting it warms those toolkits -//! too. +//! A real screen reader turns the Chromium signal on. Doing that ourselves is +//! unsafe on GNOME: its settings daemon treats the signal as a user request and +//! launches Orca. GNOME therefore gets only the generic `IsEnabled` signal by +//! default. Other desktops retain the Chromium signal for compatibility, and a +//! caller can choose either policy explicitly with +//! `CUA_DRIVER_RS_A11Y_ADVERTISE_MODE`. //! //! Everything here is best-effort. A session without an accessibility bus (some //! headless or minimal setups) just yields an error we log and ignore; enabling @@ -42,6 +40,13 @@ const SCREEN_READER_ENABLED_PROPERTY: &str = "ScreenReaderEnabled"; /// Companion property GTK/Qt watch to load their AT-SPI bridges. const ACCESSIBILITY_IS_ENABLED_PROPERTY: &str = "IsEnabled"; +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum AdvertiseMode { + All, + IsEnabledOnly, + None, +} + /// Advertise an assistive technology to the session exactly once per daemon /// process, so Chromium/Electron (including Electron AppImages), GTK, and Qt /// expose their accessibility trees to [`crate::atspi`]. Idempotent and @@ -50,17 +55,24 @@ const ACCESSIBILITY_IS_ENABLED_PROPERTY: &str = "IsEnabled"; pub fn ensure_chromium_accessibility_enabled() { static ADVERTISED: Once = Once::new(); ADVERTISED.call_once(|| { - // Opt-out for the rare session that wants its accessibility status left - // untouched (e.g. one already driven by a real screen reader the user - // configured deliberately). - if std::env::var_os("CUA_DRIVER_RS_DISABLE_A11Y_ADVERTISE").is_some() { + let mode = advertise_mode_from( + std::env::var_os("CUA_DRIVER_RS_DISABLE_A11Y_ADVERTISE").is_some(), + std::env::var("CUA_DRIVER_RS_A11Y_ADVERTISE_MODE") + .ok() + .as_deref(), + std::env::var("XDG_CURRENT_DESKTOP") + .or_else(|_| std::env::var("XDG_SESSION_DESKTOP")) + .or_else(|_| std::env::var("DESKTOP_SESSION")) + .ok() + .as_deref(), + ); + if mode == AdvertiseMode::None { tracing::debug!( - "CUA_DRIVER_RS_DISABLE_A11Y_ADVERTISE set; leaving session \ - accessibility status untouched" + "accessibility advertisement disabled; leaving session status untouched" ); return; } - if let Err(error) = advertise_screen_reader_to_session() { + if let Err(error) = advertise_accessibility_to_session(mode) { tracing::debug!( "skipped advertising accessibility to the session \ (Chromium/Electron trees may stay empty): {error:#}" @@ -69,25 +81,25 @@ pub fn ensure_chromium_accessibility_enabled() { }); } -fn advertise_screen_reader_to_session() -> anyhow::Result<()> { +fn advertise_accessibility_to_session(mode: AdvertiseMode) -> anyhow::Result<()> { // The daemon's tokio runtime is already driving this thread when the tool // registry is built, and `block_on` panics if called from within a runtime. // Run the one-shot bus work on a dedicated OS thread that owns a small // runtime of its own, then join it — no nesting, torn down once it returns. std::thread::Builder::new() .name("cua-a11y-advertise".into()) - .spawn(|| { + .spawn(move || { let runtime = tokio::runtime::Builder::new_current_thread() .enable_all() .build()?; - runtime.block_on(advertise_screen_reader()) + runtime.block_on(advertise_accessibility(mode)) }) .context("spawning the accessibility-advertise thread")? .join() .map_err(|_| anyhow!("accessibility-advertise thread panicked"))? } -async fn advertise_screen_reader() -> anyhow::Result<()> { +async fn advertise_accessibility(mode: AdvertiseMode) -> anyhow::Result<()> { let session_bus = zbus::Connection::session().await?; let status = zbus::Proxy::new( &session_bus, @@ -100,7 +112,7 @@ async fn advertise_screen_reader() -> anyhow::Result<()> { // Don't clobber a screen reader the user is already running: only write when // a flag is currently false, so an active Orca session stays authoritative // and we avoid emitting a redundant PropertiesChanged. - if !is_flag_set(&status, SCREEN_READER_ENABLED_PROPERTY).await { + if mode == AdvertiseMode::All && !is_flag_set(&status, SCREEN_READER_ENABLED_PROPERTY).await { status .set_property(SCREEN_READER_ENABLED_PROPERTY, true) .await?; @@ -113,8 +125,93 @@ async fn advertise_screen_reader() -> anyhow::Result<()> { Ok(()) } +fn advertise_mode_from( + disabled: bool, + configured: Option<&str>, + desktop: Option<&str>, +) -> AdvertiseMode { + if disabled { + return AdvertiseMode::None; + } + match configured + .map(str::trim) + .map(str::to_ascii_lowercase) + .as_deref() + { + Some("all") => AdvertiseMode::All, + Some("is_enabled_only") => AdvertiseMode::IsEnabledOnly, + Some("none") => AdvertiseMode::None, + Some(other) => { + tracing::warn!( + mode = other, + "unknown CUA_DRIVER_RS_A11Y_ADVERTISE_MODE; using desktop default" + ); + desktop_default_mode(desktop) + } + None => desktop_default_mode(desktop), + } +} + +fn desktop_default_mode(desktop: Option<&str>) -> AdvertiseMode { + let is_gnome = desktop.is_some_and(|desktop| { + desktop + .split([':', ';']) + .any(|part| part.trim().eq_ignore_ascii_case("gnome")) + }); + if is_gnome { + AdvertiseMode::IsEnabledOnly + } else { + AdvertiseMode::All + } +} + /// Read a boolean status property, treating an unreadable property as unset so /// the caller falls through to writing it. async fn is_flag_set(status: &zbus::Proxy<'_>, property: &str) -> bool { status.get_property::(property).await.unwrap_or(false) } + +#[cfg(test)] +mod tests { + use super::{advertise_mode_from, AdvertiseMode}; + + #[test] + fn gnome_default_does_not_claim_a_screen_reader() { + assert_eq!( + advertise_mode_from(false, None, Some("ubuntu:GNOME")), + AdvertiseMode::IsEnabledOnly + ); + } + + #[test] + fn non_gnome_default_preserves_chromium_compatibility() { + assert_eq!( + advertise_mode_from(false, None, Some("KDE")), + AdvertiseMode::All + ); + } + + #[test] + fn explicit_mode_overrides_desktop_default() { + assert_eq!( + advertise_mode_from(false, Some("all"), Some("GNOME")), + AdvertiseMode::All + ); + assert_eq!( + advertise_mode_from(false, Some("is_enabled_only"), Some("KDE")), + AdvertiseMode::IsEnabledOnly + ); + assert_eq!( + advertise_mode_from(false, Some("none"), Some("KDE")), + AdvertiseMode::None + ); + } + + #[test] + fn legacy_disable_wins_over_explicit_mode() { + assert_eq!( + advertise_mode_from(true, Some("all"), Some("KDE")), + AdvertiseMode::None + ); + } +} diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs index 6678910d0b..e83970012f 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/mod.rs @@ -14,6 +14,7 @@ use anyhow::Result; pub mod cache; pub mod native; pub use cache::ElementCache; +pub use native::ensure_listener_active; #[derive(Clone, Debug)] pub struct AtspiNode { @@ -23,7 +24,7 @@ pub struct AtspiNode { pub value: Option, pub description: Option, pub actions: Vec, - /// For pyatspi path: element_key = element_index as u64. + /// For AT-SPI: element_key = element_index as u64. /// For X11 fallback: element_key = xid. pub element_key: u64, /// Depth in the markdown tree (0 = top-level window child). @@ -37,6 +38,7 @@ pub struct AtspiNode { pub struct AtspiTreeResult { pub tree_markdown: String, pub nodes: Vec, + pub bounds: Vec<(usize, i32, i32, u32, u32)>, } /// Walk the AT-SPI tree for a window identified by (pid, xid). @@ -65,7 +67,9 @@ pub fn walk_tree_bounded( // after launch returns the real tree instead of an empty one. See #1927. const MAX_ATTEMPTS: usize = 4; for attempt in 0..MAX_ATTEMPTS { - if let Ok(Some((raw_md, nodes))) = native::walk_tree_bounded(pid, max_elements, max_depth) { + if let Ok(Some((raw_md, nodes, bounds))) = + native::walk_tree_bounded(pid, xid, max_elements, max_depth) + { // `nodes.len() <= 1` == only the root window resolved: the // cold-registry symptom. Accept any real tree immediately; only // keep waiting on the degenerate case, and accept it anyway on the @@ -79,6 +83,7 @@ pub fn walk_tree_bounded( return AtspiTreeResult { tree_markdown: md, nodes, + bounds, }; } } @@ -146,68 +151,12 @@ pub fn perform_action_at_screen_point( /// For Qt5, which doesn't expose widgets when unfocused, this will return Err. /// Returns Ok if an editable was found and text was set, Err otherwise. pub fn type_into_editable(pid: u32, text: &str) -> Result<()> { - let safe_text = text.replace('\\', "\\\\").replace('\'', "\\'"); - let script = format!( - r#" -import pyatspi, sys - -def find_editable(acc, depth=0): - # Try to find any EditableText interface, regardless of role - try: - et = acc.queryEditableText() - # If we can query it, return this node - return acc - except: - pass - - # Recursively search children - try: - for child in acc: - result = find_editable(child, depth + 1) - if result is not None: - return result - except: - pass - - return None - -desktop = pyatspi.Registry.getDesktop(0) -editable = None -for app in desktop: - try: - if app.get_process_id() == {pid}: - for win in app: - editable = find_editable(win) - if editable: - break - break - except: - pass - -if editable is None: - print("ERROR: No editable found", file=sys.stderr) - sys.exit(1) - -try: - et = editable.queryEditableText() - et.setTextContents('{safe_text}') - print("ok:atspi") -except Exception as e: - print(f"ERROR: {{e}}", file=sys.stderr) - sys.exit(1) -"#, - pid = pid, - safe_text = safe_text - ); - - let out = std::process::Command::new("python3") - .arg("-c") - .arg(&script) - .output()?; - if !out.status.success() { - anyhow::bail!("{}", String::from_utf8_lossy(&out.stderr).trim().to_owned()); - } - Ok(()) + native::type_into_editable(pid, text) +} + +/// Type into the exact indexed editable from the caller's accessibility snapshot. +pub fn type_into_editable_at(pid: u32, idx: usize, text: &str) -> Result<()> { + native::type_into_editable_at(pid, idx, text) } /// Set the text value of element `idx` within pid's app tree via AT-SPI. @@ -235,15 +184,6 @@ pub fn focused_is_editable(pid: u32) -> Result> { native::focused_is_editable(pid) } -/// Get the screen-coordinate bounding box (x, y, width, height) of element `idx`. -/// Screen-coordinate bounds for every action node in pid's AT-SPI tree, keyed -/// by `element_index`. Best-effort: nodes whose bounds can't be read are -/// omitted rather than erroring the whole call. Returns `(element_index, x, y, -/// width, height)` tuples in screen coordinates. -pub fn get_all_element_bounds(pid: u32, xid: u64) -> Result> { - native::get_all_element_bounds(pid, xid) -} - pub fn get_element_bounds(pid: u32, idx: usize) -> Result<(i32, i32, u32, u32)> { native::get_element_bounds(pid, idx) } @@ -260,6 +200,7 @@ fn walk_via_x11_properties(xid: u64, query: Option<&str>) -> AtspiTreeResult { return AtspiTreeResult { tree_markdown: String::new(), nodes: vec![], + bounds: vec![], } } }; @@ -310,6 +251,7 @@ fn walk_via_x11_properties(xid: u64, query: Option<&str>) -> AtspiTreeResult { AtspiTreeResult { tree_markdown, nodes, + bounds: vec![], } } diff --git a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs index 5f2c6feab5..1c2e7628cf 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/atspi/native.rs @@ -82,6 +82,43 @@ fn runtime() -> &'static tokio::runtime::Runtime { }) } +static SHARED_CONNECTION: tokio::sync::OnceCell = + tokio::sync::OnceCell::const_new(); + +/// Keep one AT-SPI connection and registry registration alive for the daemon +/// lifetime. WebKitGTK only publishes its WebProcess accessibility subtree +/// while the registry reports an interested listener. +async fn shared_connection() -> Result<&'static AccessibilityConnection> { + SHARED_CONNECTION + .get_or_try_init(|| async { + let conn = AccessibilityConnection::new() + .await + .map_err(|error| anyhow!("AT-SPI connect failed: {error}"))?; + if let Err(error) = conn.add_registry_event::().await { + dlog!("AT-SPI object-event registration failed: {error}"); + } + Ok(conn) + }) + .await +} + +/// Establish the process-lifetime listener before accessibility-aware apps are +/// launched. Idempotent; later calls reuse the same connection. +pub fn ensure_listener_active() -> Result<()> { + let connect = || runtime().block_on(async { shared_connection().await.map(|_| ()) }); + if tokio::runtime::Handle::try_current().is_ok() { + // The daemon builds its registry from its Tokio entry-point. Calling + // Runtime::block_on there panics even though this module owns a separate + // runtime, so initialize the AT-SPI connection on a plain thread and + // wait for it before accessibility-aware apps can launch. + std::thread::spawn(connect) + .join() + .map_err(|_| anyhow!("AT-SPI listener initialization thread panicked"))? + } else { + connect() + } +} + /// A node discovered during the pre-order walk, with its proxy retained so the /// per-index operations can act on it without re-walking the tree. struct Visited<'a> { @@ -133,22 +170,24 @@ async fn accessible_for<'a>( conn: &'a AccessibilityConnection, oref: &RawObjectRef, ) -> Result> { - // Keep the atspi crate's peer-to-peer path for normal AT-SPI unique names. - // Electron/Chromium uses this path for focused-child input delivery. The - // raw string path below is only needed for WebKitGTK's well-known - // WebProcess references, which cannot be represented as ObjectRef names. + // Keep the atspi crate's peer-to-peer path when this connection actually + // knows the peer. Late WebKit WebProcess children are not in the initial + // peer snapshot; object_as_accessible's bus fallback omits their destination + // and targets the Accessible interface name instead. Build an explicit bus + // proxy below for those late peers and for well-known references. if oref.name.starts_with(':') { let name = atspi::zbus::names::UniqueName::try_from(oref.name.clone()) .map_err(|e| anyhow!("bad a11y unique name: {e}"))?; - let path = atspi::zbus::zvariant::ObjectPath::try_from(oref.path.clone()) - .map_err(|e| anyhow!("bad a11y path: {e}"))?; - let object = atspi::ObjectRef::new_owned(name, path); - // `object_as_accessible` chooses a P2P peer when the toolkit exposes - // one and falls back to the shared accessibility bus otherwise. - return conn - .object_as_accessible(&object) - .await - .map_err(|e| anyhow!("AccessibleProxy build failed: {e}")); + let bus_name = atspi::zbus::names::BusName::Unique(name.as_ref()); + if conn.get_peer(&bus_name).is_some() { + let path = atspi::zbus::zvariant::ObjectPath::try_from(oref.path.clone()) + .map_err(|e| anyhow!("bad a11y path: {e}"))?; + let object = atspi::ObjectRef::new_owned(name, path); + return conn + .object_as_accessible(&object) + .await + .map_err(|e| anyhow!("AccessibleProxy build failed: {e}")); + } } AccessibleProxy::builder(conn.connection()) .cache_properties(atspi::zbus::proxy::CacheProperties::No) @@ -336,7 +375,7 @@ async fn collect_visited_bounded<'a>( // AT-SPI (most commonly because it holds a modal grab and isn't servicing // D-Bus), every per-node `call()` burns the full CALL_TIMEOUT before being // skipped, so the walk would otherwise grind for minutes. Callers that lack - // their own OP_TIMEOUT (get_all_element_bounds, insert_text) relied on this + // their own OP_TIMEOUT (snapshot bounds, insert_text) relied on this // never happening — bound it here so the walk returns partial within // OP_TIMEOUT for every caller, instead of hanging get_window_state/type_text // on modal dialogs (#1936). @@ -364,7 +403,7 @@ async fn collect_visited_bounded<'a>( // await that actually hangs, so it MUST carry the per-call timeout — // otherwise the loop never returns to the deadline check at the top and // the walk stalls past OP_TIMEOUT for callers without an outer guard - // (get_all_element_bounds, insert_text). That was the residual #1936 hang. + // (snapshot bounds, insert_text). That was the residual #1936 hang. let acc = match call(accessible_for(conn, &oref)).await { Some(Ok(a)) => a, Some(Err(error)) => { @@ -611,24 +650,27 @@ fn format_value(v: f64) -> String { /// Historically this was "the node advertises AT-SPI Actions" (buttons, menu /// items, links). That silently dropped every **Value**-only widget — GTK /// `GtkScale` sliders, scroll bars, spin buttons, progress bars expose the -/// `Value` interface but NO `Action`, so they never got an `element_index` and -/// were invisible to `get_window_state`/`set_value` even though the driver can -/// drive them (`set_value` already handles `has_value`). We now also index any -/// node carrying the Value interface so sliders and scroll regions surface as -/// usable elements. +/// `Value` interface but NO `Action`, while some text fields expose +/// `EditableText` without either. Omitting those interfaces makes controls the +/// driver can operate impossible to address by `element_index`. /// /// This predicate is the single source of truth for the element-index space and /// MUST be applied identically in `render` and in every `action_nodes` filter -/// (`perform_action`, `set_value`, `get_element_bounds`, `get_all_element_bounds`); +/// (`perform_action`, `set_value`, `get_element_bounds`, snapshot bounds); /// any divergence would desync indices between the snapshot and the operations. fn is_indexable(v: &Visited) -> bool { - !v.actions.is_empty() || v.has_value + is_indexable_capabilities(!v.actions.is_empty(), v.has_editable, v.has_value) +} + +fn is_indexable_capabilities(has_action: bool, has_editable: bool, has_value: bool) -> bool { + has_action || has_editable || has_value } // ── Public (sync) entry points ─────────────────────────────────────────────── pub fn walk_tree(pid: u32) -> Result)>> { - walk_tree_bounded(pid, None, None) + walk_tree_bounded(pid, 0, None, None) + .map(|snapshot| snapshot.map(|(markdown, nodes, _)| (markdown, nodes))) } /// Walk the AT-SPI tree with caller-supplied node + depth caps. @@ -636,26 +678,28 @@ pub fn walk_tree(pid: u32) -> Result)>> { /// = None` keeps the historical unbounded depth. Issue #22865. pub fn walk_tree_bounded( pid: u32, + xid: u64, max_elements: Option, max_depth: Option, -) -> Result)>> { +) -> Result, Vec<(usize, i32, i32, u32, u32)>)>> { runtime().block_on(async { - let work = async { - let conn = AccessibilityConnection::new() - .await - .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?; - match collect_visited_bounded(&conn, pid, max_elements, max_depth).await? { - Some(visited) => Ok(Some(render(&visited))), - None => Ok(None), - } + let walk = async { + let conn = shared_connection().await?; + collect_visited_bounded(conn, pid, max_elements, max_depth).await }; - match tokio::time::timeout(OP_TIMEOUT, work).await { - Ok(r) => r, + let visited = match tokio::time::timeout(OP_TIMEOUT, walk).await { + Ok(result) => result?, Err(_) => { dlog!("walk_tree timed out for pid {pid}"); - Ok(None) + return Ok(None); } - } + }; + let Some(visited) = visited else { + return Ok(None); + }; + let (markdown, nodes) = render(&visited); + let bounds = element_bounds_for_visited(&visited, pid, xid).await; + Ok(Some((markdown, nodes, bounds))) }) } @@ -672,12 +716,19 @@ pub fn walk_tree_bounded( /// dereference the xid against X11, so the synthetic value only needs to be /// non-zero and to round-trip back from the caller. pub fn list_windows(filter_pid: Option) -> Vec { + if tokio::runtime::Handle::try_current().is_ok() { + return std::thread::spawn(move || list_windows_blocking(filter_pid)) + .join() + .unwrap_or_default(); + } + list_windows_blocking(filter_pid) +} + +fn list_windows_blocking(filter_pid: Option) -> Vec { use crate::x11::WindowInfo; runtime().block_on(async { let work = async { - let conn = AccessibilityConnection::new() - .await - .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?; + let conn = shared_connection().await?; let zconn = conn.connection(); let root = match call(conn.root_accessible_on_registry()).await { Some(Ok(r)) => r, @@ -707,7 +758,7 @@ pub fn list_windows(filter_pid: Option) -> Vec { Some(app_ref) => app_ref, None => continue, }; - let app = match call(accessible_for(&conn, &app_ref)).await { + let app = match call(accessible_for(conn, &app_ref)).await { Some(Ok(a)) => a, _ => continue, }; @@ -725,7 +776,7 @@ pub fn list_windows(filter_pid: Option) -> Vec { Some(frame_ref) => frame_ref, None => continue, }; - let frame = match call(accessible_for(&conn, &frame_ref)).await { + let frame = match call(accessible_for(conn, &frame_ref)).await { Some(Ok(f)) => f, _ => continue, }; @@ -744,16 +795,34 @@ pub fn list_windows(filter_pid: Option) -> Vec { .and_then(|r| r.ok()) .filter(|s| !s.is_empty()) .unwrap_or_else(|| app_name.clone()); + let geometry = match call(frame.proxies()).await { + Some(Ok(proxies)) => match call(proxies.component()).await { + Some(Ok(component)) => call(component.get_extents(CoordType::Screen)) + .await + .and_then(|result| result.ok()), + _ => None, + }, + _ => None, + }; + let (x, y, width, height) = geometry + .filter(|(_, _, width, height)| *width > 0 && *height > 0) + .map(|(x, y, width, height)| { + (x, y, width.max(0) as u32, height.max(0) as u32) + }) + .unwrap_or((0, 0, 0, 0)); // Stable, non-zero, unique per (pid, frame ordinal). let xid = (((cpid as u64) << 16) | (i as u64)).max(1); out.push(WindowInfo { xid, pid: Some(cpid), + app_name: app_name.clone(), title, - x: 0, - y: 0, - width: 0, - height: 0, + is_on_screen: width > 0 && height > 0, + z_index: None, + x, + y, + width, + height, }); emitted += 1; } @@ -763,7 +832,10 @@ pub fn list_windows(filter_pid: Option) -> Vec { out.push(WindowInfo { xid: (cpid as u64).max(1), pid: Some(cpid), + app_name: app_name.clone(), title: app_name, + is_on_screen: true, + z_index: None, x: 0, y: 0, width: 0, @@ -803,14 +875,17 @@ fn pick_editable<'v, 'a>(visited: &'v [Visited<'a>]) -> Option<&'v Visited<'a>> } /// Try to write `text` into the best editable node in `visited` via AT-SPI -/// EditableText (GrabFocus first so the toolkit exposes the field on an -/// unfocused window's focused widget). Returns `Ok(true)` if the write landed, +/// EditableText. Returns `Ok(true)` if the write landed, /// `Ok(false)` if no editable was found / the EditableText write was rejected. async fn write_into_editable(visited: &[Visited<'_>], text: &str) -> Result { let target = match pick_editable(visited) { Some(t) => t, None => return Ok(false), }; + write_into_editable_target(target, text).await +} + +async fn write_into_editable_target(target: &Visited<'_>, text: &str) -> Result { dlog!( "insert target: role={:?} in_web_doc={} focused={} has_component={}", target.role, @@ -825,6 +900,13 @@ async fn write_into_editable(visited: &[Visited<'_>], text: &str) -> Result], text: &str) -> Result, + text: &str, +) -> Result { let et = proxies .editable_text() .await @@ -864,13 +953,76 @@ async fn write_into_editable(visited: &[Visited<'_>], text: &str) -> Result Result<()> { + bounded( + async { + let conn = shared_connection().await?; + let visited = collect_visited(conn, pid) + .await? + .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?; + if write_into_editable(&visited, text).await? { + Ok(()) + } else { + Err(anyhow!("no writable AT-SPI element found for pid {pid}")) + } + }, + || Err(anyhow!("AT-SPI editable lookup timed out for pid {pid}")), + ) +} + +/// Write into the exact indexed editable exposed by the caller's snapshot. +pub fn type_into_editable_at(pid: u32, idx: usize, text: &str) -> Result<()> { + bounded( + async { + let conn = shared_connection().await?; + let visited = collect_visited(conn, pid) + .await? + .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?; + let target = visited + .iter() + .filter(|node| is_indexable(node)) + .nth(idx) + .ok_or_else(|| anyhow!("element {idx} not found (total: {})", visited.len()))?; + if write_into_editable_target(target, text).await? { + Ok(()) + } else { + // GrabFocus can rebuild WebKitGTK's accessibility object. Walk + // the same index space again and retry only that exact element; + // never fall through to a different focused/first editable. + let refreshed = collect_visited(conn, pid) + .await? + .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?; + let refreshed_target = refreshed + .iter() + .filter(|node| is_indexable(node)) + .nth(idx) + .ok_or_else(|| { + anyhow!("element {idx} disappeared after AT-SPI focus refresh") + })?; + if write_into_editable_target(refreshed_target, text).await? { + Ok(()) + } else { + Err(anyhow!( + "element {idx} is not writable through AT-SPI EditableText" + )) + } + } + }, + || { + Err(anyhow!( + "AT-SPI editable write timed out for element {idx} in pid {pid}" + )) + }, + ) +} + pub fn insert_text(pid: u32, text: &str) -> Result { bounded( async { - let conn = AccessibilityConnection::new() - .await - .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?; - let visited = match collect_visited(&conn, pid).await? { + let conn = shared_connection().await?; + let visited = match collect_visited(conn, pid).await? { Some(v) => v, None => return Ok(false), }; @@ -981,10 +1133,8 @@ pub fn insert_text(pid: u32, text: &str) -> Result { pub fn focused_is_editable(pid: u32) -> Result> { bounded( async { - let conn = AccessibilityConnection::new() - .await - .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?; - let visited = match collect_visited(&conn, pid).await? { + let conn = shared_connection().await?; + let visited = match collect_visited(conn, pid).await? { Some(v) => v, None => return Ok(None), }; @@ -1029,10 +1179,8 @@ fn screen_to_window_coords(xid: u64, screen_x: i32, screen_y: i32) -> Option<(i3 pub fn perform_action(pid: u32, idx: usize) -> Result<(String, bool)> { bounded( async { - let conn = AccessibilityConnection::new() - .await - .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?; - let visited = collect_visited(&conn, pid) + let conn = shared_connection().await?; + let visited = collect_visited(conn, pid) .await? .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?; let action_nodes: Vec<&Visited> = visited.iter().filter(|v| is_indexable(v)).collect(); @@ -1084,10 +1232,8 @@ pub fn perform_action(pid: u32, idx: usize) -> Result<(String, bool)> { pub fn scroll_element(pid: u32, idx: usize, direction: &str, amount: usize) -> Result<()> { bounded( async { - let conn = AccessibilityConnection::new() - .await - .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?; - let visited = collect_visited(&conn, pid) + let conn = shared_connection().await?; + let visited = collect_visited(conn, pid) .await? .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?; let target = visited @@ -1100,46 +1246,86 @@ pub fn scroll_element(pid: u32, idx: usize, direction: &str, amount: usize) -> R .proxies() .await .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?; - let action = proxies - .action() - .await - .map_err(|e| anyhow!("Action interface unavailable: {e}"))?; let wanted = match direction { "up" => ["scrollup", "scrollbackward"], "left" => ["scrollleft", "scrollbackward"], "right" => ["scrollright", "scrollforward"], _ => ["scrolldown", "scrollforward"], }; - let count = call(action.n_actions()) - .await - .and_then(|result| result.ok()) - .unwrap_or(0); let mut selected = None; - for action_index in 0..count { - if let Some(Ok(name)) = call(action.get_name(action_index)).await { - let normalized: String = name - .chars() - .filter(|ch| ch.is_ascii_alphanumeric()) - .flat_map(|ch| ch.to_lowercase()) - .collect(); - if wanted.iter().any(|candidate| *candidate == normalized) { - selected = Some(action_index); - break; + let mut action_proxy = None; + if let Ok(action) = proxies.action().await { + let count = call(action.n_actions()) + .await + .and_then(|result| result.ok()) + .unwrap_or(0); + for action_index in 0..count { + if let Some(Ok(name)) = call(action.get_name(action_index)).await { + let normalized: String = name + .chars() + .filter(|ch| ch.is_ascii_alphanumeric()) + .flat_map(|ch| ch.to_lowercase()) + .collect(); + if wanted.iter().any(|candidate| *candidate == normalized) { + selected = Some(action_index); + break; + } } } + action_proxy = Some(action); } - let action_index = selected.ok_or_else(|| { - anyhow!("element {idx} exposes no directional scroll action for {direction}") - })?; - for _ in 0..amount.max(1) { - match call(action.do_action(action_index)).await { - Some(Ok(true)) => {} - Some(Ok(false)) => return Err(anyhow!("scroll action returned false")), - Some(Err(e)) => return Err(anyhow!("scroll action failed: {e}")), - None => return Err(anyhow!("scroll action timed out")), + + if let (Some(action), Some(action_index)) = (action_proxy, selected) { + for _ in 0..amount.max(1) { + match call(action.do_action(action_index)).await { + Some(Ok(true)) => {} + Some(Ok(false)) => return Err(anyhow!("scroll action returned false")), + Some(Err(e)) => return Err(anyhow!("scroll action failed: {e}")), + None => return Err(anyhow!("scroll action timed out")), + } } + return Ok(()); } - Ok(()) + + if target.has_value { + let value = proxies + .value() + .await + .map_err(|e| anyhow!("Value interface unavailable: {e}"))?; + let current = call(value.current_value()) + .await + .and_then(|result| result.ok()) + .ok_or_else(|| anyhow!("scroll value lookup timed out"))?; + let minimum = call(value.minimum_value()) + .await + .and_then(|result| result.ok()) + .unwrap_or(current); + let maximum = call(value.maximum_value()) + .await + .and_then(|result| result.ok()) + .unwrap_or(current); + let increment = call(value.minimum_increment()) + .await + .and_then(|result| result.ok()) + .filter(|increment| *increment > 0.0) + .unwrap_or(1.0); + let sign = if matches!(direction, "up" | "left") { + -1.0 + } else { + 1.0 + }; + let next = + (current + sign * increment * amount.max(1) as f64).clamp(minimum, maximum); + call(value.set_current_value(next)) + .await + .and_then(|result| result.ok()) + .ok_or_else(|| anyhow!("scroll value update timed out"))?; + return Ok(()); + } + + Err(anyhow!( + "element {idx} exposes neither directional scroll actions nor Value" + )) }, || Err(anyhow!("scroll_element timed out for pid {pid}")), ) @@ -1150,10 +1336,8 @@ pub fn scroll_element(pid: u32, idx: usize, direction: &str, amount: usize) -> R pub fn focus_element(pid: u32, idx: usize) -> Result { bounded( async { - let conn = AccessibilityConnection::new() - .await - .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?; - let visited = collect_visited(&conn, pid) + let conn = shared_connection().await?; + let visited = collect_visited(conn, pid) .await? .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?; let target = visited @@ -1200,10 +1384,8 @@ pub fn focus_element(pid: u32, idx: usize) -> Result { pub fn perform_action_at_point(pid: u32, win_x: i32, win_y: i32) -> Result> { bounded( async { - let conn = AccessibilityConnection::new() - .await - .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?; - let visited = match collect_visited(&conn, pid).await? { + let conn = shared_connection().await?; + let visited = match collect_visited(conn, pid).await? { Some(v) => v, None => return Ok(None), }; @@ -1266,12 +1448,12 @@ pub fn perform_action_at_point(pid: u32, win_x: i32, win_y: i32) -> Result Result> { bounded( async { - let conn = AccessibilityConnection::new() - .await - .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?; - let visited = match collect_visited(&conn, pid).await? { + let conn = shared_connection().await?; + let visited = match collect_visited(conn, pid).await? { Some(v) => v, None => return Ok(None), }; @@ -1298,7 +1478,7 @@ pub fn perform_action_at_screen_point( // correctly; Screen is (0,0)) plus the window's screen origin (the // GNOME Shell helper on Wayland, _GTK_FRAME_EXTENTS on X11). When no // offset resolves, fall back to CoordType::Screen (correct on Qt/GTK3). - let offset = window_to_screen_offset(pid, xid); + let offset = window_to_screen_offset(pid, xid, None); let coord = if offset.is_some() { CoordType::Window } else { @@ -1403,10 +1583,8 @@ fn select_click_target( pub fn set_value(pid: u32, idx: usize, value: &str) -> Result<()> { bounded( async { - let conn = AccessibilityConnection::new() - .await - .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?; - let visited = collect_visited(&conn, pid) + let conn = shared_connection().await?; + let visited = collect_visited(conn, pid) .await? .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?; let action_nodes: Vec<&Visited> = visited.iter().filter(|v| is_indexable(v)).collect(); @@ -1420,20 +1598,11 @@ pub fn set_value(pid: u32, idx: usize, value: &str) -> Result<()> { .await .map_err(|e| anyhow!("interface proxies unavailable: {e}"))?; - // EditableText write. We don't gate on the cached `has_editable` flag: - // GTK4 (and similar toolkits) only advertise the EditableText interface - // on a widget once it holds keyboard focus, so the interface list - // captured during the unfocused tree walk can be missing it even though - // the element is a real editable text box. GrabFocus first (internal - // widget focus, no window activation — same trick as `type_text`'s - // EditableText path), then resolve the EditableText proxy live over - // D-Bus and try to write. If the proxy genuinely isn't there the - // `editable_text()` resolve fails and we fall through to Value below. - if target.has_component { - if let Ok(comp) = proxies.component().await { - let _ = call(comp.grab_focus()).await; - } - } + // SetValue is a focus-free accessibility operation. Do not call + // Component.GrabFocus here: GTK may activate and raise the entire + // toplevel in response, violating the background contract. Toolkits + // that expose EditableText only while focused must return an honest + // unsupported error rather than changing desktop focus implicitly. if let Ok(et) = proxies.editable_text().await { // Replace whole contents (parity with the Windows/macOS set_value, // which overwrite rather than insert at the caret). @@ -1479,10 +1648,8 @@ pub fn set_value(pid: u32, idx: usize, value: &str) -> Result<()> { pub fn get_element_bounds(pid: u32, idx: usize) -> Result<(i32, i32, u32, u32)> { bounded( async { - let conn = AccessibilityConnection::new() - .await - .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?; - let visited = collect_visited(&conn, pid) + let conn = shared_connection().await?; + let visited = collect_visited(conn, pid) .await? .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?; let action_nodes: Vec<&Visited> = visited.iter().filter(|v| is_indexable(v)).collect(); @@ -1503,7 +1670,7 @@ pub fn get_element_bounds(pid: u32, idx: usize) -> Result<(i32, i32, u32, u32)> // Prefer WINDOW coords + a deterministic screen offset — fixes GTK4, // whose CoordType::Screen collapses every element to (0,0). Fall back to // Screen on Wayland / when no X11 window resolves (offset is None). - match window_to_screen_offset(pid, 0) { + match window_to_screen_offset(pid, 0, None) { Some((ox, oy)) => { let (x, y, w, h) = comp .get_extents(CoordType::Window) @@ -1611,7 +1778,7 @@ fn parse_gtk_frame_extents(vals: &[u32]) -> Option<(i32, i32)> { /// path and the WINDOW reconstruction can never regress a toolkit that was /// already correct. Also returns `None` on native Wayland (clients may not /// query screen origins, by design) or when no X11 window resolves. -fn window_to_screen_offset(pid: u32, xid: u64) -> Option<(i32, i32)> { +fn window_to_screen_offset(pid: u32, xid: u64, title: Option<&str>) -> Option<(i32, i32)> { if crate::wayland::is_wayland() { // Native Wayland: clients can't query a window's screen origin, and // AT-SPI CoordType::Screen collapses to (0,0) on Mutter. The bundled @@ -1621,7 +1788,21 @@ fn window_to_screen_offset(pid: u32, xid: u64) -> Option<(i32, i32)> { // this reconstructs real screen coords — the GNOME analogue of the X11 // `_GTK_FRAME_EXTENTS` path below. `None` (no extension) keeps the // legacy Screen path (still (0,0), but no worse than before). - return crate::wayland::shell_helper::window_origin_for_pid(pid); + return crate::wayland::inject_accessibility_offset(pid) + .or_else(|| crate::wayland::observed_window_origin(pid)) + .or_else(|| { + crate::wayland::sway_ipc::window_for_id(xid) + .map(|window| (window.x, window.y)) + }) + .or_else(|| { + crate::wayland::sway_ipc::list_windows().and_then(|_| { + crate::wayland::window_geometry(xid) + .map(|(window_x, window_y, _, _)| (window_x, window_y)) + }) + }) + .or_else(|| crate::wayland::shell_helper::window_origin_for_pid(pid)) + .or_else(|| crate::wayland::sway_ipc::window_origin_for_pid(pid)) + .or_else(|| title.and_then(crate::wayland::sway_ipc::window_origin_for_title)); } // Resolve a usable window xid. `xid == 0` means "no hint" (get_element_bounds // has no window context); fall back to this pid's first window — the same @@ -1639,14 +1820,47 @@ fn window_to_screen_offset(pid: u32, xid: u64) -> Option<(i32, i32)> { Some((ox + fl, oy + ft)) } -/// Screen-coordinate bounds for every action node in the tree, keyed by the -/// same `element_index` used by [`walk_tree`]/`get_element_bounds`. -/// -/// Walks the application once (unlike calling `get_element_bounds` per node, -/// which would reconnect and re-walk every time) and queries each node's -/// `Component.GetExtents(Screen)`. Nodes without a usable Component interface, -/// or whose extents query fails/times out, are silently skipped — the result is -/// best-effort and never errors on a per-node hiccup. +fn screen_extent_rebase( + x11_origin: (i32, i32), + accessible_frame_origin: (i32, i32), +) -> Option<(i32, i32)> { + // Chromium's broken "Screen" provider is rooted at the renderer-local + // origin. A legitimate screen provider may differ from the X11 client + // origin by title-bar/CSD extents; rebasing that small decoration delta + // would move otherwise-correct GTK coordinates off their controls. + if accessible_frame_origin.0.abs() <= 2 && accessible_frame_origin.1.abs() <= 2 { + Some(( + x11_origin.0 - accessible_frame_origin.0, + x11_origin.1 - accessible_frame_origin.1, + )) + } else { + None + } +} + +fn rebase_renderer_window_offset( + mut offset: (i32, i32), + frame_origin: Option<(i32, i32)>, +) -> (i32, i32) { + if let Some((frame_x, frame_y)) = frame_origin { + // Chromium may expose a negative renderer-local frame origin. Rebase + // that shape, but keep positive content insets: its descendants are + // already relative to the content origin and subtracting the inset + // moves first-row controls above the captured Wayland window. + if frame_x < 0 { + offset.0 = offset.0.saturating_sub(frame_x); + } + if frame_y < 0 { + offset.1 = offset.1.saturating_sub(frame_y); + } + } + offset +} + +/// Screen-coordinate bounds for the exact visited sequence rendered into the +/// current snapshot. Nodes without a usable Component interface, or whose +/// extents query fails/times out, are omitted rather than borrowing another +/// live traversal's ordinal. /// /// GTK4 caveat: GTK4's AT-SPI bridge returns `GetExtents(Screen)` as `(0,0)` /// for every element (issue #1564 / the #1739 a11y rework), so a screen query @@ -1657,95 +1871,187 @@ fn window_to_screen_offset(pid: u32, xid: u64) -> Option<(i32, i32)> { /// offset is just the X11 origin and the result matches the old screen path. /// /// Returns `(element_index, x, y, width, height)` tuples. -pub fn get_all_element_bounds(pid: u32, xid: u64) -> Result> { - bounded( - async { - let conn = AccessibilityConnection::new() - .await - .map_err(|e| anyhow!("AT-SPI connect failed: {e}"))?; - let visited = collect_visited(&conn, pid) - .await? - .ok_or_else(|| anyhow!("no AT-SPI application for pid {pid}"))?; - - // Query WINDOW-relative extents and add a deterministic screen offset - // (X11 window origin + GTK4 CSD inset). This fixes GTK4 — whose - // CoordType::Screen reports every element at (0,0) — by using the - // distinct per-widget WINDOW coords instead. On Wayland / when no X11 - // window resolves, `offset` is None and we keep the legacy Screen path - // so non-X11 behaviour is unchanged. - let offset = window_to_screen_offset(pid, xid); - let coord = if offset.is_some() { - CoordType::Window - } else { - CoordType::Screen +async fn element_bounds_for_visited( + visited: &[Visited<'_>], + pid: u32, + xid: u64, +) -> Vec<(usize, i32, i32, u32, u32)> { + // Query WINDOW-relative extents and add a deterministic screen offset + // (X11 window origin + GTK4 CSD inset). This fixes GTK4 — whose + // CoordType::Screen reports every element at (0,0) — by using the + // distinct per-widget WINDOW coords instead. On Wayland / when no X11 + // window resolves, `offset` is None and we keep the legacy Screen path + // so non-X11 behaviour is unchanged. + let window_title = visited.iter().find_map(|node| { + matches!( + node.role.to_ascii_lowercase().as_str(), + "frame" | "window" | "dialog" | "alert" | "file chooser" + ) + .then_some(node.name.as_str()) + }); + let offset = window_to_screen_offset(pid, xid, window_title); + let coord = if offset.is_some() { + CoordType::Window + } else { + CoordType::Screen + }; + // Chromium on X11 labels its component extents as Screen while + // returning coordinates relative to the renderer frame. Rebase + // those values by comparing the top-level accessible frame with + // the actual X11 window origin. Correct screen-coordinate providers + // produce a zero delta; Chromium's local (0,0) frame produces the + // required window-origin delta. GTK's explicit Window-coordinate + // path above remains authoritative when available. + let screen_rebase = if offset.is_none() && !crate::wayland::is_wayland() && xid != 0 { + let x11_origin = x11_window_origin(xid); + let frame = visited.iter().find(|node| { + node.has_component + && matches!( + node.role.to_ascii_lowercase().as_str(), + "frame" | "window" | "dialog" | "alert" | "file chooser" + ) + }); + if let (Some(origin), Some(frame)) = (x11_origin, frame) { + let accessible_origin = match call(frame.acc.proxies()).await { + Some(Ok(proxies)) => match call(proxies.component()).await { + Some(Ok(component)) => { + match call(component.get_extents(CoordType::Screen)).await { + Some(Ok((x, y, _, _))) => Some((x, y)), + _ => None, + } + } + _ => None, + }, + _ => None, }; - let (offset_x, offset_y) = offset.unwrap_or((0, 0)); - if let Some((ox, oy)) = offset { - dlog!("element bounds: WINDOW coords + screen offset ({ox},{oy})"); + accessible_origin.and_then(|frame_origin| screen_extent_rebase(origin, frame_origin)) + } else { + None + } + } else { + None + }; + // Renderer bridges can expose Window coordinates relative to an internal + // frame whose origin is not (0,0) (Chromium commonly reports a negative + // title-bar offset). Normalize that frame to the compositor window origin + // before adding the screen offset. Native GTK reports (0,0), so this is a + // no-op there. + let window_frame_origin = if offset.is_some() { + let frame = visited.iter().find(|node| { + node.has_component + && matches!( + node.role.to_ascii_lowercase().as_str(), + "frame" | "window" | "dialog" | "alert" | "file chooser" + ) + }); + if let Some(frame) = frame { + match call(frame.acc.proxies()).await { + Some(Ok(proxies)) => match call(proxies.component()).await { + Some(Ok(component)) => { + match call(component.get_extents(CoordType::Window)).await { + Some(Ok((x, y, _, _))) => Some((x, y)), + _ => None, + } + } + _ => None, + }, + _ => None, } + } else { + None + } + } else { + None + }; + let (offset_x, offset_y) = rebase_renderer_window_offset( + offset.or(screen_rebase).unwrap_or((0, 0)), + window_frame_origin, + ); + if let Some((ox, oy)) = offset { + dlog!("element bounds: WINDOW coords + screen offset ({ox},{oy})"); + } else if let Some((ox, oy)) = screen_rebase { + dlog!("element bounds: SCREEN coords + X11 frame rebase ({ox},{oy})"); + } - let action_nodes: Vec<&Visited> = visited.iter().filter(|v| is_indexable(v)).collect(); - // Each element costs ~3 D-Bus round-trips (proxies + component + - // GetExtents). Big trees (geany exposes ~787 nodes) would grind for - // minutes and time out callers, so cap the walk; pre-order means the - // first nodes are the window chrome / toolbars that are actually - // visible, which is what bounds consumers (overlays, targeting) need. - const MAX_BOUNDS_NODES: usize = 150; - // Hard wall-clock budget for the whole collection: on pathological - // trees individual D-Bus calls each burn up to CALL_TIMEOUT (geany's - // unrealized nodes did exactly that), so a per-node cap alone can - // still add up to minutes. Return whatever was collected in time. - let deadline = std::time::Instant::now() + Duration::from_secs(20); - let mut out = Vec::with_capacity(action_nodes.len().min(MAX_BOUNDS_NODES)); - for (idx, node) in action_nodes.iter().enumerate().take(MAX_BOUNDS_NODES) { - if std::time::Instant::now() >= deadline { - dlog!("get_all_element_bounds: 20s budget exhausted at node {idx}; returning {} bound(s)", out.len()); - break; - } - if !node.has_component { - continue; - } - let proxies = match call(node.acc.proxies()).await { - Some(Ok(p)) => p, - _ => continue, - }; - let comp = match call(proxies.component()).await { - Some(Ok(c)) => c, - _ => continue, - }; - if let Some(Ok((x, y, w, h))) = call(comp.get_extents(coord)).await { - // Unrealized widgets (e.g. items inside closed menus/popovers) - // report GetExtents as the i32::MIN sentinel and/or a degenerate - // 0x0 / 1x1 size. Emitting those poisons downstream consumers - // (overlay renderers, click targeting), so keep only elements - // with plausible on-screen geometry. (Validate the raw extents, - // before applying the screen offset, so the sentinel check still - // catches unrealized widgets.) - if x == i32::MIN - || y == i32::MIN - || x < -16384 - || y < -16384 - || w <= 1 - || h <= 1 - { - continue; - } - out.push((idx, x + offset_x, y + offset_y, w as u32, h as u32)); - } + let action_nodes: Vec<&Visited> = visited.iter().filter(|v| is_indexable(v)).collect(); + // Hard wall-clock budget for the whole collection: on pathological + // trees individual D-Bus calls each burn up to CALL_TIMEOUT (geany's + // unrealized nodes did exactly that). Return whatever was collected + // in time, but do not impose an index-based node cap: a cap silently + // stripped frames from valid controls later in renderer trees and + // made PX targeting depend on DOM order. + let deadline = std::time::Instant::now() + Duration::from_secs(20); + let mut out = Vec::with_capacity(action_nodes.len()); + for (idx, node) in action_nodes.iter().enumerate() { + if std::time::Instant::now() >= deadline { + dlog!( + "snapshot bounds: 20s budget exhausted at node {idx}; returning {} bound(s)", + out.len() + ); + break; + } + if !node.has_component { + continue; + } + let proxies = match call(node.acc.proxies()).await { + Some(Ok(p)) => p, + _ => continue, + }; + let comp = match call(proxies.component()).await { + Some(Ok(c)) => c, + _ => continue, + }; + if let Some(Ok((x, y, w, h))) = call(comp.get_extents(coord)).await { + // Unrealized widgets (e.g. items inside closed menus/popovers) + // report GetExtents as the i32::MIN sentinel and/or a degenerate + // 0x0 / 1x1 size. Emitting those poisons downstream consumers + // (overlay renderers, click targeting), so keep only elements + // with plausible on-screen geometry. (Validate the raw extents, + // before applying the screen offset, so the sentinel check still + // catches unrealized widgets.) + if x == i32::MIN || y == i32::MIN || x < -16384 || y < -16384 || w <= 1 || h <= 1 { + continue; } - Ok(out) - }, - || { - dlog!("get_all_element_bounds timed out for pid {pid}; returning no bounds"); - Ok(Vec::new()) - }, - ) + out.push((idx, x + offset_x, y + offset_y, w as u32, h as u32)); + } + } + out } #[cfg(test)] mod coord_tests { use super::parse_gtk_frame_extents; - use super::{is_passive_role, select_click_target}; + use super::{ + is_indexable_capabilities, is_passive_role, rebase_renderer_window_offset, + screen_extent_rebase, select_click_target, + }; + + #[test] + fn editable_only_nodes_are_addressable() { + assert!(is_indexable_capabilities(false, true, false)); + assert!(is_indexable_capabilities(true, false, false)); + assert!(is_indexable_capabilities(false, false, true)); + assert!(!is_indexable_capabilities(false, false, false)); + } + + #[test] + fn screen_extents_are_rebased_from_accessible_frame_to_x11_origin() { + assert_eq!(screen_extent_rebase((604, 80), (0, 0)), Some((604, 80))); + assert_eq!(screen_extent_rebase((604, 100), (604, 80)), None); + assert_eq!(screen_extent_rebase((604, 80), (604, 80)), None); + } + + #[test] + fn renderer_window_offset_only_rebases_negative_frame_origins() { + assert_eq!( + rebase_renderer_window_offset((100, 50), Some((-8, -29))), + (108, 79) + ); + assert_eq!( + rebase_renderer_window_offset((100, 50), Some((0, 29))), + (100, 50) + ); + } #[test] fn click_target_prefers_button_over_its_inner_label() { diff --git a/libs/cua-driver/rust/crates/platform-linux/src/health_report.rs b/libs/cua-driver/rust/crates/platform-linux/src/health_report.rs index 4e60dcdef0..1b7d71c743 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/health_report.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/health_report.rs @@ -304,10 +304,36 @@ async fn check_wayland_backend() -> CheckEntry { ); } }; + let remote_desktop_portal_reachable = if crate::wayland::PORTAL_INPUT_ENABLED { + tokio::task::spawn_blocking(probe_portal_remote_desktop) + .await + .ok() + .and_then(|r| r.ok()) + .unwrap_or(false) + } else { + false + }; + classify_wayland_backend( + &snap, + crate::wayland::PORTAL_INPUT_ENABLED, + remote_desktop_portal_reachable, + crate::wayland::shell_helper::list_windows(None).is_some(), + ) +} + +fn classify_wayland_backend( + snap: &crate::wayland::WaylandManagers, + portal_libei_enabled: bool, + remote_desktop_portal_reachable: bool, + target_activation_available: bool, +) -> CheckEntry { let msg = format!( - "foreign-toplevel={ftl}, screencopy={cap}, virtual-pointer={vp}, wl_shm={shm}", + "foreign-toplevel={ftl}, screencopy={cap}, ext-image-copy={ext_cap}, \ + ext-output-source={ext_src}, virtual-pointer={vp}, wl_shm={shm}", ftl = snap.foreign_toplevel, cap = snap.screencopy, + ext_cap = snap.ext_image_copy_capture, + ext_src = snap.ext_output_image_capture_source, vp = snap.virtual_pointer, shm = snap.wl_shm, ); @@ -317,15 +343,52 @@ async fn check_wayland_backend() -> CheckEntry { format!("All wlroots manager globals advertised ({msg})."), ); } - // Input-injection backend check (#1982). A non-wlroots compositor - // (KWin/Plasma, Mutter/GNOME) advertises no zwlr_virtual_pointer; on those - // the ONLY working input path is libei via xdg-desktop-portal. If this - // binary was built without `portal-libei` (the published tarball is — see - // #1967), input injection has no backend and silently no-ops: the agent - // cursor renders but clicks/keys are never delivered, while list_windows - // and capture still work. Report that explicitly instead of the misleading - // "input may fall back" partial-pass below. - if !snap.virtual_pointer && !crate::wayland::PORTAL_LIBEI_ENABLED { + if !snap.virtual_pointer { + if remote_desktop_portal_reachable && target_activation_available { + return CheckEntry::pass( + NAME_WAYLAND_BACKEND, + format!( + "No wlroots virtual-pointer advertised ({msg}), but this \ + portal/libei build can reach the RemoteDesktop portal \ + (proxy reachability only — the full create_session → \ + select_devices → start → connect_to_eis handshake is NOT \ + exercised here, to avoid a consent prompt on every doctor \ + run, so this is not a guarantee that injection succeeds). \ + The compositor helper also provides verified target \ + activation before focus-bound portal input." + ), + ); + } + if remote_desktop_portal_reachable { + return CheckEntry::fail( + NAME_WAYLAND_BACKEND, + format!( + "The RemoteDesktop portal is reachable, but this compositor \ + has no verified target-activation adapter ({msg}). Portal/libei \ + input is global and would otherwise affect whichever window is \ + focused, potentially the wrong application, so cua-driver \ + refuses foreground dispatch." + ), + "On GNOME, install and enable the bundled WinRects Shell helper, then \ + log out and back in. KDE foreground input remains unavailable until \ + a target-addressable KWin activation adapter is installed; AX actions \ + and exact background refusals remain usable.", + ); + } + if portal_libei_enabled { + return CheckEntry::fail( + NAME_WAYLAND_BACKEND, + format!( + "No wlroots virtual-pointer advertised ({msg}) and the \ + portal/libei RemoteDesktop backend is compiled in but not \ + reachable on this session; clicks and key presses have no \ + native Wayland input backend." + ), + "Ensure xdg-desktop-portal and a desktop backend such as \ + xdg-desktop-portal-gnome or xdg-desktop-portal-kde are running \ + on the session bus, or run under XWayland.", + ); + } return CheckEntry::fail( NAME_WAYLAND_BACKEND, format!( @@ -336,31 +399,31 @@ async fn check_wayland_backend() -> CheckEntry { screen capture are unaffected." ), "Use the portal-enabled Linux build (compiled with --features \ - portal-libei) for input on KDE Plasma / GNOME, or a wlroots \ + portal-input) for input on KDE Plasma / GNOME, or a wlroots \ compositor (sway, labwc, hyprland) where zwlr_virtual_pointer exists.", ); } - // Partial-pass: list_windows + capture both work, but virtual-pointer - // input is missing. Require `wl_shm` here too — `check_screen_capture_capability` - // gates on both `screencopy && wl_shm`, so excluding `wl_shm` from the - // partial-pass verdict would let the matrices disagree on degenerate - // compositors that omit it. + // Partial-pass: list_windows + capture both work, but some optional + // wlroots globals are absent. Require `wl_shm` here too — + // `check_screen_capture_capability` gates on both `screencopy && wl_shm`, + // so excluding `wl_shm` from the partial-pass verdict would let the + // matrices disagree on degenerate compositors that omit it. if snap.foreign_toplevel && snap.screencopy && snap.wl_shm { return CheckEntry::pass( NAME_WAYLAND_BACKEND, format!( - "Core wlroots manager globals available; some optional globals missing ({msg}). \ - Input may fall back where virtual-pointer is absent." + "Core wlroots manager globals available; some optional globals missing ({msg})." ), ); } CheckEntry::fail( NAME_WAYLAND_BACKEND, format!( - "Compositor does not advertise the wlroots manager globals cua-driver \ - needs ({msg})." + "Compositor does not advertise a complete native Wayland backend \ + set ({msg})." ), - "Use a wlroots-based compositor (sway, labwc, hyprland) or run under XWayland.", + "Use a wlroots-based compositor (sway, labwc, hyprland), a portal/libei \ + build on GNOME/KDE, or run under XWayland.", ) } @@ -471,6 +534,45 @@ fn probe_portal_screenshot() -> anyhow::Result { Ok(false) } +#[cfg(target_os = "linux")] +fn probe_portal_remote_desktop() -> anyhow::Result { + #[cfg(feature = "portal-input")] + { + use ashpd::desktop::remote_desktop::RemoteDesktop; + + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|e| anyhow::anyhow!("failed to build tokio runtime for RemoteDesktop probe: {e}"))?; + + rt.block_on(async { + match RemoteDesktop::new().await { + Ok(_) => Ok(true), + Err(e) => { + let msg = format!("{e}"); + if msg.contains("ServiceUnknown") + || msg.contains("NameHasNoOwner") + || msg.contains("NotFound") + { + Ok(false) + } else { + Err(anyhow::anyhow!("portal RemoteDesktop probe failed: {e}")) + } + } + } + }) + } + #[cfg(not(feature = "portal-input"))] + { + Ok(false) + } +} + +#[cfg(not(target_os = "linux"))] +fn probe_portal_remote_desktop() -> anyhow::Result { + Ok(false) +} + /// Stub of `wayland::WaylandManagers` so off-Linux builds compile. Always /// reports nothing advertised — non-Linux code paths never call this. #[cfg(not(target_os = "linux"))] @@ -541,6 +643,60 @@ mod tests { } } + #[test] + fn wayland_backend_passes_on_non_wlroots_when_portal_libei_backend_is_reachable() { + let snap = crate::wayland::WaylandManagers { + foreign_toplevel: false, + screencopy: false, + ext_image_copy_capture: false, + ext_output_image_capture_source: false, + virtual_pointer: false, + wl_shm: true, + }; + + let entry = classify_wayland_backend(&snap, true, true, true); + + assert_eq!(entry.status, CheckStatus::Pass); + assert!(entry.message.contains("portal/libei"), "{}", entry.message); + assert!(entry.message.contains("RemoteDesktop"), "{}", entry.message); + } + + #[test] + fn wayland_backend_fails_on_non_wlroots_when_portal_libei_backend_is_unreachable() { + let snap = crate::wayland::WaylandManagers { + foreign_toplevel: false, + screencopy: false, + ext_image_copy_capture: false, + ext_output_image_capture_source: false, + virtual_pointer: false, + wl_shm: true, + }; + + let entry = classify_wayland_backend(&snap, true, false, false); + + assert_eq!(entry.status, CheckStatus::Fail); + assert!(entry.message.contains("portal/libei"), "{}", entry.message); + assert!(entry.hint.as_deref().unwrap_or("").contains("xdg-desktop-portal")); + } + + #[test] + fn wayland_backend_fails_closed_when_portal_input_cannot_target_a_window() { + let snap = crate::wayland::WaylandManagers { + foreign_toplevel: false, + screencopy: false, + ext_image_copy_capture: false, + ext_output_image_capture_source: false, + virtual_pointer: false, + wl_shm: true, + }; + + let entry = classify_wayland_backend(&snap, true, true, false); + + assert_eq!(entry.status, CheckStatus::Fail); + assert!(entry.message.contains("target-activation"), "{}", entry.message); + assert!(entry.message.contains("wrong application"), "{}", entry.message); + } + #[tokio::test] async fn invoke_full_run_produces_linux_check_set() { let provider = Arc::new(LinuxHealthProvider); diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/delivery.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/delivery.rs index 258c392cdb..33f6e26e67 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/input/delivery.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/input/delivery.rs @@ -15,7 +15,7 @@ //! specific non-focused window the way X11/macOS/Windows can (this is a //! platform constraint, reported honestly, like macOS pixel input being //! driver-unverifiable). When no libei backend is available -//! (`PORTAL_LIBEI_ENABLED == false`) the tool returns a structured +//! (`PORTAL_INPUT_ENABLED == false`) the tool returns a structured //! `background_unavailable` error so the caller can escalate to foreground. //! //! - `foreground` — activate the target first, inject, then restore the prior @@ -92,31 +92,46 @@ pub fn delivery_mode_schema() -> Value { /// Reason a `background` delivery cannot be performed on Wayland. #[derive(Copy, Clone, Debug)] pub enum BackgroundUnavailable { - /// No libei backend (built without `portal-libei`, or the portal session + /// No libei backend (built without `portal-input`, or the portal session /// was denied / unavailable). Input has no actuator at all. NoLibeiBackend, - /// X11/Chromium does not accept a key chord addressed to an unfocused - /// renderer without briefly moving focus, which background delivery forbids. - ChromiumHotkey, + /// X11/Chromium does not accept synthetic pointer or keyboard input + /// addressed to an occluded, unfocused renderer without briefly moving + /// focus, which background delivery forbids. + ChromiumInput, + /// The remaining backend can only inject into the globally focused widget. + FocusedInputOnly, + /// WebKitGTK rejects synthetic XSendEvent input and no real target-addressed + /// pointer backend is available in this session. + WebKitSyntheticInput, } impl BackgroundUnavailable { fn code(self) -> &'static str { match self { Self::NoLibeiBackend => "background_unavailable", - Self::ChromiumHotkey => "background_unavailable", + Self::ChromiumInput => "background_unavailable", + Self::FocusedInputOnly => "background_unavailable", + Self::WebKitSyntheticInput => "background_unavailable", } } fn detail(self) -> &'static str { match self { Self::NoLibeiBackend => { "no libei input backend on this Wayland compositor (built without \ - portal-libei, or the xdg-desktop-portal RemoteDesktop session was \ + portal-input, or the xdg-desktop-portal RemoteDesktop session was \ unavailable/denied): synthetic input has no actuator" } - Self::ChromiumHotkey => { - "Chromium/Electron does not accept a key chord addressed to an \ - unfocused renderer through X11 background injection" + Self::ChromiumInput => { + "Chromium/Electron does not accept pointer or keyboard input \ + addressed to an occluded, unfocused renderer through X11 \ + background injection" + } + Self::FocusedInputOnly => { + "the requested target has no focus-free input backend; the remaining XTest/X11 route can only deliver to the globally focused widget" + } + Self::WebKitSyntheticInput => { + "WebKitGTK rejects synthetic XSendEvent input and this session has no real target-addressed pointer backend" } } } diff --git a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs index 6f5ced3118..063923db88 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/input/mod.rs @@ -31,6 +31,7 @@ use x11rb::protocol::xproto::*; use x11rb::rust_connection::RustConnection; const CLICK_DELAY_MS: u64 = 35; +const DOUBLE_CLICK_DELAY_MS: u64 = 50; const KEY_DELAY_MS: u64 = 10; #[derive(Clone, Debug)] @@ -2018,9 +2019,17 @@ pub fn send_click_xtest_desktop(x: i32, y: i32, button: u8, count: usize) -> Res // Absolute pointer warp (MotionNotify, detail=0 => absolute) so the button // events that follow are delivered at (x, y). conn.xtest_fake_input(MOTION_NOTIFY_EVENT, 0, 0, root, x as i16, y as i16, 0)?; - for _ in 0..count.max(1) { + let count = count.max(1); + for click_index in 0..count { conn.xtest_fake_input(BUTTON_PRESS_EVENT, button, 0, root, x as i16, y as i16, 0)?; conn.xtest_fake_input(BUTTON_RELEASE_EVENT, button, 0, root, x as i16, y as i16, 0)?; + if click_index + 1 < count { + // Chromium needs the first pair to reach the server before the + // second pair. A zero-gap batch produces two click events but no + // DOM dblclick event under Xvfb/Openbox. + conn.flush()?; + sleep(Duration::from_millis(DOUBLE_CLICK_DELAY_MS)); + } } conn.flush()?; // Round-trip so the server processes the warp+button events before this @@ -2031,6 +2040,75 @@ pub fn send_click_xtest_desktop(x: i32, y: i32, button: u8, count: usize) -> Res Ok(()) } +/// Screen-absolute drag via XTest. The caller activates the target first; XTest +/// then supplies one real press, interpolated pointer motion, and one release. +/// This is the foreground counterpart to the window-addressed XSendEvent drag. +pub fn send_drag_xtest_desktop( + from_x: i32, + from_y: i32, + to_x: i32, + to_y: i32, + button: u8, + duration_ms: u64, + steps: usize, +) -> Result<()> { + use x11rb::protocol::xtest::ConnectionExt as _; + let (conn, screen_num) = connect_x11_for_input()?; + let root = conn.setup().roots[screen_num].root; + let steps = steps.max(1); + let delay = duration_ms / steps as u64; + + conn.xtest_fake_input( + MOTION_NOTIFY_EVENT, + 0, + 0, + root, + from_x as i16, + from_y as i16, + 0, + )?; + conn.xtest_fake_input( + BUTTON_PRESS_EVENT, + button, + 0, + root, + from_x as i16, + from_y as i16, + 0, + )?; + conn.flush()?; + for step in 1..=steps { + let t = step as f64 / steps as f64; + let x = from_x as f64 + (to_x - from_x) as f64 * t; + let y = from_y as f64 + (to_y - from_y) as f64 * t; + conn.xtest_fake_input( + MOTION_NOTIFY_EVENT, + 0, + 0, + root, + x.round() as i16, + y.round() as i16, + 0, + )?; + conn.flush()?; + if delay > 0 { + sleep(Duration::from_millis(delay)); + } + } + conn.xtest_fake_input( + BUTTON_RELEASE_EVENT, + button, + 0, + root, + to_x as i16, + to_y as i16, + 0, + )?; + conn.flush()?; + let _ = conn.get_input_focus()?.reply(); + Ok(()) +} + /// Send a named key press to a window. pub fn send_key(xid: u64, key: &str, modifiers: &[&str]) -> Result<()> { send_key_to_target(xid, None, key, modifiers) diff --git a/libs/cua-driver/rust/crates/platform-linux/src/lib.rs b/libs/cua-driver/rust/crates/platform-linux/src/lib.rs index f6b41cb766..928525b0e5 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/lib.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/lib.rs @@ -16,6 +16,7 @@ pub mod tools; pub mod overlay; pub mod pip; pub mod health_report; +pub mod recording_hooks; #[cfg(target_os = "linux")] pub mod x11; @@ -44,6 +45,9 @@ pub mod a11y; #[cfg(target_os = "linux")] pub mod wayland; +#[cfg(target_os = "linux")] +pub mod video_wayland; + // `terminal` is OS-independent (pure string matching + a thin x11 hook). // Keeping it un-gated lets the unit tests run on any host. pub mod terminal; diff --git a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs index 80544f5157..1935550e62 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/overlay.rs @@ -5,7 +5,9 @@ //! from XComposite. The window covers the full display area. //! - A background thread renders frames at ~60 Hz using tiny-skia and XShmPutImage //! (or XPutImage fallback) with XRender ARGB compositing. -//! - Mouse events pass through via `XShapeSelectInput(ShapeInput, empty-region)`. +//! - XShape clips both input and visible pixels. The visible shape follows the +//! rendered alpha mask so bare X11 window managers do not show a black +//! full-screen ARGB window when no compositor is present. //! - Z-ordering: `XRaiseWindow` every 80ms to stay above normal windows. //! - Wayland: when WAYLAND_DISPLAY is set but DISPLAY is also available (XWayland), //! the X11 path is used. Pure Wayland support is a TODO. @@ -140,22 +142,21 @@ pub fn send_command_for(key: CursorKey, cmd: OverlayCommand) { #[cfg(target_os = "linux")] { if crate::wayland::is_wayland() { - let _ = crate::wayland::overlay::forward(&msg); - // Non-wlroots compositors (GNOME Mutter / KDE) expose no - // `zwlr_layer_shell_v1`, so the forward above is a no-op there. Drive - // the agent cursor through the WinRects shell extension instead - // (no-op if it isn't installed). Only the SINGLE positioning commands - // are forwarded — never the interpolated `MoveTo` stream (the - // extension does its own easing; the glide target is sent once from - // `overlay_glide_to_for`). - match &cmd { - cursor_overlay::OverlayCommand::ClickPulse { x, y } => { - crate::wayland::shell_helper::click_pulse(*x as i32, *y as i32); - } - cursor_overlay::OverlayCommand::SnapTo { x, y, .. } => { - crate::wayland::shell_helper::move_cursor(*x as i32, *y as i32); + if crate::wayland::shell_helper::available() { + // GNOME has no layer-shell. Drive only the final positioning + // commands through the compositor helper; it performs its own + // easing and avoids starting a worker that must fail. + match &cmd { + cursor_overlay::OverlayCommand::ClickPulse { x, y } => { + crate::wayland::shell_helper::click_pulse(*x as i32, *y as i32); + } + cursor_overlay::OverlayCommand::SnapTo { x, y, .. } => { + crate::wayland::shell_helper::move_cursor(*x as i32, *y as i32); + } + _ => {} } - _ => {} + } else { + let _ = crate::wayland::overlay::forward(&msg); } } } @@ -196,6 +197,21 @@ pub fn current_position_for(key: &str) -> (f64, f64) { .unwrap_or((-200.0, -200.0)) } +pub fn current_motion_for(key: &str) -> cursor_overlay::MotionConfig { + RENDER + .lock() + .ok() + .and_then(|guard| { + guard.as_ref().and_then(|map| { + map.cursors + .get(key) + .or_else(|| map.cursors.get("default")) + .map(|state| state.core.motion.clone()) + }) + }) + .unwrap_or_default() +} + fn seed_start_if_sentinel(key: &CursorKey, target_x: f64, target_y: f64) -> bool { const SEED_OFFSET: f64 = 140.0; let mut guard = RENDER.lock().unwrap(); @@ -616,6 +632,7 @@ fn paint_x11( _visual_id: u32, pm: &tiny_skia::Pixmap, ) { + use x11rb::protocol::shape::{ConnectionExt as ShapeConnectionExt, SK, SO}; use x11rb::protocol::xproto::{ConnectionExt as XprotoConnectionExt, CreateGCAux, ImageFormat}; if pm.width() == 0 || pm.height() == 0 { return; @@ -632,15 +649,21 @@ fn paint_x11( return; } - // Convert RGBA premult → BGRA premult for X11. - let src = pm.data(); - let mut bgra: Vec = Vec::with_capacity(src.len()); - for chunk in src.chunks_exact(4) { - bgra.push(chunk[2]); // B - bgra.push(chunk[1]); // G - bgra.push(chunk[0]); // R - bgra.push(chunk[3]); // A - } + let (bgra, visible_shape) = bgra_and_visible_shape(pm); + + // A 32-bit ARGB window needs a compositor to blend transparent pixels. + // Without one, zero-alpha pixels display as opaque black. Clip the native + // window to the rendered non-zero alpha runs so the overlay remains usable + // under bare Openbox/i3/Xvfb sessions as well as composited desktops. + let _ = conn.shape_rectangles( + SO::SET, + SK::BOUNDING, + x11rb::protocol::xproto::ClipOrdering::UNSORTED, + win, + 0, + 0, + &visible_shape, + ); // XPutImage (ZPixmap). let _ = conn.put_image( @@ -660,5 +683,79 @@ fn paint_x11( conn.flush().ok(); } +#[cfg(target_os = "linux")] +fn bgra_and_visible_shape( + pm: &tiny_skia::Pixmap, +) -> (Vec, Vec) { + use x11rb::protocol::xproto::Rectangle; + + let width = pm.width() as usize; + let height = pm.height() as usize; + let src = pm.data(); + let mut bgra = Vec::with_capacity(src.len()); + let mut rectangles = Vec::new(); + + for y in 0..height { + let mut run_start = None; + for x in 0..width { + let offset = (y * width + x) * 4; + let pixel = &src[offset..offset + 4]; + bgra.extend_from_slice(&[pixel[2], pixel[1], pixel[0], pixel[3]]); + + if pixel[3] != 0 { + run_start.get_or_insert(x); + } else if let Some(start) = run_start.take() { + rectangles.push(Rectangle { + x: start as i16, + y: y as i16, + width: (x - start) as u16, + height: 1, + }); + } + } + if let Some(start) = run_start { + rectangles.push(Rectangle { + x: start as i16, + y: y as i16, + width: (width - start) as u16, + height: 1, + }); + } + } + + (bgra, rectangles) +} + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use super::bgra_and_visible_shape; + + #[test] + fn visible_shape_contains_only_nontransparent_runs() { + let mut pixmap = tiny_skia::Pixmap::new(4, 2).unwrap(); + pixmap.data_mut().copy_from_slice(&[ + 1, 2, 3, 0, 10, 20, 30, 255, 11, 21, 31, 128, 4, 5, 6, 0, 7, 8, 9, 64, 1, 1, 1, 0, 2, + 2, 2, 0, 12, 22, 32, 255, + ]); + + let (bgra, rectangles) = bgra_and_visible_shape(&pixmap); + + assert_eq!(&bgra[4..8], &[30, 20, 10, 255]); + assert_eq!(rectangles.len(), 3); + assert_eq!( + (rectangles[0].x, rectangles[0].y, rectangles[0].width), + (1, 0, 2) + ); + assert_eq!( + (rectangles[1].x, rectangles[1].y, rectangles[1].width), + (0, 1, 1) + ); + assert_eq!( + (rectangles[2].x, rectangles[2].y, rectangles[2].width), + (3, 1, 1) + ); + } +} + #[cfg(not(target_os = "linux"))] fn run_overlay_thread(_cfg: CursorConfig, _rx: std::sync::mpsc::Receiver) {} diff --git a/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs b/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs new file mode 100644 index 0000000000..e77960d3c8 --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-linux/src/recording_hooks.rs @@ -0,0 +1,95 @@ +//! Application-state snapshots used by trajectory recording on Linux. + +#[cfg(target_os = "linux")] +pub fn app_state_json_for(window_id: Option, pid: Option) -> Option> { + if tokio::runtime::Handle::try_current().is_ok() { + return std::thread::spawn(move || app_state_json_for_blocking(window_id, pid)) + .join() + .ok() + .flatten(); + } + app_state_json_for_blocking(window_id, pid) +} + +#[cfg(target_os = "linux")] +fn app_state_json_for_blocking(window_id: Option, pid: Option) -> Option> { + let pid = u32::try_from(pid?).ok()?; + let window_id = resolve_window_for_recording(pid, window_id)?.xid; + let result = crate::atspi::walk_tree(pid, window_id, None); + if result.nodes.is_empty() || result.tree_markdown.trim().is_empty() { + return None; + } + let element_count = result + .nodes + .iter() + .filter(|node| node.element_index.is_some()) + .count(); + let payload = serde_json::json!({ + "pid": pid, + "window_id": window_id, + "element_count": element_count, + "tree_markdown": result.tree_markdown, + }); + serde_json::to_vec_pretty(&payload).ok() +} + +#[cfg(target_os = "linux")] +pub fn element_window_local_xy(window_id: u64, pid: i64, element_index: u32) -> Option<(f64, f64)> { + if tokio::runtime::Handle::try_current().is_ok() { + return std::thread::spawn(move || { + element_window_local_xy_blocking(window_id, pid, element_index) + }) + .join() + .ok() + .flatten(); + } + element_window_local_xy_blocking(window_id, pid, element_index) +} + +#[cfg(target_os = "linux")] +fn element_window_local_xy_blocking( + window_id: u64, + pid: i64, + element_index: u32, +) -> Option<(f64, f64)> { + let pid = u32::try_from(pid).ok()?; + let (screen_x, screen_y, width, height) = + crate::atspi::get_element_bounds(pid, element_index as usize).ok()?; + let window = resolve_window_for_recording(pid, Some(window_id))?; + Some(( + f64::from(screen_x - window.x) + f64::from(width) / 2.0, + f64::from(screen_y - window.y) + f64::from(height) / 2.0, + )) +} + +#[cfg(target_os = "linux")] +fn resolve_window_for_recording( + pid: u32, + window_id: Option, +) -> Option { + let windows = crate::wayland::list_windows_dispatch(Some(pid)); + if crate::wayland::is_wayland() { + // Foreign-toplevel protocol object ids are scoped to one Wayland + // connection. Recording hooks open a fresh connection, so re-resolve + // the target by pid instead of comparing an id from the action call. + windows.into_iter().next() + } else if let Some(window_id) = window_id { + windows.into_iter().find(|window| window.xid == window_id) + } else { + windows.into_iter().next() + } +} + +#[cfg(not(target_os = "linux"))] +pub fn app_state_json_for(_window_id: Option, _pid: Option) -> Option> { + None +} + +#[cfg(not(target_os = "linux"))] +pub fn element_window_local_xy( + _window_id: u64, + _pid: i64, + _element_index: u32, +) -> Option<(f64, f64)> { + None +} diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs index 4a5b528569..ebc5693251 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/impl_.rs @@ -402,10 +402,17 @@ impl Tool for ListWindowsTool { async fn invoke(&self, args: Value) -> ToolResult { use cua_driver_core::tool_args::ArgsExt; let filter_pid = args.opt_u64("pid").map(|v| v as u32); - let windows = + let on_screen_only = args.bool_or("on_screen_only", false); + let mut windows = tokio::task::spawn_blocking(move || crate::wayland::list_windows_dispatch(filter_pid)) .await .unwrap_or_default(); + if on_screen_only { + windows.retain(|window| window.is_on_screen); + } + if crate::wayland::is_wayland() { + crate::wayland::remember_observed_window_origins(&windows); + } let mut lines = vec![format!("Found {} windows:", windows.len())]; for w in &windows { lines.push(format!( @@ -428,19 +435,16 @@ impl Tool for ListWindowsTool { /// existing Linux callers don't break. Fully additive; no field removed, /// no schema_version bump. /// -/// The Linux `WindowInfo` struct (see `crate::x11::WindowInfo`) exposes -/// neither an app name nor a visibility flag, so `app_name` is an empty -/// string and `is_on_screen` defaults to `true` — the same best-effort -/// default the Windows backend uses. fn window_record_json(w: &crate::x11::WindowInfo) -> Value { json!({ "window_id": w.xid, "pid": w.pid, - "app_name": "", + "app_name": w.app_name, "title": w.title, // Canonical cross-platform geometry (macOS/Windows parity). "bounds": { "x": w.x, "y": w.y, "width": w.width, "height": w.height }, - "is_on_screen": true, + "is_on_screen": w.is_on_screen, + "z_index": w.z_index, // Legacy alias: flat fields kept inline for pre-existing callers. "x": w.x, "y": w.y, "width": w.width, "height": w.height, @@ -456,7 +460,10 @@ mod list_windows_tests { let w = crate::x11::WindowInfo { xid: 42, pid: Some(1234), + app_name: "example-app".to_owned(), title: "Example".to_owned(), + is_on_screen: true, + z_index: Some(3), x: 10, y: 20, width: 300, @@ -480,11 +487,20 @@ mod list_windows_tests { assert_eq!(rec["height"], json!(400)); // Cross-platform companions. - assert_eq!(rec["app_name"], json!("")); + assert_eq!(rec["app_name"], json!("example-app")); assert_eq!(rec["is_on_screen"], json!(true)); + assert_eq!(rec["z_index"], json!(3)); assert_eq!(rec["window_id"], json!(42)); assert_eq!(rec["title"], json!("Example")); } + + #[test] + fn chromium_launch_detection_uses_executable_basename() { + assert!(chromium_family_program("/usr/bin/google-chrome-stable")); + assert!(chromium_family_program("CuaTestHarness.Electron")); + assert!(chromium_family_program("chromium-browser")); + assert!(!chromium_family_program("/usr/bin/gnome-text-editor")); + } } // ── get_window_state ───────────────────────────────────────────────────────── @@ -604,9 +620,13 @@ impl Tool for GetWindowStateTool { max_elements, max_depth, )); - // Best-effort per-element screen bounds (AT-SPI Component.GetExtents). - // Tolerant: an empty/missing map never fails the call. - let bounds = crate::atspi::get_all_element_bounds(pid, xid).unwrap_or_default(); + // Bounds and element indices come from the same captured AT-SPI + // traversal. Joining two live walks by ordinal mis-associated + // Chromium controls when its lazy subtree changed between walks. + let bounds = tree_result + .as_ref() + .map(|tree| tree.bounds.clone()) + .unwrap_or_default(); // Capture and DELIVER the screenshot alongside the tree by default — the // grounding frame the agent cross-checks the tree against. With // screenshot_out_file set, write to disk and surface the path instead @@ -630,7 +650,11 @@ impl Tool for GetWindowStateTool { Some((Some(B64.encode(&png)), None, w, h, original_w)) } } - Err(_) => None, + Err(error) => { + return Err(anyhow::anyhow!( + "window screenshot failed for window {xid}: {error}" + )); + } } } else { None @@ -851,7 +875,21 @@ impl Tool for LaunchAppTool { let mut parts = cmd.split_whitespace(); let prog = parts.next().unwrap_or(cmd); let rest: Vec<&str> = parts.collect(); - match std::process::Command::new(prog).args(&rest).spawn() { + let mut launch = std::process::Command::new(prog); + launch + .args(&rest) + // Enable accessibility for this child without toggling + // GNOME's global ScreenReaderEnabled setting (which can + // launch Orca). Native toolkits ignore these when they + // do not need them. + .env("ACCESSIBILITY_ENABLED", "1") + .env("NO_AT_BRIDGE", "0"); + if chromium_family_program(prog) + && !rest.iter().any(|arg| *arg == "--force-renderer-accessibility") + { + launch.arg("--force-renderer-accessibility"); + } + match launch.spawn() { Ok(child) => { let pid = child.id(); return Ok(( @@ -881,10 +919,15 @@ impl Tool for LaunchAppTool { Ok(Ok((message, pid_opt, name))) => { if let Some(pid) = pid_opt { let windows = tokio::task::spawn_blocking(move || { - crate::x11::list_windows(Some(pid)) - .iter() - .map(window_record_json) - .collect::>() + let deadline = std::time::Instant::now() + + std::time::Duration::from_secs(3); + loop { + let windows = crate::wayland::list_windows_dispatch(Some(pid)); + if !windows.is_empty() || std::time::Instant::now() >= deadline { + return windows.iter().map(window_record_json).collect::>(); + } + std::thread::sleep(std::time::Duration::from_millis(100)); + } }) .await .unwrap_or_default(); @@ -913,14 +956,26 @@ impl Tool for LaunchAppTool { } } +fn chromium_family_program(program: &str) -> bool { + let basename = std::path::Path::new(program) + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(program) + .to_ascii_lowercase(); + ["chrome", "chromium", "electron", "brave", "edge"] + .iter() + .any(|needle| basename.contains(needle)) +} + // ── shared helpers ──────────────────────────────────────────────────────────── -/// Resolve an AT-SPI element's center in window-local X11 coordinates. +/// Resolve an AT-SPI element's center in window-local coordinates. /// /// Returns `(xid, window_local_x, window_local_y)`. -/// Looks up element bounds via pyatspi subprocess, finds the owning window +/// Looks up element bounds via native AT-SPI, finds the owning window /// (via `xid_hint` or the first window for `pid`), then converts screen-absolute -/// → window-local coords via X11 translate_coordinates. +/// → window-local coords via compositor metadata on Wayland or +/// XTranslateCoordinates on X11. fn resolve_element_local_coords( pid: u32, idx: usize, @@ -932,6 +987,12 @@ fn resolve_element_local_coords( let xid = if let Some(x) = xid_hint { x + } else if crate::wayland::wayland_input_enabled() { + crate::wayland::list_windows_dispatch(Some(pid)) + .into_iter() + .next() + .map(|window| window.xid) + .ok_or_else(|| anyhow::anyhow!("No Wayland windows for pid {pid}"))? } else { crate::x11::list_windows(Some(pid)) .into_iter() @@ -940,6 +1001,20 @@ fn resolve_element_local_coords( .ok_or_else(|| anyhow::anyhow!("No windows for pid {pid}"))? }; + if crate::wayland::wayland_input_enabled() { + let (window_x, window_y, window_width, window_height) = + crate::wayland::window_geometry(xid) + .ok_or_else(|| anyhow::anyhow!("No Wayland geometry for window {xid}"))?; + if window_width == 0 || window_height == 0 { + anyhow::bail!("Wayland window {xid} has no usable geometry"); + } + return Ok(( + xid, + screen_cx - window_x as f64, + screen_cy - window_y as f64, + )); + } + use x11rb::connection::Connection; use x11rb::protocol::xproto::ConnectionExt as _; use x11rb::rust_connection::RustConnection; @@ -1150,6 +1225,159 @@ fn is_chromium_embedder(pid: u32) -> bool { false } +fn is_webkitgtk_embedder(pid: u32) -> bool { + fn argv_is_webkit_helper(pid: u32) -> bool { + fs::read(format!("/proc/{pid}/cmdline")) + .map(|raw| { + let cmdline = String::from_utf8_lossy(&raw); + cmdline.contains("WebKitWebProcess") || cmdline.contains("WebKitNetworkProcess") + }) + .unwrap_or(false) + } + + let mut children: std::collections::HashMap> = std::collections::HashMap::new(); + let Ok(entries) = fs::read_dir("/proc") else { + return false; + }; + for entry in entries.flatten() { + let Ok(child) = entry.file_name().to_string_lossy().parse::() else { + continue; + }; + let Ok(status) = fs::read_to_string(format!("/proc/{child}/status")) else { + continue; + }; + if let Some(ppid) = status + .lines() + .find(|line| line.starts_with("PPid:")) + .and_then(|line| line[5..].trim().parse::().ok()) + { + children.entry(ppid).or_default().push(child); + } + } + let mut queue = std::collections::VecDeque::from([pid]); + let mut seen = std::collections::HashSet::new(); + while let Some(current) = queue.pop_front() { + if !seen.insert(current) { + continue; + } + if argv_is_webkit_helper(current) { + return true; + } + if let Some(descendants) = children.get(¤t) { + queue.extend(descendants.iter().copied()); + } + } + false +} + +fn maps_indicate_gtk(maps: &str) -> bool { + maps.contains("libgtk-3.so") || maps.contains("libgtk-4.so") +} + +fn is_gtk_process(pid: u32) -> bool { + fs::read_to_string(format!("/proc/{pid}/maps")) + .map(|maps| maps_indicate_gtk(&maps)) + .unwrap_or(false) +} + +fn unavailable_webkit_background( + pid: u32, + delivery: crate::input::delivery::DeliveryMode, +) -> Option { + (!delivery.is_foreground() + && is_webkitgtk_embedder(pid) + && !crate::wayland::is_inject_mode() + && !crate::input::real_pointer_input_available()) + .then(|| { + crate::input::delivery::background_unavailable_error( + crate::input::delivery::BackgroundUnavailable::WebKitSyntheticInput, + ) + }) +} + +fn unavailable_webkit_keyboard_background( + pid: u32, + delivery: crate::input::delivery::DeliveryMode, +) -> Option { + (!delivery.is_foreground() + && is_webkitgtk_embedder(pid) + && !crate::wayland::is_inject_mode()) + .then(|| { + crate::input::delivery::background_unavailable_error( + crate::input::delivery::BackgroundUnavailable::FocusedInputOnly, + ) + }) +} + +fn unavailable_gtk_keyboard_background( + pid: u32, + delivery: crate::input::delivery::DeliveryMode, +) -> Option { + (!delivery.is_foreground() && is_gtk_process(pid) && !crate::wayland::is_inject_mode()).then(|| { + crate::input::delivery::background_unavailable_error( + crate::input::delivery::BackgroundUnavailable::FocusedInputOnly, + ) + }) +} + +fn unavailable_gtk_pointer_background( + pid: u32, + delivery: crate::input::delivery::DeliveryMode, +) -> Option { + (!delivery.is_foreground() + && is_gtk_process(pid) + && !crate::wayland::is_inject_mode() + && !crate::input::real_pointer_input_available()) + .then(|| { + crate::input::delivery::background_unavailable_error( + crate::input::delivery::BackgroundUnavailable::FocusedInputOnly, + ) + }) +} + +fn unavailable_wayland_focused_input_background( + delivery: crate::input::delivery::DeliveryMode, + focus_free_inject_supported: bool, +) -> Option { + (crate::wayland::wayland_input_enabled() + && !(focus_free_inject_supported && crate::wayland::is_inject_mode()) + && !delivery.is_foreground()) + .then(|| { + crate::input::delivery::background_unavailable_error( + crate::input::delivery::BackgroundUnavailable::FocusedInputOnly, + ) + }) +} + +/// Chromium's X11 renderer drops synthetic input sent to an occluded, +/// unfocused toplevel. Returning success here would be a silent loss, so all +/// input tools expose the same typed refusal and leave foreground activation +/// as the explicit escalation. +fn unavailable_chromium_background( + pid: u32, + delivery: crate::input::delivery::DeliveryMode, +) -> Option { + if chromium_background_must_refuse( + delivery.is_foreground(), + crate::wayland::is_inject_mode(), + is_chromium_embedder(pid), + ) { + Some(crate::input::delivery::background_unavailable_error( + crate::input::delivery::BackgroundUnavailable::ChromiumInput, + )) + } else { + None + } +} + +fn chromium_background_must_refuse( + foreground: bool, + focus_free_inject_mode: bool, + chromium: bool, +) -> bool { + chromium && !foreground && !focus_free_inject_mode +} + /// Screen-absolute center of a window (top-left from translate_coordinates plus /// half its geometry). Used to position the no-focus-steal scroll over the /// window's content. Blocking — call inside spawn_blocking. @@ -1552,17 +1780,21 @@ impl Tool for ClickTool { // sees the cursor "click somewhere else." overlay_glide_to_for(&cursor_id, sx as f64, sy as f64).await; let r = tokio::task::spawn_blocking(move || { - crate::input::send_click_xtest_desktop(sx, sy, button, n) + if crate::wayland::wayland_input_enabled() { + crate::wayland::click_desktop(sx, sy, n as u32, button) + } else { + crate::input::send_click_xtest_desktop(sx, sy, button, n) + } }) .await; return match r { - // Screen-absolute XTEST click — never driver-verifiable (no + // Screen-absolute click — never driver-verifiable (no // read-back); the caller confirms via screenshot. Ok(Ok(())) => ToolResult::text(format!( "✅ Sent screen-absolute click at ({sx},{sy}) (desktop scope)." )) .with_structured( - json!({ "path": "xtest_desktop", "verified": false, "effect": "unverifiable" }), + json!({ "path": if crate::wayland::wayland_input_enabled() { "wayland_desktop" } else { "xtest_desktop" }, "verified": false, "effect": "unverifiable" }), ), Ok(Err(e)) => ToolResult::error(format!("desktop-scope click failed: {e}")), Err(e) => ToolResult::error(format!("task error: {e}")), @@ -1573,6 +1805,7 @@ impl Tool for ClickTool { Ok(v) => v, Err(e) => return e, }; + let delivery = crate::input::delivery::DeliveryMode::from_args(&args); let count = args.u64_or("count", 1) as usize; // Surface 5: reject unknown buttons so a typo can't silently fall through // to a left-click. Empty string keeps back-compat with old clients. @@ -1613,7 +1846,7 @@ impl Tool for ClickTool { }; let window_id_resolved: Option = match &resolved { cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => { - window_id.map(|v| v as u64) + window_id_arg.or_else(|| window_id.map(|v| v as u64)) } cua_driver_core::element_token::ResolvedElement::None => window_id_arg, }; @@ -1651,36 +1884,45 @@ impl Tool for ClickTool { cursor_overlay::OverlayCommand::ClickPulse { x: sx, y: sy }, ); - // Now perform the actual click: AT-SPI doAction(0) first (background- - // safe, no focus steal), else XSendEvent at window-local coords. The - // AT-SPI rung also reports whether the actuated element looked like a - // silent no-op (passive role / no advertised action) — see - // perform_action — so the response can flag `effect: "suspected_noop"`. - let result = - tokio::task::spawn_blocking(move || -> anyhow::Result<(&'static str, bool)> { - if let Ok((_action, suspected_noop)) = crate::atspi::perform_action(pid, idx) { - return Ok(("ax", suspected_noop)); - } - let (xid2, lx, ly) = resolve_element_local_coords(pid, idx, xid_hint)?; - crate::input::send_click(xid2, lx as i32, ly as i32, count, button)?; - Ok(("x11_pixel", false)) - }) - .await; + // Chromium can execute a genuine AT-SPI action without focus. Try + // that route before applying its background synthetic-input gate. + let ax_result = + tokio::task::spawn_blocking(move || crate::atspi::perform_action(pid, idx)).await; + if let Ok(Ok((_action, suspected_noop))) = ax_result { + let mut structured = json!({ + "path": "ax", + "verified": false, + "effect": if suspected_noop { "suspected_noop" } else { "unverifiable" }, + }); + if suspected_noop { + structured["escalation"] = non_ax_escalation(); + } + return ToolResult::text(format!("Clicked element [{idx}] (pid {pid}).")) + .with_structured(structured); + } + if let Some(refusal) = unavailable_chromium_background(pid, delivery) { + return refusal; + } + + // The AX route was unavailable. Fall back to a target-addressed + // X11 event for toolkits that accept it. + let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { + let (xid2, lx, ly) = resolve_element_local_coords(pid, idx, xid_hint)?; + crate::input::send_click(xid2, lx as i32, ly as i32, count, button) + }) + .await; return match result { // An element click is never driver-verifiable (no read-back) — // verified:false; the caller confirms via screenshot. `effect` is // the richer signal: a passive/role-mismatched AT-SPI actuation is // a likely no-op (→ cross to vision/pixel), otherwise the dispatch // was fine but unconfirmable. - Ok(Ok((path, suspected_noop))) => { - let mut structured = json!({ - "path": path, + Ok(Ok(())) => { + let structured = json!({ + "path": "x11_pixel", "verified": false, - "effect": if suspected_noop { "suspected_noop" } else { "unverifiable" }, + "effect": "unverifiable", }); - if suspected_noop { - structured["escalation"] = non_ax_escalation(); - } ToolResult::text(format!("Clicked element [{idx}] (pid {pid}).")) .with_structured(structured) } @@ -1689,6 +1931,10 @@ impl Tool for ClickTool { }; } + if let Some(refusal) = unavailable_chromium_background(pid, delivery) { + return refusal; + } + // Coordinate-based path. let xid = match args.opt_u64("window_id") { Some(v) => v, @@ -1719,13 +1965,20 @@ impl Tool for ClickTool { cursor_id.clone(), cursor_overlay::OverlayCommand::PinAbove(xid), ); - // Resolve the screen point the cursor glides to. On native Wayland the - // agent already passes screen coordinates (the vision screenshot and - // `get_window_state` frames are screen-space, and `window_local_to_screen` - // — an X11 `translate_coordinates` call — can't run with DISPLAY unset), - // so use them directly. On X11 the coords are window-local; translate. - let glide_target = if crate::wayland::is_wayland() { - Some((x, y)) + // Resolve the screen point the cursor glides to. Tool coordinates are + // always window-local screenshot pixels; native Wayland translates + // through compositor/AT-SPI geometry while X11 uses XTranslateCoordinates. + let wayland_output_point = if crate::wayland::wayland_input_enabled() { + Some(crate::wayland::window_local_to_output( + xid, + x.round() as i32, + y.round() as i32, + )) + } else { + None + }; + let glide_target = if let Some((sx, sy)) = wayland_output_point { + Some((sx as f64, sy as f64)) } else { tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)) .await @@ -1741,13 +1994,13 @@ impl Tool for ClickTool { } let (xi, yi) = (x as i32, y as i32); + let (output_x, output_y) = wayland_output_point.unwrap_or((xi, yi)); let cursor_id_for_task = cursor_id.clone(); // delivery_mode: background (default) = no-focus-steal injection; // foreground = activate the target window (EWMH) first, then inject, // then restore prior active. Mirrors macOS/Windows. - let delivery = crate::input::delivery::DeliveryMode::from_args(&args); let result = tokio::task::spawn_blocking(move || -> anyhow::Result<&'static str> { - if crate::wayland::is_wayland() { + if crate::wayland::wayland_input_enabled() { // Vision/pixel click on native Wayland. Mutter drops synthetic // virtual-pointer events (the `wayland::click` warp doesn't land), // so for a plain left single click resolve the screen pixel to the @@ -1755,21 +2008,29 @@ impl Tool for ClickTool { // `element_index` — the coordinate-free path already verified // working. (x,y) are screen coords here, matching the frames in // `get_window_state`. Miss → fall through to the injection paths. - if button == 1 && count == 1 { + if !delivery.is_foreground() && button == 1 && count == 1 { if let Ok(Some(_)) = - crate::atspi::perform_action_at_screen_point(pid, xid, xi, yi) + crate::atspi::perform_action_at_screen_point( + pid, + xid, + output_x, + output_y, + ) { return Ok("wayland_atspi"); } } if crate::wayland::is_inject_mode() { crate::wayland::inject_click(xid, x, y, count as u32, button)?; - return Ok("wayland_libei"); + return Ok("wayland_cua_compositor"); + } + if !delivery.is_foreground() { + return Ok("background_unavailable"); } // Native Wayland: focus+raise the target toplevel // (foreign-toplevel `activate`), then drive `count` virtual-pointer // button events. Wayland injection routes to the compositor focus. - crate::wayland::click(xid, xi, yi, count as u32, button)?; + crate::wayland::click(xid, output_x, output_y, count as u32, button)?; return Ok("wayland_activate"); } // X11 injection. Tiered no-focus-steal delivery (background): @@ -1819,6 +2080,11 @@ impl Tool for ClickTool { "background" }; match result { + Ok(Ok("background_unavailable")) => { + crate::input::delivery::background_unavailable_error( + crate::input::delivery::BackgroundUnavailable::FocusedInputOnly, + ) + } // A pixel/coordinate click is never driver-verifiable (no read-back) — // verified:false, effect:"unverifiable"; the caller confirms via // screenshot. path reports the rung taken. @@ -1882,6 +2148,43 @@ async fn focus_by_pixel( Ok(()) } +/// Establish widget-local focus inside a nested-compositor target without +/// changing the compositor's focused toplevel. AX uses Component.GrabFocus; +/// PX sends a private per-surface left click at the requested local point. +async fn focus_nested_inject_target( + pid: u32, + window_id: u64, + element_index: Option, + pixel: Option<(f64, f64)>, +) -> Result<(), ToolResult> { + if let Some(index) = element_index { + return match tokio::task::spawn_blocking(move || { + crate::atspi::focus_element(pid, index) + }) + .await + { + Ok(Ok(true)) => Ok(()), + Ok(Ok(false)) => Err(ToolResult::error(format!( + "AT-SPI Component.GrabFocus returned false for element {index}" + ))), + Ok(Err(error)) => Err(ToolResult::error(error.to_string())), + Err(error) => Err(ToolResult::error(format!("Task error: {error}"))), + }; + } + if let Some((x, y)) = pixel { + return match tokio::task::spawn_blocking(move || { + crate::wayland::inject_click(window_id, x, y, 1, 1) + }) + .await + { + Ok(Ok(())) => Ok(()), + Ok(Err(error)) => Err(ToolResult::error(error.to_string())), + Err(error) => Err(ToolResult::error(format!("Task error: {error}"))), + }; + } + Ok(()) +} + // ── type_text ───────────────────────────────────────────────────────────────── pub struct TypeTextTool { @@ -1944,7 +2247,7 @@ impl Tool for TypeTextTool { } => (Some(*element_index), window_id.map(|v| v as u64)), cua_driver_core::element_token::ResolvedElement::None => (None, None), }; - let xid_opt = resolved_window_id.or_else(|| args.opt_u64("window_id")); + let xid_opt = args.opt_u64("window_id").or(resolved_window_id); // Resolve XID: use window_id if given, else first window for pid. let xid = match xid_opt { @@ -1964,6 +2267,74 @@ impl Tool for TypeTextTool { } } }; + let delivery = crate::input::delivery::DeliveryMode::from_args(&args); + if let Some(refusal) = unavailable_chromium_background(pid, delivery) { + return refusal; + } + if resolved_elem_idx.is_none() { + if let Some(refusal) = unavailable_wayland_focused_input_background(delivery, true) { + return refusal; + } + } + + let px = args.get("x").and_then(|value| value.as_f64()); + let py = args.get("y").and_then(|value| value.as_f64()); + if px.is_some() != py.is_some() { + return ToolResult::error("Pass both x and y to type_text, or neither."); + } + if px.is_some() && resolved_elem_idx.is_some() { + return ToolResult::error( + "Pass either element_index (ax) or x,y (px) to type_text, not both.", + ); + } + + let text_len = text.chars().count(); + // Native toolkit editables have a stronger focus-free route than raw + // compositor keyboard injection. Keep Chromium/WebKit on real key events + // because their accessibility bridges may echo a write that never reaches + // renderer-owned state. + if crate::wayland::is_inject_mode() + && resolved_elem_idx.is_some() + && !is_chromium_embedder(pid) + && !is_webkitgtk_embedder(pid) + { + let idx = resolved_elem_idx.expect("checked above"); + let text_at = text.clone(); + let targeted = tokio::task::spawn_blocking(move || { + crate::atspi::type_into_editable_at(pid, idx, &text_at) + }) + .await; + if let Ok(Ok(())) = targeted { + return type_text_ax_confirm_result(pid, text_len, "via targeted AT-SPI"); + } + } + // The private nested compositor can target the owning Wayland client + // directly. Establish widget-local focus first, without changing the + // compositor's focused toplevel, so keys reach the addressed control. + if crate::wayland::is_inject_mode() { + if let Err(error) = focus_nested_inject_target( + pid, + xid, + resolved_elem_idx, + px.zip(py), + ) + .await + { + return error; + } + let text_w = text.clone(); + let result = + tokio::task::spawn_blocking(move || crate::wayland::inject_type_text(xid, &text_w)) + .await; + return match result { + Ok(Ok(())) => ToolResult::text(format!( + "Typed {text_len} character(s) (focus-free via cua-compositor)." + )) + .with_structured(type_text_structured("key_events", text_len, false)), + Ok(Err(e)) => ToolResult::error(e.to_string()), + Err(e) => ToolResult::error(format!("Task error: {e}")), + }; + } // ── px form: focus by pixel-click, then type into the now-focused element ── // Pass x,y (no element_index/token) for an *element px action*: pixel-click @@ -1971,16 +2342,10 @@ impl Tool for TypeTextTool { // can't, then fall through to the focused-element type path below (it // escalates AT-SPI → key events and lands once focused). Reuses ClickTool's // exact coordinate translation + delivery_mode. - if let (Some(cx), Some(cy)) = ( - args.get("x").and_then(|v| v.as_f64()), - args.get("y").and_then(|v| v.as_f64()), - ) { - if resolved_elem_idx.is_some() { - return ToolResult::error( - "Pass either element_index (ax) or x,y (px) to type_text, not both.", - ); + if let (Some(cx), Some(cy)) = (px, py) { + if let Some(refusal) = unavailable_webkit_keyboard_background(pid, delivery) { + return refusal; } - let fg = crate::input::delivery::DeliveryMode::from_args(&args).is_foreground(); let from_zoom = args.bool_or("from_zoom", false); if let Err(e) = focus_by_pixel( &self.state, @@ -1988,7 +2353,7 @@ impl Tool for TypeTextTool { Some(xid), cx, cy, - fg, + delivery.is_foreground(), args.opt_str("session"), from_zoom, ) @@ -2000,36 +2365,92 @@ impl Tool for TypeTextTool { // focused element via the background key / AT-SPI rung. } - // EIS nested compositor: focus-FREE per-surface typing into window_id - // (the target need not be focused). Routed over the inject control socket. - if crate::wayland::is_inject_mode() { - let text_len = text.chars().count(); - let text_w = text.clone(); - let result = - tokio::task::spawn_blocking(move || crate::wayland::inject_type_text(xid, &text_w)) - .await; - return match result { - Ok(Ok(())) => ToolResult::text(format!( - "Typed {text_len} character(s) (focus-free via EIS compositor)." - )) - .with_structured(type_text_structured( - "key_events", - text_len, - false, - )), - Ok(Err(e)) => ToolResult::error(e.to_string()), - Err(e) => ToolResult::error(format!("Task error: {e}")), - }; + if resolved_elem_idx.is_some() { + if let Some(refusal) = unavailable_webkit_keyboard_background(pid, delivery) { + return refusal; + } + } + + // Renderer EditableText writes can update the accessible value without + // emitting the DOM input event. For an explicit foreground request on + // native Wayland, focus the named field and send real keyboard input so + // Chromium/WebKit observe the same event sequence as a user. + if delivery.is_foreground() + && crate::wayland::wayland_input_enabled() + && (is_chromium_embedder(pid) || is_webkitgtk_embedder(pid)) + { + if let Some(idx) = resolved_elem_idx { + let focused = tokio::task::spawn_blocking(move || { + crate::atspi::focus_element(pid, idx) + }) + .await; + match focused { + Ok(Ok(true)) => {} + Ok(Ok(false)) => { + return ToolResult::error(format!( + "AT-SPI Component.GrabFocus returned false for element {idx}" + )) + } + Ok(Err(error)) => return ToolResult::error(error.to_string()), + Err(error) => return ToolResult::error(format!("Task error: {error}")), + } + + let text_w = text.clone(); + let result = tokio::task::spawn_blocking(move || { + crate::wayland::type_text(xid, &text_w) + }) + .await; + return match result { + Ok(Ok(())) => ToolResult::text(format!( + "Typed {text_len} character(s) (via Wayland virtual-keyboard)." + )) + .with_structured(type_text_structured( + "key_events", + text_len, + false, + )), + Ok(Err(error)) => ToolResult::error(error.to_string()), + Err(error) => ToolResult::error(format!("Task error: {error}")), + }; + } + } + + // AX addressing names one exact editable. Try this focus-free route + // before native Wayland keyboard injection, which can only target the + // compositor's globally focused surface. + if let Some(idx) = resolved_elem_idx { + let text_at = text.clone(); + let targeted = tokio::task::spawn_blocking(move || { + crate::atspi::type_into_editable_at(pid, idx, &text_at) + }) + .await; + match targeted { + Ok(Ok(())) => { + return type_text_ax_confirm_result(pid, text_len, "via targeted AT-SPI"); + } + Ok(Err(_)) | Err(_) + if !delivery.is_foreground() && crate::wayland::wayland_input_enabled() => + { + return crate::input::delivery::background_unavailable_error( + crate::input::delivery::BackgroundUnavailable::FocusedInputOnly, + ); + } + _ => {} + } } // Native Wayland: keys go to the *focused* surface (no pid/window // targeting in the protocol). Type via the virtual-keyboard tool; pair // with a prior `click`/`activate` to focus the intended window. - if crate::wayland::is_wayland() { - let text_len = text.chars().count(); + if crate::wayland::wayland_input_enabled() { + if !delivery.is_foreground() { + return crate::input::delivery::background_unavailable_error( + crate::input::delivery::BackgroundUnavailable::FocusedInputOnly, + ); + } let text_w = text.clone(); let result = - tokio::task::spawn_blocking(move || crate::wayland::type_text(&text_w)).await; + tokio::task::spawn_blocking(move || crate::wayland::type_text(xid, &text_w)).await; return match result { Ok(Ok(())) => ToolResult::text(format!( "Typed {text_len} character(s) (via Wayland virtual-keyboard)." @@ -2060,25 +2481,34 @@ impl Tool for TypeTextTool { if pid_is_terminal || wm_class_is_terminal { let text_len = text.chars().count(); let text_t = text.clone(); - let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { + let foreground = delivery.is_foreground(); + let result = tokio::task::spawn_blocking(move || -> anyhow::Result<&'static str> { // pty-master injection is preferred — it skips the X event // queue entirely. Falls through to XTest if the terminal // isn't reachable that way (descendant pty unresolvable). if inject_terminal_input(pid, xid, &text_t)? { - return Ok(()); + return Ok("pty"); + } + if foreground { + crate::input::with_x11_foreground(xid, 80, || { + crate::input::send_type_text_xtest(&text_t) + })?; + Ok("key_events_fg") + } else { + Ok("background_unavailable") } - crate::input::send_type_text_xtest(&text_t) }) .await; return match result { - Ok(Ok(())) => ToolResult::text(format!( + Ok(Ok("background_unavailable")) => { + crate::input::delivery::background_unavailable_error( + crate::input::delivery::BackgroundUnavailable::FocusedInputOnly, + ) + } + Ok(Ok(path)) => ToolResult::text(format!( "Typed {text_len} character(s) (terminal emulator: pty/XTest key events)." )) - .with_structured(type_text_structured( - "key_events", - text_len, - false, - )), + .with_structured(type_text_structured(path, text_len, false)), Ok(Err(e)) => ToolResult::error(e.to_string()), Err(e) => ToolResult::error(format!("Task error: {e}")), }; @@ -2098,7 +2528,46 @@ impl Tool for TypeTextTool { }); } } - let text_len = text.chars().count(); + // Foreground means the caller explicitly permits activation. Chromium + // and WebKitGTK can acknowledge an accessibility write without + // producing the renderer input event, so web embedders use real XTest + // key events. Native toolkits keep their verifiable AT-SPI path below. + if delivery.is_foreground() && (is_chromium_embedder(pid) || is_webkitgtk_embedder(pid)) { + if let Some(idx) = resolved_elem_idx { + let focused = + tokio::task::spawn_blocking(move || crate::atspi::focus_element(pid, idx)) + .await; + match focused { + Ok(Ok(true)) => {} + Ok(Ok(false)) => { + return ToolResult::error(format!( + "AT-SPI Component.GrabFocus returned false for element {idx}" + )) + } + Ok(Err(e)) => return ToolResult::error(e.to_string()), + Err(e) => return ToolResult::error(format!("Task error: {e}")), + } + } + let text_f = text.clone(); + let result = tokio::task::spawn_blocking(move || { + crate::input::with_x11_foreground(xid, 80, || { + crate::input::send_type_text_xtest(&text_f) + }) + }) + .await; + return match result { + Ok(Ok(())) => ToolResult::text(format!( + "Typed {text_len} character(s) (via X11, delivery_mode=foreground)." + )) + .with_structured(type_text_structured( + "key_events_fg", + text_len, + false, + )), + Ok(Err(e)) => ToolResult::error(e.to_string()), + Err(e) => ToolResult::error(format!("Task error: {e}")), + }; + } // Prefer the focused widget — the element the user just clicked. If a // NON-editable input holds keyboard focus (a spreadsheet cell, a @@ -2115,6 +2584,11 @@ impl Tool for TypeTextTool { .ok() .flatten(); if focus_kind == Some(false) { + if !delivery.is_foreground() { + return crate::input::delivery::background_unavailable_error( + crate::input::delivery::BackgroundUnavailable::FocusedInputOnly, + ); + } let text_f = text.clone(); let result = tokio::task::spawn_blocking(move || { if inject_terminal_input(pid, xid, &text_f)? { @@ -2124,7 +2598,9 @@ impl Tool for TypeTextTool { // drop synthetic key events, so a spreadsheet cell / canvas would // stay empty. The click that gave this widget focus already put it // under the X input focus, so XTest-to-focus lands correctly. - crate::input::send_type_text_xtest(&text_f) + crate::input::with_x11_foreground(xid, 80, || { + crate::input::send_type_text_xtest(&text_f) + }) }) .await; return match result { @@ -2200,19 +2676,11 @@ impl Tool for TypeTextTool { // foreground = activate the window (EWMH), then synthesize REAL key // events to it via XTest — the escalation when background didn't land // (e.g. a GTK dialog whose widget ignores synthetic XSendEvent keys). - let delivery = crate::input::delivery::DeliveryMode::from_args(&args); let result = tokio::task::spawn_blocking(move || -> anyhow::Result<&'static str> { // Terminals: write to the pty master (focus-free, below the toolkit). if inject_terminal_input(pid, xid, &text)? { return Ok("key_events"); } - if delivery.is_foreground() { - // Activate target → XTest real keystrokes → restore prior active. - return crate::input::with_x11_foreground(xid, 80, || { - crate::input::send_type_text_xtest(&text)?; - Ok("key_events_fg") - }); - } // GUI apps: X11 only routes keystrokes to the *focused* toplevel's // focused widget, so background XSendEvent typing doesn't land. Fill // the editable field via AT-SPI instead — focus-free and toolkit- @@ -2225,8 +2693,14 @@ impl Tool for TypeTextTool { if crate::input::inject_tk_send(&text).unwrap_or(false) { return Ok("key_events"); } - crate::input::send_type_text(xid, &text)?; - Ok("key_events") + if delivery.is_foreground() { + crate::input::with_x11_foreground(xid, 80, || { + crate::input::send_type_text_xtest(&text) + })?; + Ok("key_events_fg") + } else { + Ok("background_unavailable") + } }) .await; let mode_label = if delivery.is_foreground() { @@ -2248,6 +2722,11 @@ impl Tool for TypeTextTool { text_len, &format!("via X11, delivery_mode={mode_label}"), ), + Ok(Ok("background_unavailable")) => { + crate::input::delivery::background_unavailable_error( + crate::input::delivery::BackgroundUnavailable::FocusedInputOnly, + ) + } Ok(Ok(path)) => ToolResult::text(format!( "Typed {text_len} character(s) (via X11, delivery_mode={mode_label})." )) @@ -2317,7 +2796,7 @@ impl Tool for PressKeyTool { }; let xid_opt = match &resolved { cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => { - window_id.map(|v| v as u64).or(window_id_arg) + window_id_arg.or_else(|| window_id.map(|v| v as u64)) } cua_driver_core::element_token::ResolvedElement::None => window_id_arg, }; @@ -2344,6 +2823,59 @@ impl Tool for PressKeyTool { // background path (the focus-click already handled fronting when fg). Pass x,y // (no element_index) for Chromium/Electron surfaces the AX path can't focus. let delivery = crate::input::delivery::DeliveryMode::from_args(&args); + if let Some(refusal) = unavailable_chromium_background(pid, delivery) { + return refusal; + } + + if let Some(refusal) = unavailable_webkit_keyboard_background(pid, delivery) { + return refusal; + } + if let Some(refusal) = unavailable_gtk_keyboard_background(pid, delivery) { + return refusal; + } + if let Some(refusal) = unavailable_wayland_focused_input_background(delivery, true) { + return refusal; + } + + let px = args.get("x").and_then(|value| value.as_f64()); + let py = args.get("y").and_then(|value| value.as_f64()); + if px.is_some() != py.is_some() { + return ToolResult::error("Pass both x and y to press_key, or neither."); + } + if px.is_some() && element_index_arg.is_some() { + return ToolResult::error( + "Pass either element_index (ax) or x,y (px) to press_key, not both.", + ); + } + + // Nested cua-compositor addresses the owning Wayland client directly. + // Preserve legacy modifiers by promoting the request to a chord. + if crate::wayland::is_inject_mode() { + if let Err(error) = + focus_nested_inject_target(pid, xid, element_index_arg, px.zip(py)).await + { + return error; + } + let result = if mods.is_empty() { + let key_w = key.clone(); + tokio::task::spawn_blocking(move || { + crate::wayland::inject_press_key(xid, &key_w) + }) + .await + } else { + let mut chord = mods.clone(); + chord.push(key.clone()); + tokio::task::spawn_blocking(move || crate::wayland::inject_hotkey(xid, &chord)) + .await + }; + return match result { + Ok(Ok(())) => ToolResult::text(format!( + "Pressed key '{key}' (focus-free via cua-compositor)." + )), + Ok(Err(e)) => ToolResult::error(e.to_string()), + Err(e) => ToolResult::error(format!("Task error: {e}")), + }; + } // An element-addressed keypress needs to establish the target's // focus before the window-level X11 key event is sent. AT-SPI's @@ -2368,14 +2900,7 @@ impl Tool for PressKeyTool { } let px_target = { - let px = args.get("x").and_then(|v| v.as_f64()); - let py = args.get("y").and_then(|v| v.as_f64()); if let (Some(cx), Some(cy)) = (px, py) { - if element_index_arg.is_some() { - return ToolResult::error( - "Pass either element_index (ax) or x,y (px) to press_key, not both.", - ); - } let from_zoom = args.bool_or("from_zoom", false); if let Err(e) = focus_by_pixel( &self.state, @@ -2397,26 +2922,11 @@ impl Tool for PressKeyTool { } }; - // EIS nested compositor: focus-free named-key into window_id. - if crate::wayland::is_inject_mode() { - let key_w = key.clone(); - let result = - tokio::task::spawn_blocking(move || crate::wayland::inject_press_key(xid, &key_w)) - .await; - return match result { - Ok(Ok(())) => ToolResult::text(format!( - "Pressed key '{key}' (focus-free via EIS compositor)." - )), - Ok(Err(e)) => ToolResult::error(e.to_string()), - Err(e) => ToolResult::error(format!("Task error: {e}")), - }; - } - // Native Wayland: send the key to the focused surface via virtual-keyboard. - if crate::wayland::is_wayland() { + if crate::wayland::wayland_input_enabled() { let key_w = key.clone(); let result = - tokio::task::spawn_blocking(move || crate::wayland::press_key(&key_w)).await; + tokio::task::spawn_blocking(move || crate::wayland::press_key(xid, &key_w)).await; return match result { Ok(Ok(())) => ToolResult::text(format!( "Pressed key '{key}' (via Wayland virtual-keyboard)." @@ -2427,9 +2937,10 @@ impl Tool for PressKeyTool { } let key_for_task = key.clone(); - // px-focus already clicked (and fronted, when fg) the target → deliver via - // the plain background path. Otherwise honor the requested delivery_mode. - let deliver_fg = delivery.is_foreground() && px_target.is_none(); + // Foreground delivery is one atomic activate-and-XTest transaction. + // A preceding PX click establishes internal widget focus, but that + // click restores the prior top-level before returning. + let deliver_fg = delivery.is_foreground(); let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { if mods.is_empty() && key_for_task.eq_ignore_ascii_case("enter") { if inject_terminal_input(pid, xid, "\n")? { @@ -2507,6 +3018,8 @@ impl Tool for HotkeyTool { "window_id":{"type":"integer"}, "keys":{"type":"array","items":{"type":"string"},"minItems":2, "description":"Modifier(s) + one non-modifier key, e.g. [\"ctrl\",\"c\"]."}, + "element_index": cua_driver_core::tool_schema::element_index_schema(), + "element_token": cua_driver_core::tool_schema::element_token_schema(), "x":{"type":"number","description":"Screenshot-pixel X — the element px action form: pixel-click there to focus, then send the combo (so e.g. Ctrl+V pastes into that field). Pass with y. Use for Chromium/Electron surfaces the background combo can't reach."}, "y":{"type":"number","description":"Screenshot-pixel Y (see x)."}, "delivery_mode": crate::input::delivery::delivery_mode_schema() @@ -2519,7 +3032,31 @@ impl Tool for HotkeyTool { async fn invoke(&self, args: Value) -> ToolResult { use cua_driver_core::tool_args::ArgsExt; let pid = args.u64_or("pid", 0) as u32; - let xid_opt = args.opt_u64("window_id"); + let window_id_arg = args.opt_u64("window_id"); + let element_index_arg = args.opt_u64("element_index").map(|value| value as usize); + let resolved = match cua_driver_core::element_token::resolve_element_args( + pid as i32, + element_index_arg, + args.opt_str("element_token").as_deref(), + window_id_arg.map(|value| value as u32), + "hotkey", + ) { + Ok(resolved) => resolved, + Err(error) => return error, + }; + let resolved_element_index = match &resolved { + cua_driver_core::element_token::ResolvedElement::Element { + element_index, + .. + } => Some(*element_index), + cua_driver_core::element_token::ResolvedElement::None => None, + }; + let xid_opt = match &resolved { + cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => { + window_id_arg.or_else(|| window_id.map(|value| value as u64)) + } + cua_driver_core::element_token::ResolvedElement::None => window_id_arg, + }; // Resolve XID: use window_id if given, else first window for pid. let xid = match xid_opt { @@ -2566,18 +3103,100 @@ impl Tool for HotkeyTool { let mods_for_wayland = mods.clone(); let delivery = crate::input::delivery::DeliveryMode::from_args(&args); - // X11 can address an unfocused native window with synthetic events, but - // Chromium's renderer only processes modifier chords for its focused - // input surface. Do not report the XSendEvent attempt as success: the - // explicit foreground rung is the only truthful fallback unless an - // EIS compositor can target the surface without focus. - if !delivery.is_foreground() + if let Some(refusal) = unavailable_chromium_background(pid, delivery) { + return refusal; + } + if let Some(refusal) = unavailable_webkit_keyboard_background(pid, delivery) { + return refusal; + } + if let Some(refusal) = unavailable_gtk_keyboard_background(pid, delivery) { + return refusal; + } + if let Some(refusal) = unavailable_wayland_focused_input_background(delivery, true) { + return refusal; + } + + let px = args.get("x").and_then(|value| value.as_f64()); + let py = args.get("y").and_then(|value| value.as_f64()); + if px.is_some() != py.is_some() { + return ToolResult::error("Pass both x and y to hotkey, or neither."); + } + if px.is_some() && resolved_element_index.is_some() { + return ToolResult::error( + "Pass either element_index (ax) or x,y (px) to hotkey, not both.", + ); + } + + if crate::wayland::is_inject_mode() { + if let Err(error) = + focus_nested_inject_target(pid, xid, resolved_element_index, px.zip(py)).await + { + return error; + } + let mut chord = mods.clone(); + chord.push(key.clone()); + let result = tokio::task::spawn_blocking(move || { + crate::wayland::inject_hotkey(xid, &chord) + }) + .await; + return match result { + Ok(Ok(())) => ToolResult::text(format!( + "Pressed hotkey '{key_display}' (focus-free via cua-compositor)." + )), + Ok(Err(error)) => ToolResult::error(error.to_string()), + Err(error) => ToolResult::error(format!("Task error: {error}")), + }; + } + + // Chromium renderer shortcuts need DOM focus established by a real + // pointer event. Component.GrabFocus alone can report success while + // the renderer still drops the virtual-keyboard chord. + if delivery.is_foreground() + && crate::wayland::wayland_input_enabled() && is_chromium_embedder(pid) - && !crate::wayland::is_inject_mode() { - return crate::input::delivery::background_unavailable_error( - crate::input::delivery::BackgroundUnavailable::ChromiumHotkey, - ); + if let Some(element_index) = resolved_element_index { + let coordinates = tokio::task::spawn_blocking(move || { + resolve_element_local_coords(pid, element_index, Some(xid)) + }) + .await; + let (_, x, y) = match coordinates { + Ok(Ok(value)) => value, + Ok(Err(error)) => return ToolResult::error(error.to_string()), + Err(error) => return ToolResult::error(format!("Task error: {error}")), + }; + if let Err(error) = focus_by_pixel( + &self.state, + pid, + Some(xid), + x, + y, + true, + args.opt_str("session"), + false, + ) + .await + { + return error; + } + } + } + + if let Some(element_index) = resolved_element_index { + let focused = tokio::task::spawn_blocking(move || { + crate::atspi::focus_element(pid, element_index) + }) + .await; + match focused { + Ok(Ok(true)) => {} + Ok(Ok(false)) => { + return ToolResult::error(format!( + "AT-SPI Component.GrabFocus returned false for element {element_index}" + )) + } + Ok(Err(error)) => return ToolResult::error(error.to_string()), + Err(error) => return ToolResult::error(format!("Task error: {error}")), + } } // ── px form: pixel-click to focus, then the combo acts on the focused field ── @@ -2585,8 +3204,6 @@ impl Tool for HotkeyTool { // delivery_mode; after it, deliver the combo via the plain background path // (the focus-click already fronted when fg). let px_target = { - let px = args.get("x").and_then(|v| v.as_f64()); - let py = args.get("y").and_then(|v| v.as_f64()); if let (Some(cx), Some(cy)) = (px, py) { let from_zoom = args.bool_or("from_zoom", false); if let Err(e) = focus_by_pixel( @@ -2608,16 +3225,16 @@ impl Tool for HotkeyTool { None } }; - let deliver_fg = delivery.is_foreground() && px_target.is_none(); + let deliver_fg = delivery.is_foreground(); let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - if crate::wayland::is_wayland() { + if crate::wayland::wayland_input_enabled() { // Native Wayland: route the modifier combo through wtype's // -M/-k/-m sequence — the closest equivalent to the X11 // state-mask path. window_id is irrelevant once focused. let mut combo: Vec = mods_for_wayland.clone(); combo.push(key_for_wayland.clone()); - return crate::wayland::hotkey(&combo); + return crate::wayland::hotkey(xid, &combo); } let m: Vec<&str> = mods.iter().map(String::as_str).collect(); // foreground: activate the target first, then inject the accelerator @@ -2761,6 +3378,8 @@ impl Tool for ScrollTool { "window_id":{"type":"integer"}, "element_index": cua_driver_core::tool_schema::element_index_schema(), "element_token": cua_driver_core::tool_schema::element_token_schema(), + "x":{"type":"number","description":"Window-local screenshot-pixel X of the scroll target. Pass with y and without element_index."}, + "y":{"type":"number","description":"Window-local screenshot-pixel Y of the scroll target. Pass with x and without element_index."}, "delivery_mode": crate::input::delivery::delivery_mode_schema() },"additionalProperties":false }), @@ -2821,28 +3440,117 @@ impl Tool for ScrollTool { }; let delivery = crate::input::delivery::DeliveryMode::from_args(&args); - if !delivery.is_foreground() { - if let cua_driver_core::element_token::ResolvedElement::Element { - element_index, .. - } = &resolved - { - let idx = *element_index; - let direction_for_ax = direction.clone(); - let ax_result = tokio::task::spawn_blocking(move || { - crate::atspi::scroll_element(pid, idx, &direction_for_ax, amount) - }) - .await; - if matches!(ax_result, Ok(Ok(()))) { - return ToolResult::text(format!( - "Scrolled {direction} {amount} ticks via AT-SPI (delivery_mode:background)." - )) - .with_structured(json!({ - "path": "atspi", - "verified": false, - "delivery_mode": "background" - })); - } + if let Some(refusal) = unavailable_chromium_background(pid, delivery) { + return refusal; + } + if let cua_driver_core::element_token::ResolvedElement::Element { element_index, .. } = + &resolved + { + let idx = *element_index; + let direction_for_ax = direction.clone(); + let ax_result = tokio::task::spawn_blocking(move || { + crate::atspi::scroll_element(pid, idx, &direction_for_ax, amount) + }) + .await; + if matches!(ax_result, Ok(Ok(()))) { + let mode = if delivery.is_foreground() { + "foreground" + } else { + "background" + }; + return ToolResult::text(format!( + "Scrolled {direction} {amount} ticks via AT-SPI (delivery_mode:{mode})." + )) + .with_structured(json!({ + "path": "atspi", + "verified": false, + "delivery_mode": mode + })); + } + } + + let pixel_target = match ( + args.get("x").and_then(|value| value.as_f64()), + args.get("y").and_then(|value| value.as_f64()), + ) { + (Some(x), Some(y)) => Some((x, y)), + (None, None) => None, + _ => return ToolResult::error("Pass both x and y to pixel-target scroll."), + }; + if pixel_target.is_some() + && matches!( + &resolved, + cua_driver_core::element_token::ResolvedElement::Element { .. } + ) + { + return ToolResult::error( + "Pass either element_index (ax) or x,y (px) to scroll, not both.", + ); + } + + if crate::wayland::is_inject_mode() { + let Some((x, y)) = pixel_target else { + return crate::input::delivery::background_unavailable_error( + crate::input::delivery::BackgroundUnavailable::FocusedInputOnly, + ); + }; + let direction_for_inject = direction.clone(); + let result = tokio::task::spawn_blocking(move || { + crate::wayland::inject_scroll( + xid, + x, + y, + &direction_for_inject, + amount as u32, + ) + }) + .await; + return match result { + Ok(Ok(())) => ToolResult::text(format!( + "Scrolled {direction} {amount} ticks (focus-free via cua-compositor)." + )) + .with_structured(json!({ + "verified": false, + "delivery_mode": if delivery.is_foreground() { "foreground" } else { "background" }, + "route": "cua_compositor_inject" + })), + Ok(Err(error)) => ToolResult::error(error.to_string()), + Err(error) => ToolResult::error(format!("Task error: {error}")), + }; + } + + if crate::wayland::wayland_input_enabled() { + if let Some(refusal) = unavailable_wayland_focused_input_background(delivery, false) { + return refusal; } + let direction_for_wayland = direction.clone(); + let output_point = pixel_target.map(|(x, y)| { + crate::wayland::window_local_to_output(xid, x.round() as i32, y.round() as i32) + }); + let result = tokio::task::spawn_blocking(move || { + crate::wayland::scroll_at( + xid, + output_point, + &direction_for_wayland, + amount as u32, + ) + }) + .await; + return match result { + Ok(Ok(())) => ToolResult::text(format!( + "Scrolled {direction} {amount} ticks (delivery_mode=foreground)." + )) + .with_structured(json!({ "verified": false, "delivery_mode": "foreground" })), + Ok(Err(error)) => ToolResult::error(error.to_string()), + Err(error) => ToolResult::error(format!("Task error: {error}")), + }; + } + + if let Some(refusal) = unavailable_webkit_background(pid, delivery) { + return refusal; + } + if let Some(refusal) = unavailable_gtk_pointer_background(pid, delivery) { + return refusal; } // An element-addressed scroll must land over the element. The old @@ -2873,7 +3581,10 @@ impl Tool for ScrollTool { } } } - cua_driver_core::element_token::ResolvedElement::None => None, + cua_driver_core::element_token::ResolvedElement::None => pixel_target.map(|local| { + let screen = window_local_to_screen(xid, local.0, local.1).ok(); + (local, screen) + }), }; // X11 scroll buttons: 4=up, 5=down, 6=left, 7=right @@ -2884,11 +3595,11 @@ impl Tool for ScrollTool { "right" => 7, _ => 5, }; + let cursor_id_for_task = cursor_id.clone(); let direction_for_wayland = direction.clone(); let amount_u32 = amount as u32; - let cursor_id_for_task = cursor_id.clone(); let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - if crate::wayland::is_wayland() { + if crate::wayland::wayland_input_enabled() { return crate::wayland::scroll(xid, &direction_for_wayland, amount_u32); } // foreground: activate the window, then scroll, then restore — for @@ -2932,7 +3643,22 @@ impl Tool for ScrollTool { crate::input::send_click(xid, local_x as i32, local_y as i32, amount, button) }; if delivery.is_foreground() { - crate::input::with_x11_foreground(xid, 80, x11_scroll) + let point = element_point + .and_then(|(_, screen)| screen) + .or_else(|| { + window_screen_center(xid) + .ok() + .map(|(x, y)| (x as f64, y as f64)) + }) + .ok_or_else(|| anyhow::anyhow!("could not resolve foreground scroll point"))?; + crate::input::with_x11_foreground(xid, 80, || { + crate::input::send_click_xtest_desktop( + point.0 as i32, + point.1 as i32, + button, + amount, + ) + }) } else { x11_scroll() } @@ -2996,6 +3722,19 @@ impl Tool for DoubleClickTool { Ok(v) => v, Err(e) => return e, }; + let delivery = crate::input::delivery::DeliveryMode::from_args(&args); + if let Some(refusal) = unavailable_chromium_background(pid, delivery) { + return refusal; + } + if let Some(refusal) = unavailable_webkit_background(pid, delivery) { + return refusal; + } + if let Some(refusal) = unavailable_gtk_pointer_background(pid, delivery) { + return refusal; + } + if let Some(refusal) = unavailable_wayland_focused_input_background(delivery, true) { + return refusal; + } // Surface 6: element_token / element_index precedence. let resolved = match cua_driver_core::element_token::resolve_element_args( pid as i32, @@ -3015,7 +3754,8 @@ impl Tool for DoubleClickTool { }; let window_id_resolved: Option = match &resolved { cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => { - window_id.map(|v| v as u64) + args.opt_u64("window_id") + .or_else(|| window_id.map(|v| v as u64)) } cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"), }; @@ -3042,10 +3782,27 @@ impl Tool for DoubleClickTool { } let lxi = lx as i32; let lyi = ly as i32; + let wayland_point = crate::wayland::wayland_input_enabled() + .then(|| crate::wayland::window_local_to_output(xid, lxi, lyi)); let cursor_id_for_task = cursor_id.clone(); let click_result = tokio::task::spawn_blocking(move || { - if crate::wayland::is_wayland() { - return crate::wayland::click(xid, lxi, lyi, 2, 1); + if crate::wayland::is_inject_mode() { + return crate::wayland::inject_click(xid, lx, ly, 2, 1); + } + if crate::wayland::wayland_input_enabled() { + let (output_x, output_y) = wayland_point.unwrap_or((lxi, lyi)); + return crate::wayland::click(xid, output_x, output_y, 2, 1); + } + if delivery.is_foreground() { + return crate::input::with_x11_foreground(xid, 80, || { + let (sx, sy) = window_local_to_screen(xid, lxi as f64, lyi as f64)?; + crate::input::send_click_xtest_desktop( + sx.round() as i32, + sy.round() as i32, + 1, + 2, + ) + }); } x11_pixel_click_no_focus_steal(&cursor_id_for_task, xid, lxi, lyi, 1, 2) }) @@ -3090,13 +3847,17 @@ impl Tool for DoubleClickTool { cursor_id.clone(), cursor_overlay::OverlayCommand::PinAbove(xid), ); - // Resolve the screen point the cursor glides to. On native Wayland the - // agent already passes screen coordinates (the vision screenshot and - // `get_window_state` frames are screen-space, and `window_local_to_screen` - // — an X11 `translate_coordinates` call — can't run with DISPLAY unset), - // so use them directly. On X11 the coords are window-local; translate. - let glide_target = if crate::wayland::is_wayland() { - Some((x, y)) + let wayland_output_point = if crate::wayland::wayland_input_enabled() { + Some(crate::wayland::window_local_to_output( + xid, + x.round() as i32, + y.round() as i32, + )) + } else { + None + }; + let glide_target = if let Some((sx, sy)) = wayland_output_point { + Some((sx as f64, sy as f64)) } else { tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)) .await @@ -3112,10 +3873,13 @@ impl Tool for DoubleClickTool { } let (xi, yi) = (x as i32, y as i32); let cursor_id_for_task = cursor_id.clone(); - let delivery = crate::input::delivery::DeliveryMode::from_args(&args); let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - if crate::wayland::is_wayland() { - return crate::wayland::click(xid, xi, yi, 2, 1); + if crate::wayland::is_inject_mode() { + return crate::wayland::inject_click(xid, x, y, 2, 1); + } + if crate::wayland::wayland_input_enabled() { + let (output_x, output_y) = wayland_output_point.unwrap_or((xi, yi)); + return crate::wayland::click(xid, output_x, output_y, 2, 1); } if delivery.is_foreground() { return crate::input::with_x11_foreground(xid, 80, || { @@ -3191,6 +3955,19 @@ impl Tool for RightClickTool { Ok(v) => v, Err(e) => return e, }; + let delivery = crate::input::delivery::DeliveryMode::from_args(&args); + if let Some(refusal) = unavailable_chromium_background(pid, delivery) { + return refusal; + } + if let Some(refusal) = unavailable_webkit_background(pid, delivery) { + return refusal; + } + if let Some(refusal) = unavailable_gtk_pointer_background(pid, delivery) { + return refusal; + } + if let Some(refusal) = unavailable_wayland_focused_input_background(delivery, true) { + return refusal; + } // Surface 6: element_token / element_index precedence. let resolved = match cua_driver_core::element_token::resolve_element_args( pid as i32, @@ -3210,7 +3987,8 @@ impl Tool for RightClickTool { }; let window_id_resolved: Option = match &resolved { cua_driver_core::element_token::ResolvedElement::Element { window_id, .. } => { - window_id.map(|v| v as u64) + args.opt_u64("window_id") + .or_else(|| window_id.map(|v| v as u64)) } cua_driver_core::element_token::ResolvedElement::None => args.opt_u64("window_id"), }; @@ -3237,10 +4015,27 @@ impl Tool for RightClickTool { } let lxi = lx as i32; let lyi = ly as i32; + let wayland_point = crate::wayland::wayland_input_enabled() + .then(|| crate::wayland::window_local_to_output(xid, lxi, lyi)); let cursor_id_for_task = cursor_id.clone(); let click_result = tokio::task::spawn_blocking(move || { - if crate::wayland::is_wayland() { - return crate::wayland::click(xid, lxi, lyi, 1, 3); + if crate::wayland::is_inject_mode() { + return crate::wayland::inject_click(xid, lx, ly, 1, 3); + } + if crate::wayland::wayland_input_enabled() { + let (output_x, output_y) = wayland_point.unwrap_or((lxi, lyi)); + return crate::wayland::click(xid, output_x, output_y, 1, 3); + } + if delivery.is_foreground() { + return crate::input::with_x11_foreground(xid, 80, || { + let (sx, sy) = window_local_to_screen(xid, lxi as f64, lyi as f64)?; + crate::input::send_click_xtest_desktop( + sx.round() as i32, + sy.round() as i32, + 3, + 1, + ) + }); } x11_pixel_click_no_focus_steal(&cursor_id_for_task, xid, lxi, lyi, 3, 1) }) @@ -3285,13 +4080,17 @@ impl Tool for RightClickTool { cursor_id.clone(), cursor_overlay::OverlayCommand::PinAbove(xid), ); - // Resolve the screen point the cursor glides to. On native Wayland the - // agent already passes screen coordinates (the vision screenshot and - // `get_window_state` frames are screen-space, and `window_local_to_screen` - // — an X11 `translate_coordinates` call — can't run with DISPLAY unset), - // so use them directly. On X11 the coords are window-local; translate. - let glide_target = if crate::wayland::is_wayland() { - Some((x, y)) + let wayland_output_point = if crate::wayland::wayland_input_enabled() { + Some(crate::wayland::window_local_to_output( + xid, + x.round() as i32, + y.round() as i32, + )) + } else { + None + }; + let glide_target = if let Some((sx, sy)) = wayland_output_point { + Some((sx as f64, sy as f64)) } else { tokio::task::spawn_blocking(move || window_local_to_screen(xid, x, y)) .await @@ -3307,10 +4106,13 @@ impl Tool for RightClickTool { } let (xi, yi) = (x as i32, y as i32); let cursor_id_for_task = cursor_id.clone(); - let delivery = crate::input::delivery::DeliveryMode::from_args(&args); let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { - if crate::wayland::is_wayland() { - return crate::wayland::click(xid, xi, yi, 1, 3); + if crate::wayland::is_inject_mode() { + return crate::wayland::inject_click(xid, x, y, 1, 3); + } + if crate::wayland::wayland_input_enabled() { + let (output_x, output_y) = wayland_output_point.unwrap_or((xi, yi)); + return crate::wayland::click(xid, output_x, output_y, 1, 3); } if delivery.is_foreground() { return crate::input::with_x11_foreground(xid, 80, || { @@ -3376,7 +4178,8 @@ impl Tool for DragTool { "steps":{"type":"integer","minimum":1,"maximum":200,"description":"Intermediate MotionNotify events. Default: 20."}, "modifier": cua_driver_core::tool_schema::modifier_schema(), "button": cua_driver_core::tool_schema::button_schema(), - "from_zoom":{"type":"boolean"} + "from_zoom":{"type":"boolean"}, + "delivery_mode": crate::input::delivery::delivery_mode_schema() },"additionalProperties":false}), read_only: false, destructive: true, idempotent: false, open_world: true, }) @@ -3391,6 +4194,19 @@ impl Tool for DragTool { Some(v) => v, None => return ToolResult::error("window_id is required on Linux."), }; + let delivery = crate::input::delivery::DeliveryMode::from_args(&args); + if let Some(refusal) = unavailable_chromium_background(pid, delivery) { + return refusal; + } + if let Some(refusal) = unavailable_webkit_background(pid, delivery) { + return refusal; + } + if let Some(refusal) = unavailable_gtk_pointer_background(pid, delivery) { + return refusal; + } + if let Some(refusal) = unavailable_wayland_focused_input_background(delivery, true) { + return refusal; + } let coerce = |key: &str| -> Option { args.opt_f64(key) @@ -3446,9 +4262,29 @@ impl Tool for DragTool { cursor_id.clone(), cursor_overlay::OverlayCommand::PinAbove(xid), ); - if let Ok(Ok((sx_from, sy_from))) = - tokio::task::spawn_blocking(move || window_local_to_screen(xid, from_x, from_y)).await - { + let wayland_points = crate::wayland::wayland_input_enabled().then(|| { + ( + crate::wayland::window_local_to_output( + xid, + from_x.round() as i32, + from_y.round() as i32, + ), + crate::wayland::window_local_to_output( + xid, + to_x.round() as i32, + to_y.round() as i32, + ), + ) + }); + let screen_from = if let Some((from, _)) = wayland_points { + Some((from.0 as f64, from.1 as f64)) + } else { + tokio::task::spawn_blocking(move || window_local_to_screen(xid, from_x, from_y)) + .await + .ok() + .and_then(|result| result.ok()) + }; + if let Some((sx_from, sy_from)) = screen_from { overlay_glide_to_for(&cursor_id, sx_from, sy_from).await; self.state .cursor_registry @@ -3468,16 +4304,31 @@ impl Tool for DragTool { ); // Native Wayland: emit press + interpolated motion + release as one - // virtual-pointer sequence (output-relative coords). Returns early so - // we don't fall into the X11 XSendEvent loop below. - if crate::wayland::is_wayland() { - let (fxi, fyi) = (from_x.round() as i32, from_y.round() as i32); - let (txi, tyi) = (to_x.round() as i32, to_y.round() as i32); + // virtual-pointer (wlroots) or libei (GNOME/KDE) sequence, output-relative + // coords. Returns early so we don't fall into the X11 XSendEvent loop below. + if crate::wayland::wayland_input_enabled() { let steps_u32 = steps as u32; - let drag_result = tokio::task::spawn_blocking(move || { - crate::wayland::drag(xid, fxi, fyi, txi, tyi, steps_u32, button) - }) - .await; + let drag_result = if crate::wayland::is_inject_mode() { + tokio::task::spawn_blocking(move || { + crate::wayland::inject_drag( + xid, + (from_x, from_y), + (to_x, to_y), + steps, + button as u32, + ) + }) + .await + } else { + let ((fxi, fyi), (txi, tyi)) = wayland_points.unwrap_or(( + (from_x.round() as i32, from_y.round() as i32), + (to_x.round() as i32, to_y.round() as i32), + )); + tokio::task::spawn_blocking(move || { + crate::wayland::drag(xid, fxi, fyi, txi, tyi, steps_u32, button) + }) + .await + }; crate::overlay::send_command_for( cursor_id.clone(), cursor_overlay::OverlayCommand::SetPressed(false), @@ -3493,6 +4344,53 @@ impl Tool for DragTool { }; } + if delivery.is_foreground() { + let screen_points = tokio::task::spawn_blocking(move || { + Ok::<_, anyhow::Error>(( + window_local_to_screen(xid, from_x, from_y)?, + window_local_to_screen(xid, to_x, to_y)?, + )) + }) + .await; + let ((screen_from_x, screen_from_y), (screen_to_x, screen_to_y)) = match screen_points { + Ok(Ok(points)) => points, + Ok(Err(e)) => return ToolResult::error(e.to_string()), + Err(e) => return ToolResult::error(format!("Task error: {e}")), + }; + let drag_result = tokio::task::spawn_blocking(move || { + crate::input::with_x11_foreground(xid, 80, || { + crate::input::send_drag_xtest_desktop( + screen_from_x.round() as i32, + screen_from_y.round() as i32, + screen_to_x.round() as i32, + screen_to_y.round() as i32, + button, + duration_ms, + steps, + ) + }) + }) + .await; + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::SetPressed(false), + ); + return match drag_result { + Ok(Ok(())) => ToolResult::text(format!( + "Dragged ({button_str}) to pid {pid} from ({from_x:.0}, {from_y:.0}) \ + to ({to_x:.0}, {to_y:.0}) in {duration_ms}ms / {steps} steps \ + (delivery_mode=foreground)." + )) + .with_structured(json!({ + "path": "x11_xtest_fg", + "verified": false, + "delivery_mode": "foreground" + })), + Ok(Err(e)) => ToolResult::error(e.to_string()), + Err(e) => ToolResult::error(format!("Task error: {e}")), + }; + } + let press_result = tokio::task::spawn_blocking(move || { crate::input::send_button_down( xid, @@ -4097,7 +4995,7 @@ pub struct ParallelMouseDragTool { } static PMDRAG_DEF: std::sync::OnceLock = std::sync::OnceLock::new(); -/// EIS-compositor path for parallel_mouse_drag: build window-local drag paths +/// cua-compositor path for parallel_mouse_drag: build window-local drag paths /// and run them as concurrent multi-cursor injections over the control socket. /// Coordinates stay window-local (the compositor maps them per app_id), so no /// X11 geometry/MPX is needed — the X11 path's hard blocker on Wayland. @@ -4145,11 +5043,14 @@ async fn parallel_drag_inject(args: &Value) -> ToolResult { .and_then(|v| v.as_str()) .unwrap_or("left"), ) as u32; - let app = match tokio::task::spawn_blocking(move || crate::wayland::app_id_for_window(xid)) - .await + let app = match tokio::task::spawn_blocking(move || { + crate::wayland::inject_target_for_window(xid) + }) + .await { - Ok(Some(a)) => a, - _ => return ToolResult::error(format!("no Wayland app_id for window {xid}")), + Ok(Ok(target)) => target, + Ok(Err(error)) => return ToolResult::error(error.to_string()), + Err(error) => return ToolResult::error(format!("Task error: {error}")), }; drags.push(crate::wayland::InjectDrag { app_id: app, @@ -4162,7 +5063,7 @@ async fn parallel_drag_inject(args: &Value) -> ToolResult { let n = drags.len(); match tokio::task::spawn_blocking(move || crate::wayland::inject_parallel_drags(&drags)).await { Ok(Ok(())) => ToolResult::text(format!( - "Ran {n} concurrent drags (multi-cursor via EIS compositor)." + "Ran {n} concurrent drags (multi-cursor via cua-compositor)." )), Ok(Err(e)) => ToolResult::error(e.to_string()), Err(e) => ToolResult::error(format!("Task error: {e}")), @@ -4203,7 +5104,7 @@ impl Tool for ParallelMouseDragTool { } async fn invoke(&self, args: Value) -> ToolResult { - // EIS nested compositor: run the drags as concurrent multi-cursor + // Nested cua-compositor: run the drags as concurrent multi-cursor // injections (window-local, no X11 MPX/geometry needed). if crate::wayland::is_inject_mode() { return parallel_drag_inject(&args).await; @@ -4715,7 +5616,7 @@ impl Tool for MoveCursorTool { // Off-thread because the wayland-client roundtrip is blocking. Best-effort // — overlay update + registry write already succeeded; surface a warning // only if the warp itself failed. - let real_warp_note = if crate::wayland::is_wayland() { + let real_warp_note = if crate::wayland::wayland_input_enabled() { let xi = x.round() as i32; let yi = y.round() as i32; match tokio::task::spawn_blocking(move || { @@ -4787,13 +5688,19 @@ impl Tool for SetAgentCursorMotionTool { fn def(&self) -> &ToolDef { CURSOR_DEF.get_or_init(|| ToolDef { name: "set_agent_cursor_motion".into(), - description: format!("Configure the visual appearance of an agent cursor instance.\n\n\ + description: format!("Configure the visual appearance and motion curve of an agent cursor instance.\n\n\ - cursor_id: instance name (default='default')\n\ - cursor_icon: built-in ({}) or a path to a PNG/JPEG/SVG/ICO file; '' reverts to the default cursor\n\ - cursor_color: hex color e.g. '#00FFFF' or CSS name\n\ - cursor_label: short text shown near the cursor\n\ - cursor_size: dot radius in points (default=16)\n\ - - cursor_opacity: 0.0–1.0 (default=0.85)", + - cursor_opacity: 0.0–1.0 (default=0.85)\n\n\ + Motion curve (Bezier):\n\ + - arc_size: perpendicular deflection as fraction of path length [0,1]. Default 0.25\n\ + - spring: settle damping [0.3,1.0]; 1.0=no overshoot. Default 0.72\n\ + - glide_duration_ms: fixed flight duration per move [50,5000]; omit for speed-based\n\ + - dwell_after_click_ms: pause after click ripple [0,5000]. Default 80\n\ + - idle_hide_ms: auto-hide delay [0,60000]; 0=never. Default 20000", cursor_overlay::BuiltinShape::names_help()), input_schema: json!({ "type":"object","properties":{ @@ -4803,13 +5710,30 @@ impl Tool for SetAgentCursorMotionTool { "cursor_color":{"type":"string"}, "cursor_label":{"type":"string"}, "cursor_size":{"type":"number"}, - "cursor_opacity":{"type":"number"} + "cursor_opacity":{"type":"number"}, + "start_handle":{"type":"number","description":"Start-handle fraction [0,1]. Default 0.3."}, + "end_handle":{"type":"number","description":"End-handle fraction [0,1]. Default 0.3."}, + "arc_size":{"type":"number","description":"Arc deflection as fraction of path length [0,1]. Default 0.25."}, + "arc_flow":{"type":"number","description":"Asymmetry bias [-1,1]. Default 0.0."}, + "spring":{"type":"number","description":"Settle damping [0.3,1.0]. Default 0.72."}, + "glide_duration_ms":{"type":"number","minimum":50,"maximum":5000}, + "dwell_after_click_ms":{"type":"number","minimum":0,"maximum":5000}, + "idle_hide_ms":{"type":"number","minimum":0,"maximum":60000}, + "turn_radius":{"type":"number","minimum":1,"maximum":1000} },"additionalProperties":false }), read_only: false, destructive: false, idempotent: true, open_world: false, }) } async fn invoke(&self, args: Value) -> ToolResult { + fn num(value: Option<&Value>) -> Option { + value.and_then(|value| { + value + .as_f64() + .or_else(|| value.as_i64().map(|integer| integer as f64)) + }) + } + let cursor_id = resolve_cursor_key(&args); // Resolve `cursor_icon` (built-in name or image path — same vocabulary as // the CLI flags) to a shape override and dispatch it, so the overlay @@ -4849,7 +5773,44 @@ impl Tool for SetAgentCursorMotionTool { if let Some(cmd) = shape_cmd { crate::overlay::send_command_for(cursor_id.clone(), cmd); } - ToolResult::text(format!("Cursor '{cursor_id}' config updated.")).with_structured(args) + + let motion = crate::overlay::current_motion_for(&cursor_id).with_overrides( + num(args.get("start_handle")), + num(args.get("end_handle")), + num(args.get("arc_size")), + num(args.get("arc_flow")), + num(args.get("spring")), + num(args.get("glide_duration_ms")), + num(args.get("dwell_after_click_ms")), + num(args.get("idle_hide_ms")), + None, + num(args.get("turn_radius")), + ); + crate::overlay::send_command_for( + cursor_id.clone(), + cursor_overlay::OverlayCommand::SetMotion(motion.clone()), + ); + + ToolResult::text(format!( + "Cursor '{cursor_id}' config updated. Motion: arc={:.2} spring={:.2} glide={}ms dwell={}ms idle={}ms", + motion.arc_size, + motion.spring, + motion.glide_duration_ms as u32, + motion.dwell_after_click_ms as u32, + motion.idle_hide_ms as u32, + )) + .with_structured(json!({ + "cursor_id": cursor_id, + "start_handle": motion.start_handle, + "end_handle": motion.end_handle, + "arc_size": motion.arc_size, + "arc_flow": motion.arc_flow, + "spring": motion.spring, + "glide_duration_ms": motion.glide_duration_ms, + "dwell_after_click_ms": motion.dwell_after_click_ms, + "idle_hide_ms": motion.idle_hide_ms, + "turn_radius": motion.turn_radius, + })) } } @@ -5597,14 +6558,14 @@ impl Tool for TypeTextCharsTool { }; let text_len = text.chars().count(); let result = tokio::task::spawn_blocking(move || { - if crate::wayland::is_wayland() { + if crate::wayland::wayland_input_enabled() { // Per-char `wtype` loop with the requested delay — mirrors the // X11 XSendEvent per-char path. Sleeping here is fine because // we're inside spawn_blocking. let mut buf = [0u8; 4]; for ch in text.chars() { let s = ch.encode_utf8(&mut buf); - crate::wayland::type_text(s)?; + crate::wayland::type_text(xid, s)?; if delay_ms > 0 { std::thread::sleep(std::time::Duration::from_millis(delay_ms)); } @@ -5691,11 +6652,10 @@ impl Tool for BringToFrontTool { proper timestamp handling to beat focus-stealing prevention) — call \ it before `delivery_mode:\"foreground\"` input to avoid a per-call \ flash, or to escalate when background injection didn't land. \ - Wayland: a standalone activate is NOT exposed — the compositor's \ - security model bundles activation into the virtual-pointer/click \ - path, so use `delivery_mode:\"foreground\"` on the input call \ - itself; this reports that constraint on Wayland rather than \ - faking it. Matches the macOS / Windows bring_to_front rung." + Wayland: activates through a target-addressable compositor adapter \ + (wlroots foreign-toplevel or the GNOME Shell helper) and refuses \ + when the compositor offers no safe adapter. Matches the macOS / \ + Windows bring_to_front rung." .into(), input_schema: serde_json::json!({ "type":"object","required":["pid"],"properties":{ @@ -5709,33 +6669,41 @@ impl Tool for BringToFrontTool { async fn invoke(&self, args: Value) -> ToolResult { use cua_driver_core::tool_args::ArgsExt; - // Wayland: no standalone external activate (compositor security model - // bundles it into the vptr/click path). Report honestly; the agent - // escalates via delivery_mode:"foreground" on the input call instead. + let pid = args.u64_or("pid", 0) as u32; if crate::wayland::is_wayland() { - return ToolResult::error( - "bring_to_front: Wayland has no standalone window-activation API for \ - external clients — the compositor bundles activation into the \ - virtual-pointer/click path. Use delivery_mode:\"foreground\" on the \ - click/type_text call itself (it activates the target as part of the \ - injection)." - .to_string(), - ) - .with_structured(serde_json::json!({ - "code": "bring_to_front_wayland_bundled", - "platform": "linux", - "session": "wayland", - "suggestion": - "On Wayland, pass delivery_mode:\"foreground\" to click / type_text — \ - activation is performed as part of the injection.", - })); + let window_id = match args.opt_u64("window_id") { + Some(window_id) => window_id, + None => match crate::wayland::list_windows_dispatch(Some(pid)).first() { + Some(window) => window.xid, + None => { + return ToolResult::error(format!( + "bring_to_front: no window_id given and no Wayland windows found for pid {pid}." + )) + } + }, + }; + let result = tokio::task::spawn_blocking(move || { + crate::wayland::activate_window_for_input_target(window_id, Some(pid)) + }) + .await; + return match result { + Ok(Ok(())) => ToolResult::text(format!( + "Brought Wayland window {window_id} to front." + )) + .with_structured(serde_json::json!({ + "window_id": window_id, + "platform": "linux", + "session": "wayland", + })), + Ok(Err(error)) => ToolResult::error(error.to_string()), + Err(error) => ToolResult::error(format!("Task error: {error}")), + }; } // X11: resolve the target xid (window_id, else first window for pid). let xid = match args.opt_u64("window_id") { Some(x) => x, None => { - let pid = args.u64_or("pid", 0) as u32; let windows = tokio::task::spawn_blocking(move || crate::x11::list_windows(Some(pid))) .await @@ -5871,7 +6839,9 @@ pub fn build_registry(compat: bool) -> ToolRegistry { r.register(Box::new(ZoomTool { state: state.clone(), })); - r.register(Box::new(TypeTextCharsTool)); + // `type_text_chars` is a deprecated invoke-time alias for `type_text`. + // Keep it out of tools/list, matching the macOS and Windows registries. + let _: &TypeTextCharsTool = &TypeTextCharsTool; // Cross-platform `page` tool definition lives in mcp-server; Linux plugs // in its AT-SPI + CDP backend here. r.register(Box::new(cua_driver_core::page::PageTool::new(Arc::new( @@ -5884,7 +6854,7 @@ pub fn build_registry(compat: bool) -> ToolRegistry { #[cfg(test)] mod click_button_schema_tests { - use super::ClickTool; + use super::{chromium_background_must_refuse, maps_indicate_gtk, ClickTool}; use cua_driver_core::tool::Tool; /// Surface 5: schema must advertise the three canonical button values and @@ -5920,6 +6890,27 @@ mod click_button_schema_tests { "description should call out wayland fallback" ); } + + #[test] + fn chromium_background_requires_focus_free_inject_mode() { + assert!(chromium_background_must_refuse(false, false, true)); + assert!(!chromium_background_must_refuse(false, true, true)); + assert!(!chromium_background_must_refuse(true, false, true)); + assert!(!chromium_background_must_refuse(false, false, false)); + } + + #[test] + fn gtk_process_maps_are_detected_without_matching_unrelated_libraries() { + assert!(maps_indicate_gtk( + "7f00-7f01 r-xp /usr/lib/x86_64-linux-gnu/libgtk-3.so.0.2404.32" + )); + assert!(maps_indicate_gtk( + "7f00-7f01 r-xp /nix/store/hash-gtk4/lib/libgtk-4.so.1" + )); + assert!(!maps_indicate_gtk( + "7f00-7f01 r-xp /usr/lib/x86_64-linux-gnu/libgdk_pixbuf-2.0.so" + )); + } } #[cfg(test)] diff --git a/libs/cua-driver/rust/crates/platform-linux/src/tools/stubs.rs b/libs/cua-driver/rust/crates/platform-linux/src/tools/stubs.rs index 00d959dc93..aff490f272 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/tools/stubs.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/tools/stubs.rs @@ -235,7 +235,9 @@ pub fn build_registry() -> cua_driver_core::tool::ToolRegistry { r.register(Box::new(SetConfigTool)); r.register(Box::new(GetAccessibilityTreeTool)); r.register(Box::new(ZoomTool)); - r.register(Box::new(TypeTextCharsTool)); + // `type_text_chars` remains accepted as an invoke-time alias, but is not + // advertised as a separate tool on any platform. + let _: &TypeTextCharsTool = &TypeTextCharsTool; r.register_recording_tools(); r.register_session_tools(); r diff --git a/libs/cua-driver/rust/crates/platform-linux/src/video_wayland.rs b/libs/cua-driver/rust/crates/platform-linux/src/video_wayland.rs new file mode 100644 index 0000000000..291e7cb7fb --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-linux/src/video_wayland.rs @@ -0,0 +1,141 @@ +//! Native Wayland full-desktop video capture through `wf-recorder`. +//! +//! FFmpeg's Linux input backend is X11-only. On a native Wayland session, +//! `wf-recorder` consumes the compositor's screencopy protocol and writes the +//! same MP4 artifact shape expected by the shared recording session. + +use std::io::Read; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +use cua_driver_core::video::{VideoBackend, VideoBackendFactory, VideoMetadata}; + +pub struct WfRecorderVideoBackendFactory; + +impl VideoBackendFactory for WfRecorderVideoBackendFactory { + fn start(&self, output_path: &Path) -> anyhow::Result> { + WfRecorderVideoBackend::start(output_path) + .map(|backend| Box::new(backend) as Box) + } +} + +struct WfRecorderVideoBackend { + child: Child, + output_path: PathBuf, + started_at: Instant, + stderr_thread: Option>>, +} + +impl WfRecorderVideoBackend { + fn start(output_path: &Path) -> anyhow::Result { + if std::env::var_os("WAYLAND_DISPLAY").is_none() { + anyhow::bail!("wf-recorder requires WAYLAND_DISPLAY"); + } + let available = Command::new("wf-recorder") + .arg("--help") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map(|status| status.success()) + .unwrap_or(false); + if !available { + anyhow::bail!( + "wf-recorder not found on PATH. Install wf-recorder for native Wayland video." + ); + } + if let Some(parent) = output_path.parent() { + std::fs::create_dir_all(parent)?; + } + + let started_at = Instant::now(); + let mut command = Command::new("wf-recorder"); + command + .arg("-f") + .arg(output_path) + .args(["--no-damage", "-c", "libx264", "-x", "yuv420p"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()); + if let Some(output) = std::env::var_os("CUA_WAYLAND_RECORDING_OUTPUT") { + command.arg("-o").arg(output); + } + let mut child = command + .spawn() + .map_err(|error| anyhow::anyhow!("failed to start wf-recorder: {error}"))?; + let stderr_thread = child.stderr.take().map(|mut stderr| { + std::thread::spawn(move || { + let mut output = Vec::new(); + let _ = stderr.read_to_end(&mut output); + if output.len() > 4096 { + let excess = output.len() - 4096; + output.drain(..excess); + } + output + }) + }); + + let probe_deadline = Instant::now() + Duration::from_millis(1500); + while Instant::now() < probe_deadline { + if let Some(status) = child.try_wait()? { + let stderr = stderr_thread + .map(|worker| worker.join().unwrap_or_default()) + .unwrap_or_default(); + anyhow::bail!( + "wf-recorder exited during startup ({status}): {}", + String::from_utf8_lossy(&stderr) + ); + } + std::thread::sleep(Duration::from_millis(100)); + } + + Ok(Self { + child, + output_path: output_path.to_path_buf(), + started_at, + stderr_thread, + }) + } +} + +impl VideoBackend for WfRecorderVideoBackend { + fn stop(mut self: Box) -> anyhow::Result { + let elapsed = self.started_at.elapsed(); + unsafe { + libc::kill(self.child.id() as i32, libc::SIGINT); + } + let deadline = Instant::now() + Duration::from_secs(10); + let finalized = loop { + if let Some(status) = self.child.try_wait()? { + break status.success(); + } + if Instant::now() >= deadline { + let _ = self.child.kill(); + let _ = self.child.wait(); + break false; + } + std::thread::sleep(Duration::from_millis(80)); + }; + let stderr = self + .stderr_thread + .take() + .and_then(|worker| worker.join().ok()) + .unwrap_or_default(); + let has_video = self + .output_path + .metadata() + .map(|metadata| metadata.len() > 0) + .unwrap_or(false); + if !finalized || !has_video { + anyhow::bail!( + "wf-recorder did not finalize a playable artifact: {}", + String::from_utf8_lossy(&stderr) + ); + } + Ok(VideoMetadata { + path: self.output_path, + duration_ms: elapsed.as_millis() as u64, + finalized: true, + }) + } +} diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/ext_toplevel.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/ext_toplevel.rs new file mode 100644 index 0000000000..e5e6071a25 --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/ext_toplevel.rs @@ -0,0 +1,431 @@ +//! Generic staging `ext-foreign-toplevel-list-v1` window enumeration. + +use std::collections::HashMap; +use std::sync::{Mutex, OnceLock}; + +use wayland_client::{ + event_created_child, protocol::wl_registry, Connection, Dispatch, Proxy, QueueHandle, +}; +use wayland_protocols::ext::foreign_toplevel_list::v1::client::{ + ext_foreign_toplevel_handle_v1::{self as ext_handle, ExtForeignToplevelHandleV1}, + ext_foreign_toplevel_list_v1::{ + self as ext_list, ExtForeignToplevelListV1, EVT_TOPLEVEL_OPCODE, + }, +}; + +use crate::x11::WindowInfo; + +/// Synthetic ext-toplevel IDs must survive the existing `window_id as u32` +/// element-token paths. Keep them in a high, nonzero u32 namespace and reserve +/// `u32::MAX` as an exhaustion sentinel rather than emitting it. +const EXT_ID_NAMESPACE_START: u32 = 0xF000_0000; +const EXT_ID_NAMESPACE_END: u32 = 0xFFFF_FFFE; + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct ToplevelRecord { + identifier: String, + title: String, + app_id: String, +} + +#[derive(Default)] +struct PendingRecord { + identifier: Option, + title: Option, + app_id: Option, +} + +#[derive(Default)] +struct State { + manager: Option, + pending: HashMap, + toplevels: HashMap, +} + +impl Dispatch for State { + fn event( + state: &mut Self, + registry: &wl_registry::WlRegistry, + event: wl_registry::Event, + _: &(), + _: &Connection, + qh: &QueueHandle, + ) { + if let wl_registry::Event::Global { + name, + interface, + version, + } = event + { + if interface == ExtForeignToplevelListV1::interface().name { + state.manager = Some(registry.bind::( + name, + version.min(1), + qh, + (), + )); + } + } + } +} + +impl Dispatch for State { + fn event( + _: &mut Self, + _: &ExtForeignToplevelListV1, + _: ext_list::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + } + + event_created_child!(State, ExtForeignToplevelListV1, [ + EVT_TOPLEVEL_OPCODE => (ExtForeignToplevelHandleV1, ()), + ]); +} + +impl Dispatch for State { + fn event( + state: &mut Self, + handle: &ExtForeignToplevelHandleV1, + event: ext_handle::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + let protocol_id = handle.id().protocol_id(); + match event { + ext_handle::Event::Title { title } => { + state.pending.entry(protocol_id).or_default().title = Some(title); + } + ext_handle::Event::AppId { app_id } => { + state.pending.entry(protocol_id).or_default().app_id = Some(app_id); + } + ext_handle::Event::Identifier { identifier } => { + state.pending.entry(protocol_id).or_default().identifier = Some(identifier); + } + ext_handle::Event::Done => { + let pending = state.pending.remove(&protocol_id).unwrap_or_default(); + let record = state.toplevels.entry(protocol_id).or_default(); + if let Some(identifier) = pending.identifier { + record.identifier = identifier; + } + if let Some(title) = pending.title { + record.title = title; + } + if let Some(app_id) = pending.app_id { + record.app_id = app_id; + } + } + ext_handle::Event::Closed => { + state.pending.remove(&protocol_id); + state.toplevels.remove(&protocol_id); + handle.destroy(); + } + _ => {} + } + } +} + +struct IdRegistry { + by_identifier: HashMap, + by_id: HashMap, + next: Option, +} + +impl Default for IdRegistry { + fn default() -> Self { + Self { + by_identifier: HashMap::new(), + by_id: HashMap::new(), + next: Some(EXT_ID_NAMESPACE_START), + } + } +} + +impl IdRegistry { + fn id_for(&mut self, identifier: &str) -> anyhow::Result { + if let Some(id) = self.by_identifier.get(identifier) { + if self.by_id.get(id).is_some_and(|known| known == identifier) { + return Ok(u64::from(*id)); + } + anyhow::bail!("ext toplevel id registry collision for identifier {identifier:?}"); + } + + let mut id = self + .next + .ok_or_else(|| anyhow::anyhow!("ext toplevel numeric id space exhausted"))?; + while self.by_id.contains_key(&id) { + id = id + .checked_add(1) + .filter(|candidate| *candidate <= EXT_ID_NAMESPACE_END) + .ok_or_else(|| anyhow::anyhow!("ext toplevel numeric id space exhausted"))?; + } + self.next = (id < EXT_ID_NAMESPACE_END).then_some(id + 1); + self.by_identifier.insert(identifier.to_owned(), id); + self.by_id.insert(id, identifier.to_owned()); + Ok(u64::from(id)) + } +} + +fn id_registry() -> &'static Mutex { + static REGISTRY: OnceLock> = OnceLock::new(); + REGISTRY.get_or_init(|| Mutex::new(IdRegistry::default())) +} + +fn stable_id(identifier: &str) -> anyhow::Result { + id_registry() + .lock() + .map_err(|_| anyhow::anyhow!("ext toplevel id registry lock poisoned"))? + .id_for(identifier) +} + +/// Enumerate ext toplevels and enrich them with AT-SPI pid and geometry data. +pub fn list_windows() -> anyhow::Result> { + let conn = Connection::connect_to_env()?; + let mut queue = conn.new_event_queue::(); + let qh = queue.handle(); + conn.display().get_registry(&qh, ()); + + let mut state = State::default(); + queue.roundtrip(&mut state)?; + if state.manager.is_none() { + anyhow::bail!("compositor does not expose ext_foreign_toplevel_list_v1"); + } + for _ in 0..3 { + queue.roundtrip(&mut state)?; + } + + let mut native = Vec::with_capacity(state.toplevels.len()); + for record in state.toplevels.into_values() { + if record.identifier.is_empty() { + tracing::warn!("ignoring ext toplevel whose done batch had no identifier"); + continue; + } + native.push((stable_id(&record.identifier)?, record)); + } + native.sort_unstable_by_key(|(id, _)| *id); + Ok(merge_atspi_records( + native, + crate::atspi::list_windows(None), + )) +} + +fn merge_atspi_records( + native: Vec<(u64, ToplevelRecord)>, + atspi: Vec, +) -> Vec { + let mut claimed = vec![false; atspi.len()]; + let mut out = Vec::with_capacity(native.len() + atspi.len()); + + for (id, record) in native { + let exact_title = atspi.iter().enumerate().find_map(|(index, window)| { + (!claimed[index] && !record.title.is_empty() && window.title == record.title) + .then_some(index) + }); + let matching_atspi = exact_title.or_else(|| { + unique_match(&atspi, &claimed, |window| { + !record.app_id.is_empty() && window.app_name == record.app_id + }) + }); + + let mut window = WindowInfo { + xid: id, + pid: None, + app_name: record.app_id, + title: record.title, + is_on_screen: true, + z_index: None, + x: 0, + y: 0, + width: 0, + height: 0, + }; + if let Some(index) = matching_atspi { + claimed[index] = true; + let enrichment = &atspi[index]; + window.pid = enrichment.pid; + window.is_on_screen = enrichment.is_on_screen; + window.z_index = enrichment.z_index; + window.x = enrichment.x; + window.y = enrichment.y; + window.width = enrichment.width; + window.height = enrichment.height; + if window.app_name.is_empty() { + window.app_name = enrichment.app_name.clone(); + } + if window.title.is_empty() { + window.title = enrichment.title.clone(); + } + } + out.push(window); + } + + out.extend( + atspi + .into_iter() + .enumerate() + .filter_map(|(index, window)| (!claimed[index]).then_some(window)), + ); + out +} + +fn unique_match( + windows: &[WindowInfo], + claimed: &[bool], + predicate: impl Fn(&WindowInfo) -> bool, +) -> Option { + let mut matches = windows + .iter() + .enumerate() + .filter(|(index, window)| !claimed[*index] && predicate(window)); + let (index, _) = matches.next()?; + matches.next().is_none().then_some(index) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn record(identifier: &str, title: &str, app_id: &str) -> ToplevelRecord { + ToplevelRecord { + identifier: identifier.to_owned(), + title: title.to_owned(), + app_id: app_id.to_owned(), + } + } + + fn atspi(xid: u64, pid: u32, title: &str, app_name: &str, x: i32) -> WindowInfo { + WindowInfo { + xid, + pid: Some(pid), + app_name: app_name.to_owned(), + title: title.to_owned(), + is_on_screen: true, + z_index: None, + x, + y: 20, + width: 800, + height: 600, + } + } + + #[test] + fn id_registry_is_stable_and_bijective() { + let mut registry = IdRegistry::default(); + let first = registry.id_for("opaque-a").unwrap(); + let second = registry.id_for("opaque-b").unwrap(); + assert_eq!(registry.id_for("opaque-a").unwrap(), first); + assert_ne!(first, second); + assert_eq!( + registry + .by_id + .get(&u32::try_from(first).unwrap()) + .map(String::as_str), + Some("opaque-a") + ); + assert_eq!( + registry + .by_id + .get(&u32::try_from(second).unwrap()) + .map(String::as_str), + Some("opaque-b") + ); + } + + #[test] + fn merge_recovers_metadata_and_preserves_unmatched_records() { + let native = vec![ + ( + u64::from(EXT_ID_NAMESPACE_START), + record("a", "Editor", "org.editor"), + ), + ( + u64::from(EXT_ID_NAMESPACE_START + 1), + record("b", "Other", "org.viewer"), + ), + ( + u64::from(EXT_ID_NAMESPACE_START + 2), + record("c", "Native only", "org.native"), + ), + ]; + let merged = merge_atspi_records( + native, + vec![ + atspi(11, 101, "Editor", "unrelated-name", 10), + atspi(12, 202, "Different title", "org.viewer", 30), + atspi(13, 303, "AT-SPI only", "org.extra", 50), + ], + ); + + assert_eq!(merged.len(), 4); + assert_eq!( + (merged[0].xid, merged[0].pid, merged[0].x), + (u64::from(EXT_ID_NAMESPACE_START), Some(101), 10) + ); + assert_eq!(merged[1].pid, Some(202)); + assert_eq!(merged[2].pid, None); + assert_eq!((merged[3].xid, merged[3].pid), (13, Some(303))); + } + + #[test] + fn merge_does_not_guess_ambiguous_app_ids() { + let merged = merge_atspi_records( + vec![( + u64::from(EXT_ID_NAMESPACE_START), + record("a", "", "org.same"), + )], + vec![ + atspi(11, 101, "One", "org.same", 10), + atspi(12, 202, "Two", "org.same", 30), + ], + ); + assert_eq!(merged.len(), 3); + assert_eq!(merged[0].pid, None); + assert_eq!(merged[1].pid, Some(101)); + assert_eq!(merged[2].pid, Some(202)); + } + + #[test] + fn merge_pairs_repeated_exact_titles_one_to_one() { + let merged = merge_atspi_records( + vec![ + ( + u64::from(EXT_ID_NAMESPACE_START), + record("a", "Document", ""), + ), + ( + u64::from(EXT_ID_NAMESPACE_START + 1), + record("b", "Document", ""), + ), + ], + vec![ + atspi(11, 101, "Document", "first", 10), + atspi(12, 202, "Document", "second", 30), + ], + ); + assert_eq!(merged.len(), 2); + assert_eq!(merged[0].pid, Some(101)); + assert_eq!(merged[1].pid, Some(202)); + } + + #[test] + fn id_registry_skips_collisions_and_reports_exhaustion() { + let mut registry = IdRegistry::default(); + registry + .by_id + .insert(EXT_ID_NAMESPACE_START, "already-used".to_owned()); + assert_eq!( + registry.id_for("next").unwrap(), + u64::from(EXT_ID_NAMESPACE_START + 1) + ); + + registry.next = Some(EXT_ID_NAMESPACE_END); + assert_eq!( + registry.id_for("last").unwrap(), + u64::from(EXT_ID_NAMESPACE_END) + ); + assert!(registry.id_for("exhausted").is_err()); + } +} diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs index 2265ba74cb..745c0732f4 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/libei.rs @@ -7,8 +7,8 @@ //! `zwlr_virtual_pointer_v1` / `zwlr_virtual_keyboard_v1` protocols. //! //! Sequence: -//! 1. `ei::Context::connect_to_env()` — fast path when a $LIBEI_SOCKET is -//! already exported (cua-compositor / inject mode). +//! 1. `ei::Context::connect_to_env()` — fast path when a compositor or test +//! environment already exports `$LIBEI_SOCKET`. //! 2. Fallback: ashpd `RemoteDesktop::create_session` → //! `select_devices(KEYBOARD|POINTER)` → //! `start(session, parent)` (user consent dialog the first time) → @@ -20,10 +20,10 @@ //! until the worker reports the request was flushed to the EIS //! server. //! -//! Persistence: ashpd `PersistMode::Application` caches the consent grant -//! for the lifetime of the requesting binary; restore_token written to -//! `~/.config/cua-driver/libei.token` survives reboots (the worker -//! reads it on startup). +//! Persistence: ashpd `PersistMode::ExplicitlyRevoked` keeps the user's consent +//! until they revoke it in desktop settings. The restore token is stored at +//! `~/.config/cua-driver/libei-persistent.token` and reused across daemon +//! restarts. //! //! Coordinates: ei_pointer_absolute uses LOGICAL PIXELS inside an //! announced ei_device.Region — collected between device creation and @@ -36,6 +36,7 @@ use std::sync::OnceLock; use std::thread; use crossbeam_channel::{bounded, Receiver, Sender}; +use xkbcommon::xkb; /// Buttons the public API exposes. Mapped to evdev codes in the worker /// thread so the libei surface only sees evdev integers. @@ -56,9 +57,52 @@ impl Button { } } +/// One evdev keyboard state transition in a [`key_sequence`] request. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KeyTransition { + Press(u32), + Release(u32), +} + +const MAX_KEY_SEQUENCE_TRANSITIONS: usize = 64; + +/// EIS frame timestamps are CLOCK_MONOTONIC microseconds, not per-command +/// sequence numbers. Mutter discards transactions carrying the old 0,1,2... +/// placeholders. Keep them strictly increasing even when several frames are +/// emitted inside one microsecond. +fn event_time_us() -> u64 { + use std::sync::atomic::{AtomicU64, Ordering}; + + static LAST: AtomicU64 = AtomicU64::new(0); + let mut ts = unsafe { + let mut value: libc::timespec = std::mem::zeroed(); + if libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut value) == 0 { + (value.tv_sec as u64) + .saturating_mul(1_000_000) + .saturating_add((value.tv_nsec as u64) / 1_000) + } else { + 0 + } + }; + loop { + let previous = LAST.load(Ordering::Relaxed); + ts = ts.max(previous.saturating_add(1)); + if LAST + .compare_exchange_weak(previous, ts, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + { + return ts; + } + } +} + /// Commands the worker thread accepts. Each carries a reply channel so /// the caller blocks until the EIS server has received the event. enum Cmd { + WaitReady { + interface: &'static str, + reply: Sender>, + }, Click { x: f64, y: f64, @@ -83,6 +127,19 @@ enum Cmd { keycode: u32, reply: Sender>, }, + KeySequence { + transitions: Vec, + reply: Sender>, + }, + Drag { + from_x: f64, + from_y: f64, + to_x: f64, + to_y: f64, + steps: u32, + button: Button, + reply: Sender>, + }, Shutdown, } @@ -92,9 +149,21 @@ fn tx() -> anyhow::Result<&'static Sender> { TX.get().ok_or_else(|| anyhow::anyhow!("libei worker not started; call ensure_started() first")) } +fn wait_for_reply(rx: Receiver>) -> anyhow::Result<()> { + rx.recv_timeout(std::time::Duration::from_secs(20)) + .map_err(|error| match error { + crossbeam_channel::RecvTimeoutError::Timeout => anyhow::anyhow!( + "libei input backend did not become ready within 20s; the desktop portal may be waiting for Remote Desktop consent or its EIS session may be wedged" + ), + crossbeam_channel::RecvTimeoutError::Disconnected => { + anyhow::anyhow!("libei worker reply channel closed") + } + })? +} + fn restore_token_path() -> Option { let base = dirs::config_dir()?; - Some(base.join("cua-driver").join("libei.token")) + Some(base.join("cua-driver").join("libei-persistent.token")) } fn read_restore_token() -> Option { @@ -110,8 +179,21 @@ fn write_restore_token(token: &str) -> anyhow::Result<()> { anyhow::anyhow!("failed to create {} for restore_token: {e}", parent.display()) })?; } - std::fs::write(&path, token) - .map_err(|e| anyhow::anyhow!("failed to write libei restore_token to {}: {e}", path.display())) + use std::io::Write; + use std::os::unix::fs::OpenOptionsExt; + + let mut file = std::fs::OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .mode(0o600) + .open(&path) + .map_err(|e| { + anyhow::anyhow!("failed to open libei restore_token at {}: {e}", path.display()) + })?; + file.write_all(token.as_bytes()).map_err(|e| { + anyhow::anyhow!("failed to write libei restore_token to {}: {e}", path.display()) + }) } /// Spawn the libei worker thread (idempotent — safe to call from every @@ -132,6 +214,34 @@ pub fn ensure_started() -> anyhow::Result<()> { Ok(()) } +fn wait_until_ready(interface: &'static str) -> anyhow::Result<()> { + ensure_started()?; + let (tx_r, rx_r) = bounded(1); + tx()? + .send(Cmd::WaitReady { + interface, + reply: tx_r, + }) + .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?; + wait_for_reply(rx_r) +} + +/// Negotiate a resumed absolute-pointer device without emitting input. +/// Callers can then restore target activation after a portal consent dialog. +pub fn wait_pointer_ready() -> anyhow::Result<()> { + wait_until_ready("ei_pointer_absolute") +} + +/// Negotiate a resumed scroll device without emitting input. +pub fn wait_scroll_ready() -> anyhow::Result<()> { + wait_until_ready("ei_scroll") +} + +/// Negotiate a resumed keyboard device without emitting input. +pub fn wait_keyboard_ready() -> anyhow::Result<()> { + wait_until_ready("ei_keyboard") +} + // ── Public API ─────────────────────────────────────────────────────────── /// Move the cursor to absolute (x, y) within the announced device region, @@ -142,7 +252,7 @@ pub fn click(x: f64, y: f64, button: Button) -> anyhow::Result<()> { let (tx_r, rx_r) = bounded(1); tx()?.send(Cmd::Click { x, y, button, reply: tx_r }) .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?; - rx_r.recv().map_err(|e| anyhow::anyhow!("libei reply closed: {e}"))? + wait_for_reply(rx_r) } /// Move the cursor to absolute (x, y) inside the device region (no @@ -152,7 +262,7 @@ pub fn move_absolute(x: f64, y: f64) -> anyhow::Result<()> { let (tx_r, rx_r) = bounded(1); tx()?.send(Cmd::MoveAbsolute { x, y, reply: tx_r }) .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?; - rx_r.recv().map_err(|e| anyhow::anyhow!("libei reply closed: {e}"))? + wait_for_reply(rx_r) } /// Scroll by (dx, dy) logical units. Positive y scrolls down. @@ -161,7 +271,7 @@ pub fn scroll(dx: f64, dy: f64) -> anyhow::Result<()> { let (tx_r, rx_r) = bounded(1); tx()?.send(Cmd::Scroll { dx, dy, reply: tx_r }) .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?; - rx_r.recv().map_err(|e| anyhow::anyhow!("libei reply closed: {e}"))? + wait_for_reply(rx_r) } /// Inject a UTF-8 string via the `ei_text` interface (libei 1.6+). @@ -171,7 +281,7 @@ pub fn type_text(text: &str) -> anyhow::Result<()> { let (tx_r, rx_r) = bounded(1); tx()?.send(Cmd::TypeText { text: text.to_string(), reply: tx_r }) .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?; - rx_r.recv().map_err(|e| anyhow::anyhow!("libei reply closed: {e}"))? + wait_for_reply(rx_r) } /// Press + release a key by evdev code (e.g. `KEY_ENTER` = 28). For @@ -181,7 +291,74 @@ pub fn press_key(keycode: u32) -> anyhow::Result<()> { let (tx_r, rx_r) = bounded(1); tx()?.send(Cmd::PressKey { keycode, reply: tx_r }) .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?; - rx_r.recv().map_err(|e| anyhow::anyhow!("libei reply closed: {e}"))? + wait_for_reply(rx_r) +} + +/// Submit an ordered sequence of evdev key press/release transitions. +/// +/// The worker emits every transition in one emulation session and frames each +/// transition separately, allowing callers to hold modifiers while pressing a +/// key. Requests are capped to keep the synchronous worker responsive. +pub fn key_sequence(transitions: &[KeyTransition]) -> anyhow::Result<()> { + validate_key_sequence(transitions)?; + if transitions.is_empty() { + return Ok(()); + } + + ensure_started()?; + let (tx_r, rx_r) = bounded(1); + tx()? + .send(Cmd::KeySequence { + transitions: transitions.to_vec(), + reply: tx_r, + }) + .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?; + wait_for_reply(rx_r) +} + +fn validate_key_sequence(transitions: &[KeyTransition]) -> anyhow::Result<()> { + if transitions.len() > MAX_KEY_SEQUENCE_TRANSITIONS { + anyhow::bail!( + "libei key sequence has {} transitions; maximum is {}", + transitions.len(), + MAX_KEY_SEQUENCE_TRANSITIONS + ); + } + Ok(()) +} + +fn emit_key_transitions(transitions: &[KeyTransition], mut emit: impl FnMut(KeyTransition, u64)) { + for (frame, transition) in transitions.iter().copied().enumerate() { + emit(transition, frame as u64); + } +} + +/// Press `button` at (from_x, from_y), move through `steps` interpolated points +/// to (to_x, to_y), then release — a genuine button-held drag (text selection / +/// drag-and-drop / slider). Coordinates are absolute within the announced +/// device region, like [`click`]. +pub fn drag( + from_x: f64, + from_y: f64, + to_x: f64, + to_y: f64, + steps: u32, + button: Button, +) -> anyhow::Result<()> { + ensure_started()?; + let (tx_r, rx_r) = bounded(1); + tx()? + .send(Cmd::Drag { + from_x, + from_y, + to_x, + to_y, + steps, + button, + reply: tx_r, + }) + .map_err(|e| anyhow::anyhow!("libei worker channel closed: {e}"))?; + wait_for_reply(rx_r) } /// Cleanly stop the worker thread. @@ -197,9 +374,31 @@ pub fn shutdown() { // the negotiated input region. It runs a calloop event loop that handles // both the EIS protocol and the inbound command channel. +/// Owns the resources that must outlive a single libei call so the portal +/// RemoteDesktop + EIS session stays alive for the whole worker-thread lifetime. +/// +/// GNOME/Mutter (and KDE) tie the RemoteDesktop session — and therefore the +/// handed-off EIS fd — to the D-Bus connection that created it. The previous +/// code dropped the ashpd proxy/session (and the tokio runtime backing their +/// zbus connection) at the end of `open_eis_context`, before the calloop loop +/// even started, so the session died immediately on those compositors (#2105). +/// Holding these for the worker's lifetime keeps the connection — and the +/// session — open. The direct `$LIBEI_SOCKET` fast path has no portal +/// session, so it carries `None`. +#[allow(dead_code)] // fields are keep-alive only; never read +enum PortalKeepAlive { + None, + Portal { + _rt: tokio::runtime::Runtime, + _proxy: ashpd::desktop::remote_desktop::RemoteDesktop, + _session: ashpd::desktop::Session, + }, +} + fn worker(rx: Receiver) -> anyhow::Result<()> { - // Phase 1 — acquire an EIS context. - let context = open_eis_context() + // Phase 1 — acquire an EIS context (plus, on the portal path, a keep-alive + // that owns the ashpd RemoteDesktop session + its tokio runtime). + let (context, _portal_keepalive) = open_eis_context() .map_err(|e| anyhow::anyhow!("failed to obtain EIS connection: {e}"))?; // Phase 2 — handshake. The reis API requires we call context.handshake() @@ -210,17 +409,19 @@ fn worker(rx: Receiver) -> anyhow::Result<()> { // Phase 3 — run the calloop event loop. We use a calloop Generic source // wrapping the ei::Context's underlying socket so the loop wakes up on // EIS messages. The command channel is polled at the top of each loop - // iteration. + // iteration. `_portal_keepalive` stays bound until this fn returns — i.e. + // across the entire loop — so the portal session is never dropped early (#2105). run_calloop(context, rx) } -/// Open the EIS context. Fast path: $LIBEI_SOCKET (cua-compositor / -/// inject mode). Fallback: ashpd portal RemoteDesktop handshake. -fn open_eis_context() -> anyhow::Result { +/// Open the EIS context. Fast path: `$LIBEI_SOCKET` supplied by the environment. +/// Fallback: ashpd portal RemoteDesktop handshake. +fn open_eis_context() -> anyhow::Result<(reis::ei::Context, PortalKeepAlive)> { if let Some(ctx) = reis::ei::Context::connect_to_env() .map_err(|e| anyhow::anyhow!("env ei socket open failed: {e}"))? { - return Ok(ctx); + // Direct $LIBEI_SOCKET fast path: no portal session to keep alive. + return Ok((ctx, PortalKeepAlive::None)); } // Portal RemoteDesktop fallback. @@ -233,12 +434,18 @@ fn open_eis_context() -> anyhow::Result { use ashpd::enumflags2::BitFlags; use std::os::unix::net::UnixStream; - let rt = tokio::runtime::Builder::new_current_thread() + // A multi-threaded runtime (1 worker) so the zbus connection backing the + // RemoteDesktop session keeps being driven after this fn returns. The + // session must stay live for the whole worker lifetime (#2105); a + // current-thread runtime would freeze the connection the moment we stop + // calling `block_on`, and GNOME/KDE would tear the session down. + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) .enable_all() .build() .map_err(|e| anyhow::anyhow!("failed to build tokio runtime for ashpd: {e}"))?; - let fd = rt.block_on(async { + let (fd, proxy, session) = rt.block_on(async { let proxy = RemoteDesktop::new() .await .map_err(|e| anyhow::anyhow!("portal RemoteDesktop proxy unreachable: {e}. Install xdg-desktop-portal-gnome / xdg-desktop-portal-kde."))?; @@ -249,7 +456,7 @@ fn open_eis_context() -> anyhow::Result { let mut select_opts = SelectDevicesOptions::default() .set_devices(BitFlags::::from(DeviceType::Keyboard) | DeviceType::Pointer) - .set_persist_mode(PersistMode::Application); + .set_persist_mode(PersistMode::ExplicitlyRevoked); if let Some(tok) = read_restore_token() { select_opts = select_opts.set_restore_token(Some(tok.as_str())); } @@ -270,9 +477,9 @@ fn open_eis_context() -> anyhow::Result { // Best-effort persist of the restore_token. Without this, every // process restart re-prompts the user for consent — with it, ashpd - // 0.13's PersistMode::Application + the stored token together + // 0.13's explicitly-revoked mode and the stored token together // skip the dialog for the lifetime of the persistence grant - // (typically per login session on GNOME, indefinite on KDE). + // until the user explicitly revokes the grant in desktop settings. if let Some(tok) = started.restore_token() { let _ = write_restore_token(tok); } @@ -281,13 +488,13 @@ fn open_eis_context() -> anyhow::Result { .connect_to_eis(&session, ConnectToEISOptions::default()) .await .map_err(|e| anyhow::anyhow!("portal connect_to_eis failed: {e}"))?; - anyhow::Ok(fd) + anyhow::Ok((fd, proxy, session)) })?; let stream = UnixStream::from(fd); let ctx = reis::ei::Context::new(stream) .map_err(|e| anyhow::anyhow!("reis ei::Context::new failed: {e}"))?; - Ok(ctx) + Ok((ctx, PortalKeepAlive::Portal { _rt: rt, _proxy: proxy, _session: session })) } // ── calloop dispatch ──────────────────────────────────────────────────── @@ -302,7 +509,7 @@ fn open_eis_context() -> anyhow::Result { // Region selection picks the first announced ei_device::Region. fn run_calloop(context: reis::ei::Context, rx: Receiver) -> anyhow::Result<()> { - use calloop::{generic::Generic, EventLoop, Interest, Mode, PostAction}; + use calloop::{generic::Generic, EventLoop, Interest, Mode}; let mut event_loop: EventLoop = EventLoop::try_new() .map_err(|e| anyhow::anyhow!("calloop EventLoop::try_new failed: {e}"))?; @@ -318,13 +525,21 @@ fn run_calloop(context: reis::ei::Context, rx: Receiver) -> anyhow::Result< let mut state = EisState::default(); - // Command channel: drain at the top of each loop iteration with a - // bounded timeout so the EIS source stays responsive. + // Inbound commands can arrive before the EIS handshake has negotiated a + // usable input device — seat → device → interface → Resumed is async and + // only advances while `dispatch` runs, and the portal consent-and-connect + // path (#2105) returns before the device is live. Running a command against + // a not-yet-negotiated device fails with "no EIS device negotiated yet". So + // queue commands and run each only once `input_ready()`; fail any that wait + // longer than READY_TIMEOUT (handle_command then replies with the natural + // "no device" error) so a caller blocked on the reply never hangs forever. + let mut pending: Vec<(Cmd, std::time::Instant)> = Vec::new(); + const READY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(20); + loop { - // Process pending commands. match rx.try_recv() { Ok(Cmd::Shutdown) => break, - Ok(cmd) => state.handle_command(cmd), + Ok(cmd) => pending.push((cmd, std::time::Instant::now())), Err(crossbeam_channel::TryRecvError::Empty) => {} Err(crossbeam_channel::TryRecvError::Disconnected) => break, } @@ -333,6 +548,21 @@ fn run_calloop(context: reis::ei::Context, rx: Receiver) -> anyhow::Result< .dispatch(Some(std::time::Duration::from_millis(20)), &mut state) .map_err(|e| anyhow::anyhow!("calloop dispatch failed: {e}"))?; + // Run each queued command once the device IT needs is negotiated; time + // stale ones out. Per-command (not one global gate) so a keyboard-only + // session doesn't make keyboard commands wait 20s for a pointer device. + if !pending.is_empty() { + let mut still = Vec::with_capacity(pending.len()); + for (cmd, queued) in pending.drain(..) { + if state.cmd_ready(&cmd) || queued.elapsed() >= READY_TIMEOUT { + state.handle_command(cmd); + } else { + still.push((cmd, queued)); + } + } + pending = still; + } + // The handle_command path may have queued frame() requests; flush // them once per loop iteration so the EIS server sees them. if let Some(ctx) = state.context.as_ref() { @@ -349,7 +579,21 @@ struct EisState { seats: HashMap, devices: HashMap, sequence: u32, + /// The latest serial emitted by the EIS connection. Request serials are + /// connection-global (libei's `ei_get_serial`), while readiness is tracked + /// per device below. last_serial: u32, + /// Last device lifecycle event. Mutter 46 may advertise a provisional + /// absolute pointer before seat binding, then a second live device. Waiting + /// briefly for this stream to settle avoids dispatching to the transient one. + last_device_event: Option, + /// `char -> (evdev keycode, needs_shift)` built from the compositor's active + /// xkb keymap (delivered by the `ei_keyboard` device). Lets keycode typing + /// follow the user's real layout instead of assuming US. Empty until the + /// `Keymap` event arrives; typing then falls back to [`char_to_evdev`]. + keymap_chars: HashMap, + /// evdev keycode that produces `Shift_L` in the active keymap (else 42). + keymap_shift: Option, } #[derive(Default)] @@ -388,6 +632,8 @@ struct DeviceData { /// one Region. regions: Vec, interfaces: HashMap, + resumed: bool, + resumed_serial: Option, } impl EisState { @@ -421,11 +667,20 @@ impl EisState { handshake.handshake_version(1); handshake.name("cua-driver"); handshake.context_type(reis::ei::handshake::ContextType::Sender); + // Advertise the interface set + versions that reis 0.7's own + // EIS client examples use against real compositors. The + // previous list advertised ei_device v2 and omitted + // ei_pingpong, which Mutter rejects — it closes the EIS + // connection immediately after our handshake `finish()` + // (observed: one Handshake event, then read() errors). reis's + // handshaker also requires ei_pingpong / ei_callback / + // ei_connection. See reis 0.7 examples/type-text.rs. for (iface, ver) in [ ("ei_callback", 1u32), ("ei_connection", 1), + ("ei_pingpong", 1), ("ei_seat", 1), - ("ei_device", 2), + ("ei_device", 1), ("ei_pointer", 1), ("ei_pointer_absolute", 1), ("ei_button", 1), @@ -438,11 +693,17 @@ impl EisState { handshake.finish(); } } - Event::Connection(_conn, ev) => { - if let reis::ei::connection::Event::Seat { seat } = ev { + Event::Connection(_conn, ev) => match ev { + reis::ei::connection::Event::Seat { seat } => { self.seats.insert(seat, SeatData::default()); } - } + // The EIS server pings to check liveness; failing to answer + // makes it drop the connection. Mirror reis's examples. + reis::ei::connection::Event::Ping { ping } => { + ping.done(0); + } + _ => {} + }, Event::Seat(seat, ev) => { let data = self.seats.entry(seat.clone()).or_default(); match ev { @@ -478,6 +739,12 @@ impl EisState { } } Event::Device(device, ev) => { + self.last_device_event = Some(std::time::Instant::now()); + if let reis::ei::device::Event::Destroyed { serial } = &ev { + self.last_serial = *serial; + self.devices.remove(&device); + return; + } let data = self.devices.entry(device.clone()).or_default(); match ev { reis::ei::device::Event::DeviceType { device_type } => { @@ -499,89 +766,317 @@ impl EisState { } } reis::ei::device::Event::Resumed { serial } => { + data.resumed = true; + data.resumed_serial = Some(serial); + self.last_serial = serial; + // A RemoteDesktop session is one emulation transaction, + // not one transaction per click. Mutter can neutralize a + // device stopped in the same batch as its frame. Match + // libei's reference client: start on Resumed and keep it + // active until the device is paused or disconnected. + self.sequence = self.sequence.wrapping_add(1); + device.start_emulating(serial, self.sequence); + } + reis::ei::device::Event::Paused { serial } => { + data.resumed = false; + data.resumed_serial = None; self.last_serial = serial; } _ => {} } } + Event::Keyboard( + _kb, + reis::ei::keyboard::Event::Keymap { + keymap_type, + size, + keymap, + }, + ) => { + if keymap_type == reis::ei::keyboard::KeymapType::Xkb { + self.load_xkb_keymap(keymap, size); + } + } _ => {} } } + /// Compile the compositor's xkb keymap (from the `ei_keyboard` `Keymap` fd) + /// into a `char -> (evdev keycode, shift)` table + the Shift keycode, so + /// keycode typing follows the user's real layout instead of assuming US. + /// Best-effort: on any failure the table stays empty and typing falls back + /// to [`char_to_evdev`]. + fn load_xkb_keymap(&mut self, fd: std::os::unix::io::OwnedFd, size: u32) { + use std::io::{Read, Seek, SeekFrom}; + const XKB_EVDEV_OFFSET: u32 = 8; + const SHIFT_L: u32 = 0xffe1; + + // Sanity-cap the size from the (trusted, but possibly buggy) compositor + // event: real xkb keymaps are a few KiB; refuse absurd values rather + // than attempt a multi-GB allocation. + if size == 0 || size as usize > 16 * 1024 * 1024 { + return; + } + let mut file = std::fs::File::from(fd); + let mut buf = vec![0u8; size as usize]; + let _ = file.seek(SeekFrom::Start(0)); + if file.read_exact(&mut buf).is_err() { + return; + } + // The keymap buffer is NUL-terminated text. + let text = match buf.iter().position(|&b| b == 0) { + Some(n) => String::from_utf8_lossy(&buf[..n]).into_owned(), + None => String::from_utf8_lossy(&buf).into_owned(), + }; + let ctx = xkb::Context::new(xkb::CONTEXT_NO_FLAGS); + let keymap = match xkb::Keymap::new_from_string( + &ctx, + text, + xkb::KEYMAP_FORMAT_TEXT_V1, + xkb::KEYMAP_COMPILE_NO_FLAGS, + ) { + Some(k) => k, + None => return, + }; + + let mut chars: HashMap = HashMap::new(); + let mut shift = None; + for kc in keymap.min_keycode().raw()..=keymap.max_keycode().raw() { + let keycode = xkb::Keycode::new(kc); + let evdev = kc.wrapping_sub(XKB_EVDEV_OFFSET); + // Level 0 = unshifted, 1 = shifted; scan 0 first so the unshifted + // mapping wins when a char is reachable both ways. + for level in 0..=1u32 { + for sym in keymap.key_get_syms_by_level(keycode, 0, level) { + if shift.is_none() && sym.raw() == SHIFT_L { + shift = Some(evdev); + } + if let Some(ch) = sym.key_char() { + chars.entry(ch).or_insert((evdev, level == 1)); + } + } + } + } + self.keymap_chars = chars; + self.keymap_shift = shift; + } + + /// True once the device THIS command needs has been negotiated and + /// `Resumed`. Gating per-command (rather than on one global pointer check) + /// means a keyboard-only session doesn't stall keyboard commands waiting for + /// an absolute-pointer device that never arrives — and vice versa. Commands + /// run before their device exists fail with "no EIS device negotiated yet", + /// so `run_calloop` queues them until this holds (or READY_TIMEOUT elapses). + fn cmd_ready(&self, cmd: &Cmd) -> bool { + const DEVICE_QUIET_PERIOD: std::time::Duration = + std::time::Duration::from_millis(200); + if self + .last_device_event + .is_none_or(|last| last.elapsed() < DEVICE_QUIET_PERIOD) + { + return false; + } + match cmd { + Cmd::WaitReady { interface, .. } => self.device_with_interface(interface).is_some(), + Cmd::Click { .. } | Cmd::MoveAbsolute { .. } | Cmd::Drag { .. } => { + self.device_with_interface("ei_pointer_absolute").is_some() + } + Cmd::Scroll { .. } => self.device_with_interface("ei_scroll").is_some(), + Cmd::PressKey { .. } | Cmd::KeySequence { .. } => { + self.device_with_interface("ei_keyboard").is_some() + } + Cmd::TypeText { .. } => { + self.device_with_interface("ei_text").is_some() + || self.device_with_interface("ei_keyboard").is_some() + } + Cmd::Shutdown => true, + } + } + fn handle_command(&mut self, cmd: Cmd) { let result = self.run_command(&cmd); // Send the reply on whichever channel this command carries. match cmd { + Cmd::WaitReady { reply, .. } => { let _ = reply.send(result); } Cmd::Click { reply, .. } => { let _ = reply.send(result); } Cmd::MoveAbsolute { reply, .. } => { let _ = reply.send(result); } Cmd::Scroll { reply, .. } => { let _ = reply.send(result); } Cmd::TypeText { reply, .. } => { let _ = reply.send(result); } Cmd::PressKey { reply, .. } => { let _ = reply.send(result); } + Cmd::KeySequence { reply, .. } => { let _ = reply.send(result); } + Cmd::Drag { reply, .. } => { let _ = reply.send(result); } Cmd::Shutdown => {} } } fn run_command(&mut self, cmd: &Cmd) -> anyhow::Result<()> { match cmd { + Cmd::WaitReady { interface, .. } => { + self.device_with_interface(interface).ok_or_else(|| { + anyhow::anyhow!("no resumed EIS {interface} device negotiated within 20s") + })?; + } Cmd::Click { x, y, button, .. } => { let (device, rel_x, rel_y) = self.pointer_device_for(*x, *y)?; - if let Some(ptr_abs) = device_interface::(&self.devices, &device) { - device.start_emulating(self.sequence, self.last_serial); - self.sequence = self.sequence.wrapping_add(1); - ptr_abs.motion_absolute(rel_x, rel_y); - } - if let Some(btn) = device_interface::(&self.devices, &device) { - btn.button(button.to_evdev(), reis::ei::button::ButtonState::Press); - device.frame(self.last_serial, 0); - btn.button(button.to_evdev(), reis::ei::button::ButtonState::Released); - device.frame(self.last_serial, 1); + let serial = self.last_serial; + let ptr_abs = require_device_interface::(&self.devices, &device)?; + let btn = require_device_interface::(&self.devices, &device)?; + + ptr_abs.motion_absolute(rel_x, rel_y); + btn.button(button.to_evdev(), reis::ei::button::ButtonState::Press); + device.frame(serial, event_time_us()); + btn.button(button.to_evdev(), reis::ei::button::ButtonState::Released); + device.frame(serial, event_time_us()); + } + Cmd::Drag { + from_x, + from_y, + to_x, + to_y, + steps, + button, + .. + } => { + let (device, from_rx, from_ry) = self.pointer_device_for(*from_x, *from_y)?; + let serial = self.last_serial; + let ptr_abs = + require_device_interface::(&self.devices, &device)?; + let btn = require_device_interface::(&self.devices, &device)?; + // Map the drag end into the SAME device's region (a single EIS + // emulation session can't span devices). Fall back to the start + // region's offset when no region of this device contains the end + // (single-region device, or a point just off-screen) — this is + // correct only within one region, but avoids the cross-monitor + // mis-mapping of blindly reusing the start offset. + let (to_rx, to_ry) = + self.region_local_on(&device, *to_x, *to_y).unwrap_or_else(|| { + let off_x = *from_x as f32 - from_rx; + let off_y = *from_y as f32 - from_ry; + (*to_x as f32 - off_x, *to_y as f32 - off_y) + }); + + ptr_abs.motion_absolute(from_rx, from_ry); + device.frame(serial, event_time_us()); + btn.button(button.to_evdev(), reis::ei::button::ButtonState::Press); + device.frame(serial, event_time_us()); + // Bound the interpolation: run_command executes synchronously in + // the worker loop, so a huge step count would block EIS event + // processing (incl. Ping) and balloon the unflushed request queue. + let n = (*steps).max(1).min(500); + for s in 1..=n { + let t = s as f32 / n as f32; + let ix = from_rx + (to_rx - from_rx) * t; + let iy = from_ry + (to_ry - from_ry) * t; + ptr_abs.motion_absolute(ix, iy); + device.frame(serial, event_time_us()); } - device.stop_emulating(self.last_serial); + btn.button(button.to_evdev(), reis::ei::button::ButtonState::Released); + device.frame(serial, event_time_us()); } Cmd::MoveAbsolute { x, y, .. } => { let (device, rel_x, rel_y) = self.pointer_device_for(*x, *y)?; - if let Some(ptr_abs) = device_interface::(&self.devices, &device) { - device.start_emulating(self.sequence, self.last_serial); - self.sequence = self.sequence.wrapping_add(1); - ptr_abs.motion_absolute(rel_x, rel_y); - device.frame(self.last_serial, 0); - device.stop_emulating(self.last_serial); - } + let serial = self.last_serial; + let ptr_abs = require_device_interface::(&self.devices, &device)?; + + ptr_abs.motion_absolute(rel_x, rel_y); + device.frame(serial, event_time_us()); } Cmd::Scroll { dx, dy, .. } => { - let device = self.any_pointer_device() - .ok_or_else(|| anyhow::anyhow!("no EIS pointer device negotiated yet — wait for handshake"))?; - if let Some(scroll) = device_interface::(&self.devices, &device) { - device.start_emulating(self.sequence, self.last_serial); - self.sequence = self.sequence.wrapping_add(1); - scroll.scroll(*dx as f32, *dy as f32); - device.frame(self.last_serial, 0); - device.stop_emulating(self.last_serial); - } + let device = self.device_with_interface("ei_scroll") + .ok_or_else(|| anyhow::anyhow!("no EIS scroll device negotiated yet — wait for handshake"))?; + let serial = self.last_serial; + let scroll = require_device_interface::(&self.devices, &device)?; + + scroll.scroll(*dx as f32, *dy as f32); + device.frame(serial, event_time_us()); } Cmd::TypeText { text, .. } => { - let device = self.any_pointer_device() - .ok_or_else(|| anyhow::anyhow!("no EIS device negotiated yet — wait for handshake"))?; - if let Some(text_iface) = device_interface::(&self.devices, &device) { - device.start_emulating(self.sequence, self.last_serial); - self.sequence = self.sequence.wrapping_add(1); + // Prefer ei_text (libei 1.6+ — a UTF-8 string, layout-correct). + // Mutter does NOT advertise ei_text, so fall back to keycode + // typing via ei_keyboard: map each char to a US-layout evdev + // keycode + shift and emit press/release. Chars with no US-layout + // keycode are skipped (a documented limitation — full layout + // support needs an xkb keymap, like reis's type-text example). + if let Some(device) = self.device_with_interface("ei_text") { + let serial = self.last_serial; + let text_iface = require_device_interface::(&self.devices, &device)?; text_iface.utf8(text); - device.frame(self.last_serial, 0); - device.stop_emulating(self.last_serial); + device.frame(serial, event_time_us()); + } else { + use reis::ei::keyboard::KeyState; + let device = self.device_with_interface("ei_keyboard").ok_or_else(|| { + anyhow::anyhow!("no EIS keyboard device negotiated yet — wait for handshake") + })?; + let serial = self.last_serial; + let kb = require_device_interface::(&self.devices, &device)?; + let shift_code = self.keymap_shift.unwrap_or(42); + let mut skipped = 0usize; + for ch in text.chars() { + // Prefer the compositor's actual keymap (layout-correct); + // fall back to the US-layout table if no keymap arrived. + let mapped = self + .keymap_chars + .get(&ch) + .copied() + .or_else(|| char_to_evdev(ch)); + let Some((code, shift)) = mapped else { + // No keycode for this char in the keymap/US table + // (e.g. CJK/emoji via the keycode path). Skip it, but + // don't pretend the whole string was typed. + skipped += 1; + continue; + }; + if shift { + kb.key(shift_code, KeyState::Press); + device.frame(serial, event_time_us()); + } + kb.key(code, KeyState::Press); + device.frame(serial, event_time_us()); + kb.key(code, KeyState::Released); + device.frame(serial, event_time_us()); + if shift { + kb.key(shift_code, KeyState::Released); + device.frame(serial, event_time_us()); + } + } + if skipped > 0 { + tracing::warn!( + "libei keycode typing skipped {skipped} char(s) with no \ + keycode in the active keymap (e.g. non-Latin/emoji) — \ + the ei_text path (libei 1.6+) would type these verbatim" + ); + } } } Cmd::PressKey { keycode, .. } => { - let device = self.any_pointer_device() - .ok_or_else(|| anyhow::anyhow!("no EIS device negotiated yet — wait for handshake"))?; - if let Some(kb) = device_interface::(&self.devices, &device) { - device.start_emulating(self.sequence, self.last_serial); - self.sequence = self.sequence.wrapping_add(1); - kb.key(*keycode, reis::ei::keyboard::KeyState::Press); - device.frame(self.last_serial, 0); - kb.key(*keycode, reis::ei::keyboard::KeyState::Released); - device.frame(self.last_serial, 1); - device.stop_emulating(self.last_serial); - } + let device = self.device_with_interface("ei_keyboard") + .ok_or_else(|| anyhow::anyhow!("no EIS keyboard device negotiated yet — wait for handshake"))?; + let serial = self.last_serial; + let kb = require_device_interface::(&self.devices, &device)?; + + kb.key(*keycode, reis::ei::keyboard::KeyState::Press); + device.frame(serial, event_time_us()); + kb.key(*keycode, reis::ei::keyboard::KeyState::Released); + device.frame(serial, event_time_us()); + } + Cmd::KeySequence { transitions, .. } => { + let device = self.device_with_interface("ei_keyboard") + .ok_or_else(|| anyhow::anyhow!("no EIS keyboard device negotiated yet — wait for handshake"))?; + let serial = self.last_serial; + let kb = require_device_interface::(&self.devices, &device)?; + + emit_key_transitions(transitions, |transition, _frame| { + let (keycode, state) = match transition { + KeyTransition::Press(keycode) => { + (keycode, reis::ei::keyboard::KeyState::Press) + } + KeyTransition::Release(keycode) => { + (keycode, reis::ei::keyboard::KeyState::Released) + } + }; + kb.key(keycode, state); + device.frame(serial, event_time_us()); + }); } Cmd::Shutdown => {} } @@ -603,12 +1098,12 @@ impl EisState { /// back to the first pointer device with the raw coordinates, which /// matches the pre-multi-monitor behaviour. fn pointer_device_for(&self, x: f64, y: f64) -> anyhow::Result<(reis::ei::Device, f32, f32)> { - let is_pointer = |data: &DeviceData| matches!( - data.device_type, - Some(reis::ei::device::DeviceType::Virtual) | Some(reis::ei::device::DeviceType::Physical) - ); + // Absolute motion needs the device that actually carries + // ei_pointer_absolute — Mutter/KWin split the relative and absolute + // pointers onto separate devices, and only one of them accepts an + // absolute warp. for (device, data) in self.devices.iter() { - if !is_pointer(data) { + if !data.resumed || !data.interfaces.contains_key("ei_pointer_absolute") { continue; } for region in &data.regions { @@ -619,20 +1114,140 @@ impl EisState { } } } - // Fallback: first pointer device, raw coords (single-region behaviour). - let device = self.any_pointer_device() - .ok_or_else(|| anyhow::anyhow!("no EIS pointer device negotiated yet — wait for handshake"))?; + // Fallback: first absolute-pointer device, raw coords (single-region). + let device = self.device_with_interface("ei_pointer_absolute") + .ok_or_else(|| anyhow::anyhow!("no EIS absolute-pointer device negotiated yet — wait for handshake"))?; Ok((device, x as f32, y as f32)) } - fn any_pointer_device(&self) -> Option { - self.devices.iter() - .find(|(_, data)| matches!( - data.device_type, - Some(reis::ei::device::DeviceType::Virtual) | Some(reis::ei::device::DeviceType::Physical) - )) + /// Region-local coordinates for absolute `(x, y)` within one of `device`'s + /// announced regions, or `None` when no region of that device contains the + /// point. Used to map a drag endpoint into the same device the drag start + /// resolved to (a single EIS emulation session can't span devices). + fn region_local_on(&self, device: &reis::ei::Device, x: f64, y: f64) -> Option<(f32, f32)> { + let data = self.devices.get(device)?; + data.regions.iter().find(|r| r.contains(x, y)).map(|r| { + (x as f32 - r.offset_x, y as f32 - r.offset_y) + }) + } + + /// The negotiated device that exposes `iface` (e.g. `ei_pointer_absolute`, + /// `ei_scroll`, `ei_keyboard`, `ei_text`). Mutter/KWin announce one device + /// per capability group — a relative pointer, an absolute pointer, a + /// keyboard — so a command must pick the device carrying the interface it + /// needs, not just "the first virtual device" (which may be the relative + /// pointer that has no `ei_pointer_absolute`). + fn device_with_interface(&self, iface: &str) -> Option { + self.devices + .iter() + .filter(|(_, data)| { + data.resumed && data.interfaces.contains_key(iface) + }) + .max_by_key(|(_, data)| data.resumed_serial.unwrap_or(0)) .map(|(d, _)| d.clone()) } + +} + +/// Map a character to a US-layout evdev keycode + whether Shift is needed. +/// Used by the keyboard-typing fallback when the compositor's EIS exposes no +/// `ei_text` interface (e.g. Mutter). Returns `None` for characters outside the +/// US-ASCII printable set — those are skipped rather than mistyped. Full +/// layout-correct typing would need an xkb keymap (see reis `type-text` example). +fn char_to_evdev(c: char) -> Option<(u32, bool)> { + // Letter codes follow the QWERTY scancode layout (input-event-codes.h). + fn letter(c: char) -> u32 { + // KEY_A..KEY_Z follow the QWERTY scancode layout, not the alphabet. + match c { + 'a' => 30, + 'b' => 48, + 'c' => 46, + 'd' => 32, + 'e' => 18, + 'f' => 33, + 'g' => 34, + 'h' => 35, + 'i' => 23, + 'j' => 36, + 'k' => 37, + 'l' => 38, + 'm' => 50, + 'n' => 49, + 'o' => 24, + 'p' => 25, + 'q' => 16, + 'r' => 19, + 's' => 31, + 't' => 20, + 'u' => 22, + 'v' => 47, + 'w' => 17, + 'x' => 45, + 'y' => 21, + 'z' => 44, + _ => 0, + } + } + // (keycode, needs_shift). Digit/symbol codes and their shifted variants + // follow the US QWERTY row (input-event-codes.h). + Some(match c { + 'a'..='z' => (letter(c), false), + 'A'..='Z' => (letter(c.to_ascii_lowercase()), true), + '1' => (2, false), + '2' => (3, false), + '3' => (4, false), + '4' => (5, false), + '5' => (6, false), + '6' => (7, false), + '7' => (8, false), + '8' => (9, false), + '9' => (10, false), + '0' => (11, false), + '!' => (2, true), + '@' => (3, true), + '#' => (4, true), + '$' => (5, true), + '%' => (6, true), + '^' => (7, true), + '&' => (8, true), + '*' => (9, true), + '(' => (10, true), + ')' => (11, true), + ' ' => (57, false), + '\n' => (28, false), + '\t' => (15, false), + '-' => (12, false), + '_' => (12, true), + '=' => (13, false), + '+' => (13, true), + '[' => (26, false), + '{' => (26, true), + ']' => (27, false), + '}' => (27, true), + '\\' => (43, false), + '|' => (43, true), + ';' => (39, false), + ':' => (39, true), + '\'' => (40, false), + '"' => (40, true), + '`' => (41, false), + '~' => (41, true), + ',' => (51, false), + '<' => (51, true), + '.' => (52, false), + '>' => (52, true), + '/' => (53, false), + '?' => (53, true), + _ => return None, + }) +} + +fn require_device_interface( + devices: &HashMap, + device: &reis::ei::Device, +) -> anyhow::Result { + device_interface::(devices, device) + .ok_or_else(|| anyhow::anyhow!("EIS device lacks required {} interface", T::NAME)) } fn device_interface( @@ -645,3 +1260,52 @@ fn device_interface( .clone() .downcast() } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn key_sequence_accepts_empty_and_maximum_length_requests() { + assert!(validate_key_sequence(&[]).is_ok()); + + let transitions = vec![KeyTransition::Press(29); MAX_KEY_SEQUENCE_TRANSITIONS]; + assert!(validate_key_sequence(&transitions).is_ok()); + } + + #[test] + fn key_sequence_rejects_requests_over_the_worker_bound() { + let transitions = vec![KeyTransition::Release(29); MAX_KEY_SEQUENCE_TRANSITIONS + 1]; + let error = validate_key_sequence(&transitions).unwrap_err(); + + assert_eq!( + error.to_string(), + "libei key sequence has 65 transitions; maximum is 64" + ); + } + + #[test] + fn key_sequence_preserves_transition_order_and_assigns_frames() { + let transitions = [ + KeyTransition::Press(29), + KeyTransition::Press(46), + KeyTransition::Release(46), + KeyTransition::Release(29), + ]; + let mut emitted = Vec::new(); + + emit_key_transitions(&transitions, |transition, frame| { + emitted.push((transition, frame)); + }); + + assert_eq!( + emitted, + vec![ + (KeyTransition::Press(29), 0), + (KeyTransition::Press(46), 1), + (KeyTransition::Release(46), 2), + (KeyTransition::Release(29), 3), + ] + ); + } +} diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs index f214fa9e54..6c82b48715 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/mod.rs @@ -1,8 +1,8 @@ //! Native-Wayland backend. //! -//! Used when running under a Wayland compositor with no X11 (WAYLAND_DISPLAY -//! set, DISPLAY unset). Enumerates toplevels via -//! `zwlr_foreign_toplevel_manager_v1`, captures per-output screenshots via +//! Used when the experimental backend is enabled under a Wayland compositor. +//! Enumerates toplevels via `zwlr_foreign_toplevel_manager_v1` or the generic +//! staging `ext_foreign_toplevel_list_v1`, captures per-output screenshots via //! `zwlr_screencopy_manager_v1` + `wl_shm` (native — `grim` remains a //! fallback), and synthesises pointer / scroll / drag input via //! `zwlr_virtual_pointer_v1`. Per-window image capture is deferred until @@ -11,33 +11,24 @@ //! typed error on pure Wayland. pub mod ext_screencopy; +pub mod ext_toplevel; pub mod overlay; pub mod persistent_vptr; pub mod portal_screenshot; pub mod shell_helper; -// `portal_screencast` (PipeWire per-window capture) and `libei` (GNOME/KDE -// input via xdg-desktop-portal RemoteDesktop) need libpipewire-0.3 and reis -// at build time, which the cross-platform release container (debian:11, -// GLIBC_2.31 floor) can't satisfy without bumping the floor. They're behind -// the `portal-libei` feature so the published binaries stay portable; the -// Nix build (which already has modern PipeWire + libei from nixpkgs) -// enables it. Wlroots screencopy + virtual-pointer remain unconditional. -#[cfg(feature = "portal-libei")] +pub mod sway_ipc; +// RemoteDesktop/libei input is portable and ships in release binaries. +// PipeWire ScreenCast capture remains separately gated for modern/Nix builds. +#[cfg(feature = "portal-input")] pub mod libei; -#[cfg(feature = "portal-libei")] +#[cfg(feature = "portal-capture")] pub mod portal_screencast; -/// Whether this binary was compiled with the `portal-libei` feature — the -/// xdg-desktop-portal RemoteDesktop + libei input path. It is the ONLY input -/// backend that works on non-wlroots compositors (KWin/Plasma, Mutter/GNOME), -/// which do not implement `zwlr_virtual_pointer_v1`. The published -/// curl-pipe-bash tarball is built WITHOUT it (#1967 — debian:11 CD container -/// lacks a new-enough PipeWire/libei), so on those compositors input injection -/// has no backend and silently no-ops. Consulted by the doctor and the input -/// dispatch so that failure is reported instead of hidden. See #1982. -pub const PORTAL_LIBEI_ENABLED: bool = cfg!(feature = "portal-libei"); +/// Whether the GNOME/KDE RemoteDesktop + libei input backend is compiled in. +pub const PORTAL_INPUT_ENABLED: bool = cfg!(feature = "portal-input"); -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; +use std::sync::{Mutex, OnceLock}; use wayland_client::{ event_created_child, @@ -96,15 +87,25 @@ pub fn wayland_enabled() -> bool { } } -/// True when we should drive Wayland rather than X11: the experimental backend -/// is opted in ([`wayland_enabled`]), a Wayland display is present, and there is -/// no X11 DISPLAY to fall back to. Without the opt-in this returns false even on -/// a pure-Wayland session, so the backend treats it as unsupported rather than -/// silently engaging an incomplete code path. +/// True when this is an opted-in Wayland desktop session. +/// +/// GNOME and KDE export `DISPLAY` for XWayland even when the target and the +/// desktop are native Wayland. Treating that compatibility variable as proof +/// of an X11 session routed native windows, capture, video, geometry, and focus +/// through invalid XIDs. Backend selection below is capability based; the mere +/// presence of `DISPLAY` must not disable Wayland. pub fn is_wayland() -> bool { - wayland_enabled() - && std::env::var_os("WAYLAND_DISPLAY").is_some() - && std::env::var_os("DISPLAY").is_none() + wayland_enabled() && std::env::var_os("WAYLAND_DISPLAY").is_some() +} + +/// True when input tools should attempt the Wayland input path (wlroots +/// virtual-pointer, falling back to libei/portal via [`with_libei_fallback`]). +/// +/// Input-specific alias retained to make dispatch intent explicit. The +/// wlroots-vs-portal decision is made from live compositor capabilities inside +/// the `wayland::*` input functions, not from XWayland's `DISPLAY` variable. +pub fn wayland_input_enabled() -> bool { + wayland_enabled() && std::env::var_os("WAYLAND_DISPLAY").is_some() } /// Reason string when X11 input injection cannot possibly work, so callers @@ -208,6 +209,99 @@ struct Toplevel { closed: bool, } +#[derive(Clone, Debug)] +struct ToplevelIdentity { + title: String, + app_id: String, +} + +fn identity_registry() -> &'static Mutex> { + static REGISTRY: OnceLock>> = OnceLock::new(); + REGISTRY.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn observed_origin_registry() -> &'static Mutex> { + static REGISTRY: OnceLock>> = OnceLock::new(); + REGISTRY.get_or_init(|| Mutex::new(HashMap::new())) +} + +pub fn remember_observed_window_origins(windows: &[WindowInfo]) { + if let Ok(mut registry) = observed_origin_registry().lock() { + for window in windows { + if let Some(pid) = window.pid { + // Generic foreign-toplevel and AT-SPI fallbacks use (0,0) when + // they do not know compositor geometry. Do not let that + // placeholder erase a previously observed real origin or + // prevent the caller from falling through to Sway/GNOME data. + if (window.x, window.y) != (0, 0) { + registry.insert(pid, (window.x, window.y)); + } + } + } + } +} + +pub fn observed_window_origin(pid: u32) -> Option<(i32, i32)> { + observed_origin_registry() + .lock() + .ok() + .and_then(|registry| registry.get(&pid).copied()) +} + +fn remember_identity(id: u64, toplevel: &Toplevel) { + if let Ok(mut registry) = identity_registry().lock() { + registry.insert( + id, + ToplevelIdentity { + title: toplevel.title.clone(), + app_id: toplevel.app_id.clone(), + }, + ); + } +} + +fn identity_for(id: u64) -> Option { + identity_registry() + .lock() + .ok() + .and_then(|registry| registry.get(&id).cloned()) + .or_else(|| { + sway_ipc::window_for_id(id).map(|window| ToplevelIdentity { + title: window.title, + app_id: window.app_id, + }) + }) + .or_else(|| { + crate::atspi::list_windows(None) + .into_iter() + .find(|window| window.xid == id || u64::from(window.xid as u32) == id) + .map(|window| ToplevelIdentity { + title: window.title, + app_id: window.app_name, + }) + }) +} + +fn matching_handle(state: &State, id: u64) -> Option { + if let Some(identity) = identity_for(id) { + let by_title = state.toplevels.iter().find_map(|(protocol_id, toplevel)| { + (!identity.title.is_empty() && toplevel.title == identity.title) + .then(|| state.handles.get(protocol_id).cloned()) + .flatten() + }); + return by_title.or_else(|| { + state.toplevels.iter().find_map(|(protocol_id, toplevel)| { + (!identity.app_id.is_empty() && toplevel.app_id == identity.app_id) + .then(|| state.handles.get(protocol_id).cloned()) + .flatten() + }) + }); + } + + let protocol_id = u32::try_from(id).ok()?; + state.handles.get(&protocol_id).cloned() +} + /// Per-capture in-flight state populated by the screencopy frame Dispatch. #[derive(Default)] struct CaptureState { @@ -474,8 +568,9 @@ impl Dispatch for State { } /// Enumerate native Wayland toplevels via wlr-foreign-toplevel-management. -/// `xid` is the foreign-toplevel handle's protocol id (a stable per-session -/// window id); pid is unknown (not exposed by the protocol); geometry is 0 +/// `xid` begins as the foreign-toplevel handle's connection-scoped protocol id. +/// The dispatcher replaces it with a stable compositor or AT-SPI identity when +/// available. pid is unknown (not exposed by the protocol); geometry is 0 /// (the protocol does not surface position/size). app_id is folded into the /// title (`" [<app_id>]"`) so callers matching on either still match. pub fn list_windows() -> anyhow::Result<Vec<WindowInfo>> { @@ -495,6 +590,8 @@ pub fn list_windows() -> anyhow::Result<Vec<WindowInfo>> { queue.roundtrip(&mut state)?; } + let sway_windows = sway_ipc::list_windows().unwrap_or_default(); + let mut used_sway_ids = HashSet::new(); let mut out = Vec::new(); for (id, tl) in &state.toplevels { if tl.closed { @@ -505,14 +602,36 @@ pub fn list_windows() -> anyhow::Result<Vec<WindowInfo>> { } else { format!("{} [{}]", tl.title, tl.app_id) }; + let sway = sway_windows + .iter() + .find(|window| { + !used_sway_ids.contains(&window.id) + && !tl.title.is_empty() + && window.title == tl.title + }) + .or_else(|| { + sway_windows.iter().find(|window| { + !used_sway_ids.contains(&window.id) + && !tl.app_id.is_empty() + && window.app_id == tl.app_id + }) + }); + let stable_id = sway.map(|window| window.id).unwrap_or(*id as u64); + if let Some(window) = sway { + used_sway_ids.insert(window.id); + } + remember_identity(stable_id, tl); out.push(WindowInfo { - xid: *id as u64, - pid: None, + xid: stable_id, + pid: sway.map(|window| window.pid), + app_name: tl.app_id.clone(), title, - x: 0, - y: 0, - width: 0, - height: 0, + is_on_screen: sway.map(|window| window.visible).unwrap_or(true), + z_index: None, + x: sway.map(|window| window.x).unwrap_or(0), + y: sway.map(|window| window.y).unwrap_or(0), + width: sway.map(|window| window.width).unwrap_or(0), + height: sway.map(|window| window.height).unwrap_or(0), }); } Ok(out) @@ -756,12 +875,53 @@ pub(crate) unsafe fn borrowed_fd(fd: i32) -> std::os::fd::OwnedFd { /// output-level path used by `get_window_state`'s vision payload. pub fn screenshot_dispatch(xid: u64) -> anyhow::Result<Vec<u8>> { if is_wayland() { - screenshot_bytes() + let bytes = screenshot_display_dispatch()?; + if let Some((x, y, width, height)) = window_geometry(xid) { + crop_png_to_rect( + &bytes, + x, + y, + width, + height, + &format!("Wayland window {xid}"), + ) + } else { + Ok(bytes) + } } else { crate::capture::screenshot_window_bytes(xid) } } +fn crop_png_to_rect( + output_png: &[u8], + rect_x: i32, + rect_y: i32, + rect_width: u32, + rect_height: u32, + label: &str, +) -> anyhow::Result<Vec<u8>> { + let image = image::load_from_memory(output_png)?; + let image_width = image.width(); + let image_height = image.height(); + let x = rect_x.max(0) as u32; + let y = rect_y.max(0) as u32; + if x >= image_width || y >= image_height { + anyhow::bail!( + "{label} origin ({x},{y}) is outside captured output {image_width}x{image_height}" + ); + } + let width = rect_width.min(image_width - x); + let height = rect_height.min(image_height - y); + if width == 0 || height == 0 { + anyhow::bail!("{label} has empty capture geometry"); + } + let cropped = image.crop_imm(x, y, width, height); + let mut cursor = std::io::Cursor::new(Vec::new()); + cropped.write_to(&mut cursor, image::ImageFormat::Png)?; + Ok(cursor.into_inner()) +} + /// Display-level capture dispatcher. Cascade: /// 1. Native Wayland on wlroots: zwlr_screencopy_manager_v1 (fast, zero /// consent). @@ -771,7 +931,12 @@ pub fn screenshot_dispatch(xid: u64) -> anyhow::Result<Vec<u8>> { /// 3. X11: existing root-window path. pub fn screenshot_display_dispatch() -> anyhow::Result<Vec<u8>> { if is_wayland() { - // Tier 1: native wlroots screencopy (fast, zero consent). + // Tier 1: the opt-in GNOME compositor helper. It avoids probing + // wlroots-only protocols and captures the Shell stage without consent. + if let Some(bytes) = shell_helper::screenshot_display() { + return Ok(bytes); + } + // Tier 2: native wlroots screencopy (fast, zero consent). match screenshot_bytes() { Ok(bytes) => return Ok(bytes), Err(e) => { @@ -780,7 +945,7 @@ pub fn screenshot_display_dispatch() -> anyhow::Result<Vec<u8>> { ); } } - // Tier 2: ext-image-copy-capture-v1 (sway 1.10+, labwc 0.8+, niri, + // Tier 3: ext-image-copy-capture-v1 (sway 1.10+, labwc 0.8+, niri, // hyprland, KDE 6.2+, GNOME 47+). match ext_screencopy::screenshot_via_ext_copy() { Ok(bytes) => return Ok(bytes), @@ -790,7 +955,7 @@ pub fn screenshot_display_dispatch() -> anyhow::Result<Vec<u8>> { ); } } - // Tier 3: xdg-desktop-portal (GNOME, KDE, COSMIC fallback). + // Tier 4: xdg-desktop-portal (GNOME, KDE, COSMIC fallback). match portal_screenshot::screenshot_via_portal() { Ok(bytes) => return Ok(bytes), Err(e) => { @@ -813,6 +978,16 @@ pub fn screenshot_display_dispatch() -> anyhow::Result<Vec<u8>> { /// crop with. pub fn screenshot_window_dispatch(xid: u64) -> anyhow::Result<Vec<u8>> { if is_wayland() { + if let Some((x, y, width, height)) = window_geometry(xid) { + return crop_png_to_rect( + &screenshot_display_dispatch()?, + x, + y, + width, + height, + &format!("Wayland window {xid}"), + ); + } anyhow::bail!( "per-window screenshot is not yet supported on native Wayland — \ zwlr_screencopy_manager_v1 is output-only and ext-image-copy-capture-v1 \ @@ -825,6 +1000,50 @@ pub fn screenshot_window_dispatch(xid: u64) -> anyhow::Result<Vec<u8>> { // ── Input session helper ───────────────────────────────────────────────────── +/// Sentinel substring carried by the `open_vptr_session` error when the +/// compositor exposes no `zwlr_virtual_pointer_manager_v1` (KWin/Plasma, +/// Mutter/GNOME). The input dispatch matches on this to decide whether the +/// libei/portal fallback ([`libei`]) can recover the call. Kept as a string +/// marker (rather than a typed error) so the existing `anyhow::Result` +/// signatures of every input fn are unchanged. See #1982. +pub const NO_VPTR_MARKER: &str = "no-zwlr-virtual-pointer"; + +/// True when `err` is the "compositor has no wlroots virtual-pointer" failure +/// from [`open_vptr_session`] — i.e. the point where a non-wlroots compositor +/// needs the libei fallback rather than a hard error. +fn is_no_vptr(err: &anyhow::Error) -> bool { + err.to_string().contains(NO_VPTR_MARKER) +} + +/// Run the wlroots virtual-pointer closure `f`; if it fails specifically +/// because the compositor exposes no `zwlr_virtual_pointer_manager_v1` and this +/// binary carries the `portal-input` feature, run the libei `fallback` instead. +/// Any other wlroots error (and the no-vptr error in a build without the +/// feature) propagates unchanged. This is the single seam through which #1982's +/// KDE/GNOME input recovery flows. +fn with_libei_fallback<T>( + f: impl FnOnce() -> anyhow::Result<T>, + #[allow(unused_variables)] fallback: impl FnOnce() -> anyhow::Result<T>, +) -> anyhow::Result<T> { + match f() { + Ok(v) => Ok(v), + Err(e) if is_no_vptr(&e) => { + #[cfg(feature = "portal-input")] + { + tracing::info!( + "wlroots virtual-pointer unavailable ({e}); falling back to libei/portal" + ); + return fallback(); + } + #[cfg(not(feature = "portal-input"))] + { + Err(e) + } + } + Err(e) => Err(e), + } +} + /// Live virtual-pointer session: connection + queue + the bound objects every /// pointer op (click, scroll, drag) needs. Returned by [`open_vptr_session`]. pub struct VptrSession { @@ -843,7 +1062,7 @@ pub struct VptrSession { /// client from knowing another window's on-screen geometry, so we drive every /// pointer event in *output* coordinates and rely on the activated toplevel /// covering the centre. -pub fn open_vptr_session(activate_window_id: Option<u32>) -> anyhow::Result<VptrSession> { +pub fn open_vptr_session(activate_window_id: Option<u64>) -> anyhow::Result<VptrSession> { let conn = Connection::connect_to_env()?; let mut queue = conn.new_event_queue::<State>(); let qh = queue.handle(); @@ -851,38 +1070,53 @@ pub fn open_vptr_session(activate_window_id: Option<u32>) -> anyhow::Result<Vptr let mut state = State::default(); queue.roundtrip(&mut state)?; - if state.manager.is_none() { - anyhow::bail!("compositor does not expose zwlr_foreign_toplevel_manager_v1"); - } for _ in 0..4 { queue.roundtrip(&mut state)?; } - let seat = state.seat.clone().ok_or_else(|| { - anyhow::anyhow!("compositor exposed no wl_seat for virtual-pointer input") - })?; + // Evaluate the virtual-pointer / NO_VPTR_MARKER path FIRST: on compositors + // that expose neither zwlr_virtual_pointer nor zwlr_foreign_toplevel + // (KWin/Plasma, Mutter/GNOME) we must surface the marker so + // `with_libei_fallback` re-routes through libei/portal. Requiring + // foreign-toplevel up front would mask the marker and leave the libei + // fallback dead. See #1982. let mgr = state.vptr_manager.clone().ok_or_else(|| { - if PORTAL_LIBEI_ENABLED { - anyhow::anyhow!("compositor does not expose zwlr_virtual_pointer_manager_v1") + if PORTAL_INPUT_ENABLED { + // The caller (via `with_libei_fallback`) recognises NO_VPTR_MARKER + // and re-routes the op through the libei/portal backend, which DOES + // reach KWin/Plasma and Mutter/GNOME. Keep the marker in the text. + anyhow::anyhow!( + "compositor does not expose zwlr_virtual_pointer_manager_v1 \ + ({NO_VPTR_MARKER})" + ) } else { // KWin/Plasma and Mutter/GNOME don't implement zwlr_virtual_pointer, // and this build has no libei/portal fallback — so input has no - // backend at all rather than silently no-op'ing. See #1982. + // backend at all rather than silently no-op'ing. The marker still + // lets the dispatch layer classify the failure uniformly. See #1982. anyhow::anyhow!( - "no input backend for this compositor: it exposes no \ - zwlr_virtual_pointer_manager_v1 and this build was compiled \ - without libei/portal support (#1982). Use the portal-enabled \ - Linux build for input on KDE Plasma / GNOME, or a wlroots \ - compositor (sway, labwc, hyprland)." + "no input backend for this compositor ({NO_VPTR_MARKER}): it \ + exposes no zwlr_virtual_pointer_manager_v1 and this build was \ + compiled without libei/portal support (#1982). Use the \ + portal-enabled Linux build for input on KDE Plasma / GNOME, or \ + a wlroots compositor (sway, labwc, hyprland)." ) } })?; + // foreign-toplevel is only needed to activate a specific window before + // synthesising input; require it only when a caller actually asks for that. + if activate_window_id.is_some() && state.manager.is_none() { + anyhow::bail!("compositor does not expose zwlr_foreign_toplevel_manager_v1"); + } + + let seat = state + .seat + .clone() + .ok_or_else(|| anyhow::anyhow!("compositor exposed no wl_seat for virtual-pointer input"))?; + if let Some(id) = activate_window_id { - let handle = state - .handles - .get(&id) - .cloned() + let handle = matching_handle(&state, id) .ok_or_else(|| anyhow::anyhow!("no native Wayland toplevel for window_id {id}"))?; handle.activate(&seat); queue.roundtrip(&mut state)?; @@ -901,6 +1135,106 @@ pub fn open_vptr_session(activate_window_id: Option<u32>) -> anyhow::Result<Vptr }) } +/// Focus and raise a specific native Wayland toplevel before focus-bound +/// keyboard or portal/libei input. wlroots exposes an activation request on its +/// foreign-toplevel protocol; GNOME uses the bundled compositor helper. Other +/// compositors must refuse until they provide an equally target-addressable +/// adapter, because global injection without this gate can affect the wrong app. +pub fn activate_window_for_input(window_id: u64) -> anyhow::Result<()> { + let pid = crate::atspi::list_windows(None) + .into_iter() + .find(|window| window.xid == window_id) + .and_then(|window| window.pid); + activate_window_for_input_target(window_id, pid) +} + +/// Activate a Wayland target with an explicit process identity when available. +/// The bundled compositor does not depend on connection-local Wayland object +/// ids: its control protocol resolves the one mapped toplevel owned by `pid`. +pub fn activate_window_for_input_target( + window_id: u64, + target_pid: Option<u32>, +) -> anyhow::Result<()> { + if is_inject_mode() { + let pid = target_pid.ok_or_else(|| { + anyhow::anyhow!( + "foreground_unavailable: cua-compositor activation requires a verified target pid" + ) + })?; + inject_send(&[format!("f {pid}")])?; + std::thread::sleep(std::time::Duration::from_millis(60)); + return Ok(()); + } + + let conn = Connection::connect_to_env()?; + let mut queue = conn.new_event_queue::<State>(); + let qh = queue.handle(); + conn.display().get_registry(&qh, ()); + let mut state = State::default(); + queue.roundtrip(&mut state)?; + for _ in 0..4 { + queue.roundtrip(&mut state)?; + } + + if let (Some(_), Some(seat), Some(handle)) = ( + state.manager.as_ref(), + state.seat.clone(), + matching_handle(&state, window_id), + ) { + handle.activate(&seat); + queue.roundtrip(&mut state)?; + std::thread::sleep(std::time::Duration::from_millis(60)); + return Ok(()); + } + + if shell_helper::activate_window(window_id) { + std::thread::sleep(std::time::Duration::from_millis(60)); + return Ok(()); + } + + anyhow::bail!( + "foreground_unavailable: this Wayland compositor does not expose a verified, \ + target-addressable activation adapter for window {window_id}; refusing global \ + input because it could affect the wrong application" + ) +} + +/// Query the first `wl_output`'s pixel dimensions via a short Wayland +/// roundtrip, independent of the virtual-pointer protocol. Used by the libei +/// fallback (which never opens a `VptrSession`) to reproduce the vptr path's +/// default-to-centre and clamp behaviour so both backends treat coordinates +/// identically. Falls back to `(1, 1)` when no output reports a mode. +#[cfg(feature = "portal-input")] +fn output_dimensions() -> anyhow::Result<(u32, u32)> { + let conn = Connection::connect_to_env()?; + let mut queue = conn.new_event_queue::<State>(); + let qh = queue.handle(); + conn.display().get_registry(&qh, ()); + let mut state = State::default(); + queue.roundtrip(&mut state)?; + for _ in 0..4 { + queue.roundtrip(&mut state)?; + } + Ok((state.output_w.max(1), state.output_h.max(1))) +} + +/// Reproduce the wlroots vptr path's coordinate handling for the libei +/// fallback: `(0, 0)` defaults to the output centre, and any value is clamped +/// to `[0, dim-1]`. Keeps `click(.., 0, 0, ..)` landing on centre rather than +/// the top-left corner across both backends. +#[cfg(feature = "portal-input")] +fn normalize_click_xy(x: i32, y: i32, w: u32, h: u32) -> (i32, i32) { + let (px, py) = if x == 0 && y == 0 { + ((w / 2) as i32, (h / 2) as i32) + } else { + (x, y) + }; + ( + px.clamp(0, (w as i32).saturating_sub(1)), + py.clamp(0, (h as i32).saturating_sub(1)), + ) +} + /// Map a cua/X11 pointer button (1=left / 2=middle / 3=right) to its evdev /// code, which is what `zwlr_virtual_pointer_v1::button` expects. pub fn evdev_pointer_button(button: u8) -> u32 { @@ -911,6 +1245,15 @@ pub fn evdev_pointer_button(button: u8) -> u32 { } } +fn event_time_ms() -> u32 { + static START: std::sync::OnceLock<std::time::Instant> = std::sync::OnceLock::new(); + START + .get_or_init(std::time::Instant::now) + .elapsed() + .as_millis() + .clamp(1, u32::MAX as u128) as u32 +} + /// Click a native Wayland toplevel identified by its `window_id` (the /// foreign-toplevel protocol id from `list_windows`) at output-relative /// `(x, y)`, with `button` (1/2/3 = left/middle/right) emitted `count` times. @@ -919,7 +1262,41 @@ pub fn evdev_pointer_button(button: u8) -> u32 { /// real coords. A short delay between iterations gives the compositor time /// to discriminate single vs. double clicks. pub fn click(window_id: u64, x: i32, y: i32, count: u32, button: u8) -> anyhow::Result<()> { - let mut sess = open_vptr_session(Some(window_id as u32))?; + with_libei_fallback( + || click_vptr(Some(window_id), x, y, count, button), + || { + libei_wait_pointer_ready()?; + activate_window_for_input(window_id)?; + libei_click(x, y, count, button) + }, + ) +} + +/// Click a desktop-absolute point without selecting or activating a toplevel. +/// This is the Wayland peer of an XTest root-window click and is used only by +/// the explicit desktop capture scope. +pub fn click_desktop(x: i32, y: i32, count: u32, button: u8) -> anyhow::Result<()> { + if is_inject_mode() { + let btn = evdev_button(button as u32); + return inject_send(&[format!("d {x} {y} {} {btn}", count.max(1))]); + } + with_libei_fallback( + || click_vptr(None, x, y, count, button), + || libei_click(x, y, count, button), + ) +} + +/// wlroots virtual-pointer implementation of [`click`]. Falls back to libei via +/// [`with_libei_fallback`] when the compositor exposes no virtual-pointer. +fn click_vptr( + window_id: Option<u64>, + x: i32, + y: i32, + count: u32, + button: u8, +) -> anyhow::Result<()> { + let mut sess = open_vptr_session(window_id)?; + std::thread::sleep(std::time::Duration::from_millis(40)); let (w, h) = (sess.output_w, sess.output_h); let (px, py) = if x == 0 && y == 0 { ((w / 2) as i32, (h / 2) as i32) @@ -933,11 +1310,17 @@ pub fn click(window_id: u64, x: i32, y: i32, count: u32, button: u8) -> anyhow:: if i > 0 { std::thread::sleep(std::time::Duration::from_millis(80)); } - sess.vptr.motion_absolute(0, px, py, w, h); + sess.vptr.motion_absolute(event_time_ms(), px, py, w, h); sess.vptr.frame(); - sess.vptr.button(0, btn, ButtonState::Pressed); + sess.queue.roundtrip(&mut sess.state)?; + std::thread::sleep(std::time::Duration::from_millis(15)); + sess.vptr + .button(event_time_ms(), btn, ButtonState::Pressed); sess.vptr.frame(); - sess.vptr.button(0, btn, ButtonState::Released); + sess.queue.roundtrip(&mut sess.state)?; + std::thread::sleep(std::time::Duration::from_millis(20)); + sess.vptr + .button(event_time_ms(), btn, ButtonState::Released); sess.vptr.frame(); sess.queue.roundtrip(&mut sess.state)?; } @@ -954,7 +1337,126 @@ pub fn click(window_id: u64, x: i32, y: i32, count: u32, button: u8) -> anyhow:: /// virtual-pointer protocol, mirroring how a real wheel notch decomposes. The /// magnitude follows wl_pointer convention: ±10 (in wl_fixed = ×256) per tick. pub fn scroll(window_id: u64, direction: &str, amount: u32) -> anyhow::Result<()> { - let mut sess = open_vptr_session(Some(window_id as u32))?; + scroll_at(window_id, None, direction, amount) +} + +/// Translate window-local screenshot coordinates into compositor output +/// coordinates when the active compositor exposes the target geometry. +pub fn window_local_to_output(window_id: u64, x: i32, y: i32) -> (i32, i32) { + window_geometry(window_id) + .map(|(window_x, window_y, _, _)| { + (window_x.saturating_add(x), window_y.saturating_add(y)) + }) + .unwrap_or((x, y)) +} + +/// Resolve geometry through stable title/app identity when a foreign-toplevel +/// object ID came from an earlier Wayland connection. Protocol object IDs are +/// connection-local, so direct equality is only a fast path. +pub fn window_geometry(window_id: u64) -> Option<(i32, i32, u32, u32)> { + if let Some(window) = sway_ipc::window_for_id(window_id) { + return Some((window.x, window.y, window.width, window.height)); + } + + let identity = identity_for(window_id); + if let Some(identity) = identity.as_ref() { + if let Some(windows) = sway_ipc::list_windows() { + let title_matches = windows + .iter() + .filter(|window| !identity.title.is_empty() && window.title == identity.title) + .collect::<Vec<_>>(); + if title_matches.len() == 1 { + let window = title_matches[0]; + return Some((window.x, window.y, window.width, window.height)); + } + let app_matches = windows + .iter() + .filter(|window| !identity.app_id.is_empty() && window.app_id == identity.app_id) + .collect::<Vec<_>>(); + if app_matches.len() == 1 { + let window = app_matches[0]; + return Some((window.x, window.y, window.width, window.height)); + } + } + } + + let windows = list_windows_dispatch(None); + if let Some(window) = windows + .iter() + .find(|window| window.xid == window_id && window.width > 0 && window.height > 0) + { + return Some((window.x, window.y, window.width, window.height)); + } + let identity = identity?; + let title_matches = windows + .iter() + .filter(|window| { + window.width > 0 + && window.height > 0 + && !identity.title.is_empty() + && undecorated_native_title(window) == identity.title + }) + .collect::<Vec<_>>(); + if title_matches.len() == 1 { + let window = title_matches[0]; + return Some((window.x, window.y, window.width, window.height)); + } + let app_matches = windows + .iter() + .filter(|window| { + window.width > 0 + && window.height > 0 + && !identity.app_id.is_empty() + && window.app_name == identity.app_id + }) + .collect::<Vec<_>>(); + (app_matches.len() == 1).then(|| { + let window = app_matches[0]; + (window.x, window.y, window.width, window.height) + }) +} + +/// Scroll after positioning the synthetic pointer over an output-relative +/// target. Wayland routes wheel events to the surface beneath the pointer, so +/// pixel-addressed scrolls must not inherit an unrelated cursor position. +pub fn scroll_at( + window_id: u64, + point: Option<(i32, i32)>, + direction: &str, + amount: u32, +) -> anyhow::Result<()> { + let direction = direction.to_string(); + with_libei_fallback( + || scroll_vptr(window_id, point, &direction, amount), + || { + libei_wait_scroll_ready()?; + activate_window_for_input(window_id)?; + if let Some((x, y)) = point { + libei_move_absolute(x, y)?; + } + libei_scroll(&direction, amount) + }, + ) +} + +/// wlroots virtual-pointer implementation of [`scroll`]. +fn scroll_vptr( + window_id: u64, + point: Option<(i32, i32)>, + direction: &str, + amount: u32, +) -> anyhow::Result<()> { + let mut sess = open_vptr_session(Some(window_id))?; + if let Some((x, y)) = point { + let px = x.clamp(0, (sess.output_w as i32).saturating_sub(1)) as u32; + let py = y.clamp(0, (sess.output_h as i32).saturating_sub(1)) as u32; + sess.vptr + .motion_absolute(event_time_ms(), px, py, sess.output_w, sess.output_h); + sess.vptr.frame(); + sess.queue.roundtrip(&mut sess.state)?; + record_synth_cursor(px as i32, py as i32); + std::thread::sleep(std::time::Duration::from_millis(15)); + } let (axis, sign): (Axis, i32) = match direction.to_ascii_lowercase().as_str() { "up" => (Axis::VerticalScroll, -1), "down" => (Axis::VerticalScroll, 1), @@ -970,7 +1472,8 @@ pub fn scroll(window_id: u64, direction: &str, amount: u32) -> anyhow::Result<() std::thread::sleep(std::time::Duration::from_millis(25)); } sess.vptr.axis_source(AxisSource::Wheel); - sess.vptr.axis_discrete(0, axis, value, sign); + sess.vptr + .axis_discrete(event_time_ms(), axis, value, sign); sess.vptr.frame(); sess.queue.roundtrip(&mut sess.state)?; } @@ -1014,11 +1517,19 @@ pub fn last_synth_cursor_pos() -> Option<(i32, i32)> { /// the compositor commits the warp before returning. Records the position in /// the synthetic-cursor registry so `last_synth_cursor_pos` can report it. pub fn move_cursor_absolute(window_id: Option<u64>, x: i32, y: i32) -> anyhow::Result<()> { - let mut sess = open_vptr_session(window_id.map(|w| w as u32))?; + with_libei_fallback( + || move_cursor_absolute_vptr(window_id, x, y), + || libei_move_absolute(x, y), + ) +} + +/// wlroots virtual-pointer implementation of [`move_cursor_absolute`]. +fn move_cursor_absolute_vptr(window_id: Option<u64>, x: i32, y: i32) -> anyhow::Result<()> { + let mut sess = open_vptr_session(window_id)?; let (w, h) = (sess.output_w, sess.output_h); let px = x.clamp(0, (w as i32).saturating_sub(1)) as u32; let py = y.clamp(0, (h as i32).saturating_sub(1)) as u32; - sess.vptr.motion_absolute(0, px, py, w, h); + sess.vptr.motion_absolute(event_time_ms(), px, py, w, h); sess.vptr.frame(); sess.queue.roundtrip(&mut sess.state)?; record_synth_cursor(px as i32, py as i32); @@ -1030,8 +1541,8 @@ pub fn move_cursor_absolute(window_id: Option<u64>, x: i32, y: i32) -> anyhow::R /// Press-drag-release on a native Wayland toplevel. Emits one button press at /// `(from_x, from_y)`, then `steps` interpolated motion events along the /// straight segment to `(to_x, to_y)`, then a release. Coordinates are -/// output-relative; window-local coords need the EIS inject socket -/// (`CUA_INJECT_SOCKET`). +/// output-relative; window-local coords need the nested cua-compositor +/// injection socket (`CUA_INJECT_SOCKET`). pub fn drag( window_id: u64, from_x: i32, @@ -1041,7 +1552,29 @@ pub fn drag( steps: u32, button: u8, ) -> anyhow::Result<()> { - let mut sess = open_vptr_session(Some(window_id as u32))?; + with_libei_fallback( + || drag_vptr(window_id, from_x, from_y, to_x, to_y, steps, button), + || { + libei_wait_pointer_ready()?; + activate_window_for_input(window_id)?; + libei_drag(from_x, from_y, to_x, to_y, steps, button) + }, + ) +} + +/// wlroots virtual-pointer implementation of [`drag`]. +#[allow(clippy::too_many_arguments)] +fn drag_vptr( + window_id: u64, + from_x: i32, + from_y: i32, + to_x: i32, + to_y: i32, + steps: u32, + button: u8, +) -> anyhow::Result<()> { + let mut sess = open_vptr_session(Some(window_id))?; + std::thread::sleep(std::time::Duration::from_millis(40)); let (w, h) = (sess.output_w, sess.output_h); let btn = evdev_pointer_button(button); let clamp_xy = |x: i32, y: i32| -> (u32, u32) { @@ -1051,9 +1584,12 @@ pub fn drag( ) }; let (fx, fy) = clamp_xy(from_x, from_y); - sess.vptr.motion_absolute(0, fx, fy, w, h); + sess.vptr.motion_absolute(event_time_ms(), fx, fy, w, h); sess.vptr.frame(); - sess.vptr.button(0, btn, ButtonState::Pressed); + sess.queue.roundtrip(&mut sess.state)?; + std::thread::sleep(std::time::Duration::from_millis(15)); + sess.vptr + .button(event_time_ms(), btn, ButtonState::Pressed); sess.vptr.frame(); sess.queue.roundtrip(&mut sess.state)?; let n = steps.max(1); @@ -1062,15 +1598,18 @@ pub fn drag( let ix = (from_x as f64 + (to_x - from_x) as f64 * t).round() as i32; let iy = (from_y as f64 + (to_y - from_y) as f64 * t).round() as i32; let (cx, cy) = clamp_xy(ix, iy); - sess.vptr.motion_absolute(0, cx, cy, w, h); + sess.vptr + .motion_absolute(event_time_ms(), cx, cy, w, h); sess.vptr.frame(); sess.queue.roundtrip(&mut sess.state)?; std::thread::sleep(std::time::Duration::from_millis(8)); } let (tx, ty) = clamp_xy(to_x, to_y); - sess.vptr.motion_absolute(0, tx, ty, w, h); + sess.vptr.motion_absolute(event_time_ms(), tx, ty, w, h); sess.vptr.frame(); - sess.vptr.button(0, btn, ButtonState::Released); + sess.queue.roundtrip(&mut sess.state)?; + sess.vptr + .button(event_time_ms(), btn, ButtonState::Released); sess.vptr.frame(); // Sync the synthetic-cursor registry with the drag endpoint so a // subsequent `get_cursor_position` reports where we left the pointer. @@ -1088,38 +1627,58 @@ pub fn drag( /// foreign-toplevel exposes no pid and Wayland delivers keys to the *focused* /// surface, so this is window_id-free; pair it with `click`/`activate` to put /// the intended window in focus first. -pub fn type_text(text: &str) -> anyhow::Result<()> { +pub fn type_text(window_id: u64, text: &str) -> anyhow::Result<()> { if text.is_empty() { return Ok(()); } + activate_window_for_input(window_id)?; // Lead with a no-op Shift_L tap: on a freshly-focused window under a headless // seat (notably sway), the compositor needs the first virtual-keyboard event // to wire up keyboard routing, and that first key is dropped. Sacrificing a // modifier tap (no character) absorbs the drop so the real text lands intact; // it's harmless where routing is already live (labwc). - let out = std::process::Command::new("wtype") + let result = std::process::Command::new("wtype") .args(["-k", "Shift_L", "--"]) .arg(text) - .output()?; - if !out.status.success() { - anyhow::bail!("wtype failed: {}", String::from_utf8_lossy(&out.stderr)); + .output(); + match result { + Ok(out) if out.status.success() => Ok(()), + // `wtype` relies on `zwp_virtual_keyboard_v1`, which KWin/Plasma and + // Mutter/GNOME don't implement (and the binary may be missing wtype + // entirely). On a portal-input build, route typing through libei's + // `ei_text` interface instead. See #1982. + other => with_wtype_libei_fallback( + || { + libei_wait_keyboard_ready()?; + activate_window_for_input(window_id)?; + libei_type_text(text) + }, + other.map(|o| String::from_utf8_lossy(&o.stderr).into_owned()), + ), } - Ok(()) } /// Press a single named key into the focused Wayland surface via `wtype -k`. -pub fn press_key(key: &str) -> anyhow::Result<()> { +pub fn press_key(window_id: u64, key: &str) -> anyhow::Result<()> { + activate_window_for_input(window_id)?; let keysym = key_to_keysym(key); - let out = std::process::Command::new("wtype") - .args(["-k", &keysym]) - .output()?; - if !out.status.success() { - anyhow::bail!( - "wtype -k {keysym} failed: {}", - String::from_utf8_lossy(&out.stderr) - ); + // Keep the sacrificial modifier and requested key in one virtual-keyboard + // lifetime. Starting a second wtype process creates a fresh protocol object, + // causing headless seats to drop the requested key as their first event. + let result = std::process::Command::new("wtype") + .args(["-k", "Shift_L", "-k", &keysym]) + .output(); + match result { + Ok(out) if out.status.success() => Ok(()), + other => with_wtype_libei_fallback( + || { + libei_wait_keyboard_ready()?; + activate_window_for_input(window_id)?; + libei_press_key(key) + }, + other.map(|o| String::from_utf8_lossy(&o.stderr).into_owned()), + ), } - Ok(()) } /// Press a key combination (modifiers + final key) via `wtype`. Each modifier @@ -1127,10 +1686,13 @@ pub fn press_key(key: &str) -> anyhow::Result<()> { /// `wtype -M ctrl -M shift -k key -m shift -m ctrl`. Unknown values pass /// straight to wtype's `-k` so single-character keys and X keysym names work /// as-is. This is the Wayland equivalent of the X11 `send_key` modifier mask. -pub fn hotkey(keys: &[String]) -> anyhow::Result<()> { +pub fn hotkey(window_id: u64, keys: &[String]) -> anyhow::Result<()> { + activate_window_for_input(window_id)?; let (mods, final_key) = partition_modifiers(keys)?; let keysym = key_to_keysym(&final_key); - let mut args: Vec<String> = Vec::new(); + // Keep the same harmless first-event primer used by `press_key`. A fresh + // virtual-keyboard object on headless seats can drop its first event. + let mut args: Vec<String> = vec!["-k".into(), "Shift_L".into()]; for m in &mods { args.push("-M".into()); args.push(m.clone()); @@ -1142,15 +1704,32 @@ pub fn hotkey(keys: &[String]) -> anyhow::Result<()> { args.push("-m".into()); args.push(m.clone()); } - let out = std::process::Command::new("wtype").args(&args).output()?; - if !out.status.success() { - anyhow::bail!( - "wtype {} failed: {}", - args.join(" "), - String::from_utf8_lossy(&out.stderr) - ); + let result = std::process::Command::new("wtype").args(&args).output(); + match result { + Ok(out) if out.status.success() => Ok(()), + other => { + let stderr = other.map(|o| String::from_utf8_lossy(&o.stderr).into_owned()); + #[cfg(feature = "portal-input")] + { + return with_wtype_libei_fallback( + || { + libei::wait_keyboard_ready()?; + activate_window_for_input(window_id)?; + libei_hotkey(&mods, &final_key) + }, + stderr, + ); + } + #[cfg(not(feature = "portal-input"))] + { + anyhow::bail!( + "wtype {} failed: {}", + args.join(" "), + stderr.unwrap_or_else(|_| "wtype unavailable".into()) + ); + } + } } - Ok(()) } /// Split a `keys` array into wtype-compatible modifier names and a single @@ -1199,29 +1778,479 @@ fn key_to_keysym(key: &str) -> String { .to_string() } -// ── EIS nested-compositor injection ──────────────────────────────────────── +// ── libei / portal fallback adapters ─────────────────────────────────────── +// +// These bridge the wlroots-shaped public input API (output-relative integer +// coordinates, cua button codes, X-keysym key names) onto the libei worker +// (`libei` module), which speaks logical device-region floats and evdev +// codes. They are the recovery path for compositors with no +// `zwlr_virtual_pointer_v1` (KWin/Plasma, Mutter/GNOME) — see #1982. +// +// In a build WITHOUT the `portal-input` feature the `libei` module does not +// exist, so each adapter compiles to an error stub. The dispatch seams above +// only ever CALL these inside `#[cfg(feature = "portal-input")]` branches, so +// the stubs are dead in that build; they exist purely so the closures passed +// to `with_libei_fallback` / `with_wtype_libei_fallback` type-check. + +/// libei recovery wrapper for the `wtype`-based typing/key functions: when the +/// virtual-keyboard shell-out failed (`wtype_err`), try the libei `run` on a +/// portal-input build, otherwise surface the original wtype failure. +fn with_wtype_libei_fallback( + #[allow(unused_variables)] run: impl FnOnce() -> anyhow::Result<()>, + wtype_err: Result<String, std::io::Error>, +) -> anyhow::Result<()> { + #[cfg(feature = "portal-input")] + { + match wtype_err { + Ok(stderr) => tracing::info!( + "wtype failed ({stderr}); falling back to libei/portal typing" + ), + Err(e) => tracing::info!( + "wtype unavailable ({e}); falling back to libei/portal typing" + ), + } + run() + } + #[cfg(not(feature = "portal-input"))] + { + let _ = run; + match wtype_err { + Ok(stderr) => anyhow::bail!("wtype failed: {stderr}"), + Err(e) => anyhow::bail!("wtype unavailable: {e}"), + } + } +} + +// Stubs for the no-feature build: the dispatch seams never call these (the +// libei branch in `with_libei_fallback` / `with_wtype_libei_fallback` is +// `#[cfg]`-d out), but the closures still need them to exist to type-check. +#[cfg(not(feature = "portal-input"))] +fn libei_wait_pointer_ready() -> anyhow::Result<()> { + unreachable!("libei fallback compiled out (no portal-input feature)") +} +#[cfg(not(feature = "portal-input"))] +fn libei_wait_scroll_ready() -> anyhow::Result<()> { + unreachable!("libei fallback compiled out (no portal-input feature)") +} +#[cfg(not(feature = "portal-input"))] +fn libei_wait_keyboard_ready() -> anyhow::Result<()> { + unreachable!("libei fallback compiled out (no portal-input feature)") +} +#[cfg(not(feature = "portal-input"))] +fn libei_click(_x: i32, _y: i32, _count: u32, _button: u8) -> anyhow::Result<()> { + unreachable!("libei fallback compiled out (no portal-input feature)") +} +#[cfg(not(feature = "portal-input"))] +fn libei_scroll(_direction: &str, _amount: u32) -> anyhow::Result<()> { + unreachable!("libei fallback compiled out (no portal-input feature)") +} +#[cfg(not(feature = "portal-input"))] +fn libei_move_absolute(_x: i32, _y: i32) -> anyhow::Result<()> { + unreachable!("libei fallback compiled out (no portal-input feature)") +} +#[cfg(not(feature = "portal-input"))] +#[allow(clippy::too_many_arguments)] +fn libei_drag( + _from_x: i32, + _from_y: i32, + _to_x: i32, + _to_y: i32, + _steps: u32, + _button: u8, +) -> anyhow::Result<()> { + unreachable!("libei fallback compiled out (no portal-input feature)") +} +#[cfg(not(feature = "portal-input"))] +fn libei_type_text(_text: &str) -> anyhow::Result<()> { + unreachable!("libei fallback compiled out (no portal-input feature)") +} +#[cfg(not(feature = "portal-input"))] +fn libei_press_key(_key: &str) -> anyhow::Result<()> { + unreachable!("libei fallback compiled out (no portal-input feature)") +} +#[cfg(feature = "portal-input")] +fn cua_button_to_libei(button: u8) -> libei::Button { + match button { + 2 => libei::Button::Middle, + 3 => libei::Button::Right, + _ => libei::Button::Left, + } +} + +#[cfg(feature = "portal-input")] +fn libei_wait_pointer_ready() -> anyhow::Result<()> { + libei::wait_pointer_ready() +} + +#[cfg(feature = "portal-input")] +fn libei_wait_scroll_ready() -> anyhow::Result<()> { + libei::wait_scroll_ready() +} + +#[cfg(feature = "portal-input")] +fn libei_wait_keyboard_ready() -> anyhow::Result<()> { + libei::wait_keyboard_ready() +} + +#[cfg(feature = "portal-input")] +fn libei_click(x: i32, y: i32, count: u32, button: u8) -> anyhow::Result<()> { + let btn = cua_button_to_libei(button); + let (w, h) = output_dimensions()?; + let (px, py) = normalize_click_xy(x, y, w, h); + libei::move_absolute(px as f64, py as f64)?; + for i in 0..count.max(1) { + if i > 0 { + std::thread::sleep(std::time::Duration::from_millis(80)); + } + libei::click(px as f64, py as f64, btn)?; + } + record_synth_cursor(px, py); + Ok(()) +} + +#[cfg(feature = "portal-input")] +fn libei_scroll(direction: &str, amount: u32) -> anyhow::Result<()> { + // libei scroll is logical-unit deltas; mirror the wlroots ±10/tick step. + let (dx, dy): (f64, f64) = match direction.to_ascii_lowercase().as_str() { + "up" => (0.0, -10.0), + "down" => (0.0, 10.0), + "left" => (-10.0, 0.0), + "right" => (10.0, 0.0), + other => anyhow::bail!("unknown scroll direction: {other}"), + }; + for i in 0..amount.max(1) { + if i > 0 { + std::thread::sleep(std::time::Duration::from_millis(25)); + } + libei::scroll(dx, dy)?; + } + Ok(()) +} + +#[cfg(feature = "portal-input")] +fn libei_move_absolute(x: i32, y: i32) -> anyhow::Result<()> { + // Match `move_cursor_absolute_vptr`: clamp to output bounds (no + // default-to-centre — an explicit (0,0) move means the top-left corner). + let (w, h) = output_dimensions()?; + let px = x.clamp(0, (w as i32).saturating_sub(1)); + let py = y.clamp(0, (h as i32).saturating_sub(1)); + libei::move_absolute(px as f64, py as f64)?; + record_synth_cursor(px, py); + Ok(()) +} + +#[cfg(feature = "portal-input")] +fn libei_drag( + from_x: i32, + from_y: i32, + to_x: i32, + to_y: i32, + steps: u32, + button: u8, +) -> anyhow::Result<()> { + // ei_button exposes separate Press/Released states, so the libei worker can + // hold the button across the interpolated motion — a genuine + // press→move→release drag. Clamp both endpoints to the output — but NOT via + // `normalize_click_xy`, whose (0,0)→centre convention (for coordinate-free + // clicks) is wrong here: a drag endpoint is always explicit and (0,0) is a + // valid top-left corner target. + let btn = cua_button_to_libei(button); + let (w, h) = output_dimensions()?; + let cx = |x: i32| x.clamp(0, (w as i32).saturating_sub(1)); + let cy = |y: i32| y.clamp(0, (h as i32).saturating_sub(1)); + libei::drag( + cx(from_x) as f64, + cy(from_y) as f64, + cx(to_x) as f64, + cy(to_y) as f64, + steps, + btn, + )?; + record_synth_cursor(cx(to_x), cy(to_y)); + Ok(()) +} + +#[cfg(feature = "portal-input")] +fn libei_type_text(text: &str) -> anyhow::Result<()> { + if text.is_empty() { + return Ok(()); + } + libei::type_text(text) +} + +#[cfg(feature = "portal-input")] +fn libei_press_key(key: &str) -> anyhow::Result<()> { + let keycode = key_to_evdev(key) + .ok_or_else(|| anyhow::anyhow!("no evdev keycode mapping for key '{key}' (libei path)"))?; + libei::press_key(keycode) +} + +#[cfg(feature = "portal-input")] +fn libei_hotkey(mods: &[String], key: &str) -> anyhow::Result<()> { + use libei::KeyTransition::{Press, Release}; + + let mut modifier_codes = Vec::with_capacity(mods.len()); + for modifier in mods { + modifier_codes.push(match modifier.as_str() { + "ctrl" => 29, + "shift" => 42, + "alt" => 56, + "logo" => 125, + other => anyhow::bail!("no evdev keycode mapping for modifier '{other}'"), + }); + } + let keycode = key_to_evdev(key) + .ok_or_else(|| anyhow::anyhow!("no evdev keycode mapping for key '{key}' (libei path)"))?; + let mut transitions = Vec::with_capacity(modifier_codes.len() * 2 + 2); + transitions.extend(modifier_codes.iter().copied().map(Press)); + transitions.push(Press(keycode)); + transitions.push(Release(keycode)); + transitions.extend(modifier_codes.iter().rev().copied().map(Release)); + libei::key_sequence(&transitions) +} + +/// Map cua key names to Linux evdev keycodes for the libei `press_key` path +/// (libei emulates raw evdev, not X keysyms). Mirrors [`key_to_keysym`] but +/// emits `linux/input-event-codes.h` values. Returns `None` for keys with no +/// known mapping so the caller can fail loudly. +#[cfg(feature = "portal-input")] +fn key_to_evdev(key: &str) -> Option<u32> { + let code = match key.to_lowercase().as_str() { + "enter" | "return" => 28, // KEY_ENTER + "tab" => 15, // KEY_TAB + "esc" | "escape" => 1, // KEY_ESC + "space" => 57, // KEY_SPACE + "backspace" => 14, // KEY_BACKSPACE + "delete" | "del" => 111, // KEY_DELETE + "up" => 103, // KEY_UP + "down" => 108, // KEY_DOWN + "left" => 105, // KEY_LEFT + "right" => 106, // KEY_RIGHT + "home" => 102, // KEY_HOME + "end" => 107, // KEY_END + "pageup" | "page_up" => 104, // KEY_PAGEUP + "pagedown" | "page_down" => 109, // KEY_PAGEDOWN + // Letters a-z. evdev codes follow the QWERTY scancode layout, not the + // alphabet, so each is listed explicitly (linux/input-event-codes.h). + "a" => 30, // KEY_A + "b" => 48, // KEY_B + "c" => 46, // KEY_C + "d" => 32, // KEY_D + "e" => 18, // KEY_E + "f" => 33, // KEY_F + "g" => 34, // KEY_G + "h" => 35, // KEY_H + "i" => 23, // KEY_I + "j" => 36, // KEY_J + "k" => 37, // KEY_K + "l" => 38, // KEY_L + "m" => 50, // KEY_M + "n" => 49, // KEY_N + "o" => 24, // KEY_O + "p" => 25, // KEY_P + "q" => 16, // KEY_Q + "r" => 19, // KEY_R + "s" => 31, // KEY_S + "t" => 20, // KEY_T + "u" => 22, // KEY_U + "v" => 47, // KEY_V + "w" => 17, // KEY_W + "x" => 45, // KEY_X + "y" => 21, // KEY_Y + "z" => 44, // KEY_Z + // Digits. KEY_1=2 .. KEY_9=10, KEY_0=11 (input-event-codes.h). + "1" => 2, // KEY_1 + "2" => 3, // KEY_2 + "3" => 4, // KEY_3 + "4" => 5, // KEY_4 + "5" => 6, // KEY_5 + "6" => 7, // KEY_6 + "7" => 8, // KEY_7 + "8" => 9, // KEY_8 + "9" => 10, // KEY_9 + "0" => 11, // KEY_0 + // Function keys. KEY_F1=59 .. KEY_F10=68, then KEY_F11=87, KEY_F12=88. + "f1" => 59, + "f2" => 60, + "f3" => 61, + "f4" => 62, + "f5" => 63, + "f6" => 64, + "f7" => 65, + "f8" => 66, + "f9" => 67, + "f10" => 68, + "f11" => 87, + "f12" => 88, + _ => return None, + }; + Some(code) +} + +// ── Nested cua-compositor injection ──────────────────────────────────────── // // When cua-driver's nested compositor is `cua-compositor` (our patched wlroots, // see nix/cua-driver/compositor/), it exposes a line-protocol control socket at // $CUA_INJECT_SOCKET for what stock Wayland forbids: focus-FREE per-surface -// keyboard injection and MULTI-cursor pointer injection, both routed to a target -// window by its xdg app_id. These helpers speak that protocol. - -/// The control socket path, when running against the EIS nested compositor. +// keyboard injection and MULTI-cursor pointer injection, routed by stable PID +// when available and xdg app_id only as a fallback. These helpers speak that +// protocol. +// +// The protocol is a simple line-based v1 exchange with per-command +// acknowledgement: the client sends `INJECT_PROTO_HELLO` and the compositor +// echoes it (or replies `err ...`), then every command line is answered by +// exactly one `ok` / `err <reason>` line. The client fails on a protocol +// mismatch, a read timeout, an EOF, or any compositor error line — an +// acknowledgement is transport evidence only, not proof the target changed. + +/// Version banner exchanged at connect time: the client sends this line and the +/// compositor must echo it back verbatim to confirm both speak v1. +const INJECT_PROTO_HELLO: &str = "cua-inject v1"; + +/// The exact named keys the nested compositor's `k` command can emit — the +/// whitelist in `cua_key_named` (cua_compositor_patch.py). Compared +/// case-insensitively, matching the compositor's `strcasecmp`. +const INJECT_NAMED_KEYS: &[&str] = &[ + "enter", "return", "tab", "escape", "esc", "backspace", "space", "up", "down", "left", "right", + "f1", "f2", "f3", "f4", "f5", "f6", "f7", "f8", "f9", "f10", "f11", "f12", +]; + +/// The control socket path, when running against the nested cua-compositor. pub fn inject_socket_path() -> Option<String> { std::env::var("CUA_INJECT_SOCKET") .ok() .filter(|s| !s.is_empty()) } -/// True when input should be routed through the EIS compositor's control socket -/// (focus-free / multi-cursor) rather than wtype / virtual-pointer. +/// True when input should be routed through the nested cua-compositor's control +/// socket (focus-free / multi-cursor) rather than wtype / virtual-pointer. pub fn is_inject_mode() -> bool { inject_socket_path().is_some() } -fn inject_send(lines: &[String]) -> anyhow::Result<()> { - use std::io::Write; +/// Reject any character the nested compositor cannot type before it reaches the +/// wire. The compositor's chartab (`cua_init_keymap`) only covers printable +/// ASCII (`0x20..=0x7E`) plus newline and tab; anything else — Unicode, other +/// control bytes — would be silently dropped, so fail loudly instead. +fn validate_injectable_text(text: &str) -> anyhow::Result<()> { + for ch in text.chars() { + let ok = ch == '\n' || ch == '\t' || (ch.is_ascii() && !ch.is_ascii_control()); + if !ok { + anyhow::bail!( + "cua-compositor cannot type {ch:?}: only printable ASCII plus newline and tab \ + are supported in the v1 injection protocol" + ); + } + } + Ok(()) +} + +/// Reject any key name outside the compositor's named-key whitelist before +/// sending. Mirrors `cua_key_named`; unsupported names must fail here rather +/// than being silently ignored by the compositor. +fn validate_injectable_key(key: &str) -> anyhow::Result<()> { + let normalized = key.trim().to_ascii_lowercase(); + if INJECT_NAMED_KEYS.contains(&normalized.as_str()) { + Ok(()) + } else { + anyhow::bail!( + "cua-compositor does not support key {key:?}; supported keys: {}", + INJECT_NAMED_KEYS.join(", ") + ); + } +} + +fn validate_injectable_hotkey(keys: &[String]) -> anyhow::Result<(String, String)> { + let (key, modifiers) = keys + .split_last() + .ok_or_else(|| anyhow::anyhow!("cua-compositor hotkey requires a non-modifier key"))?; + let key = key.trim().to_ascii_lowercase(); + if !(key.len() == 1 && key.is_ascii()) && !INJECT_NAMED_KEYS.contains(&key.as_str()) { + anyhow::bail!("cua-compositor does not support hotkey key {key:?}"); + } + let mut normalized = Vec::with_capacity(modifiers.len()); + for modifier in modifiers { + let modifier = modifier.trim().to_ascii_lowercase(); + let canonical = match modifier.as_str() { + "ctrl" | "control" => "ctrl", + "shift" => "shift", + "alt" | "option" => "alt", + "meta" | "super" | "win" | "cmd" => "meta", + _ => anyhow::bail!("cua-compositor does not support modifier {modifier:?}"), + }; + normalized.push(canonical); + } + if normalized.is_empty() { + anyhow::bail!("cua-compositor hotkey requires at least one modifier"); + } + Ok((normalized.join(","), key)) +} + +/// Interpret the compositor's handshake reply. Accepts only the verbatim v1 +/// banner; a compositor `err ...` line or anything else is a protocol mismatch. +fn parse_inject_hello(line: &str) -> anyhow::Result<()> { + let trimmed = line.trim(); + if trimmed == INJECT_PROTO_HELLO { + Ok(()) + } else if let Some(reason) = trimmed.strip_prefix("err") { + anyhow::bail!( + "cua-compositor rejected the v1 handshake:{}", + if reason.trim().is_empty() { + String::new() + } else { + format!(" {}", reason.trim()) + } + ) + } else { + anyhow::bail!( + "cua-compositor protocol mismatch: expected {INJECT_PROTO_HELLO:?}, got {trimmed:?}" + ) + } +} + +/// Interpret a single per-command acknowledgement line. `ok` succeeds; `err +/// <reason>` and any unrecognised line fail. +fn parse_inject_reply(line: &str) -> anyhow::Result<()> { + let trimmed = line.trim(); + if trimmed == "ok" { + Ok(()) + } else if let Some(reason) = trimmed.strip_prefix("err") { + let reason = reason.trim(); + if reason.is_empty() { + anyhow::bail!("cua-compositor rejected the command"); + } + anyhow::bail!("cua-compositor rejected the command: {reason}") + } else { + anyhow::bail!("unexpected cua-compositor response: {trimmed:?}") + } +} + +/// Read one newline-terminated response line, mapping timeout and EOF to clear +/// errors so the caller never blocks forever on an unresponsive compositor. +fn read_inject_line(reader: &mut impl std::io::BufRead) -> anyhow::Result<String> { + let mut line = String::new(); + let n = reader.read_line(&mut line).map_err(|e| { + if matches!( + e.kind(), + std::io::ErrorKind::WouldBlock | std::io::ErrorKind::TimedOut + ) { + anyhow::anyhow!("cua-compositor did not respond within the timeout") + } else { + anyhow::anyhow!("cua-compositor read failed: {e}") + } + })?; + if n == 0 { + anyhow::bail!("cua-compositor closed the connection before responding"); + } + Ok(line) +} + +/// Connect to the nested cua-compositor control socket, perform the v1 +/// handshake, then send each command line and require exactly one +/// acknowledgement per command. Fails on protocol mismatch, timeout, EOF, or a +/// compositor error line — the earlier fire-and-forget path hid all of these. +fn inject_exchange(lines: &[String]) -> anyhow::Result<Vec<String>> { + use std::io::{BufReader, Write}; use std::os::unix::net::UnixStream; let path = inject_socket_path().ok_or_else(|| anyhow::anyhow!("CUA_INJECT_SOCKET not set"))?; // The nested compositor may still be starting; retry the connect briefly. @@ -1235,36 +2264,80 @@ fn inject_send(lines: &[String]) -> anyhow::Result<()> { Err(_) => std::thread::sleep(std::time::Duration::from_millis(50)), } } - let mut s = + let stream = stream.ok_or_else(|| anyhow::anyhow!("could not connect to inject socket {path}"))?; - let mut buf = String::new(); + stream.set_read_timeout(Some(std::time::Duration::from_secs(5)))?; + let mut writer = stream.try_clone()?; + let mut reader = BufReader::new(stream); + + // v1 handshake: send our banner and require the compositor to echo it. + writeln!(writer, "{INJECT_PROTO_HELLO}")?; + writer.flush()?; + parse_inject_hello(&read_inject_line(&mut reader)?)?; + + let mut replies = Vec::with_capacity(lines.len()); + // One command per line; block on its response before the next. for l in lines { - buf.push_str(l); - buf.push('\n'); + writeln!(writer, "{l}")?; + writer.flush()?; + replies.push(read_inject_line(&mut reader)?); + } + Ok(replies) +} + +fn inject_send(lines: &[String]) -> anyhow::Result<()> { + for reply in inject_exchange(lines)? { + parse_inject_reply(&reply)?; } - s.write_all(buf.as_bytes())?; - s.flush()?; - // Give the compositor a moment to process before the socket closes. - std::thread::sleep(std::time::Duration::from_millis(80)); Ok(()) } -/// Resolve a window_id (foreign-toplevel protocol id) to its xdg app_id by -/// enumerating toplevels — the inject protocol addresses windows by app_id. -pub fn app_id_for_window(window_id: u64) -> Option<String> { - let conn = Connection::connect_to_env().ok()?; - let mut queue = conn.new_event_queue::<State>(); - let qh = queue.handle(); - conn.display().get_registry(&qh, ()); - let mut state = State::default(); - queue.roundtrip(&mut state).ok()?; - for _ in 0..4 { - queue.roundtrip(&mut state).ok()?; +fn parse_inject_geometry(line: &str) -> anyhow::Result<((i32, i32), (i32, i32))> { + let fields = line.split_whitespace().collect::<Vec<_>>(); + if fields.len() == 5 && fields[0] == "geometry" { + return Ok(( + (fields[1].parse()?, fields[2].parse()?), + (fields[3].parse()?, fields[4].parse()?), + )); + } + if let Some(reason) = line.trim().strip_prefix("err") { + anyhow::bail!("cua-compositor geometry query failed: {}", reason.trim()); + } + anyhow::bail!("unexpected cua-compositor geometry response: {:?}", line.trim()) +} + +/// Return the offset that rebases native Wayland accessibility coordinates into +/// the nested compositor's root-surface/output coordinate space. +pub fn inject_accessibility_offset(pid: u32) -> Option<(i32, i32)> { + if !is_inject_mode() || pid == 0 { + return None; } - state - .toplevels - .get(&(window_id as u32)) - .map(|t| t.app_id.clone()) + let replies = inject_exchange(&[format!("g {pid}")]).ok()?; + replies + .first() + .and_then(|line| parse_inject_geometry(line).ok()) + .map(|geometry| geometry.0) +} + +fn inject_window_origin(pid: u32) -> Option<(i32, i32)> { + if !is_inject_mode() || pid == 0 { + return None; + } + let replies = inject_exchange(&[format!("g {pid}")]).ok()?; + replies + .first() + .and_then(|line| parse_inject_geometry(line).ok()) + .map(|geometry| geometry.1) +} + +/// Resolve a window_id to its xdg app_id via the stable identity registry that +/// [`list_windows`] populates (falling back to sway IPC / AT-SPI through +/// [`identity_for`]) — the nested cua-compositor injection protocol addresses +/// windows by app_id. Returns `None` when no identity is registered or the +/// resolved app_id is empty, so callers can surface a clear error. +pub fn app_id_for_window(window_id: u64) -> Option<String> { + identity_for(window_id) + .map(|identity| identity.app_id) .filter(|s| !s.is_empty()) } @@ -1295,25 +2368,77 @@ fn evdev_button(x_button: u32) -> u32 { } } -/// Focus-free type into the window's surface (no focus change). +/// Sentinel error when a window_id has no registered cua-compositor identity. +fn no_app_id(window_id: u64) -> anyhow::Error { + anyhow::anyhow!( + "no known cua-compositor app_id for window {window_id}; call list_windows first so its \ + Wayland identity is registered" + ) +} + +/// Resolve the strongest target token understood by the private nested +/// compositor. AT-SPI window IDs are synthetic on Wayland, but its process ID +/// is the same credential the compositor observes on the owning wl_client. +/// Fall back to app_id for clients whose accessibility metadata has no PID. +pub fn inject_target_for_window(window_id: u64) -> anyhow::Result<String> { + if let Some(pid) = crate::atspi::list_windows(None) + .into_iter() + .find(|window| window.xid == window_id) + .and_then(|window| window.pid) + { + return Ok(format!("pid:{pid}")); + } + app_id_for_window(window_id).ok_or_else(|| no_app_id(window_id)) +} + +/// Focus-free type into the window's surface (no focus change). Rejects any +/// character the compositor cannot emit before touching the socket. pub fn inject_type_text(window_id: u64, text: &str) -> anyhow::Result<()> { - let app = app_id_for_window(window_id) - .ok_or_else(|| anyhow::anyhow!("no Wayland app_id for window {window_id}"))?; + validate_injectable_text(text)?; + let app = inject_target_for_window(window_id)?; inject_send(&[format!("t {app} {}", to_hex(text))]) } -/// Focus-free named-key press into the window's surface. +/// Focus-free named-key press into the window's surface. Rejects any key +/// outside the compositor's whitelist before touching the socket. pub fn inject_press_key(window_id: u64, key: &str) -> anyhow::Result<()> { - let app = app_id_for_window(window_id) - .ok_or_else(|| anyhow::anyhow!("no Wayland app_id for window {window_id}"))?; - inject_send(&[format!("k {app} {key}")]) + validate_injectable_key(key)?; + let app = inject_target_for_window(window_id)?; + inject_send(&[format!("k {app} {}", key.trim())]) } -/// Focus-free click into the window's surface via the nested EIS compositor. +/// Focus-free modifier chord into the target surface. +pub fn inject_hotkey(window_id: u64, keys: &[String]) -> anyhow::Result<()> { + let (modifiers, key) = validate_injectable_hotkey(keys)?; + let app = inject_target_for_window(window_id)?; + inject_send(&[format!("h {app} {modifiers} {key}")]) +} + +/// Focus-free wheel/axis input at one target-local point. +pub fn inject_scroll( + window_id: u64, + x: f64, + y: f64, + direction: &str, + amount: u32, +) -> anyhow::Result<()> { + let app = inject_target_for_window(window_id)?; + let (axis, value) = match direction.to_ascii_lowercase().as_str() { + "up" => (0, -15.0), + "down" | "page" => (0, 15.0), + "left" => (1, -15.0), + "right" => (1, 15.0), + _ => anyhow::bail!("unsupported cua-compositor scroll direction {direction:?}"), + }; + let mut lines = vec![format!("m {app} 0 {x:.1} {y:.1}")]; + lines.extend((0..amount.max(1)).map(|_| format!("a {app} 0 {axis} {value:.1}"))); + inject_send(&lines) +} + +/// Focus-free click into the window's surface via the nested cua-compositor. /// Coordinates are window-local, matching the rest of the inject protocol. pub fn inject_click(window_id: u64, x: f64, y: f64, count: u32, button: u8) -> anyhow::Result<()> { - let app = app_id_for_window(window_id) - .ok_or_else(|| anyhow::anyhow!("no Wayland app_id for window {window_id}"))?; + let app = inject_target_for_window(window_id)?; let btn = evdev_button(button as u32); let n = count.max(1); let mut lines = Vec::with_capacity((n as usize) * 4); @@ -1417,28 +2542,98 @@ pub fn inject_parallel_drags(drags: &[InjectDrag]) -> anyhow::Result<()> { inject_send(&lines) } -/// Window-enumeration dispatcher: native Wayland when applicable, else X11. +/// Focus-free single drag using the same per-surface path as parallel drags. +pub fn inject_drag( + window_id: u64, + from: (f64, f64), + to: (f64, f64), + steps: usize, + x_button: u32, +) -> anyhow::Result<()> { + let app_id = inject_target_for_window(window_id)?; + inject_parallel_drags(&[InjectDrag { + app_id, + idx: 0, + x_button, + path: vec![from, to], + steps, + }]) +} + +fn wayland_atspi_windows(filter_pid: Option<u32>) -> Vec<WindowInfo> { + let mut windows = crate::atspi::list_windows(filter_pid); + if is_inject_mode() { + for window in &mut windows { + if let Some(pid) = window.pid { + if let Some((window_x, window_y)) = inject_window_origin(pid) { + window.x = window_x; + window.y = window_y; + } + } + } + } + windows +} + +/// Window-enumeration dispatcher: native Wayland when available, else X11. pub fn list_windows_dispatch(filter_pid: Option<u32>) -> Vec<WindowInfo> { - if is_wayland() { - // wlroots compositors expose zwlr_foreign_toplevel_management — use it - // (it has no pid, so filter_pid can't apply there). - match list_windows() { - Ok(ws) if !ws.is_empty() => return ws, + if wayland_enabled() && std::env::var_os("WAYLAND_DISPLAY").is_some() { + // Prefer the richer wlroots protocol. The generic staging protocol is + // only consulted when wlroots yields no windows (including when its + // manager global is absent). + let native = match list_windows() { + Ok(ws) if !ws.is_empty() => Ok(enrich_native_windows( + ws, + wayland_atspi_windows(filter_pid), + is_inject_mode(), + )), + Ok(_) => ext_toplevel::list_windows(), + Err(wlr_error) => ext_toplevel::list_windows().map_err(|ext_error| { + anyhow::anyhow!( + "wlr provider failed ({wlr_error}); ext provider failed ({ext_error})" + ) + }), + }; + match native { + Ok(ws) if !ws.is_empty() => { + if let Some(pid) = filter_pid { + if let Some(filtered) = native_windows_for_pid(ws, pid) { + return filtered; + } + } else { + return ws; + } + // A compositor window without pid metadata cannot satisfy a + // pid-scoped request. Continue to the AT-SPI registry. + let ws = wayland_atspi_windows(filter_pid); + if !ws.is_empty() { + return ws; + } + } Ok(_) => { - // GNOME Mutter / KDE KWin don't implement foreign-toplevel, so the - // list came back empty. Native Wayland apps have no X11 XID either, - // so fall back to enumerating windows from the AT-SPI registry - // (keyed by pid — the same tree get_window_state walks). - let ws = crate::atspi::list_windows(filter_pid); + if let Some(ws) = + shell_helper::list_windows(filter_pid).filter(|ws| !ws.is_empty()) + { + return ws; + } + let ws = wayland_atspi_windows(filter_pid); if !ws.is_empty() { return ws; } } Err(e) => { + if let Some(ws) = + shell_helper::list_windows(filter_pid).filter(|ws| !ws.is_empty()) + { + tracing::debug!( + "native Wayland protocols unavailable ({e}); using compositor helper" + ); + return ws; + } tracing::warn!( - "wayland foreign-toplevel list_windows failed: {e}; trying AT-SPI registry" + "native Wayland list_windows failed: {e}; trying AT-SPI registry" ); - let ws = crate::atspi::list_windows(filter_pid); + let ws = wayland_atspi_windows(filter_pid); if !ws.is_empty() { return ws; } @@ -1446,7 +2641,124 @@ pub fn list_windows_dispatch(filter_pid: Option<u32>) -> Vec<WindowInfo> { } // Last resort under Wayland: an Xwayland app may still have an X11 XID. } - crate::x11::list_windows(filter_pid) + // If native enumeration and its AT-SPI fallback found nothing, X11 may still + // expose XWayland clients. Merge one final AT-SPI snapshot so native windows + // remain visible on hybrid sessions even when neither foreign-toplevel + // protocol is advertised (#1978). Gated on the native-Wayland opt-in. + // + // Caveats for the merged AT-SPI entries: they carry a synthetic (non-X11) + // xid and zero geometry (x/y/w/h = 0), like the existing wlroots AT-SPI + // fallback — so `bring_to_front` / `screenshot_window` / pixel translation + // against them error cleanly rather than acting (input on GNOME/KDE routes + // by pid + screen coords, not xid, so it's unaffected). Dedup is per-pid, so + // the rare app owning BOTH an XWayland window and a separate native-Wayland + // toplevel would list only the XWayland one. + let mut ws = crate::x11::list_windows(filter_pid); + if wayland_enabled() && std::env::var_os("WAYLAND_DISPLAY").is_some() { + let seen: std::collections::HashSet<u32> = ws.iter().filter_map(|w| w.pid).collect(); + // A specific pid already resolved via X11 needs no AT-SPI walk (a full + // D-Bus enumeration of every registered app): it can only add duplicates. + let already_covered = filter_pid.map_or(false, |p| seen.contains(&p)); + if !already_covered { + merge_atspi_windows(&mut ws, &seen, wayland_atspi_windows(filter_pid)); + } + } + ws +} + +fn merge_atspi_windows( + windows: &mut Vec<WindowInfo>, + x11_pids: &std::collections::HashSet<u32>, + atspi_windows: Vec<WindowInfo>, +) { + for window in atspi_windows { + // XWayland apps appear in both lists; keep the X11 entry (real XID + + // geometry) and retain every native frame whose pid X11 did not expose. + if window.pid.is_none_or(|pid| !x11_pids.contains(&pid)) { + windows.push(window); + } + } +} + +fn enrich_native_windows( + mut native: Vec<WindowInfo>, + atspi: Vec<WindowInfo>, + adopt_atspi_ids: bool, +) -> Vec<WindowInfo> { + let mut claimed = std::collections::HashSet::new(); + for window in &mut native { + if window.pid.is_some() { + continue; + } + let native_title = undecorated_native_title(window); + let title_match = atspi.iter().enumerate().find_map(|(index, candidate)| { + (!claimed.contains(&index) + && !native_title.is_empty() + && candidate.title == native_title) + .then_some(index) + }); + let app_match = title_match.or_else(|| { + let matches = atspi + .iter() + .enumerate() + .filter(|(index, candidate)| { + !claimed.contains(index) + && !window.app_name.is_empty() + && candidate.app_name == window.app_name + }) + .map(|(index, _)| index) + .collect::<Vec<_>>(); + (matches.len() == 1).then(|| matches[0]) + }); + let Some(index) = app_match else { continue }; + claimed.insert(index); + let candidate = &atspi[index]; + window.pid = candidate.pid; + if adopt_atspi_ids { + let toplevel = Toplevel { + title: undecorated_native_title(window).to_owned(), + app_id: window.app_name.clone(), + closed: false, + }; + window.xid = candidate.xid; + remember_identity(window.xid, &toplevel); + } + if window.width == 0 || window.height == 0 { + window.x = candidate.x; + window.y = candidate.y; + window.width = candidate.width; + window.height = candidate.height; + } + if adopt_atspi_ids { + if let Some(pid) = candidate.pid { + if let Some((window_x, window_y)) = inject_window_origin(pid) { + window.x = window_x; + window.y = window_y; + } + } + } + window.is_on_screen = candidate.is_on_screen; + } + native +} + +fn undecorated_native_title(window: &WindowInfo) -> &str { + if window.app_name.is_empty() { + return &window.title; + } + let suffix = format!(" [{}]", window.app_name); + window.title.strip_suffix(&suffix).unwrap_or(&window.title) +} + +/// Return native records only when they contain a real match for a pid-scoped +/// request. Ext records whose AT-SPI merge left pid unknown must not suppress +/// the later AT-SPI and X11 fallback providers. +fn native_windows_for_pid(windows: Vec<WindowInfo>, pid: u32) -> Option<Vec<WindowInfo>> { + let matching: Vec<_> = windows + .into_iter() + .filter(|window| window.pid == Some(pid)) + .collect(); + (!matching.is_empty()).then_some(matching) } /// Snapshot of which wlroots manager globals the running compositor advertises. @@ -1542,3 +2854,199 @@ impl Dispatch<wl_registry::WlRegistry, ()> for ExtProbeState { // compatibility with earlier slice constants. #[allow(dead_code)] const _BTN_LEFT_ALIAS: u32 = BTN_LEFT; + +#[cfg(test)] +mod tests { + use super::*; + + fn window(xid: u64, pid: Option<u32>, title: &str) -> WindowInfo { + WindowInfo { + xid, + pid, + app_name: String::new(), + title: title.to_owned(), + is_on_screen: true, + z_index: None, + x: 0, + y: 0, + width: 0, + height: 0, + } + } + + #[test] + fn atspi_merge_keeps_x11_geometry_owner_and_native_only_frames() { + let mut windows = vec![window(10, Some(100), "XWayland")]; + let x11_pids = std::collections::HashSet::from([100]); + merge_atspi_windows( + &mut windows, + &x11_pids, + vec![ + window(100 << 16, Some(100), "XWayland duplicate"), + window(200 << 16, Some(200), "Native Wayland"), + window(1, None, "Unknown native frame"), + ], + ); + assert_eq!(windows.len(), 3); + assert_eq!(windows[0].xid, 10); + assert_eq!(windows[1].pid, Some(200)); + assert_eq!(windows[2].pid, None); + } + + #[test] + fn zero_geometry_does_not_replace_a_real_observed_origin() { + let pid = u32::MAX - 17; + let mut observed = window(1, Some(pid), "Observed"); + observed.x = 120; + observed.y = 80; + remember_observed_window_origins(&[observed]); + assert_eq!(observed_window_origin(pid), Some((120, 80))); + + remember_observed_window_origins(&[window(2, Some(pid), "Unknown")]); + assert_eq!(observed_window_origin(pid), Some((120, 80))); + } + + #[test] + fn native_enrichment_matches_plain_atspi_title() { + let mut native = window(42, None, "CUA Fixture [cua-fixture]"); + native.app_name = "cua-fixture".into(); + let mut accessible = window(123 << 16, Some(123), "CUA Fixture"); + accessible.x = 20; + accessible.y = 30; + accessible.width = 800; + accessible.height = 600; + + let enriched = enrich_native_windows(vec![native], vec![accessible], false); + + assert_eq!(enriched[0].xid, 42); + assert_eq!(enriched[0].pid, Some(123)); + assert_eq!((enriched[0].x, enriched[0].y), (20, 30)); + assert_eq!((enriched[0].width, enriched[0].height), (800, 600)); + } + + #[test] + fn unmatched_ext_windows_do_not_satisfy_pid_filter() { + let windows = vec![window(0xF000_0000, None, "Protocol-only")]; + assert!(native_windows_for_pid(windows, 4242).is_none()); + } + + #[test] + fn native_title_match_recovers_pid_without_replacing_native_id() { + let native = vec![window(77, None, "CuaTestHarness")]; + let mut accessible = window(123 << 16, Some(123), "CuaTestHarness"); + accessible.x = 20; + accessible.y = 30; + accessible.width = 800; + accessible.height = 600; + let enriched = enrich_native_windows(native, vec![accessible], false); + assert_eq!(enriched[0].xid, 77); + assert_eq!(enriched[0].pid, Some(123)); + assert_eq!( + (enriched[0].x, enriched[0].y, enriched[0].width, enriched[0].height), + (20, 30, 800, 600) + ); + } + + #[test] + fn nested_enrichment_adopts_stable_atspi_id() { + let native = vec![window(77, None, "CuaTestHarness")]; + let accessible = window(123 << 16, Some(123), "CuaTestHarness"); + let enriched = enrich_native_windows(native, vec![accessible], true); + assert_eq!(enriched[0].xid, 123 << 16); + assert_eq!(enriched[0].pid, Some(123)); + assert_eq!(identity_for(enriched[0].xid).unwrap().title, "CuaTestHarness"); + } + + #[test] + fn sway_window_capture_is_cropped_to_compositor_geometry() { + let source = image::DynamicImage::ImageRgba8(image::RgbaImage::from_pixel( + 8, + 6, + image::Rgba([20, 40, 60, 255]), + )); + let mut encoded = std::io::Cursor::new(Vec::new()); + source + .write_to(&mut encoded, image::ImageFormat::Png) + .expect("encode fixture PNG"); + let cropped = crop_png_to_rect(encoded.get_ref(), 2, 1, 3, 4, "fixture") + .expect("crop fixture PNG"); + let decoded = image::load_from_memory(&cropped).expect("decode cropped PNG"); + assert_eq!((decoded.width(), decoded.height()), (3, 4)); + } + + #[test] + fn injectable_text_accepts_printable_ascii_newline_and_tab() { + validate_injectable_text("Hello, World! 123 @#$%\t\n").expect("printable ASCII is typable"); + // The full printable ASCII span the compositor chartab covers. + let printable: String = (0x20u8..=0x7e).map(|b| b as char).collect(); + validate_injectable_text(&printable).expect("every printable ASCII byte is typable"); + } + + #[test] + fn injectable_text_rejects_unicode_and_other_controls() { + for bad in ["café", "emoji 😀", "bell\u{07}", "null\u{00}", "delete\u{7f}", "cr\r"] { + assert!( + validate_injectable_text(bad).is_err(), + "{bad:?} must be rejected before it reaches the compositor" + ); + } + } + + #[test] + fn injectable_key_accepts_whitelist_case_insensitively() { + for good in ["enter", "Enter", "RETURN", "tab", "Escape", "esc", "space", "up", "Left", "f1", "F12"] { + validate_injectable_key(good).unwrap_or_else(|e| panic!("{good:?} should pass: {e}")); + } + } + + #[test] + fn injectable_key_rejects_unsupported_names() { + for bad in ["f13", "ctrl", "a", "delete", "home", "pageup", ""] { + assert!( + validate_injectable_key(bad).is_err(), + "{bad:?} is not in the compositor whitelist" + ); + } + } + + #[test] + fn injectable_hotkey_normalizes_supported_chords() { + let keys = vec!["control".to_owned(), "SHIFT".to_owned(), "7".to_owned()]; + assert_eq!( + validate_injectable_hotkey(&keys).expect("supported chord"), + ("ctrl,shift".to_owned(), "7".to_owned()) + ); + assert!(validate_injectable_hotkey(&["7".to_owned()]).is_err()); + assert!(validate_injectable_hotkey(&["hyper".to_owned(), "k".to_owned()]).is_err()); + } + + #[test] + fn hello_reply_parses_exact_banner_and_rejects_mismatch() { + parse_inject_hello("cua-inject v1\n").expect("verbatim banner is accepted"); + parse_inject_hello(" cua-inject v1 ").expect("surrounding whitespace is tolerated"); + assert!(parse_inject_hello("cua-inject v2").is_err()); + assert!(parse_inject_hello("err unsupported-version").is_err()); + assert!(parse_inject_hello("garbage").is_err()); + } + + #[test] + fn command_reply_parses_ok_and_surfaces_error_reason() { + parse_inject_reply("ok\n").expect("ok is success"); + parse_inject_reply("ok").expect("ok without newline is success"); + let err = parse_inject_reply("err ambiguous-app-id\n").unwrap_err(); + assert!(err.to_string().contains("ambiguous-app-id")); + assert!(parse_inject_reply("err").is_err()); + assert!(parse_inject_reply("maybe").is_err()); + } + + #[test] + fn geometry_reply_is_strict_and_signed() { + assert_eq!( + parse_inject_geometry("geometry -4 23 10 20\n").unwrap(), + ((-4, 23), (10, 20)) + ); + assert!(parse_inject_geometry("geometry 1").is_err()); + assert!(parse_inject_geometry("err target-not-found").is_err()); + assert!(parse_inject_geometry("ok").is_err()); + } +} diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/persistent_vptr.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/persistent_vptr.rs index 4b9df956a9..9ee2718e12 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/persistent_vptr.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/persistent_vptr.rs @@ -147,7 +147,7 @@ fn handle_press( // Keep the (out_w, out_h) but drop the queue + state at end of scope; the // vptr itself remains alive (Wayland objects survive their original queue // as long as the Connection is alive). - let mut sess = open_vptr_session(Some(window_id as u32))?; + let mut sess = open_vptr_session(Some(window_id))?; let (w, h) = (sess.output_w, sess.output_h); let px = x.clamp(0, w as i32 - 1) as u32; let py = y.clamp(0, h as i32 - 1) as u32; diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/portal_screenshot.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/portal_screenshot.rs index c1f1afbc0a..887b4f8dac 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/portal_screenshot.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/portal_screenshot.rs @@ -27,6 +27,15 @@ use ashpd::desktop::screenshot::Screenshot; /// on the first call per session unless they've pre-approved the /// requesting binary in their DE's privacy settings. pub fn screenshot_via_portal() -> anyhow::Result<Vec<u8>> { + if tokio::runtime::Handle::try_current().is_ok() { + return std::thread::spawn(screenshot_via_portal_blocking) + .join() + .map_err(|_| anyhow::anyhow!("xdg-desktop-portal Screenshot worker panicked"))?; + } + screenshot_via_portal_blocking() +} + +fn screenshot_via_portal_blocking() -> anyhow::Result<Vec<u8>> { let rt = tokio::runtime::Builder::new_current_thread() .enable_all() .build() diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs index 076be30a47..4ec0524135 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/shell_helper.rs @@ -23,11 +23,22 @@ use std::process::Command; use std::time::Duration; +use crate::x11::WindowInfo; + const DEST: &str = "org.cua.WinRects"; const PATH: &str = "/org/cua/WinRects"; const IFACE: &str = "org.cua.WinRects"; +pub fn available() -> bool { + static AVAILABLE: std::sync::OnceLock<bool> = std::sync::OnceLock::new(); + *AVAILABLE.get_or_init(|| gdbus_call("GetRects", &[]).is_some()) +} + fn gdbus_call(method: &str, args: &[String]) -> Option<String> { + gdbus_call_with_timeout(method, args, Duration::from_millis(800)) +} + +fn gdbus_call_with_timeout(method: &str, args: &[String], timeout: Duration) -> Option<String> { let mut cmd = Command::new("gdbus"); cmd.arg("call") .arg("--session") @@ -47,34 +58,85 @@ fn gdbus_call(method: &str, args: &[String]) -> Option<String> { .stderr(std::process::Stdio::null()) .spawn() .ok()?; - let out = wait_timeout(child, Duration::from_millis(800))?; + let out = wait_timeout(child, timeout)?; if !out.status.success() { return None; } Some(String::from_utf8_lossy(&out.stdout).into_owned()) } +/// Capture the GNOME stage through the compositor helper. +/// +/// Mutter does not expose wlroots screencopy protocols, and its one-shot +/// Screenshot portal may reject an unregistered command-line process. The +/// opt-in helper already runs inside Shell for geometry and activation, so it +/// can use Shell's screenshot API without confusing a stable Wayland window id +/// for an X11 drawable. +pub fn screenshot_display() -> Option<Vec<u8>> { + use base64::{engine::general_purpose::STANDARD as B64, Engine as _}; + + let raw = gdbus_call_with_timeout("Capture", &[], Duration::from_secs(5))?; + let start = raw.find('\'')? + 1; + let end = raw.rfind('\'')?; + if end <= start { + return None; + } + B64.decode(&raw[start..end]).ok() +} + /// `Child::wait` with a deadline (no extra crates). Kills + reaps on timeout. fn wait_timeout(mut child: std::process::Child, dur: Duration) -> Option<std::process::Output> { + use std::io::Read; + + // Drain stdout while the child is running. Capture() returns a base64 PNG + // that readily exceeds a pipe's ~64 KiB capacity; waiting for exit before + // reading deadlocks the child on a full pipe and turns a healthy Shell + // response into a false timeout. + let stdout = child.stdout.take()?; + let reader = std::thread::spawn(move || { + let mut stdout = stdout; + let mut bytes = Vec::new(); + stdout.read_to_end(&mut bytes).ok()?; + Some(bytes) + }); let deadline = std::time::Instant::now() + dur; - loop { + let status = loop { match child.try_wait() { - Ok(Some(_)) => return child.wait_with_output().ok(), + Ok(Some(status)) => break status, Ok(None) => { if std::time::Instant::now() >= deadline { let _ = child.kill(); - let _ = child.wait(); + let status = child.wait().ok()?; + let _ = reader.join(); + if !status.success() { + return None; + } return None; } std::thread::sleep(Duration::from_millis(15)); } - Err(_) => return None, + Err(_) => { + let _ = child.kill(); + let _ = child.wait(); + let _ = reader.join(); + return None; + } } - } + }; + let stdout = reader.join().ok().flatten()?; + Some(std::process::Output { + status, + stdout, + stderr: Vec::new(), + }) } -/// Screen origin (x, y) of the window backing `pid`, from the extension's -/// `GetRects`. `None` when the extension is unavailable or no window matches. +/// Screen origin of the Wayland surface buffer backing `pid`. +/// +/// GTK's AT-SPI `CoordType::Window` includes client-side shadow extents, while +/// Mutter's frame rectangle excludes them. The buffer origin preserves those +/// extents so accessibility frames line up with pixels. Older helpers omit the +/// buffer fields and fall back to the frame origin. pub fn window_origin_for_pid(pid: u32) -> Option<(i32, i32)> { let raw = gdbus_call("GetRects", &[])?; // gdbus prints a GVariant tuple like `('[{"pid":..,"x":..}]',)`. Pull the @@ -86,14 +148,124 @@ pub fn window_origin_for_pid(pid: u32) -> Option<(i32, i32)> { let arr: Vec<serde_json::Value> = serde_json::from_str(json).ok()?; for w in &arr { if w.get("pid").and_then(|p| p.as_u64()) == Some(pid as u64) { - let x = w.get("x")?.as_i64()? as i32; - let y = w.get("y")?.as_i64()? as i32; + let x = w + .get("buffer_x") + .and_then(serde_json::Value::as_i64) + .or_else(|| w.get("x").and_then(serde_json::Value::as_i64))? + as i32; + let y = w + .get("buffer_y") + .and_then(serde_json::Value::as_i64) + .or_else(|| w.get("y").and_then(serde_json::Value::as_i64))? + as i32; return Some((x, y)); } } None } +/// Enumerate GNOME Shell toplevels when the compositor helper is available. +/// +/// AT-SPI remains the source of accessibility elements, but it is a poor +/// source of truth for desktop window discovery: one unresponsive application +/// can exhaust the bounded registry walk and hide every healthy toplevel. The +/// shell already owns the authoritative stacking list, geometry, visibility, +/// title, and PID, so use that metadata directly for `list_windows`. +pub fn list_windows(filter_pid: Option<u32>) -> Option<Vec<WindowInfo>> { + let raw = gdbus_call("GetRects", &[])?; + parse_windows(&raw, filter_pid) +} + +/// Ask GNOME Shell to focus and raise one stable-sequence window. +/// +/// Returns `false` when the helper is absent, the id is unknown, or Shell did +/// not confirm focus. Callers must not inject global libei input unless this +/// returns true: portal input is focus-bound and otherwise targets whichever +/// application the user happened to be using. +pub fn activate_window(window_id: u64) -> bool { + let Ok(window_id) = u32::try_from(window_id) else { + return false; + }; + let accepted = gdbus_call("Activate", &[window_id.to_string()]) + .is_some_and(|output| output.trim_start().starts_with("(true,")); + if !accepted { + return false; + } + std::thread::sleep(Duration::from_millis(60)); + window_is_focused(window_id) +} + +fn window_is_focused(window_id: u32) -> bool { + let Some(raw) = gdbus_call("GetRects", &[]) else { + return false; + }; + let (Some(start), Some(end)) = (raw.find('['), raw.rfind(']')) else { + return false; + }; + serde_json::from_str::<Vec<serde_json::Value>>(&raw[start..=end]) + .ok() + .and_then(|windows| { + windows.into_iter().find(|window| { + window.get("id").and_then(serde_json::Value::as_u64) == Some(window_id as u64) + }) + }) + .and_then(|window| window.get("focused").and_then(serde_json::Value::as_bool)) + .unwrap_or(false) +} + +fn parse_windows(raw: &str, filter_pid: Option<u32>) -> Option<Vec<WindowInfo>> { + let start = raw.find('[')?; + let end = raw.rfind(']')?; + let windows: Vec<serde_json::Value> = serde_json::from_str(&raw[start..=end]).ok()?; + + Some( + windows + .into_iter() + .filter_map(|window| { + let pid = u32::try_from(window.get("pid")?.as_u64()?).ok()?; + if filter_pid.is_some_and(|wanted| wanted != pid) { + return None; + } + let id = window.get("id")?.as_u64()?.max(1); + let x = i32::try_from(window.get("x")?.as_i64()?).ok()?; + let y = i32::try_from(window.get("y")?.as_i64()?).ok()?; + let width = u32::try_from(window.get("w")?.as_u64()?).ok()?; + let height = u32::try_from(window.get("h")?.as_u64()?).ok()?; + let visible = window + .get("visible") + .and_then(serde_json::Value::as_bool) + .unwrap_or(width > 0 && height > 0); + let minimized = window + .get("minimized") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false); + let title = window + .get("title") + .and_then(serde_json::Value::as_str) + .unwrap_or_default() + .to_owned(); + let z_index = window + .get("stacking") + .and_then(serde_json::Value::as_u64) + .and_then(|value| usize::try_from(value).ok()); + + Some(WindowInfo { + xid: id, + pid: Some(pid), + app_name: title.clone(), + title, + is_on_screen: visible && !minimized && width > 0 && height > 0, + z_index, + x, + y, + width, + height, + }) + }) + .collect(), + ) +} + /// Glide the agent cursor to screen `(x, y)`. pub fn move_cursor(x: i32, y: i32) { let _ = gdbus_call("MoveCursor", &[x.to_string(), y.to_string()]); @@ -108,3 +280,30 @@ pub fn click_pulse(x: i32, y: i32) { pub fn hide_cursor() { let _ = gdbus_call("HideCursor", &[]); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_and_filters_shell_windows() { + let raw = r#"('[{"id":46,"pid":6079,"title":"Sentinel's window","x":66,"y":32,"w":958,"h":736,"focused":true,"minimized":false,"visible":true,"stacking":2},{"id":47,"pid":6080,"title":"Hidden","x":0,"y":0,"w":100,"h":100,"minimized":true,"visible":false,"stacking":1}]',)"#; + let windows = parse_windows(raw, Some(6079)).expect("valid helper response"); + assert_eq!(windows.len(), 1); + assert_eq!(windows[0].xid, 46); + assert_eq!(windows[0].pid, Some(6079)); + assert_eq!(windows[0].title, "Sentinel's window"); + assert_eq!((windows[0].x, windows[0].y), (66, 32)); + assert_eq!((windows[0].width, windows[0].height), (958, 736)); + assert!(windows[0].is_on_screen); + assert_eq!(windows[0].z_index, Some(2)); + } + + #[test] + fn marks_minimized_shell_windows_off_screen() { + let raw = r#"('[{"id":47,"pid":6080,"title":"Hidden","x":0,"y":0,"w":100,"h":100,"minimized":true,"visible":false,"stacking":1}]',)"#; + let windows = parse_windows(raw, None).expect("valid helper response"); + assert_eq!(windows.len(), 1); + assert!(!windows[0].is_on_screen); + } +} diff --git a/libs/cua-driver/rust/crates/platform-linux/src/wayland/sway_ipc.rs b/libs/cua-driver/rust/crates/platform-linux/src/wayland/sway_ipc.rs new file mode 100644 index 0000000000..dea7a6ac51 --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-linux/src/wayland/sway_ipc.rs @@ -0,0 +1,173 @@ +//! Best-effort Sway/i3-compatible compositor metadata. +//! +//! The wlroots foreign-toplevel protocol exposes titles and app ids, but not +//! process ids or geometry. Sway's IPC tree supplies those missing fields and +//! a compositor-stable container id. Other compositors simply return no data. + +use std::process::{Command, Stdio}; + +use serde::Deserialize; + +#[derive(Clone, Debug, Default, Deserialize)] +struct Rect { + x: i32, + y: i32, + width: i32, + height: i32, +} + +#[derive(Clone, Debug, Default, Deserialize)] +struct Node { + id: u64, + #[serde(default)] + name: String, + #[serde(default)] + app_id: String, + pid: Option<u32>, + #[serde(default)] + rect: Rect, + #[serde(default)] + focused: bool, + #[serde(default = "default_visible")] + visible: bool, + #[serde(default)] + fullscreen_mode: i32, + #[serde(default)] + nodes: Vec<Node>, + #[serde(default)] + floating_nodes: Vec<Node>, +} + +fn default_visible() -> bool { + true +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Window { + pub id: u64, + pub pid: u32, + pub title: String, + pub app_id: String, + pub x: i32, + pub y: i32, + pub width: u32, + pub height: u32, + pub focused: bool, + pub visible: bool, + pub fullscreen: bool, +} + +fn collect(node: &Node, windows: &mut Vec<Window>) { + if let Some(pid) = node.pid { + if !node.name.is_empty() || !node.app_id.is_empty() { + windows.push(Window { + id: node.id, + pid, + title: node.name.clone(), + app_id: node.app_id.clone(), + x: node.rect.x, + y: node.rect.y, + width: node.rect.width.max(0) as u32, + height: node.rect.height.max(0) as u32, + focused: node.focused, + visible: node.visible, + fullscreen: node.fullscreen_mode != 0, + }); + } + } + for child in node.nodes.iter().chain(&node.floating_nodes) { + collect(child, windows); + } +} + +fn parse_tree(bytes: &[u8]) -> Option<Vec<Window>> { + let root: Node = serde_json::from_slice(bytes).ok()?; + let mut windows = Vec::new(); + collect(&root, &mut windows); + Some(windows) +} + +pub fn list_windows() -> Option<Vec<Window>> { + if std::env::var_os("SWAYSOCK").is_none() { + return None; + } + let output = Command::new("swaymsg") + .args(["-r", "-t", "get_tree"]) + .stdin(Stdio::null()) + .stderr(Stdio::null()) + .output() + .ok()?; + if !output.status.success() { + return None; + } + parse_tree(&output.stdout) +} + +pub fn window_for_id(id: u64) -> Option<Window> { + list_windows()?.into_iter().find(|window| window.id == id) +} + +pub fn window_origin_for_pid(pid: u32) -> Option<(i32, i32)> { + let window = list_windows()? + .into_iter() + .filter(|window| window.pid == pid && window.width > 0 && window.height > 0) + .max_by_key(|window| { + ( + window.focused, + window.visible, + u64::from(window.width) * u64::from(window.height), + ) + })?; + Some((window.x, window.y)) +} + +pub fn window_origin_for_title(title: &str) -> Option<(i32, i32)> { + let window = list_windows()? + .into_iter() + .filter(|window| { + window.width > 0 + && window.height > 0 + && (window.title == title + || (!window.title.is_empty() && title.starts_with(&window.title))) + }) + .max_by_key(|window| (window.focused, window.visible))?; + Some((window.x, window.y)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_nested_and_floating_windows() { + let tree = br#"{ + "id": 1, + "nodes": [{ + "id": 2, + "nodes": [{ + "id": 10, + "name": "Editor", + "app_id": "org.example.Editor", + "pid": 123, + "rect": {"x": 20, "y": 30, "width": 800, "height": 600}, + "focused": true, + "visible": true, + "fullscreen_mode": 1 + }], + "floating_nodes": [{ + "id": 11, + "name": "Dialog", + "pid": 124, + "rect": {"x": 100, "y": 120, "width": 300, "height": 200} + }] + }] + }"#; + let windows = parse_tree(tree).expect("parse Sway tree"); + assert_eq!(windows.len(), 2); + assert_eq!(windows[0].pid, 123); + assert_eq!((windows[0].x, windows[0].y), (20, 30)); + assert!(windows[0].focused); + assert!(windows[0].fullscreen); + assert_eq!(windows[1].title, "Dialog"); + } +} diff --git a/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs b/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs index 1dfa5f0985..d13de7967b 100644 --- a/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs +++ b/libs/cua-driver/rust/crates/platform-linux/src/x11/mod.rs @@ -13,7 +13,10 @@ pub struct WindowInfo { /// X11 Window (XID) cast to u64. pub xid: u64, pub pid: Option<u32>, + pub app_name: String, pub title: String, + pub is_on_screen: bool, + pub z_index: Option<usize>, pub x: i32, pub y: i32, pub width: u32, @@ -37,7 +40,7 @@ fn list_windows_inner(filter_pid: Option<u32>) -> Result<Vec<WindowInfo>> { let windows = get_window_list(&conn, root)?; let mut result = Vec::new(); - for xid in windows { + for (z_index, xid) in windows.into_iter().enumerate() { let pid = get_window_pid(&conn, xid).ok().flatten(); if let Some(fp) = filter_pid { if pid != Some(fp) { continue; } @@ -45,6 +48,14 @@ fn list_windows_inner(filter_pid: Option<u32>) -> Result<Vec<WindowInfo>> { let title = get_window_title(&conn, xid).unwrap_or_default(); if title.trim().is_empty() { continue; } + let app_name = get_window_class(&conn, xid) + .map(|(instance, class)| if class.is_empty() { instance } else { class }) + .unwrap_or_default(); + let is_on_screen = conn + .get_window_attributes(xid) + .ok() + .and_then(|cookie| cookie.reply().ok()) + .is_some_and(|attributes| attributes.map_state == MapState::VIEWABLE); let geom = conn.get_geometry(xid)?.reply().ok(); let (x, y, w, h) = if let Some(g) = geom { @@ -56,7 +67,18 @@ fn list_windows_inner(filter_pid: Option<u32>) -> Result<Vec<WindowInfo>> { (0, 0, 0, 0) }; - result.push(WindowInfo { xid: xid as u64, pid, title, x, y, width: w, height: h }); + result.push(WindowInfo { + xid: xid as u64, + pid, + app_name, + title, + is_on_screen, + z_index: Some(z_index), + x, + y, + width: w, + height: h, + }); } Ok(result) @@ -70,15 +92,36 @@ fn get_window_list(conn: &RustConnection, root: Window) -> Result<Vec<Window>> { let windows: Vec<Window> = reply.value32() .map(|iter| iter.collect()) .unwrap_or_default(); - if !windows.is_empty() { + if client_list_property(reply.type_, windows.as_slice()).is_some() { return Ok(windows); } } } } - // Fallback: query tree from root. + + // No EWMH client-list property means there may be no window manager. In + // that case only expose mapped root children; unmapped Electron children + // can otherwise be reported before a late-starting WM reparents them. let tree = conn.query_tree(root)?.reply()?; - Ok(tree.children) + Ok(tree + .children + .into_iter() + .filter(|window| { + conn.get_window_attributes(*window) + .ok() + .and_then(|cookie| cookie.reply().ok()) + .map(|attributes| fallback_window_is_listable(attributes.map_state)) + .unwrap_or(false) + }) + .collect()) +} + +fn client_list_property(property_type: Atom, windows: &[Window]) -> Option<&[Window]> { + (property_type != x11rb::NONE).then_some(windows) +} + +fn fallback_window_is_listable(map_state: MapState) -> bool { + map_state == MapState::VIEWABLE } fn get_atom(conn: &RustConnection, name: &str) -> Result<Atom> { @@ -119,6 +162,10 @@ fn get_window_title(conn: &RustConnection, window: Window) -> Result<String> { /// WM_CLASS atom set, or the property could not be read. pub fn wm_class_for_window(xid: u64) -> Option<(String, String)> { let (conn, _) = RustConnection::connect(None).ok()?; + get_window_class(&conn, xid as u32) +} + +fn get_window_class(conn: &RustConnection, xid: Window) -> Option<(String, String)> { let reply = conn .get_property(false, xid as u32, AtomEnum::WM_CLASS, AtomEnum::STRING, 0, 512) .ok()? @@ -133,3 +180,25 @@ pub fn wm_class_for_window(xid: u64) -> Option<(String, String)> { } Some((instance, class)) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn empty_present_client_list_does_not_fall_back_to_query_tree() { + assert_eq!(client_list_property(1, &[]), Some([].as_slice())); + } + + #[test] + fn absent_client_list_allows_query_tree_fallback() { + assert_eq!(client_list_property(x11rb::NONE, &[]), None); + } + + #[test] + fn query_tree_fallback_only_lists_viewable_windows() { + assert!(fallback_window_is_listable(MapState::VIEWABLE)); + assert!(!fallback_window_is_listable(MapState::UNMAPPED)); + assert!(!fallback_window_is_listable(MapState::UNVIEWABLE)); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs index b4e4c8902d..5391d26228 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs @@ -3,7 +3,12 @@ //! We call the C-level AX API directly rather than using a crate wrapper, //! because most available crates are incomplete or unmaintained. -#![allow(non_upper_case_globals, non_camel_case_types, non_snake_case, dead_code)] +#![allow( + non_upper_case_globals, + non_camel_case_types, + non_snake_case, + dead_code +)] use core_foundation::{ array::CFArrayRef, @@ -54,14 +59,14 @@ extern "C" { element: AXUIElementRef, names: *mut CFArrayRef, ) -> AXError; - pub fn AXUIElementCopyActionNames( - element: AXUIElementRef, - names: *mut CFArrayRef, - ) -> AXError; - pub fn AXUIElementPerformAction( - element: AXUIElementRef, - action: CFStringRef, + pub fn AXUIElementCopyActionNames(element: AXUIElementRef, names: *mut CFArrayRef) -> AXError; + pub fn AXUIElementCopyElementAtPosition( + application: AXUIElementRef, + x: f32, + y: f32, + element: *mut AXUIElementRef, ) -> AXError; + pub fn AXUIElementPerformAction(element: AXUIElementRef, action: CFStringRef) -> AXError; pub fn AXUIElementSetAttributeValue( element: AXUIElementRef, attribute: CFStringRef, @@ -73,27 +78,42 @@ extern "C" { /// `{kAXTrustedCheckOptionPrompt: true}` raises the system Accessibility /// prompt if the process isn't already trusted. Returns the post-prompt /// trust state (may still be false if the user dismissed the prompt). - pub fn AXIsProcessTrustedWithOptions(options: core_foundation::dictionary::CFDictionaryRef) -> bool; + pub fn AXIsProcessTrustedWithOptions( + options: core_foundation::dictionary::CFDictionaryRef, + ) -> bool; /// Private SPI: maps an AX window element to its CGWindowID. /// Stable since macOS 10.9; used by yabai, Hammerspoon, Accessibility Inspector. pub fn _AXUIElementGetWindow(element: AXUIElementRef, window_id: *mut u32) -> AXError; } +/// Hit-test one process's accessibility tree at a screen point. The returned +/// element is retained and must be released by the caller. +pub unsafe fn element_at_screen_position(pid: i32, x: f64, y: f64) -> Option<AXUIElementRef> { + let application = AXUIElementCreateApplication(pid); + if application.is_null() { + return None; + } + let mut element = std::ptr::null_mut(); + let error = AXUIElementCopyElementAtPosition(application, x as f32, y as f32, &mut element); + CFRelease(application as CFTypeRef); + (error == kAXErrorSuccess && !element.is_null()).then_some(element) +} + // ── AXValue functions ──────────────────────────────────────────────────────── #[link(name = "ApplicationServices", kind = "framework")] extern "C" { pub fn AXValueGetType(value: AXValueRef) -> AXValueType; - pub fn AXValueGetValue(value: AXValueRef, the_type: AXValueType, value_ptr: *mut c_void) -> bool; + pub fn AXValueGetValue( + value: AXValueRef, + the_type: AXValueType, + value_ptr: *mut c_void, + ) -> bool; } // ── Helper functions ────────────────────────────────────────────────────────── -use core_foundation::{ - array::CFArray, - base::TCFType, - string::CFString as CFStr, -}; +use core_foundation::{array::CFArray, base::TCFType, string::CFString as CFStr}; /// Copy a string attribute from an AX element. Returns `None` on any error. pub unsafe fn copy_string_attr(element: AXUIElementRef, attr_name: &str) -> Option<String> { @@ -162,7 +182,10 @@ pub unsafe fn element_screen_center(element: AXUIElementRef) -> Option<(f64, f64 return None; } #[repr(C)] - struct CGPoint { x: f64, y: f64 } + struct CGPoint { + x: f64, + y: f64, + } let mut pos = CGPoint { x: 0.0, y: 0.0 }; let ok = AXValueGetValue( pos_ref as AXValueRef, @@ -170,7 +193,9 @@ pub unsafe fn element_screen_center(element: AXUIElementRef) -> Option<(f64, f64 &mut pos as *mut _ as *mut std::ffi::c_void, ); CFRelease(pos_ref); - if !ok { return None; } + if !ok { + return None; + } // AXSize → CGSize let sz_attr = CFStr::new("AXSize"); @@ -180,7 +205,10 @@ pub unsafe fn element_screen_center(element: AXUIElementRef) -> Option<(f64, f64 return None; } #[repr(C)] - struct CGSize { w: f64, h: f64 } + struct CGSize { + w: f64, + h: f64, + } let mut sz = CGSize { w: 0.0, h: 0.0 }; let ok2 = AXValueGetValue( sz_ref as AXValueRef, @@ -188,7 +216,9 @@ pub unsafe fn element_screen_center(element: AXUIElementRef) -> Option<(f64, f64 &mut sz as *mut _ as *mut std::ffi::c_void, ); CFRelease(sz_ref); - if !ok2 || sz.w < 1.0 || sz.h < 1.0 { return None; } + if !ok2 || sz.w < 1.0 || sz.h < 1.0 { + return None; + } Some((pos.x + sz.w / 2.0, pos.y + sz.h / 2.0)) } @@ -204,7 +234,10 @@ pub unsafe fn element_screen_rect(element: AXUIElementRef) -> Option<[f64; 4]> { return None; } #[repr(C)] - struct CGPoint { x: f64, y: f64 } + struct CGPoint { + x: f64, + y: f64, + } let mut pos = CGPoint { x: 0.0, y: 0.0 }; let ok = AXValueGetValue( pos_ref as AXValueRef, @@ -212,7 +245,9 @@ pub unsafe fn element_screen_rect(element: AXUIElementRef) -> Option<[f64; 4]> { &mut pos as *mut _ as *mut std::ffi::c_void, ); CFRelease(pos_ref); - if !ok { return None; } + if !ok { + return None; + } // AXSize → CGSize let sz_attr = CFStr::new("AXSize"); @@ -222,7 +257,10 @@ pub unsafe fn element_screen_rect(element: AXUIElementRef) -> Option<[f64; 4]> { return None; } #[repr(C)] - struct CGSize { w: f64, h: f64 } + struct CGSize { + w: f64, + h: f64, + } let mut sz = CGSize { w: 0.0, h: 0.0 }; let ok2 = AXValueGetValue( sz_ref as AXValueRef, @@ -230,7 +268,9 @@ pub unsafe fn element_screen_rect(element: AXUIElementRef) -> Option<[f64; 4]> { &mut sz as *mut _ as *mut std::ffi::c_void, ); CFRelease(sz_ref); - if !ok2 || sz.w < 1.0 || sz.h < 1.0 { return None; } + if !ok2 || sz.w < 1.0 || sz.h < 1.0 { + return None; + } Some([pos.x, pos.y, sz.w, sz.h]) } @@ -287,6 +327,25 @@ pub unsafe fn copy_children(element: AXUIElementRef) -> Vec<AXUIElementRef> { .collect() } +/// Copy an AX element-valued attribute. The returned element is retained and +/// must be released by the caller. +pub unsafe fn copy_element_attr( + element: AXUIElementRef, + attr_name: &str, +) -> Option<AXUIElementRef> { + let attr = CFStr::new(attr_name); + let mut value: CFTypeRef = std::ptr::null(); + let err = AXUIElementCopyAttributeValue(element, attr.as_concrete_TypeRef(), &mut value); + if err != kAXErrorSuccess || value.is_null() { + return None; + } + if core_foundation::base::CFGetTypeID(value) != AXUIElementGetTypeID() { + CFRelease(value); + return None; + } + Some(value as AXUIElementRef) +} + /// Perform an AX action using a string attribute name. pub unsafe fn perform_action(element: AXUIElementRef, action_name: &str) -> AXError { let action = CFStr::new(action_name); @@ -352,7 +411,11 @@ pub unsafe fn enable_chromium_accessibility(app_element: AXUIElementRef) -> bool pub unsafe fn ax_get_window_id(element: AXUIElementRef) -> Option<u32> { let mut wid: u32 = 0; let err = _AXUIElementGetWindow(element, &mut wid); - if err == kAXErrorSuccess && wid != 0 { Some(wid) } else { None } + if err == kAXErrorSuccess && wid != 0 { + Some(wid) + } else { + None + } } /// Read the `AXWindows` attribute of an application element. diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs index 47c30f9775..83a01a78a1 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/input/mouse.rs @@ -202,9 +202,8 @@ fn click_at_xy_inner( /// Full Chromium-compatible left-click recipe matching Swift's `clickViaAuthSignedPost`. /// -/// The focus-without-raise prologue makes the target window key without changing -/// its z-order, which Chromium requires before it accepts a background pixel -/// mouseDown. The cursor overlay is re-pinned by the click tool after dispatch. +/// The sequence stays PID/window-routed throughout. It must not make the target +/// key: changing key-window ownership violates background delivery. /// 1. Stamped `mouseMoved` at target coords (f0=2, cursor-state primer). /// 2. Off-screen primer down/up at (-1, -1) (f0=1/2) — satisfies Chromium's /// user-activation gate without hitting any DOM element. @@ -234,13 +233,6 @@ pub fn click_at_xy_chromium( ) -> anyhow::Result<()> { use std::time::{SystemTime, UNIX_EPOCH}; - // Chromium's first-mouse handling rejects a background click delivered to - // a non-key window. This SkyLight focus record keys the requested window - // without raising it or moving the user's cursor. - if crate::input::skylight::activate_without_raise(pid as libc::pid_t, wid) { - std::thread::sleep(std::time::Duration::from_millis(50)); - } - let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState) .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?; let target = CGPoint::new(screen_x, screen_y); diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs index f18b8a36c8..7cef8a554f 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/click.rs @@ -23,11 +23,12 @@ use std::sync::Arc; use crate::apps; use crate::ax::bindings::{ - copy_action_names, copy_children, copy_string_attr, element_screen_rect, AXUIElementRef, + copy_action_names, copy_children, copy_string_attr, element_at_screen_position, + element_screen_rect, kAXErrorSuccess, AXUIElementPerformAction, AXUIElementRef, }; use crate::focus_guard; use crate::window_change_detector::WindowChangeDetector; -use core_foundation::base::CFRelease; +use core_foundation::base::{CFRelease, TCFType}; use super::ToolState; @@ -111,7 +112,7 @@ fn def() -> &'static ToolDef { "delivery_mode": { "type": "string", "enum": ["background", "foreground"], - "description": "Best-effort-background ladder rung for a PIXEL click (default \"background\"). \"background\": post the CGEvent to the pid without fronting. \"foreground\": briefly front the window, click, restore the prior frontmost — the explicit last resort for surfaces that drop background synthetic clicks. Requires window_id. A click is never driver-verifiable (no read-back), so both report verified:false — confirm the effect via screenshot. Use the agent loop: background AX (element_index) → screenshot → background pixel (x/y) → screenshot → delivery_mode:\"foreground\"." + "description": "Best-effort-background ladder rung (default \"background\"). \"background\": perform the AX action or post the CGEvent without fronting. \"foreground\": briefly front the window, act, let transient UI settle, then restore the prior frontmost app. Requires window_id. A click is never driver-verifiable (no read-back), so both report verified:false — confirm the effect via screenshot. Use the agent loop: background AX (element_index) → screenshot → background pixel (x/y) → screenshot → delivery_mode:\"foreground\"." }, "scope": { "type": "string", @@ -224,85 +225,13 @@ impl Tool for ClickTool { .cursor_registry .update_position(&cursor_key, sx, sy); - // Resolve the frontmost on-screen window under the point (the macOS - // peer of Windows' WindowFromPoint). When found, click THAT pid via - // the proven SkyLight path (`click_at_xy`, screen coords) — reliable - // on AppKit/Chromium where a bare HID post can miss. Only when no - // app window owns the pixel (desktop background, etc.) fall back to - // the cursor-warp + HID post. - // Resolve as (pid, window_id, win_origin_x, win_origin_y) so the - // click can stamp the window-LOCAL point — AppKit hit-tests the - // stamped window-local coordinate, not the bare screen point, so a - // plain screen-coord post misses. - // Exclude our OWN windows (the agent-cursor overlay we just glided to - // the point sits on top of the target — never resolve the click to it). - let own_pid = std::process::id() as i32; - let target = { - let mut wins = crate::windows::visible_windows(); - // visible_windows() assigns HIGHER z_index = MORE FRONT - // (z_index = total - idx over CGWindowList's front-to-back order). - // Sort DESCENDING so the first match is the FRONTMOST window under - // the point — the one the agent actually sees in the screenshot. - // (Ascending picked the BACKMOST occluded window — a real miss when - // windows overlap, e.g. resolving a click to a buried app.) - wins.sort_by(|a, b| b.z_index.cmp(&a.z_index)); // front-to-back - wins.into_iter() - .find(|w| { - w.layer == 0 - && w.pid != own_pid - && sx >= w.bounds.x - && sx < w.bounds.x + w.bounds.width - && sy >= w.bounds.y - && sy < w.bounds.y + w.bounds.height - }) - .map(|w| (w.pid, w.window_id, w.bounds.x, w.bounds.y)) - }; let btn = button.clone(); - let result = tokio::task::spawn_blocking(move || -> anyhow::Result<Option<i32>> { - match target { - Some((pid, wid, ox, oy)) => { - let (wx, wy) = (sx - ox, sy - oy); - // Honor `btn` on the window-resolved path too: a windowless - // right/middle click over an app window must stay a - // right/middle click, not silently degrade to left. Route to - // the window-local right/middle primitives (single-pair, same - // as the pixel path); `count` only repeats on the left path. - match btn.as_str() { - "right" => crate::input::mouse::right_click_at_xy_with_window_local( - pid, - sx, - sy, - wx, - wy, - wid, - &[], - )?, - "middle" => crate::input::mouse::middle_click_at_xy_with_window_local( - pid, - sx, - sy, - wx, - wy, - &[], - )?, - _ => crate::input::mouse::click_at_xy_with_window_local( - pid, - sx, - sy, - wx, - wy, - wid, - count, - &[], - )?, - } - Ok(Some(pid)) - } - None => { - crate::input::mouse::click_at_xy_desktop(sx, sy, count, &btn)?; - Ok(None) - } - } + let result = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { + // Desktop scope is explicitly foreground and vision-driven: post + // at the global HID tap so WindowServer delivers to the window + // actually visible at this point. PID-posting here would silently + // turn the foreground contract back into background delivery. + crate::input::mouse::click_at_xy_desktop(sx, sy, count, &btn) }) .await; let button_label = match button.as_str() { @@ -311,18 +240,12 @@ impl Tool for ClickTool { _ => "click", }; return match result { - Ok(Ok(Some(pid))) => ToolResult::text(format!( - "✅ Sent {button_label} at desktop-pixel ({sx_shot:.0},{sy_shot:.0}) \ - → screen-point ({sx:.0},{sy:.0}) on pid {pid} (desktop scope; \ - not driver-verified — confirm via screenshot)." - )) - .with_structured(serde_json::json!({ "path": "cgevent", "verified": false, "effect": "unverifiable" })), - Ok(Ok(None)) => ToolResult::text(format!( + Ok(Ok(())) => ToolResult::text(format!( "✅ Sent screen-absolute {button_label} at desktop-pixel \ ({sx_shot:.0},{sy_shot:.0}) → screen-point ({sx:.0},{sy:.0}) \ - (desktop scope, no window under point; not driver-verified)." + (desktop scope; not driver-verified)." )) - .with_structured(serde_json::json!({ "path": "cgevent", "verified": false, "effect": "unverifiable" })), + .with_structured(serde_json::json!({ "path": "cgevent_hid", "verified": false, "effect": "unverifiable" })), Ok(Err(e)) => ToolResult::error(format!("desktop-scope click failed: {e}")), Err(e) => ToolResult::error(format!("task error: {e}")), }; @@ -374,9 +297,8 @@ impl Tool for ClickTool { // "middle" has no AX equivalent and falls back to a pixel middle-click // at the element's screen-space center. let button_str = args.str_or("button", "left").to_lowercase(); - // delivery_mode: per-call ladder rung. foreground only applies to the - // pixel path and needs a window_id to front (else it degrades to - // background). A click is never driver-verifiable either way. + // delivery_mode: per-call ladder rung. Foreground briefly activates the + // target for both AX and pixel paths, then restores the prior app. let delivery_mode = super::DeliveryMode::parse(args.opt_str("delivery_mode").as_deref()); // Reject unknown buttons explicitly so silent left-click fall-through can't // mask a typo. Keep "" → default left for old clients that never sent the field. @@ -494,7 +416,12 @@ impl Tool for ClickTool { // new-window / foreground side-effects and append a one-liner // suffix matching Swift's wording. let prior_front = apps::frontmost_pid(); - let snapshot = WindowChangeDetector::snapshot(prior_front); + let foreground = delivery_mode.is_foreground(); + let snapshot = if foreground { + WindowChangeDetector::snapshot_without_suppression(prior_front) + } else { + WindowChangeDetector::snapshot(prior_front) + }; // Run AX work on a blocking thread (can't block async executor). // Use `effective_action` so button=right rewrites press → show_menu. @@ -505,12 +432,37 @@ impl Tool for ClickTool { // and stomp default for a non-default session). let ck = cursor_key.clone(); let result = focus_guard::with_focus_suppressed( - Some(pid), + if foreground { None } else { Some(pid) }, prior_front, "click.AXPress", || async move { tokio::task::spawn_blocking(move || { - perform_ax_click(element_ptr, idx, pid, wid, &action_clone, &ck) + if foreground { + let mut outcome = None; + let fronted = crate::input::skylight::with_foreground_assist( + pid as libc::pid_t, + wid, + || { + outcome = Some(perform_ax_click( + element_ptr, + idx, + pid, + wid, + &action_clone, + &ck, + )?); + std::thread::sleep(std::time::Duration::from_millis(150)); + Ok(()) + }, + )?; + let outcome = outcome.ok_or_else(|| { + anyhow::anyhow!("foreground AX click did not execute") + })?; + Ok((outcome, fronted)) + } else { + perform_ax_click(element_ptr, idx, pid, wid, &action_clone, &ck) + .map(|outcome| (outcome, false)) + } }) .await }, @@ -521,7 +473,7 @@ impl Tool for ClickTool { let changes = snapshot.detect_async().await; match result { - Ok(Ok((mut msg, needs_webkit_delay, suspected_noop))) => { + Ok(Ok(((mut msg, needs_webkit_delay, suspected_noop), fronted))) => { // For text inputs, wait 800ms for WebKit DOM focus to settle // before returning — matches the Swift reference behaviour. if needs_webkit_delay { @@ -538,7 +490,7 @@ impl Tool for ClickTool { // * unverifiable — dispatched fine, driver just can't confirm; // the caller verifies via screenshot. let mut structured = serde_json::json!({ - "path": "ax", + "path": if fronted { "ax_fg" } else { "ax" }, "verified": false, "effect": if suspected_noop { "suspected_noop" } else { "unverifiable" }, }); @@ -677,6 +629,62 @@ impl Tool for ClickTool { (cx, cy, cx, cy) }; + // A background PX action can still use an accessibility delivery + // backend after resolving the requested screen point. This keeps + // targeting (PX) orthogonal to delivery (AX) and avoids making a + // Chromium/AppKit window key merely to satisfy first-mouse rules. + if !delivery_mode.is_foreground() + && window_id.is_some() + && button_str == "left" + && count == 1 + && modifiers.is_empty() + { + let focus_only = action == "focus"; + let ax_result = tokio::task::spawn_blocking(move || unsafe { + let Some(element) = element_at_screen_position(pid, screen_x, screen_y) else { + return Ok::<bool, anyhow::Error>(false); + }; + let delivered = if focus_only { + crate::input::ax_actions::focus_element(element as usize).is_ok() + } else { + let press = core_foundation::string::CFString::new("AXPress"); + AXUIElementPerformAction(element, press.as_concrete_TypeRef()) + == kAXErrorSuccess + }; + CFRelease(element as _); + Ok(delivered) + }) + .await; + match ax_result { + Ok(Ok(true)) => { + let label = if focus_only { "focused" } else { "pressed" }; + return ToolResult::text(format!( + "✅ PX hit-test {label} the background element via AX." + )) + .with_structured(serde_json::json!({ + "path": "ax", + "verified": false, + "effect": "unverifiable" + })); + } + Ok(Ok(false)) if focus_only => { + return ToolResult::error( + "Background PX focus is unavailable at the requested point.".to_owned(), + ) + .with_structured(serde_json::json!({ + "code": "background_unavailable" + })); + } + Ok(Err(error)) if focus_only => { + return ToolResult::error(format!("Background PX focus failed: {error}")) + .with_structured(serde_json::json!({ + "code": "background_unavailable" + })); + } + _ => {} + } + } + // Pin the overlay above the target window BEFORE animating so // the cursor is already sandwiched correctly while it glides in. if let Some(wid) = window_id { diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs index 8deff2249d..c07221adaf 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/drag.rs @@ -118,6 +118,13 @@ impl Tool for DragTool { // that drop background CGEvents), via the same skylight assist click // uses. Requires a window_id to have a window to front. let delivery_mode = super::DeliveryMode::parse(args.opt_str("delivery_mode").as_deref()); + if !delivery_mode.is_foreground() { + return ToolResult::error( + "Background drag is unavailable on macOS; use delivery_mode:\"foreground\"." + .to_owned(), + ) + .with_structured(serde_json::json!({ "code": "background_unavailable" })); + } let cursor_key = super::cursor_tools::resolve_cursor_key(&args); // Coerce integer or float from JSON for coordinate fields. diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs index ba90c36015..d8db786e73 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs @@ -117,6 +117,7 @@ pub(crate) async fn focus_by_pixel( let mut click_args = serde_json::json!({ "pid": pid, "x": x, "y": y, "delivery_mode": if foreground { "foreground" } else { "background" }, + "action": if foreground { "press" } else { "focus" }, }); if let Some(wid) = window_id { click_args["window_id"] = serde_json::json!(wid); } if let Some(s) = session { click_args["session"] = serde_json::json!(s); } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs index 38d8ae59f7..ad69f68a41 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/scroll.rs @@ -1,10 +1,17 @@ use async_trait::async_trait; -use cua_driver_core::{protocol::ToolResult, tool::{Tool, ToolDef}}; +use core_foundation::base::{CFRelease, CFTypeRef}; +use cua_driver_core::{ + protocol::ToolResult, + tool::{Tool, ToolDef}, +}; use serde_json::Value; use std::sync::Arc; use crate::apps; -use crate::ax::bindings::{element_screen_center, AXUIElementRef}; +use crate::ax::bindings::{ + copy_children, copy_element_attr, copy_string_attr, element_screen_center, kAXErrorSuccess, + perform_action, AXUIElementRef, +}; use crate::focus_guard; use crate::window_change_detector::WindowChangeDetector; @@ -30,7 +37,9 @@ pub struct ScrollTool { } impl ScrollTool { - pub fn new(state: Arc<ToolState>) -> Self { Self { state } } + pub fn new(state: Arc<ToolState>) -> Self { + Self { state } + } } static DEF: std::sync::OnceLock<ToolDef> = std::sync::OnceLock::new(); @@ -96,22 +105,37 @@ fn def() -> &'static ToolDef { #[async_trait] impl Tool for ScrollTool { - fn def(&self) -> &ToolDef { def() } + fn def(&self) -> &ToolDef { + def() + } async fn invoke(&self, args: Value) -> ToolResult { use cua_driver_core::tool_args::ArgsExt; - let pid = match args.require_i32("pid") { Ok(v) => v, Err(e) => return e }; + let pid = match args.require_i32("pid") { + Ok(v) => v, + Err(e) => return e, + }; // delivery_mode: foreground briefly fronts the window before the // pixel-wheel dispatch (the explicit last resort for surfaces that drop // background CGEvents). Only the pixel-wheel path honors it; the // keystroke path is background-by-design and untouched. let delivery_mode = super::DeliveryMode::parse(args.opt_str("delivery_mode").as_deref()); - let direction = match args.require_str("direction") { Ok(v) => v, Err(e) => return e }; + if !delivery_mode.is_foreground() && crate::browser::ElectronJs::is_electron(pid) { + return ToolResult::error( + "Background scroll is unavailable for Electron/Chromium windows on macOS." + .to_owned(), + ) + .with_structured(serde_json::json!({ "code": "background_unavailable" })); + } + let direction = match args.require_str("direction") { + Ok(v) => v, + Err(e) => return e, + }; let by = args.str_or("by", "line"); let amount = args.u64_or("amount", 3) as usize; // Surface 6: element_token / element_index precedence. let element_token_arg = args.opt_str("element_token"); - let window_id_arg = args.opt_u64("window_id").map(|v| v as u32); + let window_id_arg = args.opt_u64("window_id").map(|v| v as u32); let element_index_arg = args.opt_u64("element_index").map(|v| v as usize); let resolved = match cua_driver_core::element_token::resolve_element_args( pid, @@ -126,7 +150,9 @@ impl Tool for ScrollTool { let (element_index, window_id) = match resolved { cua_driver_core::element_token::ResolvedElement::None => (None, window_id_arg), cua_driver_core::element_token::ResolvedElement::Element { - window_id: wid, element_index: idx, via_token: _, + window_id: wid, + element_index: idx, + via_token: _, } => (Some(idx), wid), }; @@ -155,6 +181,79 @@ impl Tool for ScrollTool { } } + // AppKit exposes vertical scroll-bar buttons beneath the text area's + // AXScrollArea parent. Pressing those controls is a true + // background-safe scroll: no activation, z-order change, or cursor move. + if matches!(direction.as_str(), "up" | "down") { + if let (Some(index), Some(wid)) = (element_index, window_id) { + let native_element_guard = self + .state + .element_cache + .get_element_retained(pid, wid, index); + let direction_for_ax = direction.clone(); + let by_for_ax = by.clone(); + let foreground = delivery_mode.is_foreground(); + let ax_result = + tokio::task::spawn_blocking(move || -> anyhow::Result<(bool, bool)> { + let Some(element_guard) = native_element_guard else { + return Ok((false, false)); + }; + if foreground { + let mut delivered = false; + let fronted = crate::input::skylight::with_foreground_assist( + pid as libc::pid_t, + wid, + || { + delivered = unsafe { + scroll_native_text_area( + element_guard.as_ptr() as AXUIElementRef, + &direction_for_ax, + &by_for_ax, + amount, + ) + }; + std::thread::sleep(std::time::Duration::from_millis(100)); + Ok(()) + }, + )?; + Ok((delivered, fronted)) + } else { + Ok(( + unsafe { + scroll_native_text_area( + element_guard.as_ptr() as AXUIElementRef, + &direction_for_ax, + &by_for_ax, + amount, + ) + }, + false, + )) + } + }) + .await; + match ax_result { + Ok(Ok((true, fronted))) => { + return ToolResult::text(format!( + "✅ Scrolled native macOS control {direction} by {by} × {amount} through AX." + )) + .with_structured(serde_json::json!({ + "path": if fronted { "ax_fg" } else { "ax" }, + "verified": false, + "effect": "unverifiable" + })); + } + Ok(Ok((false, _))) => {} + Ok(Err(error)) => { + return ToolResult::error(format!("Native AX scroll failed: {error}")); + } + Err(error) => { + return ToolResult::error(format!("Native AX scroll task failed: {error}")); + } + } + } + } + // ── Targeted wheel path ───────────────────────────────────────────── // A target — element (preferred) OR window-local x,y — routes the scroll // through a synthesized mouse-wheel event at that screen point, so the @@ -162,19 +261,27 @@ impl Tool for ScrollTool { // cursor. This is the ONLY way to scroll a nested overflow:auto region // that never takes keyboard focus (the keystroke path below no-ops on // it). No user-facing flag: presence of a target IS the switch. - let x_arg = args.opt_f64("x").or_else(|| args.opt_i64("x").map(|v| v as f64)); - let y_arg = args.opt_f64("y").or_else(|| args.opt_i64("y").map(|v| v as f64)); + let x_arg = args + .opt_f64("x") + .or_else(|| args.opt_i64("x").map(|v| v as f64)); + let y_arg = args + .opt_f64("y") + .or_else(|| args.opt_i64("y").map(|v| v as f64)); // Per-notch step + direction→delta mapping (sign convention lives // here; the mouse primitive stays sign-agnostic). macOS: +y reveals // content ABOVE, -y reveals BELOW; +x reveals LEFT, -x reveals RIGHT. - let step = if by == "page" { WHEEL_STEP_PAGE_PX } else { WHEEL_STEP_LINE_PX }; + let step = if by == "page" { + WHEEL_STEP_PAGE_PX + } else { + WHEEL_STEP_LINE_PX + }; let (delta_y, delta_x): (i32, i32) = match direction.as_str() { - "down" => (-step, 0), - "up" => ( step, 0), + "down" => (-step, 0), + "up" => (step, 0), "right" => (0, -step), - "left" => (0, step), - _ => (-step, 0), + "left" => (0, step), + _ => (-step, 0), }; // Resolve a screen-space wheel target, if a target was supplied. @@ -201,7 +308,12 @@ impl Tool for ScrollTool { let win_local = wid .and_then(crate::windows::window_bounds_by_id) .map(|b| (cx - b.x, cy - b.y)); - WheelTarget { screen_x: cx, screen_y: cy, win_local, wid } + WheelTarget { + screen_x: cx, + screen_y: cy, + win_local, + wid, + } }) }) .await @@ -213,8 +325,7 @@ impl Tool for ScrollTool { // Without one, refuse rather than scrolling at screen-absolute coords. if window_id.is_none() { return ToolResult::error( - "window_id is required when scrolling by window-local x,y pixels." - .to_string(), + "window_id is required when scrolling by window-local x,y pixels.".to_string(), ); } // Pixel path: x,y are window-local screenshot pixels. Mirror the @@ -231,21 +342,39 @@ impl Tool for ScrollTool { let scale: f64 = if let Some(ref b) = bounds { if let Ok(png) = crate::capture::screenshot_window_bytes(wid) { if png.len() >= 24 { - let pw = u32::from_be_bytes([png[16], png[17], png[18], png[19]]) as f64; - if b.width > 0.0 && pw > b.width { pw / b.width } else { 1.0 } - } else { 1.0 } - } else { 1.0 } - } else { 1.0 }; + let pw = + u32::from_be_bytes([png[16], png[17], png[18], png[19]]) as f64; + if b.width > 0.0 && pw > b.width { + pw / b.width + } else { + 1.0 + } + } else { + 1.0 + } + } else { + 1.0 + } + } else { + 1.0 + }; if let Some(b) = bounds { let (wx, wy) = (cx / scale, cy / scale); return WheelTarget { - screen_x: b.x + wx, screen_y: b.y + wy, - win_local: Some((wx, wy)), wid: Some(wid), + screen_x: b.x + wx, + screen_y: b.y + wy, + win_local: Some((wx, wy)), + wid: Some(wid), }; } } // No window_id → treat x,y as screen coordinates. - WheelTarget { screen_x: cx, screen_y: cy, win_local: None, wid: None } + WheelTarget { + screen_x: cx, + screen_y: cy, + win_local: None, + wid: None, + } }) .await .ok() @@ -264,15 +393,26 @@ impl Tool for ScrollTool { ); } crate::cursor::overlay::animate_cursor_to( - cursor_key.clone(), target.screen_x, target.screen_y, - ).await; - self.state.cursor_registry - .update_position(&cursor_key, target.screen_x, target.screen_y); + cursor_key.clone(), + target.screen_x, + target.screen_y, + ) + .await; + self.state.cursor_registry.update_position( + &cursor_key, + target.screen_x, + target.screen_y, + ); let prior_front = apps::frontmost_pid(); let snapshot = WindowChangeDetector::snapshot(prior_front); - let WheelTarget { screen_x, screen_y, win_local, wid } = target; + let WheelTarget { + screen_x, + screen_y, + win_local, + wid, + } = target; let amount_ticks = amount; let fg = delivery_mode.is_foreground() && wid.is_some(); let result = focus_guard::with_focus_suppressed( @@ -283,14 +423,24 @@ impl Tool for ScrollTool { tokio::task::spawn_blocking(move || -> anyhow::Result<()> { let do_it = move || -> anyhow::Result<()> { crate::input::mouse::scroll_wheel_at_xy( - pid, screen_x, screen_y, win_local, wid, - delta_y, delta_x, amount_ticks, + pid, + screen_x, + screen_y, + win_local, + wid, + delta_y, + delta_x, + amount_ticks, ) }; // Foreground rung: brief front → wheel → restore prior frontmost. match (fg, wid) { (true, Some(w)) => { - crate::input::skylight::with_foreground_assist(pid as libc::pid_t, w, do_it)?; + crate::input::skylight::with_foreground_assist( + pid as libc::pid_t, + w, + do_it, + )?; Ok(()) } _ => do_it(), @@ -302,7 +452,11 @@ impl Tool for ScrollTool { .await; let changes = snapshot.detect_async().await; - let mode_label = if fg { " (delivery_mode:foreground)" } else { "" }; + let mode_label = if fg { + " (delivery_mode:foreground)" + } else { + "" + }; return match result { Ok(Ok(())) => ToolResult::text(format!( "✅ Sent {direction} scroll by {by} × {amount} via pixel wheel at \ @@ -319,13 +473,13 @@ impl Tool for ScrollTool { } let key = match (by.as_str(), direction.as_str()) { - ("page", "down") | (_, "down") if by == "page" => "pagedown", - ("page", "up") | (_, "up") if by == "page" => "pageup", - ("line", "down") | (_, "down") => "down", - ("line", "up") | (_, "up") => "up", - (_, "left") => "left", - (_, "right") => "right", - _ => "down", + ("page", "down") | (_, "down") if by == "page" => "pagedown", + ("page", "up") | (_, "up") if by == "page" => "pageup", + ("line", "down") | (_, "down") => "down", + ("line", "up") | (_, "up") => "up", + (_, "left") => "left", + (_, "right") => "right", + _ => "down", }; let key = key.to_owned(); @@ -350,7 +504,8 @@ impl Tool for ScrollTool { if let Some(element_ptr) = pre_focus_ptr { let _ = tokio::task::spawn_blocking(move || { crate::input::ax_actions::focus_element(element_ptr) - }).await; + }) + .await; tokio::time::sleep(std::time::Duration::from_millis(30)).await; } @@ -382,3 +537,69 @@ impl Tool for ScrollTool { } } } + +unsafe fn scroll_native_text_area( + element: AXUIElementRef, + direction: &str, + by: &str, + amount: usize, +) -> bool { + if copy_string_attr(element, "AXRole").as_deref() != Some("AXTextArea") { + return false; + } + let Some(scroll_area) = copy_element_attr(element, "AXParent") else { + return false; + }; + if copy_string_attr(scroll_area, "AXRole").as_deref() != Some("AXScrollArea") { + CFRelease(scroll_area as CFTypeRef); + return false; + } + let mut buttons = Vec::new(); + collect_ax_buttons(scroll_area, 0, &mut buttons); + CFRelease(scroll_area as CFTypeRef); + if buttons.is_empty() { + return false; + } + + let reverse = direction == "up"; + let base = if by == "page" && buttons.len() >= 4 { + 2 + } else { + 0 + }; + let index = base + usize::from(reverse); + let mut delivered = false; + if let Some(target) = buttons.get(index).copied() { + for _ in 0..amount.max(1) { + if perform_action(target, "AXPress") != kAXErrorSuccess { + break; + } + delivered = true; + std::thread::sleep(std::time::Duration::from_millis(30)); + } + } + for button in buttons { + CFRelease(button as CFTypeRef); + } + delivered +} + +unsafe fn collect_ax_buttons( + element: AXUIElementRef, + depth: usize, + buttons: &mut Vec<AXUIElementRef>, +) { + if depth >= 4 || buttons.len() >= 4 { + return; + } + for child in copy_children(element) { + if buttons.len() >= 4 { + CFRelease(child as CFTypeRef); + } else if copy_string_attr(child, "AXRole").as_deref() == Some("AXButton") { + buttons.push(child); + } else { + collect_ax_buttons(child, depth + 1, buttons); + CFRelease(child as CFTypeRef); + } + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs index 2d9ef6d610..49def31da7 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/type_text.rs @@ -476,6 +476,13 @@ fn cgevent_type_verified( std::thread::sleep(std::time::Duration::from_millis(settle_ms)); } if clear_first { + if settle_ms > 0 { + // Some renderer focus proxies discard the first printable event + // after activation even after their AX focus is visible. Prime + // that channel with disposable text, then clear it before the + // requested payload. Never do this for a nonempty field. + let _ = crate::input::keyboard::type_text_with_delay(pid, " ", delay_ms); + } let _ = crate::input::keyboard::press_key(pid, "a", &["cmd"]); let _ = crate::input::keyboard::press_key(pid, "delete", &[]); } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs b/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs index 974d21045d..8836344897 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/window_change_detector.rs @@ -179,6 +179,17 @@ impl WindowChangeDetector { /// Safe to call from any thread — `CGWindowListCopyWindowInfo` is /// documented as thread-safe. pub fn snapshot(prior_front: Option<i32>) -> Snapshot { + Self::capture(prior_front, true) + } + + /// Capture the same before-state without arming reactive focus suppression. + /// Foreground delivery owns its temporary activation and restoration, so a + /// wildcard lease would race the target while the action is settling. + pub fn snapshot_without_suppression(prior_front: Option<i32>) -> Snapshot { + Self::capture(prior_front, false) + } + + fn capture(prior_front: Option<i32>, suppress_focus: bool) -> Snapshot { let window_ids: HashSet<u32> = windows::visible_windows() .into_iter() .filter(|w| w.layer == 0) @@ -190,7 +201,7 @@ impl WindowChangeDetector { // (any other pid). If there's no frontmost (rare — screensaver, // login window), we skip the lease; foreground-change tracking // still runs. - let lease = prior_front.map(|restore_to| { + let lease = prior_front.filter(|_| suppress_focus).map(|restore_to| { focus_steal::begin_suppression( None, // wildcard restore_to, @@ -471,7 +482,10 @@ mod tests { foreground_changed: false, }; // No title → just the app name, no parentheses. - assert_eq!(c.result_suffix(), "\n\n🪟 Action opened new window(s): Finder."); + assert_eq!( + c.result_suffix(), + "\n\n🪟 Action opened new window(s): Finder." + ); } /// Regression: `snapshot(prior_front)` must store the caller's diff --git a/libs/cua-driver/rust/crates/platform-windows/src/capture.rs b/libs/cua-driver/rust/crates/platform-windows/src/capture.rs index 6237436ed2..54b7361a72 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/capture.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/capture.rs @@ -238,13 +238,13 @@ unsafe fn screenshot_window_bytes_with_occlusion_unsafe(hwnd: u64) -> Result<(Ve // The WGC sibling path at `wgc.rs:58` already short-circuits this case; // the GDI/PrintWindow fallback below + the screen-region BitBlt fallback // both happily produced the degenerate PNG. Guarding here covers both - // and matches the WGC error shape so callers can `list_windows` or - // raise the window before retrying. + // and matches the WGC error shape so callers can `list_windows` and + // restore the window before retrying. if IsIconic(hwnd).as_bool() { bail!( "cannot capture minimized window 0x{hwnd_raw:x}: it has no \ - rendered content. Restore the window first via list_windows \ - / raise_window. The PrintWindow GDI path and the screen-region \ + rendered content. Call bring_to_front with this window_id to \ + restore it first. The PrintWindow GDI path and the screen-region \ BitBlt fallback both return an all-black bitmap for iconic \ windows." ); @@ -565,4 +565,3 @@ pub fn crosshair_png_bytes(png_bytes: &[u8], cx: f64, cy: f64) -> Result<Vec<u8> pub fn png_dimensions_pub(data: &[u8]) -> Result<(u32, u32)> { cua_driver_core::image_utils::png_dimensions(data) } - diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/delivery.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/delivery.rs index 89b11c86ec..93aca9a094 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/input/delivery.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/input/delivery.rs @@ -140,21 +140,38 @@ pub fn would_be_silently_dropped(hwnd: u64, kind: EventKind) -> bool { use EventKind::*; if crate::input::is_chromium_target_window(hwnd) { // Chromium's input thread architecture requires SendInput-queue - // origin for mouse + key-combo events (#1623). Plain keystrokes and - // text input via WM_CHAR still work because they go through - // Chromium's IME path, which DOES consume Win32 messages. - return matches!(kind, MouseClick | MouseMove | MouseScroll | KeyCombo); + // origin for pointer and keyboard events (#1623). Posted WM_CHAR and + // plain key messages can return success while a background renderer + // receives nothing, so they must be refused as honestly as chords. + return matches!( + kind, + MouseClick | MouseMove | MouseScroll | Keystroke | KeyCombo | TextInput + ); + } + if crate::input::has_chromium_descendant(hwnd) { + // Embedded WebView2 hosts retain useful UIA/top-level routes for + // clicks and ValuePattern text. Their drag, wheel and modifier-chord + // paths still depend on the renderer's system input queue. + return matches!(kind, MouseMove | MouseScroll | KeyCombo); } if is_wpf_target_window(hwnd) { - // WPF ignores posted pointer messages (its input manager drops - // WM_MOUSE* unless the live system cursor is over the window). It must - // be driven by coordinate-routed system-queue input for clicks/moves. + // WPF ignores posted pointer messages unless the live system cursor is + // over the window. Its InputManager also ignores posted key messages + // while another native window owns foreground; PostMessage still + // returns success, so both routes need an honest refusal. // // Do not classify WM_VSCROLL/WM_HSCROLL here: the scroll tool posts the // scrollbar messages directly to the top-level HWND, and WPF hosts that // explicitly handle those messages (including our harness hook) can // consume them without a foreground swap. - return matches!(kind, MouseClick | MouseMove); + return wpf_drops_event(kind, target_is_foreground(hwnd)); + } + if is_tk_target_window(hwnd) { + // Tk's Windows event loop does not treat posted WM_CHAR/WM_KEYDOWN as + // genuine keyboard input for the focused widget. The messages can be + // accepted by PostMessage while the Entry receives nothing, so refuse + // instead of reporting a false background success. + return matches!(kind, Keystroke | KeyCombo | TextInput); } // NB: WinUI3 (`WinUIDesktopWin32WindowClass`) is deliberately NOT flagged // here. It looks WPF-like, but its composition input-site does NOT consume @@ -190,6 +207,25 @@ pub fn would_be_silently_dropped(hwnd: u64, kind: EventKind) -> bool { false } +fn wpf_drops_event(kind: EventKind, target_is_foreground: bool) -> bool { + matches!(kind, EventKind::MouseClick | EventKind::MouseMove) + || (!target_is_foreground && matches!(kind, EventKind::Keystroke | EventKind::KeyCombo)) +} + +fn target_is_foreground(hwnd: u64) -> bool { + use windows::Win32::Foundation::HWND; + use windows::Win32::UI::WindowsAndMessaging::{GetAncestor, GetForegroundWindow, GA_ROOT}; + if hwnd == 0 { + return false; + } + unsafe { + let target = GetAncestor(HWND(hwnd as *mut _), GA_ROOT); + let foreground = GetForegroundWindow(); + let foreground_root = GetAncestor(foreground, GA_ROOT); + !target.0.is_null() && target == foreground_root + } +} + /// Detect LibreOffice / OpenOffice (VCL framework) windows. /// /// VCL on Windows registers window classes with a `SAL` prefix (StarOffice's @@ -222,6 +258,16 @@ pub fn is_wpf_target_window(hwnd: u64) -> bool { read_class_name(hwnd).starts_with("HwndWrapper") } +/// Detect Tk/Tkinter top-level windows. Tk registers this stable class name +/// for its root and child toplevels on Windows. +pub fn is_tk_target_window(hwnd: u64) -> bool { + is_tk_class_name(&read_class_name(hwnd)) +} + +fn is_tk_class_name(class: &str) -> bool { + class == "TkTopLevel" || class.starts_with("TkTopLevel.") +} + /// Detect WinUI3 / Windows-App-SDK desktop top-level windows. The frame is a /// Win32 HWND of class `WinUIDesktopWin32WindowClass`, but — unlike WPF — that /// frame does NOT host the visual tree or consume pointer input. The XAML @@ -351,6 +397,25 @@ pub fn background_unavailable_error_with_cause( mod tests { use super::*; + #[test] + fn detects_tk_toplevel_classes_without_matching_unrelated_windows() { + assert!(is_tk_class_name("TkTopLevel")); + assert!(is_tk_class_name("TkTopLevel.1")); + assert!(!is_tk_class_name("TkChild")); + assert!(!is_tk_class_name("Chrome_WidgetWin_1")); + } + + #[test] + fn wpf_refuses_posted_pointer_and_keyboard_events() { + assert!(wpf_drops_event(EventKind::MouseClick, true)); + assert!(wpf_drops_event(EventKind::MouseMove, true)); + assert!(wpf_drops_event(EventKind::Keystroke, false)); + assert!(wpf_drops_event(EventKind::KeyCombo, false)); + assert!(!wpf_drops_event(EventKind::Keystroke, true)); + assert!(!wpf_drops_event(EventKind::KeyCombo, true)); + assert!(!wpf_drops_event(EventKind::TextInput, false)); + assert!(!wpf_drops_event(EventKind::MouseScroll, false)); + } #[test] fn delivery_mode_parses_known_values() { let j = |s: &str| serde_json::json!({"delivery_mode": s}); @@ -401,17 +466,12 @@ mod tests { ); let structured = result.structured_content.as_ref().expect("structured"); assert_eq!(result.is_error, Some(true)); - assert_eq!( - structured["code"].as_str(), - Some("background_occluded") - ); + assert_eq!(structured["code"].as_str(), Some("background_occluded")); assert_eq!(structured["event_kind"].as_str(), Some("mouse_click")); - assert!( - structured["cause"] - .as_str() - .unwrap_or_default() - .contains("occluded") - ); + assert!(structured["cause"] + .as_str() + .unwrap_or_default() + .contains("occluded")); let text = match &result.content[0] { cua_driver_core::protocol::Content::Text { text, .. } => text, diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs index 4abc077dce..ce8157fc58 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/input/inject.rs @@ -34,6 +34,7 @@ use std::thread::sleep; use std::time::Duration; use windows::Win32::Foundation::{HANDLE, HWND, POINT, RECT}; +use windows::Win32::System::Threading::{AttachThreadInput, GetCurrentThreadId}; use windows::Win32::UI::Controls::{ CreateSyntheticPointerDevice, DestroySyntheticPointerDevice, HSYNTHETICPOINTERDEVICE, POINTER_FEEDBACK_DEFAULT, POINTER_TYPE_INFO, POINTER_TYPE_INFO_0, @@ -42,13 +43,68 @@ use windows::Win32::UI::Input::Pointer::{ InjectSyntheticPointerInput, POINTER_FLAG_DOWN, POINTER_FLAG_INCONTACT, POINTER_FLAG_INRANGE, POINTER_FLAG_UP, POINTER_FLAG_UPDATE, POINTER_INFO, POINTER_PEN_INFO, POINTER_TOUCH_INFO, }; -use windows::Win32::System::Threading::{AttachThreadInput, GetCurrentThreadId}; use windows::Win32::UI::WindowsAndMessaging::{ GetAncestor, GetCursorPos, GetForegroundWindow, GetWindowLongPtrW, GetWindowThreadProcessId, - IsWindow, SetCursorPos, SetForegroundWindow, SetWindowLongPtrW, WindowFromPoint, GA_ROOT, - GWL_EXSTYLE, PT_PEN, PT_TOUCH, WS_EX_NOACTIVATE, + IsWindow, LockSetForegroundWindow, SetCursorPos, SetForegroundWindow, SetWindowLongPtrW, + WindowFromPoint, GA_ROOT, GWL_EXSTYLE, LSFW_LOCK, LSFW_UNLOCK, PT_PEN, PT_TOUCH, + WS_EX_NOACTIVATE, }; +#[derive(Default)] +struct ForegroundLockState { + holders: usize, + locked: bool, +} + +static FOREGROUND_LOCK_STATE: Mutex<ForegroundLockState> = Mutex::new(ForegroundLockState { + holders: 0, + locked: false, +}); + +/// Prevent other processes from taking the foreground during a background +/// launch. Windows automatically clears this lock on genuine user input; Drop +/// still balances the documented unlock call and overlapping driver launches. +pub struct ForegroundLockGuard { + held: bool, +} + +impl ForegroundLockGuard { + pub fn acquire() -> Self { + let mut state = FOREGROUND_LOCK_STATE.lock().unwrap(); + if state.holders == 0 { + state.locked = unsafe { LockSetForegroundWindow(LSFW_LOCK) }.is_ok(); + if state.locked { + tracing::debug!(target: "launch_app.focus_lock", "locked foreground changes during background launch"); + } else { + tracing::warn!(target: "launch_app.focus_lock", "could not lock foreground changes during background launch"); + } + } + if state.locked { + state.holders += 1; + } + Self { held: state.locked } + } + + pub fn acquired(&self) -> bool { + self.held + } +} + +impl Drop for ForegroundLockGuard { + fn drop(&mut self) { + if !self.held { + return; + } + let mut state = FOREGROUND_LOCK_STATE.lock().unwrap(); + state.holders = state.holders.saturating_sub(1); + if state.holders == 0 { + let _ = unsafe { LockSetForegroundWindow(LSFW_UNLOCK) }; + state.locked = false; + tracing::debug!(target: "launch_app.focus_lock", "unlocked foreground changes after background launch"); + } + } +} + /// Bring `target` to the foreground using the AttachThreadInput trick, which /// inherits the current foreground thread's FG-lock token so the swap is /// honored even on a foreground-locked session without UIAccess (mirrors the @@ -89,7 +145,6 @@ pub struct NoActivateGuard { // Store the handle as an integer so the guard is `Send` and can be held // across `.await` in the async tools. root_addr: isize, - prev_exstyle: isize, applied: bool, } @@ -111,7 +166,10 @@ impl NoActivateGuard { // by UIPI on higher-integrity targets). (GetWindowLongPtrW(root, GWL_EXSTYLE) & want) != 0 }; - Self { root_addr: root.0 as isize, prev_exstyle: prev, applied } + Self { + root_addr: root.0 as isize, + applied, + } } } } @@ -120,7 +178,13 @@ impl Drop for NoActivateGuard { fn drop(&mut self) { if self.applied { unsafe { - let _ = SetWindowLongPtrW(HWND(self.root_addr as *mut _), GWL_EXSTYLE, self.prev_exstyle); + let root = HWND(self.root_addr as *mut _); + let current = GetWindowLongPtrW(root, GWL_EXSTYLE); + let noactivate = WS_EX_NOACTIVATE.0 as isize; + // Clear only the bit this guard added. Restoring the full + // captured value can clobber unrelated style changes the app + // made while the background action was in flight. + SetWindowLongPtrW(root, GWL_EXSTYLE, current & !noactivate); } } } @@ -537,4 +601,3 @@ pub fn inject_drag_screen( // WPF/terminal text) is now reported as `background_unavailable`; the agent // escalates to `delivery_mode:"foreground"`, which uses the explicit // SetForegroundWindow path (send_key_synthesized / send_text_synthesized). - diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/keyboard.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/keyboard.rs index ffb23cb153..75e00bcb6e 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/input/keyboard.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/input/keyboard.rs @@ -17,18 +17,17 @@ use anyhow::{bail, Result}; use std::thread::sleep; -use std::time::Duration; -use windows::Win32::Foundation::{HWND, LPARAM, WPARAM}; -use windows::Win32::System::Threading::{AttachThreadInput, GetCurrentThreadId}; -use windows::Win32::UI::Input::KeyboardAndMouse::GetFocus; +use std::time::{Duration, Instant}; +use windows::Win32::Foundation::{BOOL, HWND, LPARAM, TRUE, WPARAM}; use windows::Win32::UI::Input::KeyboardAndMouse::{ MapVirtualKeyW, SendInput, INPUT, INPUT_0, INPUT_KEYBOARD, KEYBDINPUT, KEYBD_EVENT_FLAGS, KEYEVENTF_EXTENDEDKEY, KEYEVENTF_KEYUP, KEYEVENTF_SCANCODE, KEYEVENTF_UNICODE, MAPVK_VK_TO_VSC, VIRTUAL_KEY, }; use windows::Win32::UI::WindowsAndMessaging::{ - GetClassNameW, GetWindowThreadProcessId, IsChild, PostMessageW, WM_CHAR, WM_KEYDOWN, WM_KEYUP, - WM_SYSKEYDOWN, WM_SYSKEYUP, + EnumChildWindows, GetClassNameW, GetGUIThreadInfo, GetParent, GetWindowThreadProcessId, + IsChild, PostMessageW, GUITHREADINFO, WM_CHAR, WM_KEYDOWN, WM_KEYUP, WM_SYSKEYDOWN, + WM_SYSKEYUP, }; use windows::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, SetForegroundWindow}; @@ -127,50 +126,97 @@ pub fn is_xaml_host_hwnd(hwnd: u64) -> bool { const KEY_DELAY_MS: u64 = 4; -/// If the target's UI thread has a focused child window that's a descendant -/// of `parent`, return that child. Otherwise `None`. Used to retarget +/// If any UI thread under the target has a focused child window that's a +/// descendant of `parent`, return that child. Otherwise `None`. Used to retarget /// `PostMessage(WM_CHAR/WM_KEYDOWN)` from the top-level frame to the actual /// editor control (Scintilla in Notepad++, RichEdit in WordPad, etc.) — /// top-level WindowProcs don't forward keyboard messages to embedded editors /// automatically, so without this drill-down `type_text` silently no-ops /// against any app that puts its text surface in a child HWND. /// -/// Uses `AttachThreadInput` to read the target thread's focus state, which -/// is the standard cross-thread way to read another thread's `GetFocus()`. -/// We detach immediately after — attaching for the duration of the post -/// would change input-state visibility for the duration. +/// Embedded renderers such as WebView2 may put their focused child on a +/// different UI thread from the native top-level frame. Enumerating descendant +/// thread ids is therefore required; checking only the frame thread queues the +/// message successfully but leaves the renderer untouched. More than one of +/// those threads can retain a focused HWND, so choose the deepest focused +/// descendant rather than whichever thread happens to enumerate first. fn focused_descendant(parent: HWND) -> Option<HWND> { if parent.0.is_null() { return None; } - let mut target_pid: u32 = 0; - let target_thread = unsafe { GetWindowThreadProcessId(parent, Some(&mut target_pid)) }; - if target_thread == 0 { + let parent_thread = unsafe { GetWindowThreadProcessId(parent, None) }; + if parent_thread == 0 { return None; } - let our_thread = unsafe { GetCurrentThreadId() }; - let focused = if our_thread == target_thread { - unsafe { GetFocus() } - } else { - let _ = unsafe { AttachThreadInput(our_thread, target_thread, true) }; - let f = unsafe { GetFocus() }; - let _ = unsafe { AttachThreadInput(our_thread, target_thread, false) }; - f - }; - if focused.0.is_null() { - return None; + unsafe extern "system" fn collect_thread(child: HWND, lparam: LPARAM) -> BOOL { + let threads = &mut *(lparam.0 as *mut Vec<u32>); + let thread = GetWindowThreadProcessId(child, None); + if thread != 0 && !threads.contains(&thread) { + threads.push(thread); + } + TRUE } - if focused == parent { - return None; + + let mut target_threads = vec![parent_thread]; + unsafe { + let _ = EnumChildWindows( + parent, + Some(collect_thread), + LPARAM(&mut target_threads as *mut Vec<u32> as isize), + ); + } + let mut best: Option<(usize, HWND)> = None; + for target_thread in target_threads { + let mut info = GUITHREADINFO { + cbSize: std::mem::size_of::<GUITHREADINFO>() as u32, + ..Default::default() + }; + if unsafe { GetGUIThreadInfo(target_thread, &mut info) }.is_err() { + continue; + } + let focused = info.hwndFocus; + if focused.0.is_null() + || focused == parent + || !unsafe { IsChild(parent, focused) }.as_bool() + { + continue; + } + + let mut depth = 0usize; + let mut current = focused; + while current != parent && depth < 64 { + let Ok(next) = (unsafe { GetParent(current) }) else { + break; + }; + if next.0.is_null() { + break; + } + depth += 1; + current = next; + } + if current == parent && best.as_ref().map_or(true, |(d, _)| depth > *d) { + best = Some((depth, focused)); + } } - // Only retarget if focus is genuinely a descendant of `parent` — protects - // against accidentally posting to an unrelated window if the target is - // not the foreground app at the moment. - if unsafe { IsChild(parent, focused) }.as_bool() { - Some(focused) - } else { - None + best.map(|(_, focused)| focused) +} + +/// Wait for an element-focused embedded renderer to expose its child HWND. +/// UIA SetFocus can complete before WebView2 updates GUITHREADINFO; polling the +/// observable focus target avoids posting the key to the native frame in that +/// short interval. +pub fn wait_for_focused_descendant(hwnd: u64, timeout: Duration) -> Option<u64> { + let parent = HWND(hwnd as *mut _); + let deadline = Instant::now() + timeout; + loop { + if let Some(target) = focused_descendant(parent) { + return Some(target.0 as usize as u64); + } + if Instant::now() >= deadline { + return None; + } + sleep(Duration::from_millis(10)); } } diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs index e45be446fd..0501db47a1 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mod.rs @@ -14,11 +14,18 @@ pub mod keyboard; pub mod delivery; pub mod inject; -pub use inject::{inject_click_screen, point_in_window_bounds, NoActivateGuard}; -pub use mouse::{is_chromium_target_window, post_click, post_click_screen, send_click_synthesized, send_click_synthesized_mods, send_wheel_synthesized}; +pub use inject::{ + inject_click_screen, point_in_window_bounds, ForegroundLockGuard, NoActivateGuard, +}; +pub(crate) use inject::force_foreground_attached; +pub use mouse::{ + has_chromium_descendant, is_chromium_target_window, post_click, post_click_screen, + send_click_synthesized, send_click_synthesized_active_mods, send_click_synthesized_mods, + send_wheel_synthesized, +}; pub use keyboard::{ is_xaml_host_hwnd, post_char, post_key, post_type_text, post_type_text_with_delay, - send_key_synthesized, send_text_synthesized, + send_key_synthesized, send_text_synthesized, wait_for_focused_descendant, }; use windows::Win32::Foundation::{CloseHandle, HANDLE, HWND}; diff --git a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs index f2fb50d3c9..d370855657 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/input/mouse.rs @@ -16,13 +16,14 @@ use windows::Win32::UI::Input::KeyboardAndMouse::{ MOUSEINPUT, SendInput, }; use windows::Win32::UI::WindowsAndMessaging::{ - ChildWindowFromPointEx, CWP_SKIPDISABLED, CWP_SKIPINVISIBLE, CWP_SKIPTRANSPARENT, - GetCursorPos, GetForegroundWindow, GetSystemMetrics, GetWindowLongPtrW, PostMessageW, - SetCursorPos, SetWindowPos, GWL_EXSTYLE, HWND_NOTOPMOST, HWND_TOP, HWND_TOPMOST, - SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN, SM_XVIRTUALSCREEN, SM_YVIRTUALSCREEN, - SWP_NOACTIVATE, SWP_NOMOVE, SWP_NOSIZE, WS_EX_TOPMOST, - WM_LBUTTONDOWN, WM_LBUTTONUP, WM_MBUTTONDOWN, WM_MBUTTONUP, - WM_MOUSEMOVE, WM_RBUTTONDOWN, WM_RBUTTONUP, + ChildWindowFromPointEx, GetAncestor, GetClassLongPtrW, GetCursorPos, GetForegroundWindow, + GetSystemMetrics, GetWindowLongPtrW, PostMessageW, SetCursorPos, SetWindowPos, CS_DBLCLKS, + CWP_SKIPDISABLED, CWP_SKIPINVISIBLE, GA_ROOT, GCL_STYLE, + CWP_SKIPTRANSPARENT, GWL_EXSTYLE, HWND_NOTOPMOST, HWND_TOP, HWND_TOPMOST, + SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN, SM_XVIRTUALSCREEN, SM_YVIRTUALSCREEN, SWP_NOACTIVATE, + SWP_NOMOVE, SWP_NOSIZE, WM_LBUTTONDBLCLK, WM_LBUTTONDOWN, WM_LBUTTONUP, + WM_MBUTTONDBLCLK, WM_MBUTTONDOWN, WM_MBUTTONUP, WM_MOUSEMOVE, WM_RBUTTONDBLCLK, + WM_RBUTTONDOWN, WM_RBUTTONUP, WS_EX_TOPMOST, }; const MK_LBUTTON: u32 = 0x0001; @@ -31,6 +32,14 @@ const MK_RBUTTON: u32 = 0x0002; const CLICK_DELAY_MS: u64 = 35; +fn posted_press_message(down: u32, double: u32, click_index: usize, wants_double: bool) -> u32 { + if wants_double && click_index % 2 == 1 { + double + } else { + down + } +} + /// Walk from `root` down to the deepest visible child that contains /// `screen_pt`, mirroring trope-cua's DeepestChildFromScreenPoint. /// @@ -99,20 +108,39 @@ fn post_click_on(hwnd: HWND, x: i32, y: i32, count: usize, button: &str) -> Resu anyhow::bail!(msg); } - let (down_msg, up_msg, mk_flag) = match button { - "right" => (WM_RBUTTONDOWN, WM_RBUTTONUP, MK_RBUTTON), - "middle" => (WM_MBUTTONDOWN, WM_MBUTTONUP, MK_MBUTTON), - _ => (WM_LBUTTONDOWN, WM_LBUTTONUP, MK_LBUTTON), + let (down_msg, double_msg, up_msg, mk_flag) = match button { + "right" => (WM_RBUTTONDOWN, WM_RBUTTONDBLCLK, WM_RBUTTONUP, MK_RBUTTON), + "middle" => ( + WM_MBUTTONDOWN, + WM_MBUTTONDBLCLK, + WM_MBUTTONUP, + MK_MBUTTON, + ), + _ => (WM_LBUTTONDOWN, WM_LBUTTONDBLCLK, WM_LBUTTONUP, MK_LBUTTON), }; let lparam = make_lparam(x, y); let wdown = WPARAM(mk_flag as usize); let wup = WPARAM(0); + let wants_double = + unsafe { (GetClassLongPtrW(hwnd, GCL_STYLE) as u32 & CS_DBLCLKS.0) != 0 }; + let prev_fg = unsafe { GetForegroundWindow() }; + let target_root = unsafe { + let root = GetAncestor(hwnd, GA_ROOT); + if root.0.is_null() { hwnd } else { root } + }; + // Posted pointer messages are normally non-activating, but WebView hosts can + // call SetForegroundWindow from their event handlers. Keep the top-level + // categorically non-activatable until the complete burst has settled. + let _noact = crate::input::NoActivateGuard::arm(hwnd); for i in 0..count { + let press_msg = posted_press_message(down_msg, double_msg, i, wants_double); unsafe { // WM_MOUSEMOVE first so hover state is correct before the click. PostMessageW(hwnd, WM_MOUSEMOVE, WPARAM(0), lparam)?; - PostMessageW(hwnd, down_msg, wdown, lparam)?; + // Win32 controls do not infer a double-click from two posted DOWN + // messages. The second press must use WM_*BUTTONDBLCLK. + PostMessageW(hwnd, press_msg, wdown, lparam)?; sleep(Duration::from_millis(CLICK_DELAY_MS)); PostMessageW(hwnd, up_msg, wup, lparam)?; } @@ -120,6 +148,17 @@ fn post_click_on(hwnd: HWND, x: i32, y: i32, count: usize, button: &str) -> Resu sleep(Duration::from_millis(80)); } } + sleep(Duration::from_millis(50)); + unsafe { + if !prev_fg.0.is_null() + && prev_fg != target_root + && GetForegroundWindow() == target_root + { + crate::input::force_foreground_attached(prev_fg); + sleep(Duration::from_millis(12)); + crate::input::force_foreground_attached(prev_fg); + } + } Ok(()) } @@ -298,6 +337,46 @@ pub fn is_chromium_target_window(hwnd: u64) -> bool { is_chromium } +/// Return true when `hwnd` hosts a Chromium/WebView2 renderer child even if +/// its own top-level class is framework-specific (for example a Tauri host). +/// Keep this separate from [`is_chromium_target_window`]: embedded WebView2 +/// surfaces support some UIA/top-level background routes that direct Chromium +/// frames do not, so delivery policy needs to distinguish the two shapes. +pub fn has_chromium_descendant(hwnd: u64) -> bool { + use windows::Win32::Foundation::{BOOL, FALSE, LPARAM, TRUE}; + use windows::Win32::UI::WindowsAndMessaging::{EnumChildWindows, GetClassNameW}; + + if hwnd == 0 { + return false; + } + struct Scan { + found: bool, + } + unsafe extern "system" fn child_cb(child: HWND, lparam: LPARAM) -> BOOL { + let scan = &mut *(lparam.0 as *mut Scan); + let mut buf = [0u16; 64]; + let n = GetClassNameW(child, &mut buf); + if n > 0 { + let class = String::from_utf16_lossy(&buf[..n as usize]); + if class.starts_with("Chrome_WidgetWin_") || class.starts_with("CefBrowser") { + scan.found = true; + return FALSE; + } + } + TRUE + } + + let mut scan = Scan { found: false }; + unsafe { + let _ = EnumChildWindows( + HWND(hwnd as *mut _), + Some(child_cb), + LPARAM(&mut scan as *mut Scan as isize), + ); + } + scan.found +} + /// Click at **screen** coordinates `(sx, sy)` via `SendInput` against the /// system input queue, briefly focusing `target` so the click lands there. /// @@ -344,6 +423,33 @@ pub fn send_click_synthesized_mods( count: usize, button: &str, modifiers: &[&str], +) -> Result<()> { + send_click_synthesized_mods_impl(target, sx, sy, count, button, modifiers, false) +} + +/// SendInput click for an explicit foreground request. Unlike the historical +/// z-order-assisted path, this activates the target and does not add +/// `WS_EX_NOACTIVATE`, so retained-mode frameworks such as WPF process the +/// system-queue pointer event. The caller owns any later foreground restore. +pub fn send_click_synthesized_active_mods( + target: u64, + sx: i32, + sy: i32, + count: usize, + button: &str, + modifiers: &[&str], +) -> Result<()> { + send_click_synthesized_mods_impl(target, sx, sy, count, button, modifiers, true) +} + +fn send_click_synthesized_mods_impl( + target: u64, + sx: i32, + sy: i32, + count: usize, + button: &str, + modifiers: &[&str], + activate: bool, ) -> Result<()> { let target = HWND(target as *mut _); if target.0.is_null() { @@ -404,8 +510,8 @@ pub fn send_click_synthesized_mods( r#type: INPUT_MOUSE, Anonymous: INPUT_0 { mi: MOUSEINPUT { - dx: norm_x, dy: norm_y, mouseData: 0, - dwFlags: down_flag | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK, + dx: 0, dy: 0, mouseData: 0, + dwFlags: down_flag, time: 0, dwExtraInfo: 0, }, }, @@ -414,8 +520,8 @@ pub fn send_click_synthesized_mods( r#type: INPUT_MOUSE, Anonymous: INPUT_0 { mi: MOUSEINPUT { - dx: norm_x, dy: norm_y, mouseData: 0, - dwFlags: up_flag | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK, + dx: 0, dy: 0, mouseData: 0, + dwFlags: up_flag, time: 0, dwExtraInfo: 0, }, }, @@ -437,12 +543,20 @@ pub fn send_click_synthesized_mods( // restore" for pointer input, done the one Windows way that doesn't // need UIAccess — the technique the OG GTK path used. (Keyboard // foreground still needs *real* focus; only pointer can be z-routed.) - let _noact = crate::input::NoActivateGuard::arm(target); // Capture whether the target was ALREADY always-on-top so we don't strip // that state on restore — only demote below if WE promoted it. let was_topmost = (GetWindowLongPtrW(target, GWL_EXSTYLE) as u32) & WS_EX_TOPMOST.0 != 0; - let _ = SetWindowPos(target, HWND_TOPMOST, 0, 0, 0, 0, SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE); + let foreground_attach_failed = activate && !crate::input::force_foreground_attached(target); + let noactivate = (!activate).then(|| crate::input::NoActivateGuard::arm(target)); + if !activate || foreground_attach_failed { + let flags = if activate { + SWP_NOMOVE | SWP_NOSIZE + } else { + SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE + }; + let _ = SetWindowPos(target, HWND_TOPMOST, 0, 0, 0, 0, flags); + } // Move the cursor so the OS hover state matches before the click; the // MOUSEEVENTF_MOVE input ensures Chromium's input filter sees a @@ -462,6 +576,9 @@ pub fn send_click_synthesized_mods( let count = count.max(1); let mut sent_ok = true; for i in 0..count { + // Only the move record carries absolute coordinates. Button-only + // records act at the current pointer position; adding ABSOLUTE to + // them can prevent retained-mode controls from seeing the press. let events = [move_input, down_input, up_input]; let sent = SendInput(&events, std::mem::size_of::<INPUT>() as i32); if sent as usize != events.len() { @@ -478,21 +595,31 @@ pub fn send_click_synthesized_mods( SendInput(&mod_ups, std::mem::size_of::<INPUT>() as i32); } - // Brief settle so the target processes the click, then restore z-order: - // demote the target out of the topmost band and restack the user's - // window on top (no activation), and restore the cursor. - sleep(Duration::from_millis(40)); - if !was_topmost { + // Let the target process mouse-up before any background-route restore. + // Retained-mode frameworks establish capture/focus on mouse-down and can + // lose the click if the real cursor is warped away while those queued + // messages are still being dispatched. + sleep(Duration::from_millis(if activate { 120 } else { 40 })); + if !was_topmost && (!activate || foreground_attach_failed) { let _ = SetWindowPos(target, HWND_NOTOPMOST, 0, 0, 0, 0, SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE); } - if !prev_fg.0.is_null() && prev_fg != target { - let _ = SetWindowPos(prev_fg, HWND_TOP, 0, 0, 0, 0, SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE); + if !activate { + if !prev_fg.0.is_null() && prev_fg != target { + let _ = SetWindowPos(prev_fg, HWND_TOP, 0, 0, 0, 0, SWP_NOACTIVATE | SWP_NOMOVE | SWP_NOSIZE); + } + let _ = SetCursorPos(prev_cursor.x, prev_cursor.y); } - let _ = SetCursorPos(prev_cursor.x, prev_cursor.y); - drop(_noact); + drop(noactivate); if !sent_ok { bail!("SendInput inserted fewer mouse events than expected for the foreground click."); } + if activate { + let foreground_root = GetAncestor(GetForegroundWindow(), GA_ROOT); + let target_root = GetAncestor(target, GA_ROOT); + if foreground_root != target_root { + bail!("The foreground click did not activate its target window."); + } + } } Ok(()) @@ -709,7 +836,28 @@ pub fn send_wheel_synthesized(sx: i32, sy: i32, ticks: i32, horizontal: bool) -> #[cfg(test)] mod wheel_tests { - use super::{wheel_mouse_data, WHEEL_DELTA}; + use super::{posted_press_message, wheel_mouse_data, WHEEL_DELTA}; + use windows::Win32::UI::WindowsAndMessaging::{WM_LBUTTONDBLCLK, WM_LBUTTONDOWN}; + + #[test] + fn posted_double_click_uses_the_win32_double_click_message() { + assert_eq!( + posted_press_message(WM_LBUTTONDOWN, WM_LBUTTONDBLCLK, 0, true), + WM_LBUTTONDOWN + ); + assert_eq!( + posted_press_message(WM_LBUTTONDOWN, WM_LBUTTONDBLCLK, 1, true), + WM_LBUTTONDBLCLK + ); + assert_eq!( + posted_press_message(WM_LBUTTONDOWN, WM_LBUTTONDBLCLK, 2, true), + WM_LBUTTONDOWN + ); + assert_eq!( + posted_press_message(WM_LBUTTONDOWN, WM_LBUTTONDBLCLK, 1, false), + WM_LBUTTONDOWN + ); + } #[test] fn wheel_data_up_is_positive_one_notch() { diff --git a/libs/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs b/libs/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs index f587d311b8..4a94156bad 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/recording_hooks.rs @@ -12,6 +12,19 @@ use std::sync::{Arc, OnceLock}; #[cfg(target_os = "windows")] use crate::uia::ElementCache; +use cua_driver_core::recording::ScreenshotCapture; + +#[cfg(target_os = "windows")] +use windows::Win32::Foundation::HWND; + +#[cfg(target_os = "windows")] +use windows::Win32::UI::Input::KeyboardAndMouse::IsWindowEnabled; + +#[cfg(target_os = "windows")] +use windows::Win32::UI::WindowsAndMessaging::{ + GetLastActivePopup, GetWindowThreadProcessId, IsWindow, IsWindowVisible, +}; + #[cfg(target_os = "windows")] static ELEMENT_CACHE: OnceLock<Arc<ElementCache>> = OnceLock::new(); @@ -20,15 +33,69 @@ pub fn set_element_cache(cache: Arc<ElementCache>) { let _ = ELEMENT_CACHE.set(cache); } +/// Resolve the window whose application evidence should be captured. Keep a +/// live explicit HWND so occluded/background turns capture the exact target. +/// When an action closes a modal HWND, fall back to another top-level window +/// owned by the same pid for the post-action application state. #[cfg(target_os = "windows")] -pub fn app_state_json_for(window_id: Option<u64>, pid: Option<i64>) -> Option<Vec<u8>> { +pub fn resolve_window_for_recording(window_id: Option<u64>, pid: Option<i64>) -> Option<u64> { + if let Some(window_id) = window_id { + let hwnd = HWND(window_id as *mut _); + if unsafe { IsWindow(hwnd) }.as_bool() { + let popup = unsafe { GetLastActivePopup(hwnd) }; + if !unsafe { IsWindowEnabled(hwnd) }.as_bool() + && popup != hwnd + && unsafe { IsWindow(popup) }.as_bool() + && unsafe { IsWindowVisible(popup) }.as_bool() + { + let mut popup_pid = 0; + unsafe { GetWindowThreadProcessId(popup, Some(&mut popup_pid)) }; + if pid.and_then(|value| u32::try_from(value).ok()) == Some(popup_pid) { + return Some(popup.0 as u64); + } + } + return Some(window_id); + } + } let pid = u32::try_from(pid?).ok()?; - let hwnd = match window_id { - Some(w) => w, - None => crate::win32::list_windows(Some(pid)).first().map(|w| w.hwnd)?, + crate::win32::list_windows(Some(pid)) + .first() + .map(|window| window.hwnd) +} + +#[cfg(target_os = "windows")] +pub fn screenshot_for_recording( + window_id: Option<u64>, + pid: Option<i64>, +) -> ScreenshotCapture { + if window_id.is_none() && pid.is_none() { + return crate::capture::screenshot_display_bytes() + .map(ScreenshotCapture::captured) + .unwrap_or_else(|_| ScreenshotCapture::unavailable("capture_failed")); + } + let Some(hwnd) = resolve_window_for_recording(window_id, pid) else { + return ScreenshotCapture::unavailable("target_unavailable"); }; + match crate::capture::screenshot_window_bytes_with_occlusion(hwnd) { + Ok((_, true)) => ScreenshotCapture::unavailable("background_occluded"), + Ok((png, false)) => ScreenshotCapture::captured(png), + Err(error) if error.to_string().contains("minimized window") => { + ScreenshotCapture::unavailable("target_minimized") + } + Err(_) => ScreenshotCapture::unavailable("capture_failed"), + } +} + +#[cfg(target_os = "windows")] +pub fn app_state_json_for(window_id: Option<u64>, pid: Option<i64>) -> Option<Vec<u8>> { + let pid = u32::try_from(pid?).ok()?; + let hwnd = resolve_window_for_recording(window_id, Some(pid.into()))?; let result = crate::uia::walk_tree(hwnd, None); - let element_count = result.nodes.iter().filter(|n| n.element_index.is_some()).count(); + let element_count = result + .nodes + .iter() + .filter(|n| n.element_index.is_some()) + .count(); let payload = serde_json::json!({ "pid": pid, "window_id": hwnd, @@ -52,6 +119,25 @@ pub fn element_window_local_xy(window_id: u64, pid: i64, element_index: u32) -> } #[cfg(not(target_os = "windows"))] -pub fn app_state_json_for(_window_id: Option<u64>, _pid: Option<i64>) -> Option<Vec<u8>> { None } +pub fn app_state_json_for(_window_id: Option<u64>, _pid: Option<i64>) -> Option<Vec<u8>> { + None +} #[cfg(not(target_os = "windows"))] -pub fn element_window_local_xy(_window_id: u64, _pid: i64, _element_index: u32) -> Option<(f64, f64)> { None } +pub fn resolve_window_for_recording(_window_id: Option<u64>, _pid: Option<i64>) -> Option<u64> { + None +} +#[cfg(not(target_os = "windows"))] +pub fn screenshot_for_recording( + _window_id: Option<u64>, + _pid: Option<i64>, +) -> ScreenshotCapture { + ScreenshotCapture::unavailable("unsupported_platform") +} +#[cfg(not(target_os = "windows"))] +pub fn element_window_local_xy( + _window_id: u64, + _pid: i64, + _element_index: u32, +) -> Option<(f64, f64)> { + None +} diff --git a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs index 610ca60791..da370e98c8 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/tools/impl_.rs @@ -134,6 +134,11 @@ fn bitmap_to_screen(hwnd: u64, px: i32, py: i32) -> (i32, i32) { } } +fn screen_to_bitmap(hwnd: u64, sx: i32, sy: i32) -> (i32, i32) { + let (origin_x, origin_y) = bitmap_to_screen(hwnd, 0, 0); + (sx - origin_x, sy - origin_y) +} + /// Animate the agent cursor to (sx, sy) in screen coordinates and wait for the /// glide to finish before returning. No-op when the overlay is not enabled. /// @@ -633,11 +638,9 @@ impl Tool for ListWindowsTool { one?\".\n\n\ Per-record fields: window_id (HWND), pid + app_name, title, \ bounds {x, y, width, height}, layer (always 0), z_index (stacking order), \ - is_on_screen. The macOS-specific on_current_space / space_ids fields are \ + is_on_screen, minimized. The macOS-specific on_current_space / space_ids fields are \ omitted on Windows; current_space_id is null.\n\n\ - Inputs: pid (optional pid filter), on_screen_only (bool, default false — \ - Windows currently only enumerates visible non-minimized windows; this flag \ - is accepted but has no effect on the result set yet).".into(), + Inputs: pid (optional pid filter), on_screen_only (bool, default false).".into(), input_schema: json!({"type":"object","properties":{ "pid":{"type":"integer","description":"Optional pid filter. When set, only this pid's windows are returned."}, "on_screen_only":{"type":"boolean","description":"When true, drop windows that aren't currently on-screen. Default false."} @@ -649,8 +652,8 @@ impl Tool for ListWindowsTool { async fn invoke(&self, args: Value) -> ToolResult { use cua_driver_core::tool_args::ArgsExt; let filter_pid = args.opt_u64("pid").map(|v| v as u32); - let _on_screen_only = args.bool_or("on_screen_only", false); - let (windows, pid_to_name) = tokio::task::spawn_blocking(move || { + let on_screen_only = args.bool_or("on_screen_only", false); + let (mut windows, pid_to_name) = tokio::task::spawn_blocking(move || { let wins = crate::win32::list_windows(filter_pid); let procs = crate::win32::list_processes(); let map: std::collections::HashMap<u32, String> = @@ -659,9 +662,12 @@ impl Tool for ListWindowsTool { }) .await .unwrap_or_default(); + if on_screen_only { + windows.retain(|w| w.is_on_screen); + } // Swift surfaces a warning when a pid filter matches nothing. - if let Some(fp) = filter_pid { + let missing_pid_warning = if let Some(fp) = filter_pid { if windows.is_empty() { use windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow; use windows::Win32::UI::WindowsAndMessaging::GetWindowThreadProcessId; @@ -672,14 +678,17 @@ impl Tool for ListWindowsTool { p }; let fg_name = pid_to_name.get(&fg_pid).map(|s| s.as_str()).unwrap_or("?"); - let msg = format!( + Some(format!( "⚠️ No windows found for pid {fp}. The pid may be wrong or the app may not \ have created a window yet. The current frontmost app appears to be \ \"{fg_name}\" (pid {fg_pid})." - ); - return ToolResult::text(msg); + )) + } else { + None } - } + } else { + None + }; // z_index: list_windows merges EnumWindows first (which the Win32 // window manager returns in canonical top-to-bottom z-order), then @@ -704,7 +713,8 @@ impl Tool for ListWindowsTool { "bounds": { "x": w.x, "y": w.y, "width": w.width, "height": w.height }, "layer": 0, "z_index": z_index, - "is_on_screen": true, + "is_on_screen": w.is_on_screen, + "minimized": w.minimized, }) }) .collect(); @@ -726,6 +736,9 @@ impl Tool for ListWindowsTool { (SkyLight Space SPIs unavailable — on_current_space / space_ids omitted.)" ); let mut lines = vec![header]; + if let Some(warning) = missing_pid_warning { + lines.push(warning); + } for r in &records { let app = r["app_name"].as_str().unwrap_or("?"); let pid = r["pid"].as_u64().unwrap_or(0); @@ -754,6 +767,7 @@ impl Tool for ListWindowsTool { json!({ "window_id": w.hwnd, "pid": w.pid, "title": w.title, "x": w.x, "y": w.y, "width": w.width, "height": w.height, + "is_on_screen": w.is_on_screen, "minimized": w.minimized, }) }) .collect(); @@ -928,7 +942,7 @@ impl Tool for GetWindowStateTool { // surface *why* there's no image (the iconic-window guard from // #1973 / PR #1974 is the load-bearing case: minimized windows // legitimately can't be captured, and the caller needs to know - // to call `raise_window` / `list_windows` instead of retrying). + // to call `bring_to_front` instead of retrying). // The previous `Err(_) => None` silently dropped the error and // upstream agents saw an empty response with no signal. let (screenshot, screenshot_err) = if do_shot { @@ -1549,7 +1563,7 @@ impl Tool for LaunchAppTool { "cdp_debugging_port":{"type":"integer","description":"Accepted for cross-platform parity; currently no-op on Windows."}, "webkit_inspector_port":{"type":"integer","description":"Accepted for cross-platform parity; no-op on Windows."}, "creates_new_application_instance":{"type":"boolean","description":"Accepted for parity; no-op on Windows (ShellExecuteEx always creates a new process)."}, - "start_minimized":{"type":"boolean","description":"When true, launch the app's window minimized to the taskbar instead of restored-but-not-activated. Use this when the agent wants to drive the app entirely in the background — the user's previously-frontmost window (e.g. terminal) stays visually on top. Implementation uses SW_SHOWMINNOACTIVE for the ShellExecuteEx path and a follow-up ShowWindow(SW_MINIMIZE) on the AUMID path. UIA / background dispatch still work on a minimized window; only `screenshot` and `delivery_mode:\"foreground\"` need it restored."} + "start_minimized":{"type":"boolean","description":"When true, launch the app's window minimized to the taskbar instead of restored-but-not-activated. Use this when the agent wants to drive the app entirely in the background — the user's previously-frontmost window (e.g. terminal) stays visually on top. Desktop launches hold the foreground lock through startup and use SW_SHOWMINNOACTIVE; packaged-app activation remains broker-controlled and receives a best-effort SW_SHOWMINNOACTIVE post-pass. UIA / background dispatch still work on a minimized window; only `screenshot` and `delivery_mode:\"foreground\"` need it restored."} },"additionalProperties":false}), read_only: false, destructive: false, idempotent: true, open_world: true, }) @@ -1773,6 +1787,31 @@ impl Tool for LaunchAppTool { } }; + // Strict no-activation is available only for the desktop launch path. + // UWP activation is broker-controlled and retains its existing + // restore-after-activation behavior. + let mut foreground_lock = if start_minimized && aumid_for_uwp.is_none() { + Some(crate::input::ForegroundLockGuard::acquire()) + } else { + None + }; + if foreground_lock + .as_ref() + .is_some_and(|guard| !guard.acquired()) + { + return ToolResult::error( + "Background minimized launch is unavailable because Windows did not grant the \ + foreground lock required to prevent the new process from activating. No process \ + was started. Launch without start_minimized only when a foreground change is \ + acceptable.", + ) + .with_structured(json!({ + "code": "background_unavailable", + "delivery_mode": "background", + "event_kind": "app_launch", + })); + } + // Branch: AUMID activation if we resolved one; else legacy // ShellExecuteExW. Both branches still need to handle `urls` // (additional URLs always go through ShellExecuteExW since the @@ -1801,6 +1840,13 @@ impl Tool for LaunchAppTool { let target_for_shell = target_file_opt.clone(); let extra_for_shell = extra_joined.clone(); let n_show_for_shell = n_show; + let direct_minimized_exe = start_minimized + && target_for_shell.as_deref().is_some_and(|target| { + std::path::Path::new(target).is_file() + && std::path::Path::new(target) + .extension() + .is_some_and(|extension| extension.eq_ignore_ascii_case("exe")) + }); // Bound the shell launch with a timeout. An unregistered protocol // or file association makes `ShellExecuteExW` block on a modal shell // dialog ("you'll need a new app to open this …") on the *session* @@ -1811,9 +1857,12 @@ impl Tool for LaunchAppTool { // backstop for any *other* blocking broker dialog (SmartScreen, an // elevation/consent surface) so a bad target can't hang the daemon. let launch = tokio::task::spawn_blocking(move || -> anyhow::Result<u32> { - use windows::core::PCWSTR; + use windows::core::{PCWSTR, PWSTR}; use windows::Win32::Foundation::CloseHandle; - use windows::Win32::System::Threading::GetProcessId; + use windows::Win32::System::Threading::{ + CreateProcessW, GetProcessId, PROCESS_CREATION_FLAGS, PROCESS_INFORMATION, + STARTF_USESHOWWINDOW, STARTUPINFOW, + }; use windows::Win32::UI::Shell::{ ShellExecuteExW, SEE_MASK_FLAG_NO_UI, SEE_MASK_NOCLOSEPROCESS, SHELLEXECUTEINFOW, @@ -1830,30 +1879,63 @@ impl Tool for LaunchAppTool { }); let args_w = to_wide(&extra_for_shell); - let mut info = SHELLEXECUTEINFOW { - cbSize: std::mem::size_of::<SHELLEXECUTEINFOW>() as u32, - fMask: SEE_MASK_NOCLOSEPROCESS | SEE_MASK_FLAG_NO_UI, - lpVerb: PCWSTR(op_w.as_ptr()), - lpFile: PCWSTR(file_w.as_ptr()), - lpParameters: if extra_for_shell.is_empty() { - PCWSTR::null() + let pid = if direct_minimized_exe { + let target = target_for_shell.as_deref().expect("checked executable path"); + let mut command_line = to_wide(&if extra_for_shell.is_empty() { + format!(r#""{target}""#) } else { - PCWSTR(args_w.as_ptr()) - }, - nShow: n_show_for_shell, - ..Default::default() - }; - unsafe { - ShellExecuteExW(&mut info)?; - } - let pid = if !info.hProcess.is_invalid() { - let p = unsafe { GetProcessId(info.hProcess) }; + format!(r#""{target}" {extra_for_shell}"#) + }); + let startup = STARTUPINFOW { + cb: std::mem::size_of::<STARTUPINFOW>() as u32, + dwFlags: STARTF_USESHOWWINDOW, + wShowWindow: n_show_for_shell as u16, + ..Default::default() + }; + let mut process = PROCESS_INFORMATION::default(); unsafe { - let _ = CloseHandle(info.hProcess); + CreateProcessW( + PCWSTR(file_w.as_ptr()), + PWSTR(command_line.as_mut_ptr()), + None, + None, + false, + PROCESS_CREATION_FLAGS(0), + None, + PCWSTR::null(), + &startup, + &mut process, + )?; + let _ = CloseHandle(process.hThread); + let _ = CloseHandle(process.hProcess); } - p + process.dwProcessId } else { - 0 + let mut info = SHELLEXECUTEINFOW { + cbSize: std::mem::size_of::<SHELLEXECUTEINFOW>() as u32, + fMask: SEE_MASK_NOCLOSEPROCESS | SEE_MASK_FLAG_NO_UI, + lpVerb: PCWSTR(op_w.as_ptr()), + lpFile: PCWSTR(file_w.as_ptr()), + lpParameters: if extra_for_shell.is_empty() { + PCWSTR::null() + } else { + PCWSTR(args_w.as_ptr()) + }, + nShow: n_show_for_shell, + ..Default::default() + }; + unsafe { + ShellExecuteExW(&mut info)?; + } + if !info.hProcess.is_invalid() { + let p = unsafe { GetProcessId(info.hProcess) }; + unsafe { + let _ = CloseHandle(info.hProcess); + } + p + } else { + 0 + } }; // Open any additional URLs in the default browser (no focus @@ -2130,12 +2212,9 @@ impl Tool for LaunchAppTool { // and minimizes anything that materializes. The task runs detached // so the launch_app response isn't held up by it. // - // SW_MINIMIZE itself activates "the next top-level window in z-order" - // which would shift the user's focus, but the foreground-restore - // polling task (spawned earlier in this method via - // `restore_foreground_polling_best_effort`) flips foreground back to - // the pre-launch window, so the net effect is "minimize and leave - // the user's window where it was". + // SW_SHOWMINNOACTIVE preserves the foreground while minimizing. Using + // SW_MINIMIZE here would itself activate the next z-order window and + // force a visible restore-after-steal cycle. if start_minimized { // First, minimize anything already resolved (covers the common // single-process path where windows_json was populated). @@ -2156,17 +2235,21 @@ impl Tool for LaunchAppTool { // minimize a user's unrelated app that started during the 5 s // poll window. let parent_pid = pid; + let immediate_hwnds_for_poll = immediate_hwnds.clone(); let _ = tokio::task::spawn_blocking(move || { use windows::Win32::Foundation::HWND; - use windows::Win32::UI::WindowsAndMessaging::{ShowWindow, SW_MINIMIZE}; + use windows::Win32::UI::WindowsAndMessaging::{ShowWindow, SW_SHOWMINNOACTIVE}; for h in immediate_hwnds { unsafe { - let _ = ShowWindow(HWND(h as *mut _), SW_MINIMIZE); + let _ = ShowWindow(HWND(h as *mut _), SW_SHOWMINNOACTIVE); } } }) .await; - // Detached polling for launcher-stub late-window cases. + // Poll launcher-stub late-window cases before returning. The + // start_minimized contract is observable at response time; a + // detached task allowed callers to see a transient restored + // window immediately after a successful launch. // Strategy: every 200 ms for 5 s, find pids that // (a) weren't in the pre-launch snapshot, AND // (b) are part of the launched app's family — either a @@ -2181,17 +2264,21 @@ impl Tool for LaunchAppTool { // the regression CodeRabbit flagged. // // Loop ends early once we've minimized the first set of - // windows AND remained idle for one tick — the typical + // windows AND they remain minimized for three ticks — the typical // app has its main window up within ~2 s of launch. let pre_pids = pre_launch_pids.clone(); let basename_for_poll = stub_basename.clone(); - tokio::spawn(async move { + let launch_foreground_lock = foreground_lock.take(); + (async move { use std::collections::HashSet; use windows::Win32::Foundation::HWND; - use windows::Win32::UI::WindowsAndMessaging::{ShowWindow, SW_MINIMIZE}; - let mut minimized: HashSet<u64> = HashSet::new(); + use windows::Win32::UI::WindowsAndMessaging::{ + IsIconic, SW_SHOWMINNOACTIVE, ShowWindow, + }; + let _foreground_lock = launch_foreground_lock; + let mut minimized: HashSet<u64> = immediate_hwnds_for_poll.into_iter().collect(); let mut idle_ticks_after_any_hit: u8 = 0; - let mut hit_count_total: usize = 0; + let mut hit_count_total = minimized.len(); for _ in 0..25 { let pre_pids_clone = pre_pids.clone(); let basename_clone = basename_for_poll.clone(); @@ -2222,22 +2309,32 @@ impl Tool for LaunchAppTool { .await .unwrap_or_default(); for w in wins { - if minimized.insert(w.hwnd) { - tick_hits += 1; + let is_new = minimized.insert(w.hwnd); + if is_new { hit_count_total += 1; - let hwnd_iso = w.hwnd as usize; - let _ = tokio::task::spawn_blocking(move || unsafe { - let _ = ShowWindow(HWND(hwnd_iso as *mut _), SW_MINIMIZE); - }) - .await; + } + let hwnd_iso = w.hwnd as usize; + let restored = tokio::task::spawn_blocking(move || unsafe { + let hwnd = HWND(hwnd_iso as *mut _); + if IsIconic(hwnd).as_bool() { + false + } else { + let _ = ShowWindow(hwnd, SW_SHOWMINNOACTIVE); + true + } + }) + .await + .unwrap_or(false); + if is_new || restored { + tick_hits += 1; } } } if tick_hits == 0 { idle_ticks_after_any_hit += 1; if hit_count_total > 0 && idle_ticks_after_any_hit >= 3 { - // Stable: had hits, then 600 ms of nothing new. - // Done. + // Stable: known windows remained minimized and no + // new window appeared for 600 ms. break; } } else { @@ -2245,7 +2342,8 @@ impl Tool for LaunchAppTool { } tokio::time::sleep(std::time::Duration::from_millis(200)).await; } - }); + }) + .await; } // Match Swift text format 1:1. @@ -2413,6 +2511,25 @@ impl Tool for ClickTool { let sx = args.f64_or("x", 0.0) as i32; let sy = args.f64_or("y", 0.0) as i32; + // Resolve the application window before moving the agent cursor. + // WindowFromPoint can return a transparent layered overlay, and the + // cursor overlay is about to occupy this exact screen point. + let root = unsafe { + use windows::Win32::Foundation::POINT; + use windows::Win32::UI::WindowsAndMessaging::{ + GetAncestor, WindowFromPoint, GA_ROOT, + }; + let target = WindowFromPoint(POINT { x: sx, y: sy }); + if target.0.is_null() { + return ToolResult::error(format!( + "No window under screen point ({sx},{sy})." + )); + } + let root = GetAncestor(target, GA_ROOT); + if root.0.is_null() { target } else { root } + }; + let hwnd_u = root.0 as u64; + // Animate the agent cursor to the screen point, then click. overlay_glide_to(&cursor_key, sx as f64, sy as f64).await; crate::overlay::send_command( @@ -2423,22 +2540,12 @@ impl Tool for ClickTool { }, ); - // Resolve the HWND that owns this screen pixel and click it via - // send_click_synthesized — it does the foreground-swap + UIPI checks - // on whatever owns the pixel, which is what lands Chromium-content - // clicks. WindowFromPoint walks to the leaf window at the point. - // (send_click_synthesized restores the previous foreground + cursor - // itself ~40ms after the click, so no extra restore guard here.) + // Click the HWND that owned the pixel before the driver overlay + // moved there. The active SendInput path performs the foreground + // swap and UIPI checks needed for Chromium and retained-mode apps. let send_result = tokio::task::spawn_blocking(move || -> anyhow::Result<u64> { - use windows::Win32::Foundation::POINT; - use windows::Win32::UI::WindowsAndMessaging::WindowFromPoint; - let target = unsafe { WindowFromPoint(POINT { x: sx, y: sy }) }; - if target.0.is_null() { - anyhow::bail!("No window under screen point ({sx},{sy})."); - } - let hwnd_u = target.0 as u64; let mod_refs: Vec<&str> = modifiers.iter().map(String::as_str).collect(); - crate::input::send_click_synthesized_mods( + crate::input::send_click_synthesized_active_mods( hwnd_u, sx, sy, count, &button, &mod_refs, )?; Ok(hwnd_u) @@ -2682,6 +2789,57 @@ impl Tool for ClickTool { ); let btn = button.clone(); + // An explicit accessibility action is a semantic request, not a + // pixel gesture hint. Route expand through ExpandCollapsePattern + // even when foreground delivery was allowed; this reliably opens + // WPF/WinUI menus and tree nodes whose visual click target is + // transient or scroll-adjusted. + if action_req.as_deref() == Some("expand") { + let state = self.state.clone(); + let expand = tokio::task::spawn_blocking(move || -> anyhow::Result<()> { + use windows::core::Interface; + use windows::Win32::UI::Accessibility::{ + IUIAutomationElement, IUIAutomationExpandCollapsePattern, + UIA_ExpandCollapsePatternId, + }; + + let retained = state + .element_cache + .get_element_retained(pid, hwnd, idx) + .ok_or_else(|| { + anyhow::anyhow!("element [{idx}] is not in the UIA cache") + })?; + if !retained.is_uia() { + anyhow::bail!("element [{idx}] is not a UIA element"); + } + let element = + unsafe { IUIAutomationElement::from_raw(retained.as_ptr() as *mut _) }; + let pattern = unsafe { + element + .GetCurrentPattern(UIA_ExpandCollapsePatternId) + .and_then(|value| value.cast::<IUIAutomationExpandCollapsePattern>()) + } + .map_err(|error| { + anyhow::anyhow!("ExpandCollapsePattern unavailable: {error}") + })?; + let result = + crate::uia::fg_bypass::run_with_uwp_bypass(hwnd as isize, || unsafe { + pattern.Expand() + }); + std::mem::forget(element); + result.map_err(|error| anyhow::anyhow!("ExpandCollapse.Expand failed: {error}")) + }) + .await; + return match expand { + Ok(Ok(())) => ToolResult::text(format!( + "✅ Expanded UIA element [{idx}] via ExpandCollapsePattern." + )) + .with_structured(json!({ "path": "uia_expand_collapse", "verified": false, "effect": "unverifiable" })), + Ok(Err(error)) => ToolResult::error(error.to_string()), + Err(error) => ToolResult::error(format!("Task error: {error}")), + }; + } + // delivery_mode:"foreground" — skip UIA Invoke and use SendInput at the // cached element center. The caller explicitly chose foreground // delivery; UIA Invoke would be background-safe (which they @@ -2734,6 +2892,34 @@ impl Tool for ClickTool { return r; } } + // Chromium's UIA Invoke raises a fully occluded renderer, while a + // targeted PostMessage left click reaches the renderer without + // changing foreground, z-order, or the real cursor. Keep AX for + // target resolution and use the posted-message transport only for + // the empirically verified single-left-click shape. + if delivery == DeliveryMode::Background + && btn == "left" + && count == 1 + && crate::input::is_chromium_target_window(hwnd) + { + let posted = tokio::task::spawn_blocking(move || { + crate::input::post_click_screen(hwnd, cx, cy, count, &btn) + }) + .await; + return match posted { + Ok(Ok(())) => ToolResult::text(format!( + "✅ Posted click on Chromium element [{idx}] at screen ({cx},{cy}) \ + (background, no foreground swap)." + )) + .with_structured(json!({ + "path": "post_message", + "verified": false, + "effect": "unverifiable" + })), + Ok(Err(error)) => ToolResult::error(error.to_string()), + Err(error) => ToolResult::error(format!("Task error: {error}")), + }; + } // Try UIA Invoke first (it works for UWP / modern XAML / web // content where PostMessage(WM_LBUTTONDOWN) hits the outer // HWND but never reaches the inner XAML/composition element). @@ -2747,6 +2933,25 @@ impl Tool for ClickTool { let state_clone = self.state.clone(); let use_uia_invoke = (btn == "left" || btn == "middle") && count == 1; let result = tokio::task::spawn_blocking(move || -> anyhow::Result<String> { + // Direct Chromium UIA Invoke can return S_OK without firing a + // DOM event while occluded. Try the honest coordinate actuator + // first: it lands while visible and reports occlusion without + // raising the window when hidden. + if delivery == DeliveryMode::Background + && crate::input::is_chromium_target_window(hwnd) + { + let (cx, cy) = resolve_onscreen_point_with_scroll( + &state_clone.element_cache, pid, hwnd, idx, cx, cy, "clicking", + ) + .map_err(|message| anyhow::anyhow!(message))?; + return crate::input::inject_click_screen(hwnd, cx, cy, count, &btn) + .map(|()| format!( + "✅ Injected click on [{idx}] (screen ({cx},{cy}), background, no foreground swap)." + )) + .map_err(|error| anyhow::anyhow!( + "__CUA_BG_UNAVAILABLE_CLICK__{error}" + )); + } if use_uia_invoke { // Retain the element out of the cache (AddRef under the // cache lock) so it can't be freed by a concurrent @@ -2838,30 +3043,31 @@ impl Tool for ClickTool { std::mem::forget(elem); } } - // PostMessage fallback (legacy Win32 + non-Invokable elements). - // delivery_mode:"background" on targets that silently drop PostMessage - // clicks (Chromium content, GTK buttons): route through the - // universal coordinate-injection actuator (touch injection, no - // foreground swap, z-order preserved) so the caller never needs - // to know the target is Chromium/GTK and never sees a raise. - // Only the structured error remains as a last resort (e.g. a - // right-click, which has no clean touch mapping). if delivery == DeliveryMode::Background - && crate::input::delivery::would_be_silently_dropped(hwnd, EventKind::MouseClick) + && crate::input::delivery::would_be_silently_dropped( + hwnd, + EventKind::MouseClick, + ) { - // Coordinate injection lands at (cx,cy); scroll the element - // into view if it's off-screen, else preserve the clean - // off-screen failure. let (cx, cy) = resolve_onscreen_point_with_scroll( - &state_clone.element_cache, pid, hwnd, idx, cx, cy, "clicking", + &state_clone.element_cache, + pid, + hwnd, + idx, + cx, + cy, + "clicking", ) - .map_err(|m| anyhow::anyhow!(m))?; - match crate::input::inject_click_screen(hwnd, cx, cy, count, &btn) { - Ok(()) => return Ok(format!( - "✅ Injected click on [{idx}] (screen ({cx},{cy}), background, no foreground swap)." - )), - Err(e) => anyhow::bail!("__CUA_BG_UNAVAILABLE_CLICK__{e}"), - } + .map_err(|message| anyhow::anyhow!(message))?; + return crate::input::inject_click_screen(hwnd, cx, cy, count, &btn) + .map(|()| { + format!( + "✅ Injected click on [{idx}] (screen ({cx},{cy}), background, no foreground swap)." + ) + }) + .map_err(|error| { + anyhow::anyhow!("__CUA_BG_UNAVAILABLE_CLICK__{error}") + }); } crate::input::post_click_screen(hwnd, cx, cy, count, &btn)?; let action_name = match btn.as_str() { @@ -2965,7 +3171,7 @@ impl Tool for ClickTool { let mods_owned = modifiers.clone(); let send_result = tokio::task::spawn_blocking(move || { let mod_refs: Vec<&str> = mods_owned.iter().map(String::as_str).collect(); - crate::input::send_click_synthesized_mods( + crate::input::send_click_synthesized_active_mods( hwnd, sx as i32, sy as i32, count, &btn, &mod_refs, ) }) @@ -3001,6 +3207,56 @@ impl Tool for ClickTool { return r; } } + // Match the AX-addressed Chromium route above: the point remains + // PX-resolved, but transport uses the background-safe posted + // message path proven against the fully occluded fixture. + if delivery == DeliveryMode::Background + && btn == "left" + && count == 1 + && crate::input::is_chromium_target_window(hwnd) + { + let posted = tokio::task::spawn_blocking(move || { + crate::input::post_click_screen(hwnd, sx_i, sy_i, count, &btn) + }) + .await; + return match posted { + Ok(Ok(())) => ToolResult::text(format!( + "✅ Posted click to Chromium pid {pid} at ({sx},{sy}) \ + (background, no foreground swap)." + )) + .with_structured(json!({ + "path": "post_message", + "verified": false, + "effect": "unverifiable" + })), + Ok(Err(error)) => ToolResult::error(error.to_string()), + Err(error) => ToolResult::error(format!("Task error: {error}")), + }; + } + // As above, bypass Chromium's false-positive UIA Invoke and use + // the coordinate actuator before attempting any accessibility hit + // test. The actuator itself distinguishes visible delivery from a + // fully occluded structured refusal. + if delivery == DeliveryMode::Background && crate::input::is_chromium_target_window(hwnd) + { + let btn2 = btn.clone(); + let inj = tokio::task::spawn_blocking(move || { + crate::input::inject_click_screen(hwnd, sx as i32, sy as i32, count, &btn2) + }) + .await; + return match inj { + Ok(Ok(())) => ToolResult::text(format!( + "✅ Injected click to pid {pid} at ({sx},{sy}) (background, no foreground swap)." + )) + .with_structured(json!({ "path": "pixel", "verified": false, "effect": "unverifiable" })), + Ok(Err(error)) => crate::input::delivery::background_unavailable_error_with_cause( + hwnd, + EventKind::MouseClick, + error.to_string(), + ), + Err(error) => ToolResult::error(format!("Task error: {error}")), + }; + } let use_uia = (btn == "left" || btn == "middle") && count == 1; if use_uia { let invoked = tokio::task::spawn_blocking(move || { @@ -3022,20 +3278,8 @@ impl Tool for ClickTool { } } - // UIA hit-test didn't land. Decide between PostMessage / injection / - // SendInput based on dispatch mode. - // - // delivery_mode:"background" (the default) — never swap foreground. If the - // target silently drops PostMessage mouse events (Chromium DOM - // content, GTK button widgets), route through the universal - // coordinate-injection actuator: touch injection lands in the system - // input queue (so Chromium/Electron/WPF accept it; the OS promotes to - // WM_*BUTTON for legacy Win32) WITHOUT SetForegroundWindow, and a - // cloak+restore z-order guard keeps the target from visibly raising. - // This is what lets a caller "just target the app and play actions" - // without knowing whether it's Chromium/GTK/etc. The structured - // background_unavailable error only survives as a last resort for - // inputs injection can't express (e.g. right/middle clicks). + // UIA did not land. Known dropped surfaces other than direct + // Chromium (handled above) get one targeted injection attempt. if delivery == DeliveryMode::Background && crate::input::delivery::would_be_silently_dropped(hwnd, EventKind::MouseClick) { @@ -3045,23 +3289,16 @@ impl Tool for ClickTool { }) .await; return match inj { - Ok(Ok(())) => { - let click_word = match count { - 2 => "double-click", - 3 => "triple-click", - _ => "click", - }; - ToolResult::text(format!( - "✅ Injected {click_word} to pid {pid} at ({sx},{sy}) (background, no foreground swap)." - )) - .with_structured(json!({ "path": "pixel", "verified": false, "effect": "unverifiable" })) - } - Ok(Err(e)) => crate::input::delivery::background_unavailable_error_with_cause( + Ok(Ok(())) => ToolResult::text(format!( + "✅ Injected click to pid {pid} at ({sx},{sy}) (background, no foreground swap)." + )) + .with_structured(json!({ "path": "pixel", "verified": false, "effect": "unverifiable" })), + Ok(Err(error)) => crate::input::delivery::background_unavailable_error_with_cause( hwnd, EventKind::MouseClick, - e.to_string(), + error.to_string(), ), - Err(e) => ToolResult::error(format!("Task error: {e}")), + Err(error) => ToolResult::error(format!("Task error: {error}")), }; } @@ -3146,9 +3383,11 @@ async fn focus_by_pixel( .invoke(click_args) .await; if focus.is_error == Some(true) { - return Err(ToolResult::error(format!( - "focus pixel-click at ({x:.0},{y:.0}) failed." - ))); + // Preserve the click tool's structured background refusal (for example + // background_occluded / background_uipi_blocked). Re-wrapping it as a + // text-only error made keyboard-family PX calls lose the actionable + // capability result produced by the actual actuator. + return Err(focus); } // Brief settle so the renderer registers focus before the keystrokes. tokio::time::sleep(std::time::Duration::from_millis(120)).await; @@ -3330,12 +3569,12 @@ impl Tool for TypeTextTool { }; let text_len = text.chars().count(); - // delivery_mode:"background" — TextInput is currently never flagged as - // silently dropped (Chromium accepts WM_CHAR through its IME path), - // but call the helper so the policy stays centralised in delivery.rs - // and future targets can be added without touching this site. + // Refuse known background drops before the final WM_CHAR path. WPF is + // conditional: indexed text still has a working UIA ValuePattern route, + // while unindexed text would be posted to the top-level and disappear. if delivery == DeliveryMode::Background - && crate::input::delivery::would_be_silently_dropped(hwnd, EventKind::TextInput) + && (crate::input::delivery::would_be_silently_dropped(hwnd, EventKind::TextInput) + || (elem_idx.is_none() && crate::input::delivery::is_wpf_target_window(hwnd))) { return crate::input::delivery::background_unavailable_error( hwnd, @@ -3350,6 +3589,53 @@ impl Tool for TypeTextTool { // rejected (daemon not at UIAccess integrity), it returns an error // rather than a false success. if delivery == DeliveryMode::Foreground { + // An indexed foreground type targets that element, not whichever + // child happened to retain focus in the top-level window. UIA + // SetFocus is not sufficient for Chromium renderer controls, so + // establish real system focus with the same foreground coordinate + // actuator used by an indexed click before sending Unicode input. + if let Some(idx) = elem_idx { + let (cx, cy) = + match self + .state + .element_cache + .get_element_center(pid, hwnd, idx as usize) + { + Some(center) => center, + None => { + return ToolResult::error(format!( + "Element {idx} not in cache for hwnd={hwnd}. Call get_window_state first." + )) + } + }; + let (cx, cy) = match resolve_onscreen_point_with_scroll( + &self.state.element_cache, + pid, + hwnd, + idx as usize, + cx, + cy, + "foreground typing", + ) { + Ok(point) => point, + Err(message) => return ToolResult::error(message), + }; + let focus_result = tokio::task::spawn_blocking(move || { + crate::input::send_click_synthesized(hwnd, cx, cy, 1, "left") + }) + .await; + match focus_result { + Ok(Ok(())) => { + tokio::time::sleep(std::time::Duration::from_millis(120)).await; + } + Ok(Err(error)) => return ToolResult::error(error.to_string()), + Err(error) => { + return ToolResult::error(format!( + "foreground element-focus task failed: {error}" + )) + } + } + } let text_fg = text.clone(); let r = tokio::task::spawn_blocking(move || { crate::input::send_text_synthesized(hwnd, &text_fg) @@ -3892,50 +4178,104 @@ impl Tool for PressKeyTool { } }; + // Classify known background drops before touching UIA focus. Focusing + // first made an honest Chromium refusal transiently activate the target. + let event_kind = if mods.is_empty() { + EventKind::Keystroke + } else { + EventKind::KeyCombo + }; + if delivery == DeliveryMode::Background + && crate::input::delivery::would_be_silently_dropped(hwnd, event_kind) + { + return crate::input::delivery::background_unavailable_error(hwnd, event_kind); + } + // W1: an element-addressed key needs the control's actual focus - // target, not merely its owning top-level HWND. Keep background - // delivery non-activating while UIA establishes child focus. - let _noact = if elem_idx.is_some() && delivery == DeliveryMode::Background { + // target, not merely its owning top-level HWND. Embedded WebView hosts + // can activate their frame from UIA SetFocus even under + // WS_EX_NOACTIVATE. Their proven-safe pixel route establishes renderer + // focus with a posted click, so reuse that route at the AX element's + // cached center. + let background_webview_focus = elem_idx.is_some() + && delivery == DeliveryMode::Background + && crate::input::has_chromium_descendant(hwnd); + let mut noact = if elem_idx.is_some() && delivery == DeliveryMode::Background { Some(crate::input::NoActivateGuard::arm( windows::Win32::Foundation::HWND(hwnd as *mut _), )) } else { None }; - if let Some(idx) = elem_idx { + if let Some(idx) = elem_idx.filter(|_| background_webview_focus) { + let Some((cx, cy)) = self + .state + .element_cache + .get_element_center(pid, hwnd, idx as usize) + else { + return ToolResult::error(format!( + "Element {idx} not in cache for hwnd={hwnd}. Call get_window_state first." + )); + }; + let (cx, cy) = match resolve_onscreen_point_with_scroll( + &self.state.element_cache, + pid, + hwnd, + idx as usize, + cx, + cy, + "focusing for key delivery", + ) { + Ok(point) => point, + Err(message) => return ToolResult::error(message), + }; + let (mut px, mut py) = screen_to_bitmap(hwnd, cx, cy); + if let Some(ratio) = self.state.resize_registry.ratio(pid) { + px = (px as f64 / ratio).round() as i32; + py = (py as f64 / ratio).round() as i32; + } + // Release the outer guard before the shared pixel helper. ClickTool + // owns a guard around the click itself, then releases it before its + // renderer settle period. This is the exact route already proven by + // the PX background cell, including targeted injection fallback. + drop(noact.take()); + if let Err(error) = focus_by_pixel( + &self.state, + pid, + Some(hwnd), + px as f64, + py as f64, + false, + args.opt_str("session"), + args.opt_str("_session_id"), + false, + ) + .await + { + return error; + } + } else if let Some(idx) = elem_idx { let state = self.state.clone(); let focused = tokio::task::spawn_blocking(move || { - state.element_cache.focus_element(pid, hwnd, idx as usize) + crate::uia::fg_bypass::run_with_uwp_bypass(hwnd as isize, || { + state.element_cache.focus_element(pid, hwnd, idx as usize) + }) }) .await; match focused { - Ok(Ok(())) => {} + Ok(Ok(())) => { + if delivery == DeliveryMode::Background { + let _ = crate::input::wait_for_focused_descendant( + hwnd, + std::time::Duration::from_millis(500), + ); + } + } Ok(Err(e)) => return ToolResult::error(e.to_string()), Err(e) => return ToolResult::error(format!("UIA focus task failed: {e}")), } } let key_display = key.clone(); - // Background mode: plain keystrokes (no modifiers) go through Chromium - // and GTK fine — would_be_silently_dropped returns false for the - // Keystroke variant by design. KeyCombo (modifiers) on Chromium IS - // dropped, so check that when modifiers are present. - let event_kind = if mods.is_empty() { - EventKind::Keystroke - } else { - EventKind::KeyCombo - }; - if !px_focus - && delivery == DeliveryMode::Background - && crate::input::delivery::would_be_silently_dropped(hwnd, event_kind) - { - // macOS-aligned contract: a `background` actuation never fronts. This - // key would be silently dropped by the target's input stack - // (TranslateAccelerator-based VCL/classic Win32, or Chromium key- - // combos) and the only way to land it is a foreground/focus grab — - // which background must not do. Surface background_unavailable so the - // agent escalates to delivery_mode:"foreground" (which may front). - return crate::input::delivery::background_unavailable_error(hwnd, event_kind); - } // Foreground: send_key_synthesized takes the SetForegroundWindow path. // Skipped when px-focus already fronted/clicked the target — the key then // goes via the plain background post path below. @@ -4254,13 +4594,11 @@ impl Tool for HotkeyTool { // reaches here means the key combo is NOT silently dropped on this // target (the drop-check above returned early otherwise), so it stays // on PostMessage and the no-foreground contract holds. - // px-focus delivers the combo via PostMessage to the now-focused field, so - // it never takes the SendInput foreground swap. - let use_send_input = !px_focus - && match delivery { - DeliveryMode::Foreground => true, - DeliveryMode::Background => false, - }; + // Foreground is an explicit request for system-queue delivery. This is + // still required after a PX focus click: PostMessage does not update + // global modifier state, so Chromium never observes Ctrl+Shift+7 as a + // chord even though the renderer control is focused. + let use_send_input = delivery == DeliveryMode::Foreground; let result = tokio::task::spawn_blocking(move || { let m: Vec<&str> = mods.iter().map(String::as_str).collect(); if use_send_input { @@ -4513,8 +4851,8 @@ impl Tool for ScrollTool { "by":{"type":"string","enum":["line","page"],"description":"Scroll granularity. Default: line."}, "amount":{"type":"integer","minimum":1,"maximum":50, "description":"Number of scroll ticks. Default 3."}, - "x":{"type":"number","description":"Screen-absolute X (desktop scope only) — wheel routes to the window under (x,y). Must be paired with y and no pid/window_id."}, - "y":{"type":"number","description":"Screen-absolute Y (desktop scope only). Must be paired with x and no pid/window_id."}, + "x":{"type":"number","description":"With pid/window_id: window-local screenshot X used to target a nested scroll surface in foreground mode. Without pid/window_id: screen-absolute X for desktop scope. Must be paired with y."}, + "y":{"type":"number","description":"With pid/window_id: window-local screenshot Y used to target a nested scroll surface in foreground mode. Without pid/window_id: screen-absolute Y for desktop scope. Must be paired with x."}, "window_id":{"type":"integer","description":"HWND of the target window. Required when element_index is used; otherwise auto-resolves the pid's first visible window."}, "element_index":{"type":"integer","description":"Optional element_index. Accepted for parity; currently no-op on Windows."}, "element_token": cua_driver_core::tool_schema::element_token_schema(), @@ -4653,7 +4991,27 @@ impl Tool for ScrollTool { // UIA while their top-level HWND ignores WM_VSCROLL. Prefer the // accessibility channel for an indexed target; the message path below // remains the fallback for native Win32 scrollbars. + if delivery == DeliveryMode::Background && crate::input::is_chromium_target_window(hwnd) { + return crate::input::delivery::background_unavailable_error( + hwnd, + EventKind::MouseScroll, + ); + } if let Some(idx) = elem_idx { + let prev_fg_addr = if delivery == DeliveryMode::Background { + Some(unsafe { + windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow().0 as isize + }) + } else { + None + }; + let _noact = if delivery == DeliveryMode::Background { + Some(crate::input::NoActivateGuard::arm( + windows::Win32::Foundation::HWND(hwnd as *mut _), + )) + } else { + None + }; let state = self.state.clone(); let direction_for_uia = direction.clone(); let uia_result = tokio::task::spawn_blocking(move || { @@ -4674,6 +5032,32 @@ impl Tool for ScrollTool { }) .await; if matches!(uia_result, Ok(Ok(()))) { + if delivery == DeliveryMode::Background { + // Keep WS_EX_NOACTIVATE armed through any WebView handler + // queued by the UIA operation. + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + if let Some(previous_addr) = prev_fg_addr { + use windows::Win32::Foundation::HWND; + use windows::Win32::UI::WindowsAndMessaging::{ + GetAncestor, GetForegroundWindow, GA_ROOT, + }; + let previous = HWND(previous_addr as *mut _); + let current = unsafe { GetForegroundWindow() }; + let current_root = unsafe { GetAncestor(current, GA_ROOT) }; + let target_root = unsafe { GetAncestor(HWND(hwnd as *mut _), GA_ROOT) }; + if !previous.0.is_null() + && current != previous + && !target_root.0.is_null() + && current_root == target_root + { + unsafe { + crate::input::force_foreground_attached(previous); + std::thread::sleep(std::time::Duration::from_millis(12)); + crate::input::force_foreground_attached(previous); + } + } + } + } return ToolResult::text(format!( "Scrolled {direction} {amount} ticks via UIA (delivery_mode:background)." )) @@ -4685,6 +5069,16 @@ impl Tool for ScrollTool { } } + if delivery == DeliveryMode::Background + && args.get("x").is_some_and(serde_json::Value::is_number) + && args.get("y").is_some_and(serde_json::Value::is_number) + { + return crate::input::delivery::background_unavailable_error( + hwnd, + EventKind::MouseScroll, + ); + } + // delivery_mode:"background" — WM_VSCROLL/HSCROLL is silently dropped by // Chromium and may be by GTK. Surface the standard structured // background_unavailable error: its remediation (bring_to_front + @@ -4719,16 +5113,27 @@ impl Tool for ScrollTool { }; let per: i32 = if by == "page" { 3 } else { 1 }; let ticks = sign * (amount as i32) * per; - // Target the window's screen center so the wheel lands on it. - let center = tokio::task::spawn_blocking(move || { - crate::win32::list_windows(Some(pid)) - .into_iter() - .find(|w| w.hwnd == hwnd) - .map(|w| (w.x + w.width / 2, w.y + w.height / 2)) - }) - .await - .ok() - .flatten(); + // A supplied PX target is window-local in the get_window_state + // bitmap. Route the wheel there so nested web scrollers receive it; + // otherwise retain the whole-window center fallback. + let px = args.get("x").and_then(|value| value.as_f64()); + let py = args.get("y").and_then(|value| value.as_f64()); + if px.is_some() != py.is_some() { + return ToolResult::error("scroll requires x and y together."); + } + let center = if let (Some(x), Some(y)) = (px, py) { + Some(bitmap_to_screen(hwnd, x as i32, y as i32)) + } else { + tokio::task::spawn_blocking(move || { + crate::win32::list_windows(Some(pid)) + .into_iter() + .find(|w| w.hwnd == hwnd) + .map(|w| (w.x + w.width / 2, w.y + w.height / 2)) + }) + .await + .ok() + .flatten() + }; let (cx, cy) = match center { Some(c) => c, None => { @@ -5791,7 +6196,7 @@ impl Tool for DragTool { // no cursor move; the target is held non-activatable + cloaked for the // stroke (mirrors the click pen path). if delivery == DeliveryMode::Background - && crate::input::delivery::would_be_silently_dropped(hwnd, EventKind::MouseClick) + && crate::input::delivery::would_be_silently_dropped(hwnd, EventKind::MouseMove) { let target = hwnd; let btn = button.clone(); @@ -5832,7 +6237,11 @@ impl Tool for DragTool { (delivery_mode:background, PostMessage would have been dropped)." )) } - Ok(Err(e)) => ToolResult::error(e.to_string()), + Ok(Err(e)) => crate::input::delivery::background_unavailable_error_with_cause( + hwnd, + EventKind::MouseMove, + e.to_string(), + ), Err(e) => ToolResult::error(format!("Task error: {e}")), }; } @@ -7436,12 +7845,14 @@ impl Tool for BringToFrontTool { // trick mirrors `send_key_synthesized` (input/keyboard.rs:313-345) // and is validated by `flash-repro/16-edge-launch-fg.ps1` for the // Edge launch focus-steal recovery case. - let outcome = tokio::task::spawn_blocking(move || -> Result<(u64, u64, bool), String> { + let outcome = + tokio::task::spawn_blocking(move || -> Result<(u64, u64, bool, bool), String> { use windows::Win32::Foundation::HWND; use windows::Win32::System::Threading::{AttachThreadInput, GetCurrentThreadId}; use windows::Win32::UI::WindowsAndMessaging::{ - GetForegroundWindow, GetWindowThreadProcessId, IsWindow, SetForegroundWindow, - SetWindowPos, HWND_NOTOPMOST, HWND_TOPMOST, SWP_NOACTIVATE, SWP_NOMOVE, SWP_NOSIZE, + GetForegroundWindow, GetWindowThreadProcessId, IsIconic, IsWindow, + SetForegroundWindow, SetWindowPos, ShowWindowAsync, HWND_NOTOPMOST, HWND_TOPMOST, + SWP_NOACTIVATE, SWP_NOMOVE, SWP_NOSIZE, SW_RESTORE, }; let target = HWND(hwnd as *mut _); @@ -7452,6 +7863,25 @@ impl Tool for BringToFrontTool { let prev_fg = unsafe { GetForegroundWindow() }; let prev_fg_addr = prev_fg.0 as u64; + // Iconic windows have no rendered pixels and live at the sentinel + // (-32000, -32000) position. Restore before changing z-order so + // bring_to_front is also the advertised recovery path for capture. + let was_minimized = unsafe { IsIconic(target) }.as_bool(); + if was_minimized { + let _ = unsafe { ShowWindowAsync(target, SW_RESTORE) }; + for _ in 0..20 { + if !unsafe { IsIconic(target) }.as_bool() { + break; + } + std::thread::sleep(std::time::Duration::from_millis(25)); + } + if unsafe { IsIconic(target) }.as_bool() { + return Err(format!( + "restore request did not complete for minimized hwnd 0x{hwnd:x}" + )); + } + } + // Lock-free z-order raise FIRST: bring the window to the top of the // normal band (the HWND_TOPMOST→HWND_NOTOPMOST force-to-front trick) // so it's brought to the VISIBLE front even when the foreground-lock @@ -7499,12 +7929,12 @@ impl Tool for BringToFrontTool { let _ = unsafe { AttachThreadInput(my_tid, fg_tid, false) }; } let now_fg = unsafe { GetForegroundWindow() }; - Ok((prev_fg_addr, now_fg.0 as u64, raised)) + Ok((prev_fg_addr, now_fg.0 as u64, raised, was_minimized)) }) .await; match outcome { - Ok(Ok((prev, now, raised))) => { + Ok(Ok((prev, now, raised, restored))) => { let focused = now == hwnd; let msg = if focused { format!("✅ bring_to_front: pid {pid} hwnd 0x{hwnd:x} is now foreground (was 0x{prev:x}).") @@ -7532,6 +7962,7 @@ impl Tool for BringToFrontTool { "target_hwnd": format!("0x{hwnd:x}"), "landed_on_target": focused, "raised": raised, + "restored": restored, })) } Ok(Err(e)) => ToolResult::error(format!("bring_to_front: {e}")), diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs index e5e7fb8504..e95278b7b0 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/cache.rs @@ -262,6 +262,25 @@ impl ElementCache { result.map_err(|e| anyhow::anyhow!("UIA SetFocus failed: {e}")) } + pub fn element_has_keyboard_focus( + &self, + pid: u32, + hwnd: u64, + element_index: usize, + ) -> Option<bool> { + let retained = self.get_element_retained(pid, hwnd, element_index)?; + if !retained.is_uia() { + return None; + } + let element: IUIAutomationElement = + unsafe { IUIAutomationElement::from_raw(retained.as_ptr() as *mut _) }; + let focused = unsafe { element.CurrentHasKeyboardFocus() } + .ok() + .map(|value| value.as_bool()); + std::mem::forget(element); + focused + } + /// Cached screen rect for the element. Used by the click tool to /// compute the right-edge dispatch point for `action:"expand"` on /// MSAA BUTTONDROPDOWN. diff --git a/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs b/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs index 5973f34789..b18c2317f7 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/uia/windows_enum.rs @@ -671,6 +671,8 @@ unsafe fn window_info_from_uia_element(elem: &IUIAutomationElement) -> Option<Wi y, width: w, height: h, + is_on_screen: true, + minimized: false, }) } diff --git a/libs/cua-driver/rust/crates/platform-windows/src/wgc.rs b/libs/cua-driver/rust/crates/platform-windows/src/wgc.rs index 4569bb801c..c614a55225 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/wgc.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/wgc.rs @@ -58,7 +58,8 @@ unsafe fn wgc_capture_impl(hwnd: HWND) -> Result<(Vec<u8>, u32, u32)> { if IsIconic(hwnd).as_bool() { bail!( "WGC cannot capture a minimized window (no rendered content). \ - Restore the window first — `get_window_state` still returns the \ + Call bring_to_front with this window_id to restore it first. \ + `get_window_state` still returns the \ UIA tree for a minimized window (the screenshot is reported \ unavailable)." ); diff --git a/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs b/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs index 17b31d8e9d..8db8e571b5 100644 --- a/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs +++ b/libs/cua-driver/rust/crates/platform-windows/src/win32/windows.rs @@ -14,9 +14,10 @@ //! `FindAll(TreeScope::Children, ...)` makes no z-order guarantee, so we //! deliberately do NOT let it reorder anything Win32 already reported. //! -//! Both sources apply the same filters (visible, non-iconic, non-empty -//! title). The `filter_pid` argument is applied to the merged list so the -//! union/dedupe pipeline runs unconditionally. +//! Both sources apply the same filters (visible, non-empty title). Minimized +//! windows remain addressable and are reported as off-screen so callers can +//! restore them explicitly. The `filter_pid` argument is applied to the merged +//! list so the union/dedupe pipeline runs unconditionally. use std::collections::HashSet; use std::sync::Mutex; @@ -40,6 +41,8 @@ pub struct WindowInfo { pub y: i32, pub width: i32, pub height: i32, + pub is_on_screen: bool, + pub minimized: bool, } struct EnumState { @@ -82,7 +85,7 @@ pub fn list_windows(filter_pid: Option<u32>) -> Vec<WindowInfo> { merged } -/// Walk `EnumWindows` and collect every visible, non-iconic, non-empty-titled +/// Walk `EnumWindows` and collect every visible, non-empty-titled /// top-level window. No pid filter is applied here — the caller does that on /// the merged list. fn enumerate_via_enum_windows() -> Vec<WindowInfo> { @@ -97,10 +100,12 @@ fn enumerate_via_enum_windows() -> Vec<WindowInfo> { unsafe extern "system" fn enum_windows_cb(hwnd: HWND, lparam: LPARAM) -> BOOL { let state = &*(lparam.0 as *const Mutex<EnumState>); - // Skip invisible or minimized windows. - if IsWindowVisible(hwnd).0 == 0 || IsIconic(hwnd).0 != 0 { + // Invisible helper windows are not user-addressable. Iconic windows are: + // retain them with explicit state so callers can restore them. + if IsWindowVisible(hwnd).0 == 0 { return TRUE; } + let minimized = IsIconic(hwnd).0 != 0; // Get pid. let mut pid: u32 = 0; @@ -128,6 +133,8 @@ unsafe extern "system" fn enum_windows_cb(hwnd: HWND, lparam: LPARAM) -> BOOL { y, width: w, height: h, + is_on_screen: !minimized, + minimized, }); TRUE @@ -301,5 +308,7 @@ pub fn resolve_uwp_host_window(app_pid: u32) -> Option<WindowInfo> { y, width: w, height: h, + is_on_screen: true, + minimized: false, }) } diff --git a/libs/cua-driver/scripts/_install-local-rust.sh b/libs/cua-driver/scripts/_install-local-rust.sh index 69f1bfd2bd..dd6c9d6d3d 100755 --- a/libs/cua-driver/scripts/_install-local-rust.sh +++ b/libs/cua-driver/scripts/_install-local-rust.sh @@ -71,6 +71,7 @@ NORMAL=$(tput sgr0 2>/dev/null || true) RED=$(tput setaf 1 2>/dev/null || true) GREEN=$(tput setaf 2 2>/dev/null || true) BLUE=$(tput setaf 4 2>/dev/null || true) +YELLOW=$(tput setaf 3 2>/dev/null || true) if [ "$(id -u)" -eq 0 ] || [ -n "${SUDO_USER:-}" ]; then echo "${RED}Error: do not run this script with sudo or as root.${NORMAL}" @@ -307,6 +308,32 @@ ensure_local_signing_identity() { printf -- '-' } +# Keychain-backed codesign can wait forever for a GUI authorization prompt when +# install-local is launched from a headless shell. Keep the install bounded; +# ad-hoc signing remains a usable fallback for local development. +codesign_bounded() { + local timeout_seconds="$1" + shift + if command -v gtimeout >/dev/null 2>&1; then + gtimeout "$timeout_seconds" codesign "$@" + elif command -v perl >/dev/null 2>&1; then + perl -e 'alarm shift; exec @ARGV' "$timeout_seconds" codesign "$@" + else + codesign "$@" + fi +} + +clean_partial_bundle_signature() { + local app="$1" + # A certificate-backed codesign killed by the timeout can leave both a + # partial resource seal and `<executable>.cstemp`. Signing over that state + # seals the transient file; when the interrupted signer removes it later, + # the fallback bundle becomes invalid. Always restart fallback signing from + # the unsigned staged bundle. + rm -rf "$app/Contents/_CodeSignature" + find "$app" -type f -name '*.cstemp' -delete +} + # --- macOS: wrap the binary in CuaDriver.app for a stable TCC identity --- # # TCC keys Accessibility / Screen-Recording grants on the bundle @@ -339,27 +366,57 @@ if [ "$OS" = "Darwin" ]; then plutil -replace CFBundleVersion -string "$VERSION_TAG" \ "$APP_STAGE/Contents/Info.plist" 2>/dev/null || true fi - # Install to /Applications (user-writable for admins; no sudo — same - # as install.sh). Replace any prior bundle wholesale. - rm -rf "$APP_DEST" - ditto "$APP_STAGE" "$APP_DEST" - # Re-sign the whole bundle (--deep covers the inner binary). Required on + # Sign the staged bundle before touching the live installation. Required on # macOS 26+ where Taskgated rejects a copied binary's stale signature. # Prefer the STABLE self-signed identity so TCC grants survive rebuilds; - # fall back to ad-hoc (which works but resets grants on the next rebuild). + # never downgrade an existing certificate-signed installation to ad-hoc, + # because that would invalidate its working TCC grants. if command -v codesign >/dev/null 2>&1; then SIGN_ID="$(ensure_local_signing_identity)" if [ "$SIGN_ID" != "-" ] \ - && codesign --force --deep --sign "$SIGN_ID" "$APP_DEST" 2>/dev/null; then - echo "${GREEN}signed $APP_DEST with a stable local identity — TCC grants survive future install-local rebuilds${NORMAL}" - elif codesign --force --deep --sign - "$APP_DEST" 2>/dev/null; then - if [ "$SIGN_ID" != "-" ]; then - echo "${YELLOW}note: stable-identity signing failed; signed ad-hoc instead (Accessibility/Screen Recording will reset on the next rebuild)${NORMAL}" >&2 - fi + && codesign_bounded 20 --force --deep --sign "$SIGN_ID" "$APP_STAGE" 2>/dev/null; then + echo "${GREEN}signed staged app with a stable local identity — TCC grants survive future install-local rebuilds${NORMAL}" + elif [ -d "$APP_DEST" ] \ + && codesign -d -r- "$APP_DEST" 2>&1 | grep -q 'certificate leaf'; then + echo "${RED}Error: stable signing failed; preserving the existing certificate-signed $APP_DEST and its TCC grants.${NORMAL}" >&2 + echo "Unlock/authorize the login-keychain signing key, then rerun install-local." >&2 + exit 1 else - echo "${YELLOW}warning: codesign of $APP_DEST failed; first run may hit a Gatekeeper/Taskgated prompt${NORMAL}" >&2 + clean_partial_bundle_signature "$APP_STAGE" + if codesign_bounded 20 --force --deep --sign - "$APP_STAGE" 2>/dev/null; then + if [ "$SIGN_ID" != "-" ]; then + echo "${YELLOW}note: stable-identity signing failed; signed ad-hoc instead (Accessibility/Screen Recording will reset on the next rebuild)${NORMAL}" >&2 + fi + else + clean_partial_bundle_signature "$APP_STAGE" + echo "${RED}Error: codesign of staged CuaDriver.app failed; live installation was not changed.${NORMAL}" >&2 + exit 1 + fi + fi + if ! codesign --verify --deep --strict "$APP_STAGE" 2>/dev/null; then + echo "${RED}Error: staged CuaDriver.app failed signature verification; live installation was not changed.${NORMAL}" >&2 + exit 1 fi fi + + # Install to /Applications (user-writable for admins; no sudo — same as + # install.sh). Keep the prior bundle available until the copy completes so + # an interrupted install cannot leave a corrupt live app. + APP_BACKUP="${APP_DEST}.install-backup.$$" + rm -rf "$APP_BACKUP" + if [ -d "$APP_DEST" ]; then + mv "$APP_DEST" "$APP_BACKUP" + fi + if ditto "$APP_STAGE" "$APP_DEST"; then + rm -rf "$APP_BACKUP" + else + rm -rf "$APP_DEST" + if [ -d "$APP_BACKUP" ]; then + mv "$APP_BACKUP" "$APP_DEST" + fi + echo "${RED}Error: failed to install CuaDriver.app; restored the previous bundle.${NORMAL}" >&2 + exit 1 + fi echo "${GREEN}installed $APP_DEST${NORMAL}" # --- Clear a TCC grant pinned to a PREVIOUS signing identity ----------- diff --git a/libs/cua-driver/scripts/sync-vm-worktree.sh b/libs/cua-driver/scripts/sync-vm-worktree.sh index c110abab30..cccc9e0966 100755 --- a/libs/cua-driver/scripts/sync-vm-worktree.sh +++ b/libs/cua-driver/scripts/sync-vm-worktree.sh @@ -50,20 +50,25 @@ remote_os="${REMOTE_OS:-posix}" script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" repo_root="$(cd "$script_dir/../../.." && pwd)" +source_sha="$(git -C "$repo_root" rev-parse HEAD)" artifact_root="$repo_root/libs/cua-driver/docs/vm-artifacts" target_slug="$(printf '%s' "$target" | tr -c 'A-Za-z0-9_.-' '_')" timestamp="$(date -u +%Y%m%dT%H%M%SZ)" exclude_args=( - --exclude=.git/ + # A linked worktree stores `.git` as a file, while a normal checkout stores + # it as a directory. Neither host-specific form belongs on a verification VM. + --exclude=.git --exclude=.DS_Store --exclude='._*' --exclude=target/ + --exclude=libs/cua-driver/rust/test-apps/ --exclude=node_modules/ --exclude=.venv/ --exclude=__pycache__/ --exclude='*.pyc' --exclude=dist/ + --exclude=vm-out/ ) tar_exclude_args=( @@ -71,11 +76,13 @@ tar_exclude_args=( --exclude ./.DS_Store --exclude '._*' --exclude ./target + --exclude ./libs/cua-driver/rust/test-apps --exclude ./node_modules --exclude ./.venv --exclude '*/__pycache__' --exclude '*.pyc' --exclude ./dist + --exclude ./vm-out ) remote_mkdir() { @@ -102,8 +109,37 @@ push_rsync() { push_tar() { remote_mkdir - COPYFILE_DISABLE=1 tar "${tar_exclude_args[@]}" -czf - -C "$repo_root" . \ - | "$rsync_ssh" "$target" "tar -xzf - -C \"$remote_dir\"" + remote_path=${remote_dir/#\~/'$HOME'} + COPYFILE_DISABLE=1 tar --no-xattrs "${tar_exclude_args[@]}" -czf - -C "$repo_root" . \ + | "$rsync_ssh" "$target" "tar -xzf - -C \"$remote_path\"" +} + +write_source_marker() { + case "$remote_os" in + windows) + "$rsync_ssh" "$target" \ + "powershell -NoProfile -Command \"Set-Content -NoNewline -Path '$remote_dir/.cua-e2e-source-sha' -Value '$source_sha'\"" + ;; + posix) + marker_dir=${remote_dir/#\~/"\$HOME"} + printf '%s\n' "$source_sha" \ + | "$rsync_ssh" "$target" "mkdir -p $marker_dir && tee $marker_dir/.cua-e2e-source-sha >/dev/null" + ;; + esac +} + +ensure_remote_runtime_dirs() { + case "$remote_os" in + windows) + "$rsync_ssh" "$target" \ + "powershell -NoProfile -Command \"New-Item -ItemType Directory -Force '$remote_dir/libs/cua-driver/rust/test-apps' | Out-Null\"" + ;; + posix) + remote_path=${remote_dir/#\~/'$HOME'} + "$rsync_ssh" "$target" \ + "mkdir -p \"$remote_path/libs/cua-driver/rust/test-apps\"" + ;; + esac } pull_artifacts_rsync() { @@ -111,7 +147,8 @@ pull_artifacts_rsync() { } pull_artifacts_tar() { - "$rsync_ssh" "$target" "tar -czf - -C \"$remote_dir/$remote_artifact_dir\" ." \ + remote_path=${remote_dir/#\~/'$HOME'} + "$rsync_ssh" "$target" "tar -czf - -C \"$remote_path/$remote_artifact_dir\" ." \ | tar -xzf - -C "$dest" } @@ -121,8 +158,9 @@ pull_code_rsync() { } pull_code_tar() { - "$rsync_ssh" "$target" "tar -czf - -C \"$remote_dir\" ." \ - | COPYFILE_DISABLE=1 tar "${tar_exclude_args[@]}" -xzf - -C "$repo_root" + remote_path=${remote_dir/#\~/'$HOME'} + "$rsync_ssh" "$target" "tar -czf - -C \"$remote_path\" ." \ + | COPYFILE_DISABLE=1 tar --no-xattrs "${tar_exclude_args[@]}" -xzf - -C "$repo_root" } case "$mode" in @@ -132,6 +170,8 @@ case "$mode" in tar) push_tar ;; *) echo "SYNC_TRANSPORT must be rsync or tar, got: $transport" >&2; exit 2 ;; esac + ensure_remote_runtime_dirs + write_source_marker ;; pull-artifacts) diff --git a/libs/cua-driver/tests/fixtures/README.md b/libs/cua-driver/tests/fixtures/README.md index 74df5105a3..30d3f494f5 100644 --- a/libs/cua-driver/tests/fixtures/README.md +++ b/libs/cua-driver/tests/fixtures/README.md @@ -27,9 +27,7 @@ tests/fixtures/ │ ├── winui3/ # unpackaged WinUI3 app │ └── webview2/ # WPF + WebView2 host for shared DOM ├── build/ # host build scripts -├── smoke/ # lightweight local smoke runners -├── linux-container/ # Azure/XFCE Linux verification helpers -└── modality-recordings/ # recorder scripts and dashboards +└── smoke/ # lightweight local smoke runners ``` `shared/scenarios.json` is the source of truth for AutomationIds, AX @@ -44,6 +42,7 @@ Build scripts stage outputs into `libs/cua-driver/rust/test-apps/`. # macOS: AppKit, SwiftUI, WKWebView, Electron, Tauri libs/cua-driver/tests/fixtures/build/macos.sh libs/cua-driver/tests/fixtures/build/macos.sh --skip electron +libs/cua-driver/tests/fixtures/build/macos.sh --only wkwebview # Linux: GTK3, Electron, Tauri libs/cua-driver/tests/fixtures/build/linux.sh @@ -69,9 +68,10 @@ Host requirements: Rust tests under `libs/cua-driver/rust/crates/cua-driver/tests/` consume the staged `rust/test-apps/harness-<name>/` outputs: -- `harness_<toolkit>_test.rs`: toolkit-specific app coverage, usually ignored. -- `modality_<area>[_<os>]_test.rs`: background input, capture, and desktop - modality coverage, ignored/manual or VM-backed. +- `cross_platform_behavior_test.rs`: typed Electron/Tauri action matrix. +- `harness_<toolkit>_test.rs`: toolkit-specific app coverage. +- `capture_contract_test.rs` and `desktop_scope_<os>_test.rs`: capture and + desktop-scope contracts. - `protocol_*_test.rs` and schema tests: headless protocol coverage, default. Rust integration tests under `libs/cua-driver/rust/crates/cua-driver/tests/` drive @@ -99,5 +99,5 @@ the shared Electron/Tauri harnesses through the public MCP interface. tests. - Keep platform-specific quirks in the relevant Rust test or app source, not in the shared fixture. -- If a recorder finding graduates into an invariant, move it into a Rust test - and summarize it here. +- Record behavioral evidence through the Rust testkit and canonical OS runner; + do not add a second Python or shell assertion layer. diff --git a/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/build.ps1 b/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/build.ps1 index 3f0c4e4d2c..62d9c4e0ae 100644 --- a/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/build.ps1 +++ b/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/build.ps1 @@ -48,6 +48,7 @@ try { $appDir = Join-Path $outDir "resources\app" if (-not (Test-Path $appDir)) { New-Item -ItemType Directory $appDir -Force | Out-Null } Copy-Item (Join-Path $elecDir "main.js") $appDir -Force + Copy-Item (Join-Path $elecDir "preload.js") $appDir -Force Copy-Item (Join-Path $elecDir "package.json") $appDir -Force Copy-Item $webDir (Join-Path $appDir "web") -Recurse -Force diff --git a/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/build.sh b/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/build.sh index bf03e88064..b196919d5e 100755 --- a/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/build.sh +++ b/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/build.sh @@ -119,6 +119,7 @@ if [ "$platform" = "Darwin" ]; then rm -rf "$appDir" mkdir -p "$appDir" cp "$elecDir/main.js" "$appDir/" + cp "$elecDir/preload.js" "$appDir/" cp "$elecDir/package.json" "$appDir/" cp -R "$elecDir/web" "$appDir/web" @@ -157,10 +158,12 @@ else appDir="$outDir/resources/app" mkdir -p "$appDir" cp "$elecDir/main.js" "$appDir/" + cp "$elecDir/preload.js" "$appDir/" cp "$elecDir/package.json" "$appDir/" cp -r "$elecDir/web" "$appDir/web" - mv "$outDir/electron" "$outDir/CuaTestHarness.Electron" - chmod +x "$outDir/CuaTestHarness.Electron" + mv "$outDir/electron" "$outDir/CuaTestHarness.Electron.bin" + cp "$elecDir/launcher-linux.sh" "$outDir/CuaTestHarness.Electron" + chmod +x "$outDir/CuaTestHarness.Electron" "$outDir/CuaTestHarness.Electron.bin" cat <<EOF [OK] Staged: $outDir/CuaTestHarness.Electron diff --git a/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/launcher-linux.sh b/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/launcher-linux.sh new file mode 100644 index 0000000000..6b49c008cd --- /dev/null +++ b/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/launcher-linux.sh @@ -0,0 +1,6 @@ +#!/bin/sh +# Keep Nix's glibc-bearing library path out of the prebuilt Electron child. +unset LD_LIBRARY_PATH NIX_LD NIX_LD_LIBRARY_PATH +export ACCESSIBILITY_ENABLED=1 +export NO_AT_BRIDGE=0 +exec "$(dirname "$0")/CuaTestHarness.Electron.bin" "$@" diff --git a/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/main.js b/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/main.js index cdc48cc36c..4b00250f8e 100644 --- a/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/main.js +++ b/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/main.js @@ -3,8 +3,20 @@ // tool routes through CDP when --remote-debugging-port is set, so we // expose one here on a configurable port. -const { app, BrowserWindow } = require('electron'); +const { app, BrowserWindow, ipcMain } = require('electron'); +const fs = require('fs'); +const http = require('http'); const path = require('path'); +const sentinelMode = process.env.CUA_E2E_SENTINEL === '1'; +const fixtureJournalUrl = process.env.CUA_E2E_FIXTURE_JOURNAL_URL || ''; +const sentinelJournalPath = process.env.CUA_E2E_SENTINEL_JOURNAL || ''; +if (process.env.CUA_E2E_USER_DATA_DIR) { + app.setPath('userData', process.env.CUA_E2E_USER_DATA_DIR); +} +if (process.platform === 'linux' && process.env.WAYLAND_DISPLAY) { + app.commandLine.appendSwitch('ozone-platform', 'wayland'); + app.commandLine.appendSwitch('enable-features', 'UseOzonePlatform'); +} // Validate CUA_ELECTRON_CDP_PORT before forwarding to Chromium — // remote-debugging-port=0 means "pick an ephemeral port" which would @@ -20,19 +32,57 @@ if (!Number.isInteger(cdpPortNum) || cdpPortNum < 1 || cdpPortNum > 65535) { const CDP_PORT = String(cdpPortNum); app.commandLine.appendSwitch('remote-debugging-port', CDP_PORT); +ipcMain.on('cua-e2e-config', event => { + event.returnValue = { journalUrl: fixtureJournalUrl, sentinelMode }; +}); + +ipcMain.on('cua-e2e-fixture-state', (_event, state) => { + if (!fixtureJournalUrl) return; + const body = JSON.stringify(state); + const request = http.request(fixtureJournalUrl, { + method: 'POST', + headers: { + 'Content-Type': 'text/plain', + 'Content-Length': Buffer.byteLength(body), + }, + }); + request.on('error', () => {}); + request.end(body); +}); + +ipcMain.on('cua-e2e-sentinel-event', (_event, entry) => { + if (!sentinelMode || !sentinelJournalPath) return; + fs.appendFileSync(sentinelJournalPath, `${JSON.stringify(entry)}\n`, 'utf8'); +}); + let mainWindow; function createWindow() { - const fixedTitle = `CuaTestHarness Electron [cdp=${CDP_PORT}]`; + const fixedTitle = sentinelMode + ? `CuaTestHarness Sentinel [cdp=${CDP_PORT}]` + : `CuaTestHarness Electron [cdp=${CDP_PORT}]`; mainWindow = new BrowserWindow({ - width: 940, - height: 780, + width: sentinelMode ? 1280 : 940, + height: sentinelMode ? 900 : 780, + // Keep the normal fixture inside virtual desktops whose window manager + // has no persisted placement policy (notably Openbox under Xvfb). + x: sentinelMode ? 0 : 120, + y: sentinelMode ? 0 : 120, title: fixedTitle, - show: false, + // Map the normal harness immediately. Xvfb/Openbox can enumerate a + // deferred BrowserWindow while never painting it into the root desktop. + // The sentinel stays hidden until it has maximized and claimed focus. + show: !sentinelMode, + // A floating-level macOS window is omitted by cua-driver's deliberate + // layer-0 top-level window contract. Foreground + maximized is sufficient + // for occlusion there and lets an unexpected target raise remain visible. + alwaysOnTop: sentinelMode && process.platform !== 'darwin', autoHideMenuBar: true, webPreferences: { nodeIntegration: false, contextIsolation: true, + sandbox: !sentinelMode, + preload: path.join(__dirname, 'preload.js'), }, }); @@ -66,7 +116,21 @@ function createWindow() { // our fixedTitle and break the harness-window-discovery test. if (mainWindow && !mainWindow.isDestroyed()) { mainWindow.setTitle(fixedTitle); - mainWindow.showInactive(); + if (sentinelMode) { + if (process.platform !== 'darwin') { + mainWindow.setAlwaysOnTop(true); + } + mainWindow.maximize(); + mainWindow.show(); + mainWindow.focus(); + } else { + // Xvfb/Openbox can keep a showInactive window inspectable through + // AT-SPI while never mapping it onto the captured root desktop. + // Show it normally; background cells subsequently foreground the + // occlusion sentinel before taking their desktop snapshot. + mainWindow.show(); + mainWindow.focus(); + } } }) .catch(err => { diff --git a/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/preload.js b/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/preload.js new file mode 100644 index 0000000000..c111490cdd --- /dev/null +++ b/libs/cua-driver/tests/fixtures/apps/cross-platform/electron/preload.js @@ -0,0 +1,48 @@ +const { contextBridge, ipcRenderer } = require('electron'); + +const fixtureConfig = ipcRenderer.sendSync('cua-e2e-config'); +const fixtureJournalUrl = fixtureConfig.journalUrl || ''; + +contextBridge.exposeInMainWorld('cuaE2E', { + journalUrl: fixtureJournalUrl, + publishFixtureState(state) { + if (fixtureJournalUrl) ipcRenderer.send('cua-e2e-fixture-state', state); + }, +}); + +const sentinelMode = fixtureConfig.sentinelMode; + +function record(kind, details = {}) { + if (!sentinelMode) return; + ipcRenderer.send('cua-e2e-sentinel-event', { + kind, + at_ms: Date.now(), + ...details, + }); +} + +if (sentinelMode) { + window.addEventListener('DOMContentLoaded', () => { + document.body.innerHTML = ` + <main style="min-height:100vh;background:#146c43;color:white;display:grid;place-content:center;text-align:center;font:24px system-ui"> + <h1 style="font-size:52px;margin:0 0 16px">CUA OCCLUSION SENTINEL</h1> + <p>CUA_OCCLUSION_SENTINEL_v1</p> + </main> + `; + record('ready'); + }); + window.addEventListener('focus', () => record('focus')); + window.addEventListener('blur', () => record('blur')); + window.addEventListener('keydown', event => + record('keydown', { key: event.key, code: event.code }) + ); + window.addEventListener('pointerdown', event => + record('pointerdown', { button: event.button, x: event.clientX, y: event.clientY }) + ); + window.addEventListener('wheel', event => + record('wheel', { delta_x: event.deltaX, delta_y: event.deltaY }) + ); + window.addEventListener('contextmenu', event => + record('contextmenu', { x: event.clientX, y: event.clientY }) + ); +} diff --git a/libs/cua-driver/tests/fixtures/apps/cross-platform/tauri/src-tauri/src/main.rs b/libs/cua-driver/tests/fixtures/apps/cross-platform/tauri/src-tauri/src/main.rs index 2331526d10..f689a4d095 100644 --- a/libs/cua-driver/tests/fixtures/apps/cross-platform/tauri/src-tauri/src/main.rs +++ b/libs/cua-driver/tests/fixtures/apps/cross-platform/tauri/src-tauri/src/main.rs @@ -1,7 +1,14 @@ #![cfg_attr(not(debug_assertions), windows_subsystem = "windows")] fn main() { + let journal_url = std::env::var("CUA_E2E_FIXTURE_JOURNAL_URL").unwrap_or_default(); + let journal_plugin = tauri::plugin::Builder::<tauri::Wry, ()>::new("e2e-journal") + .js_init_script(format!( + "window.__CUA_E2E_FIXTURE_JOURNAL_URL = {journal_url:?};" + )) + .build(); tauri::Builder::default() + .plugin(journal_plugin) .run(tauri::generate_context!()) .expect("error while running CuaTestHarness.Tauri"); } diff --git a/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py b/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py index 7b13e40765..d65e1689e0 100755 --- a/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py +++ b/libs/cua-driver/tests/fixtures/apps/linux/gtk3/main.py @@ -36,17 +36,22 @@ def section(box, title): class HarnessWindow(Gtk.Window): def __init__(self): super().__init__(title="CuaTestHarness GTK3") - self.set_default_size(480, 760) + # Keep the nested scroll viewport visible on the canonical 1024x768 + # desktop; the outer scroller still exposes controls below it. + self.set_default_size(560, 720) self.counter = 0 self.clicks = 0 self._last_action = "none" + self._double_click_pending = False self._menu_action = "none" + self.key_presses = 0 + self.hotkeys = 0 # Top-level scroller so every control is reachable even on a short window. scroller = Gtk.ScrolledWindow() scroller.set_policy(Gtk.PolicyType.NEVER, Gtk.PolicyType.AUTOMATIC) self.add(scroller) - root = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=10) + root = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=4) root.set_border_width(12) scroller.add(root) @@ -82,6 +87,12 @@ def __init__(self): self.click_status = Gtk.Label(label="last_action=none clicks=0", xalign=0) root.pack_start(self.click_status, False, False, 0) + # ── keyboard delivery ───────────────────────────────────────────── + self.key_status = Gtk.Label(label="last_key=none key_presses=0", xalign=0) + root.pack_start(self.key_status, False, False, 0) + self.hotkey_status = Gtk.Label(label="last_hotkey=none hotkeys=0", xalign=0) + root.pack_start(self.hotkey_status, False, False, 0) + # ── slider ──────────────────────────────────────────────────────── section(root, "slider") adj = Gtk.Adjustment(value=0, lower=0, upper=100, step_increment=1, page_increment=10) @@ -120,11 +131,15 @@ def __init__(self): inner.set_policy(Gtk.PolicyType.NEVER, Gtk.PolicyType.AUTOMATIC) inner.set_size_request(-1, 140) tall = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=2) + scroll_target = aid(Gtk.Button(label="Scroll viewport target"), "scroll-tall-viewport") + scroll_target.set_can_focus(False) + tall.pack_start(scroll_target, False, False, 0) tall.pack_start(Gtk.Label(label="SCROLL_TOP_MARKER_v1", xalign=0), False, False, 0) for i in range(2, 41): tall.pack_start(Gtk.Label(label=f"line {i:02d}", xalign=0), False, False, 0) tall.pack_start(Gtk.Label(label="SCROLL_BOTTOM_MARKER_v1", xalign=0), False, False, 0) inner.add(tall) + aid(inner.get_vscrollbar(), "scroll-tall-vertical") root.pack_start(inner, False, False, 0) self.scroll_inner = inner self.scroll_status = Gtk.Label(label="scroll_offset=0", xalign=0) @@ -136,6 +151,8 @@ def __init__(self): open_pop = aid(Gtk.Button(label="Open Popover"), "btn-open-popover") open_pop.connect("clicked", self.on_open_popover) root.pack_start(open_pop, False, False, 0) + self.popover_status = Gtk.Label(label="popover_open=False", xalign=0) + root.pack_start(self.popover_status, False, False, 0) self.popover = Gtk.Popover.new(open_pop) self.popover.set_border_width(10) self.popover.add(Gtk.Label(label="POPOVER_MARKER_v1")) @@ -146,6 +163,7 @@ def __init__(self): root.pack_start(ext, False, False, 0) self.connect("destroy", Gtk.main_quit) + self.connect("key-press-event", self.on_key_press) # ── handlers ────────────────────────────────────────────────────────── def on_increment(self, *_): @@ -161,16 +179,22 @@ def on_entry_changed(self, entry): def on_click_target(self, *_): self.clicks += 1 - self._last_action = "click" - self.click_status.set_text(f"last_action=click clicks={self.clicks}") + if self._double_click_pending: + self._double_click_pending = False + else: + self._last_action = "click" + self.click_status.set_text(f"last_action={self._last_action} clicks={self.clicks}") def on_click_target_press(self, _w, ev): if ev.type == Gdk.EventType.DOUBLE_BUTTON_PRESS: + self._double_click_pending = True self._last_action = "double_click" self.click_status.set_text(f"last_action=double_click clicks={self.clicks}") elif ev.button == 3: self._last_action = "right_click" self.click_status.set_text(f"last_action=right_click clicks={self.clicks}") + elif ev.button == 1: + self._double_click_pending = False def on_scale(self, s): self.scale_status.set_text(f"slider_value={int(s.get_value())}") @@ -186,11 +210,28 @@ def on_ctx_item(self, _w, lbl): self._menu_action = lbl self.ctx_status.set_text(f"menu_action={lbl}") + def on_key_press(self, _w, ev): + key = (Gdk.keyval_name(ev.keyval) or "unknown").lower() + ctrl = bool(ev.state & Gdk.ModifierType.CONTROL_MASK) + shift = bool(ev.state & Gdk.ModifierType.SHIFT_MASK) + if ctrl and shift and key == "k": + self.hotkeys += 1 + self.hotkey_status.set_text( + f"last_hotkey=ctrl+shift+k hotkeys={self.hotkeys}" + ) + return True + if key == "f5": + self.key_presses += 1 + self.key_status.set_text(f"last_key=f5 key_presses={self.key_presses}") + return True + return False + def on_scroll(self, adj): self.scroll_status.set_text(f"scroll_offset={int(adj.get_value())}") def on_open_popover(self, *_): self.popover.show_all() + self.popover_status.set_text("popover_open=True") def main(): diff --git a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift index eb5bec9d1d..b97f4b3ee9 100644 --- a/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift +++ b/libs/cua-driver/tests/fixtures/apps/macos/appkit/main.swift @@ -52,7 +52,7 @@ let kMenuItemTitle = "Harness Test Item" final class HarnessWindowController: NSObject, NSTextFieldDelegate { let window: NSWindow - let counterLabel = NSTextField(labelWithString: "0") + let counterLabel = NSTextField(labelWithString: "counter=0") var counterValue = 0 let textInput = NSTextField(string: "") let textInputMirror = NSTextField(labelWithString: "") @@ -67,7 +67,7 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate { // Pinned content size — every launch MUST produce a byte-identical window // so screenshot dimensions (and the hardcoded pixel coords the harness tests // rely on) never drift. - static let kContentSize = NSSize(width: 720, height: 1080) + static let kContentSize = NSSize(width: 720, height: 860) override init() { let rect = NSRect(origin: NSPoint(x: 100, y: 100), size: HarnessWindowController.kContentSize) @@ -103,8 +103,8 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate { let content = NSStackView() content.orientation = .vertical content.alignment = .leading - content.spacing = 16 - content.edgeInsets = NSEdgeInsets(top: 20, left: 20, bottom: 20, right: 20) + content.spacing = 8 + content.edgeInsets = NSEdgeInsets(top: 12, left: 20, bottom: 12, right: 20) content.translatesAutoresizingMaskIntoConstraints = false // counter @@ -232,7 +232,7 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate { let scrollWrap = NSStackView() scrollWrap.orientation = .horizontal scrollWrap.spacing = 12 - let scroller = NSScrollView(frame: NSRect(x: 0, y: 0, width: 360, height: 200)) + let scroller = NSScrollView(frame: NSRect(x: 0, y: 0, width: 360, height: 120)) scroller.hasVerticalScroller = true scroller.borderType = .lineBorder let bodyText = NSTextView(frame: NSRect(x: 0, y: 0, width: 340, height: 600)) @@ -296,12 +296,12 @@ final class HarnessWindowController: NSObject, NSTextFieldDelegate { @objc private func onIncrement() { counterValue += 1 - counterLabel.stringValue = String(counterValue) + counterLabel.stringValue = "counter=\(counterValue)" } @objc private func onReset() { counterValue = 0 - counterLabel.stringValue = "0" + counterLabel.stringValue = "counter=0" } @objc private func onExit() { diff --git a/libs/cua-driver/tests/fixtures/apps/macos/swiftui/main.swift b/libs/cua-driver/tests/fixtures/apps/macos/swiftui/main.swift index 7ee28048da..6aa61d0ad1 100644 --- a/libs/cua-driver/tests/fixtures/apps/macos/swiftui/main.swift +++ b/libs/cua-driver/tests/fixtures/apps/macos/swiftui/main.swift @@ -46,6 +46,7 @@ let kScrollOffsetAID = "lbl-scroll-offset" let kScrollTopMarker = "SCROLL_TOP_MARKER_v1" let kScrollBottomMarker = "SCROLL_BOTTOM_MARKER_v1" let kPopupTriggerAID = "btn-open-popover" +let kPopupStateAID = "lbl-popover-state" let kPopupTextAID = "txt-popover-body" let kPopupMarker = "POPOVER_MARKER_v1" let kExitButtonAID = "btn-exit" @@ -107,7 +108,7 @@ struct HarnessRootView: View { .accessibilityIdentifier(kIncrementButtonAID) Button("Reset") { counter = 0 } .accessibilityIdentifier(kResetButtonAID) - Text("\(counter)") + Text("counter=\(counter)") .font(.system(size: 18, weight: .semibold, design: .monospaced)) .accessibilityIdentifier(kCounterLabelAID) } @@ -216,6 +217,9 @@ struct HarnessRootView: View { .padding() .accessibilityIdentifier(kPopupTextAID) } + Text("popover_open=" + String(showPopover)) + .font(.system(.body, design: .monospaced)) + .accessibilityIdentifier(kPopupStateAID) } Spacer(minLength: 24) diff --git a/libs/cua-driver/tests/fixtures/apps/macos/wkwebview/main.swift b/libs/cua-driver/tests/fixtures/apps/macos/wkwebview/main.swift index f070f20eb4..e7283ec984 100644 --- a/libs/cua-driver/tests/fixtures/apps/macos/wkwebview/main.swift +++ b/libs/cua-driver/tests/fixtures/apps/macos/wkwebview/main.swift @@ -35,6 +35,15 @@ final class HarnessAppDelegate: NSObject, NSApplicationDelegate, WKNavigationDel window.setContentSize(kContentSize) let config = WKWebViewConfiguration() + let journalURL = ProcessInfo.processInfo.environment["CUA_E2E_FIXTURE_JOURNAL_URL"] ?? "" + if let encoded = try? JSONEncoder().encode(journalURL), + let literal = String(data: encoded, encoding: .utf8) { + let source = "window.__CUA_E2E_FIXTURE_JOURNAL_URL = \(literal);" + config.userContentController.addUserScript(WKUserScript( + source: source, + injectionTime: .atDocumentStart, + forMainFrameOnly: false)) + } webView = WKWebView(frame: NSRect(origin: .zero, size: kContentSize), configuration: config) webView.autoresizingMask = [.width, .height] webView.navigationDelegate = self diff --git a/libs/cua-driver/tests/fixtures/apps/windows/webview2/MainWindow.xaml b/libs/cua-driver/tests/fixtures/apps/windows/webview2/MainWindow.xaml index 27025fbfb3..12b4acb0c2 100644 --- a/libs/cua-driver/tests/fixtures/apps/windows/webview2/MainWindow.xaml +++ b/libs/cua-driver/tests/fixtures/apps/windows/webview2/MainWindow.xaml @@ -2,7 +2,7 @@ xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:wv2="clr-namespace:Microsoft.Web.WebView2.Wpf;assembly=Microsoft.Web.WebView2.Wpf" - Title="CuaTestHarness WebView" + Title="Loading WebView2 fixture" AutomationProperties.AutomationId="wnd-main" Width="940" Height="780" WindowStartupLocation="CenterScreen"> diff --git a/libs/cua-driver/tests/fixtures/apps/windows/webview2/MainWindow.xaml.cs b/libs/cua-driver/tests/fixtures/apps/windows/webview2/MainWindow.xaml.cs index 22a8049495..65c6729126 100644 --- a/libs/cua-driver/tests/fixtures/apps/windows/webview2/MainWindow.xaml.cs +++ b/libs/cua-driver/tests/fixtures/apps/windows/webview2/MainWindow.xaml.cs @@ -1,5 +1,7 @@ using System; using System.IO; +using System.Text.Json; +using System.Threading.Tasks; using System.Windows; using Microsoft.Web.WebView2.Core; @@ -17,9 +19,6 @@ private async void OnLoaded(object sender, RoutedEventArgs e) { try { - var userData = Path.Combine(Path.GetTempPath(), "CuaTestHarness.WebView.UserData"); - Directory.CreateDirectory(userData); - // Read the CDP port from CUA_WEBVIEW_CDP_PORT (default 9222). // cua-driver's `page` tool routes JS execution through CDP when // `--remote-debugging-port` is exposed; this is the analogue of @@ -37,6 +36,14 @@ private async void OnLoaded(object sender, RoutedEventArgs e) throw new InvalidOperationException( $"Invalid CUA_WEBVIEW_CDP_PORT: '{portStr}'. Expected an integer in 1-65535."); } + // WebView2 requires every process sharing a user-data directory to + // use identical environment options. Each fixture gets a different + // CDP port, so isolate its browser environment by process and port. + var userData = Path.Combine( + Path.GetTempPath(), + "CuaTestHarness.WebView.UserData", + $"{Environment.ProcessId}-{cdpPort}"); + Directory.CreateDirectory(userData); var opts = new CoreWebView2EnvironmentOptions { AdditionalBrowserArguments = $"--remote-debugging-port={cdpPort}", @@ -44,6 +51,17 @@ private async void OnLoaded(object sender, RoutedEventArgs e) var env = await CoreWebView2Environment.CreateAsync(userDataFolder: userData, options: opts); await Wv.EnsureCoreWebView2Async(env); + // The Rust E2E harness owns the loopback receiver. Publish DOM state + // through the shared fixture script so click delivery is judged + // independently of cua-driver's UIA or CDP read-back channels. + var journalUrl = Environment.GetEnvironmentVariable("CUA_E2E_FIXTURE_JOURNAL_URL"); + if (!string.IsNullOrWhiteSpace(journalUrl)) + { + var encodedJournalUrl = JsonSerializer.Serialize(journalUrl); + await Wv.CoreWebView2.AddScriptToExecuteOnDocumentCreatedAsync( + $"window.__CUA_E2E_FIXTURE_JOURNAL_URL = {encodedJournalUrl};"); + } + var htmlPath = Path.Combine(AppContext.BaseDirectory, "web", "index.html"); if (!File.Exists(htmlPath)) { @@ -54,14 +72,30 @@ private async void OnLoaded(object sender, RoutedEventArgs e) htmlPath); } var fileUri = new Uri(htmlPath).AbsoluteUri; + var navigation = new TaskCompletionSource<CoreWebView2NavigationCompletedEventArgs>( + TaskCreationOptions.RunContinuationsAsynchronously); + void OnNavigationCompleted( + object? navigationSender, + CoreWebView2NavigationCompletedEventArgs navigationArgs) + { + Wv.NavigationCompleted -= OnNavigationCompleted; + navigation.TrySetResult(navigationArgs); + } + Wv.NavigationCompleted += OnNavigationCompleted; Wv.Source = new Uri(fileUri); + var navigationResult = await navigation.Task; + if (!navigationResult.IsSuccess) + { + throw new InvalidOperationException( + $"Web fixture navigation failed: {navigationResult.WebErrorStatus}"); + } LblPageUrl.Text = fileUri; - Title = $"CuaTestHarness WebView [cdp={cdpPort}]"; + Title = $"CuaTestHarness WebView [ready cdp={cdpPort}]"; } catch (Exception ex) { - MessageBox.Show($"WebView2 init failed: {ex.Message}", "harness", MessageBoxButton.OK, MessageBoxImage.Error); - throw; + Console.Error.WriteLine($"WebView2 init failed: {ex}"); + Environment.Exit(1); } } diff --git a/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml b/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml index 10e65205ba..c500005cac 100644 --- a/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml +++ b/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml @@ -66,7 +66,8 @@ AutomationProperties.AutomationId="border-click-target" Content="Click target (left / right / double)" Width="260" Height="40" HorizontalAlignment="Left" - MouseLeftButtonDown="OnTargetLeftDown" + PreviewMouseLeftButtonDown="OnTargetLeftDown" + Click="OnTargetClick" MouseRightButtonDown="OnTargetRightDown" MouseDoubleClick="OnTargetDoubleClick"/> <TextBlock x:Name="LblLastAction" diff --git a/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml.cs b/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml.cs index 6664135fb6..9a66907483 100644 --- a/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml.cs +++ b/libs/cua-driver/tests/fixtures/apps/windows/wpf/MainWindow.xaml.cs @@ -1,5 +1,8 @@ using System; +using System.Collections.Generic; +using System.IO; using System.Runtime.InteropServices; +using System.Text.Json; using System.Windows; using System.Windows.Automation; using System.Windows.Controls; @@ -16,11 +19,14 @@ public partial class MainWindow : Window private int _counter; private int _accelCount; private int _clickCount; + private bool _targetPointerSeen; private readonly ScenariosManifest _manifest; + private readonly string? _fixtureStatePath; public MainWindow() { _manifest = ScenariosManifest.Load(); + _fixtureStatePath = Environment.GetEnvironmentVariable("CUA_E2E_FIXTURE_STATE_PATH"); InitializeComponent(); Title = _manifest.Wpf.MainWindow.Title; @@ -48,6 +54,7 @@ private void OnLoaded(object sender, RoutedEventArgs e) // PostMessage actually arrived and was actionable. var source = HwndSource.FromHwnd(new WindowInteropHelper(this).Handle); source?.AddHook(OnWindowMessage); + PublishFixtureState(); } private const int WM_VSCROLL = 0x0115; @@ -82,12 +89,14 @@ private void OnIncrementClick(object sender, RoutedEventArgs e) { _counter++; LblCounter.Text = $"counter={_counter}"; + PublishFixtureState(); } private void OnResetClick(object sender, RoutedEventArgs e) { _counter = 0; LblCounter.Text = "counter=0"; + PublishFixtureState(); } private void OnOpenMessageBoxClick(object sender, RoutedEventArgs e) @@ -145,6 +154,7 @@ private void OnInputChanged(object sender, TextChangedEventArgs e) private void OnTargetLeftDown(object sender, MouseButtonEventArgs e) { + _targetPointerSeen = true; _clickCount++; if (e.ClickCount >= 2) { @@ -155,9 +165,29 @@ private void OnTargetLeftDown(object sender, MouseButtonEventArgs e) LblLastAction.Text = "last_action=left_click"; } LblClickCount.Text = $"clicks={_clickCount}"; + PublishFixtureState(); // Don't mark handled — let the Button's own logic still run. } + private void OnTargetClick(object sender, RoutedEventArgs e) + { + // A real pointer click already passed through PreviewMouseLeftButtonDown. + // UIA Invoke raises Click directly, so count that path here. This gives + // the PX-background row observable fixture state for both its foreground + // geometry probe and its occluded delivery action. + if (_targetPointerSeen) + { + _targetPointerSeen = false; + } + else + { + _clickCount++; + LblLastAction.Text = "last_action=left_click"; + LblClickCount.Text = $"clicks={_clickCount}"; + } + PublishFixtureState(); + } + private void OnTargetDoubleClick(object sender, MouseButtonEventArgs e) { // Belt + suspenders: Button raises MouseDoubleClick separately from @@ -165,11 +195,36 @@ private void OnTargetDoubleClick(object sender, MouseButtonEventArgs e) // back-end implementations that fire only one path still register. LblLastAction.Text = "last_action=double_click"; LblClickCount.Text = $"clicks={_clickCount}"; + PublishFixtureState(); } private void OnTargetRightDown(object sender, MouseButtonEventArgs e) { LblLastAction.Text = "last_action=right_click"; + PublishFixtureState(); + } + + private void PublishFixtureState() + { + if (string.IsNullOrWhiteSpace(_fixtureStatePath)) return; + + var state = new Dictionary<string, object> + { + ["page-marker"] = new { text = "WPF_HARNESS_MARKER_v1" }, + ["lbl-counter"] = new { text = LblCounter?.Text ?? "counter=0" }, + ["lbl-last-action"] = new { text = LblLastAction?.Text ?? "last_action=none" }, + ["lbl-click-count"] = new { text = LblClickCount?.Text ?? "clicks=0" }, + }; + try + { + var temporaryPath = $"{_fixtureStatePath}.{Environment.ProcessId}.tmp"; + File.WriteAllText(temporaryPath, JsonSerializer.Serialize(state)); + File.Move(temporaryPath, _fixtureStatePath, true); + } + catch (Exception ex) + { + Console.Error.WriteLine($"WPF fixture state publish failed: {ex.Message}"); + } } private void OnScrollChanged(object sender, ScrollChangedEventArgs e) diff --git a/libs/cua-driver/tests/fixtures/build/linux.sh b/libs/cua-driver/tests/fixtures/build/linux.sh index b82c7e87a8..4632f70fdb 100755 --- a/libs/cua-driver/tests/fixtures/build/linux.sh +++ b/libs/cua-driver/tests/fixtures/build/linux.sh @@ -11,6 +11,7 @@ # Usage: # ./linux.sh # ./linux.sh --skip gtk3 # skip one target (gtk3|electron|tauri) +# ./linux.sh --only electron,gtk3 # ./linux.sh --clean set -euo pipefail @@ -20,6 +21,7 @@ TEST_APPS_DIR="$(cd "$HARNESS_DIR/../../rust/test-apps" && pwd)" STAGE="$TEST_APPS_DIR/harness-gtk3" SRC="$HARNESS_DIR/apps/linux/gtk3" SKIP="none" +ONLY=",gtk3,electron,tauri," CLEAN=0 while [[ $# -gt 0 ]]; do @@ -32,8 +34,12 @@ while [[ $# -gt 0 ]]; do CLEAN=1 shift ;; + --only) + ONLY=",${2:-}," + shift 2 + ;; *) - echo "Usage: $0 [--skip gtk3|electron|tauri] [--clean]" >&2 + echo "Usage: $0 [--skip gtk3|electron|tauri] [--only comma-separated-targets] [--clean]" >&2 exit 2 ;; esac @@ -43,33 +49,36 @@ if [[ "$CLEAN" == "1" ]]; then rm -rf "$TEST_APPS_DIR/harness-gtk3" "$TEST_APPS_DIR/harness-electron" "$TEST_APPS_DIR/harness-tauri" fi -if [[ "$SKIP" != "gtk3" ]]; then +if [[ "$SKIP" != "gtk3" && "$ONLY" == *",gtk3,"* ]]; then rm -rf "$STAGE" mkdir -p "$STAGE" cp "$SRC/main.py" "$STAGE/main.py" cat > "$STAGE/CuaTestHarness.Gtk3" <<'LAUNCHER' #!/usr/bin/env bash -# Force the X11 backend so the window is enumerable via cua-driver's X11 -# list_windows (_NET_CLIENT_LIST) — under a Wayland session this routes through -# Xwayland; on a pure-X11 session it's a no-op. AT-SPI works over D-Bus either way. -export GDK_BACKEND=x11 +# X11 remains the default for the canonical Xvfb lane. A native Wayland lane +# exports GDK_BACKEND=wayland before launching this repo-owned fixture. +export GDK_BACKEND="${GDK_BACKEND:-x11}" exec python3 "$(dirname "$(readlink -f "$0")")/main.py" "$@" LAUNCHER chmod +x "$STAGE/CuaTestHarness.Gtk3" echo "==> Staged GTK3 harness -> $STAGE/CuaTestHarness.Gtk3" - if python3 -c "import gi; gi.require_version('Gtk','3.0'); from gi.repository import Gtk" 2>/dev/null; then + gtk_probe=(python3 -c "import gi; gi.require_version('Gtk','3.0'); from gi.repository import Gtk") + if command -v timeout >/dev/null 2>&1; then + gtk_probe=(timeout 10s "${gtk_probe[@]}") + fi + if "${gtk_probe[@]}" 2>/dev/null; then echo "==> PyGObject/GTK3 present" else echo "WARNING: PyGObject/GTK3 not importable - install python3-gi gir1.2-gtk-3.0 at-spi2-core" >&2 fi fi -if [[ "$SKIP" != "electron" ]]; then +if [[ "$SKIP" != "electron" && "$ONLY" == *",electron,"* ]]; then "$HARNESS_DIR/apps/cross-platform/electron/build.sh" fi -if [[ "$SKIP" != "tauri" ]]; then +if [[ "$SKIP" != "tauri" && "$ONLY" == *",tauri,"* ]]; then "$HARNESS_DIR/apps/cross-platform/tauri/build.sh" fi diff --git a/libs/cua-driver/tests/fixtures/build/macos.sh b/libs/cua-driver/tests/fixtures/build/macos.sh index f2747c7f54..8c50f2d375 100755 --- a/libs/cua-driver/tests/fixtures/build/macos.sh +++ b/libs/cua-driver/tests/fixtures/build/macos.sh @@ -9,7 +9,8 @@ # Usage: # ./macos.sh # build all macOS-runnable harnesses # ./macos.sh --skip swiftui # skip one target (appkit|swiftui|wkwebview|electron|tauri) -# ./macos.sh --clean # remove staged outputs first +# ./macos.sh --only wkwebview # build just one target +# ./macos.sh --clean # archive staged outputs first set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -17,10 +18,12 @@ HARNESS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" STAGE_DIR="$(cd "$HARNESS_DIR/../../rust/test-apps" && pwd)" SKIP="" +ONLY="" CLEAN=0 while [[ $# -gt 0 ]]; do case "$1" in --skip) SKIP="$2"; shift 2;; + --only) ONLY="$2"; shift 2;; --clean) CLEAN=1; shift;; -h|--help) sed -n '2,11p' "$0"; exit 0;; @@ -28,8 +31,22 @@ while [[ $# -gt 0 ]]; do esac done +archive_existing() { + local target="$1" + [[ -e "$target" ]] || return 0 + local archive_root="${TMPDIR:-/tmp}/cua-driver-fixture-build-archive" + local stamp + stamp="$(date +%Y%m%d-%H%M%S)-$$" + mkdir -p "$archive_root" + mv "$target" "$archive_root/$(basename "$target").$stamp" +} + if [[ "$CLEAN" == "1" ]]; then - rm -rf "$STAGE_DIR/harness-appkit" "$STAGE_DIR/harness-swiftui" "$STAGE_DIR/harness-wkwebview" "$STAGE_DIR/harness-electron" "$STAGE_DIR/harness-tauri" + archive_existing "$STAGE_DIR/harness-appkit" + archive_existing "$STAGE_DIR/harness-swiftui" + archive_existing "$STAGE_DIR/harness-wkwebview" + archive_existing "$STAGE_DIR/harness-electron" + archive_existing "$STAGE_DIR/harness-tauri" mkdir -p "$STAGE_DIR/harness-appkit" "$STAGE_DIR/harness-swiftui" "$STAGE_DIR/harness-wkwebview" "$STAGE_DIR/harness-electron" "$STAGE_DIR/harness-tauri" echo "==> Cleaned stage dirs" fi @@ -41,7 +58,7 @@ build_app() { local plist="$bundle/Contents/Info.plist" echo "==> Building $name" - rm -rf "$bundle" + archive_existing "$bundle" mkdir -p "$bundle/Contents/MacOS" # shellcheck disable=SC2086 # word-splitting on $frameworks is intentional @@ -77,21 +94,21 @@ EOF echo " → $bundle" } -if [[ "$SKIP" != "appkit" ]]; then +if [[ -z "$ONLY" || "$ONLY" == "appkit" ]] && [[ "$SKIP" != "appkit" ]]; then build_app "CuaTestHarness.AppKit" \ "$HARNESS_DIR/apps/macos/appkit" \ "" \ "harness-appkit" fi -if [[ "$SKIP" != "swiftui" ]]; then +if [[ -z "$ONLY" || "$ONLY" == "swiftui" ]] && [[ "$SKIP" != "swiftui" ]]; then build_app "CuaTestHarness.SwiftUI" \ "$HARNESS_DIR/apps/macos/swiftui" \ "" \ "harness-swiftui" fi -if [[ "$SKIP" != "wkwebview" ]]; then +if [[ -z "$ONLY" || "$ONLY" == "wkwebview" ]] && [[ "$SKIP" != "wkwebview" ]]; then build_app "CuaTestHarness.WKWebView" \ "$HARNESS_DIR/apps/macos/wkwebview" \ "-framework WebKit" \ @@ -105,11 +122,11 @@ if [[ "$SKIP" != "wkwebview" ]]; then echo " → bundled shared/web/index.html into Resources/web/" fi -if [[ "$SKIP" != "electron" ]]; then +if [[ -z "$ONLY" || "$ONLY" == "electron" ]] && [[ "$SKIP" != "electron" ]]; then "$HARNESS_DIR/apps/cross-platform/electron/build.sh" fi -if [[ "$SKIP" != "tauri" ]]; then +if [[ -z "$ONLY" || "$ONLY" == "tauri" ]] && [[ "$SKIP" != "tauri" ]]; then "$HARNESS_DIR/apps/cross-platform/tauri/build.sh" fi diff --git a/libs/cua-driver/tests/fixtures/build/windows.ps1 b/libs/cua-driver/tests/fixtures/build/windows.ps1 index 7e3b150522..7e0f98b194 100644 --- a/libs/cua-driver/tests/fixtures/build/windows.ps1 +++ b/libs/cua-driver/tests/fixtures/build/windows.ps1 @@ -12,7 +12,9 @@ param( [ValidateSet("none","wpf","winui3","webview","electron","tauri")] - [string]$Skip = "none" + [string]$Skip = "none", + [ValidateSet("wpf","winui3","webview","electron","tauri")] + [string[]]$Targets = @("wpf","winui3","webview","electron","tauri") ) Set-StrictMode -Version Latest @@ -31,6 +33,11 @@ if (-not (Get-Command dotnet -ErrorAction SilentlyContinue)) { New-Item -ItemType Directory -Force $testAppsDir | Out-Null +function Should-Build { + param([string]$Name) + return $Skip -ne $Name -and $Targets -contains $Name +} + function Publish-Project { param([string]$ProjPath, [string]$OutDirName) $outDir = Join-Path $testAppsDir $OutDirName @@ -54,10 +61,10 @@ function Publish-Project { Write-Host "[OK] Published: $outDir" -ForegroundColor Green } -if ($Skip -ne "wpf") { +if (Should-Build "wpf") { Publish-Project (Join-Path $harnessDir "apps\windows\wpf\CuaTestHarness.Wpf.csproj") "harness-wpf" } -if ($Skip -ne "winui3") { +if (Should-Build "winui3") { $winuiProj = Join-Path $harnessDir "apps\windows\winui3\CuaTestHarness.WinUI3.csproj" if (Test-Path $winuiProj) { Publish-Project $winuiProj "harness-winui3" @@ -65,7 +72,7 @@ if ($Skip -ne "winui3") { Write-Host "[SKIP] WinUI3 project not present yet - skipping." -ForegroundColor Yellow } } -if ($Skip -ne "webview") { +if (Should-Build "webview") { $webProj = Join-Path $harnessDir "apps\windows\webview2\CuaTestHarness.WebView.csproj" if (Test-Path $webProj) { Publish-Project $webProj "harness-webview" @@ -73,41 +80,26 @@ if ($Skip -ne "webview") { Write-Host "[SKIP] WebView project not present yet - skipping." -ForegroundColor Yellow } } -if ($Skip -ne "electron") { +if (Should-Build "electron") { $elecBuild = Join-Path $harnessDir "apps\cross-platform\electron\build.ps1" if (Test-Path $elecBuild) { Write-Host "" Write-Host "[BUILD] electron -> $testAppsDir\harness-electron\" -ForegroundColor Cyan - # Electron build sets $ErrorActionPreference=Stop internally and - # throws on npm install / publish failure. Wrap so a throw degrades - # to a warning rather than aborting the whole harness build. - try { - & $elecBuild - if ($LASTEXITCODE -ne 0) { - Write-Host "[WARN] Electron build failed (exit $LASTEXITCODE)" -ForegroundColor Yellow - } - } catch { - Write-Host "[WARN] Electron build errored: $($_.Exception.Message)" -ForegroundColor Yellow - } + & $elecBuild + if ($LASTEXITCODE -ne 0) { throw "Electron harness build failed" } } else { - Write-Host "[SKIP] Electron project not present yet - skipping." -ForegroundColor Yellow + throw "Electron build script not found: $elecBuild" } } -if ($Skip -ne "tauri") { +if (Should-Build "tauri") { $tauriBuild = Join-Path $harnessDir "apps\cross-platform\tauri\build.ps1" if (Test-Path $tauriBuild) { Write-Host "" Write-Host "[BUILD] tauri -> $testAppsDir\harness-tauri\" -ForegroundColor Cyan - try { - & $tauriBuild - if ($LASTEXITCODE -ne 0) { - Write-Host "[WARN] Tauri build failed (exit $LASTEXITCODE)" -ForegroundColor Yellow - } - } catch { - Write-Host "[WARN] Tauri build errored: $($_.Exception.Message)" -ForegroundColor Yellow - } + & $tauriBuild + if ($LASTEXITCODE -ne 0) { throw "Tauri harness build failed" } } else { - Write-Host "[SKIP] Tauri project not present yet - skipping." -ForegroundColor Yellow + throw "Tauri build script not found: $tauriBuild" } } diff --git a/libs/cua-driver/tests/fixtures/linux-container/README.md b/libs/cua-driver/tests/fixtures/linux-container/README.md deleted file mode 100644 index 3439e87e55..0000000000 --- a/libs/cua-driver/tests/fixtures/linux-container/README.md +++ /dev/null @@ -1,148 +0,0 @@ -# Linux dispatch-ladder lane — XFCE in a container - -A reproducible rig for validating the cua-driver Linux **`delivery_mode`** -background/foreground ladder and the AT-SPI modalities against a **real window -manager** (xfwm4 + EWMH) and a **live AT-SPI tree** — the two things the -existing Xvfb harness (no WM) can't exercise. - -The target is the `trycua/cua-xfce` image: a full XFCE desktop on X11 with an -accessibility bus, reachable over VNC and the computer-server API. It's a -richer target than Xvfb for exactly the rungs that need a WM (foreground / EWMH -activation) or a11y (AT-SPI element + at-point clicks). - -## What it validates - -`calc.sh` drives `galculator` through the four parity modalities; map each to a -`delivery_mode` rung and confirm the reported `path`: - -| Modality | `delivery_mode` | Expected `path` | Driver-verifiable? | -|---|---|---|---| -| Element click (`element_index`) | `background` | `x11_atspi` (AT-SPI `do_action`) | yes (a11y action) | -| Pixel / vision click | `background` | `x11_atspi` (`do_action`-at-point) for AX apps; else MPX `x11_pixel` | yes when AT-SPI-at-point lands | -| Pixel click (escalated) | `foreground` | `x11_pixel_fg` (EWMH activate → inject → restore) | no — confirm via screenshot | -| `type_text` into editable | `background` | `ax` (AT-SPI `insertText`) | yes (`verified:true`) | -| `type_text`, non-editable focus | `foreground` | `key_events_fg` | no — confirm via screenshot | - -Background pixel/vision click **lands** on X11: an AX app takes the focus-free -AT-SPI `do_action`-at-point path (`x11_atspi`) — the same background-pixel -behavior macOS/Windows have. It falls to the MPX virtual-pointer path -(`x11_pixel`, which needs a **real Xorg + `/dev/uinput`**, absent under -Xvnc/minimal containers) only for non-AX surfaces; escalate to `foreground` -there. - -## Setup recipe - -1. **Run the image** (Azure Container Instances is the recommended single- - container host; the image is public on Docker Hub): - - ``` - az container create -g <rg> -n cua-xfce --image trycua/cua-xfce \ - --ports 8000 5901 --ip-address Public --os-type Linux --cpu 2 --memory 4 - ``` - -2. **Install cua-driver as the desktop user** (`install-local` refuses to run - as root; the container exec lands as root, so use `runuser`). Sync the repo - in or build from a checkout, then: - - ``` - chown -R cua /opt/cua /home/cua/.cargo /home/cua/.rustup - runuser -u cua -- bash /opt/cua/libs/cua-driver/install-local.sh - ``` - -3. **Drive the scenario** (the daemon now self-discovers the session bus, so no - manual `DBUS_SESSION_BUS_ADDRESS` export is needed for `serve` — `calc.sh` - still exports it for the short-lived CLI calls): - - ``` - runuser -u cua -- bash calc.sh fullreset # daemon + a11y + launch galculator - runuser -u cua -- bash calc.sh doctor # AT-SPI/org.a11y.Bus probe + dbus addr - runuser -u cua -- bash calc.sh state # AT-SPI tree (shows degraded flag if empty) - runuser -u cua -- bash calc.sh prep # resolve window id (needed before pxclick) - runuser -u cua -- bash calc.sh click 7 background # element click, bg - runuser -u cua -- bash calc.sh pxclick 3 background # pixel/vision click, bg - runuser -u cua -- bash calc.sh type 789 foreground # foreground EWMH type - runuser -u cua -- bash calc.sh btf # bring_to_front (EWMH) - ``` - -## Gotchas (learned the hard way) - -- **AT-SPI needs the session bus.** A daemon started outside the desktop - session has no `DBUS_SESSION_BUS_ADDRESS` → the AT-SPI tree comes back empty - and `get_window_state` reports `degraded:true`. The daemon now auto-discovers - it (`platform-linux/src/session_bus.rs`); if it still can't, the session has - no a11y bus or the daemon isn't running as the desktop user. -- **Run as the desktop user.** Root-against-a-user-session can't read that - user's session-process environ or `/run/user/<uid>/bus`. This is the Linux - analogue of the Windows Session 0 isolation problem. -- **Zombie children pollute `pgrep`.** A single-instance app (galculator) - launched by the daemon can leave an unreaped zombie child after a kill; it - shows in `pgrep` and makes `list_windows` look empty. `fullreset` restarts - the daemon, which reaps it. -- **Stale daemon → stale schema.** A long-lived daemon proxies a cached tool - schema. To assert the *freshly built* binary's schema, shell - `cua-driver describe <tool>` (computes the ToolDef locally) instead of a - daemon round-trip — this is what `modality_dispatch_linux_test.rs` does. -- **VNC screenshots over the computer-server `/cmd` SSE endpoint** are `data:`- - prefixed JSON; strip the `data: ` prefix before `json.loads`. - -## Second lane: trycua/cua-ubuntu (Kasm desktop) - -The same harness validates the Kasm-based `trycua/cua-ubuntu` image (XFCE under -Kasm/VNC, user `kasm-user`). Two image-specific gotchas: - -- **Startup hangs without a Kasm orchestrator.** `vnc_startup.sh` blocks in - `wait_for_network_devices`, which loops until `ip link show type veth` returns - an `eth*` interface — an interface type the Kasm orchestrator's networking - provides but plain ACI does not, so it waits forever (no `NET_ADMIN` to create - one, and the script's filesystem is read-only). Bypass it at container start - with an override command-line that patches the wait to a no-op (runs as the - main process, so no restart loop): - - ``` - az container create -g <rg> -n cua-ubuntu --image trycua/cua-ubuntu:latest \ - --cpu 2 --memory 4 --ports 6901 8000 --ip-address Public \ - --os-type Linux --restart-policy OnFailure \ - --command-line '/bin/bash -c "sed s/^wait_for_network_devices/true/ /dockerstartup/vnc_startup.sh>/tmp/vs.sh && exec /bin/bash /tmp/vs.sh /dockerstartup/kasm_startup.sh --wait"' - ``` - -- **Drive it as `kasm-user`** with `$HOME=/home/kasm-user`. The harness derives - its paths from `$HOME`, so no edits are needed: - `bash modality_matrix.sh setup` / `matrix` just work once cua-driver is built - to `~/.local/bin`. noVNC is on `:6901` (password `vncpassword`). - -Both lanes produce the same modality matrix and both prove the session-bus -auto-discovery fix (daemon started with `DBUS_SESSION_BUS_ADDRESS` unset → 58 -AT-SPI elements, `degraded` unset). - -## Third lane: KDE / GNOME VMs (`derec.sh`) - -The container lanes above are both **XFCE** (GTK3 galculator) over `az exec`. -The genuinely different desktops/toolkits run on full **Azure VMs** over SSH, -driven by **`derec.sh`** (also in this directory): - -| VM | Desktop | App | Toolkit | Why a VM, not a container | -|---|---|---|---|---| -| `cua-kde` | KDE Plasma (X11) | `kcalc` | **Qt** | Plasma X11 renders fine over VNC | -| `cua-gnome` | GNOME Shell (X11) | `gnome-calculator` | **GTK4** | gnome-shell needs **real console Xorg** (software GLX); it won't run over Xvnc (no GLX) | - -`derec.sh setup` / `record` / `env` mirror the container harness; **`derec.sh -verify`** asserts the **GTK4 coordinate invariant** — per-button `frame`s are -distinct (not collapsed to the window corner, the GTK4 `(0,0)` regression) and -every button center lies inside the window's X11 rect. That's the runtime guard -the schema-only `modality_dispatch_linux_test` can't provide. - -## CI coverage - -Pure logic is unit-tested **display-free** in `platform-linux`: -- session-bus discovery — the comm-truncation matcher *and* the - `/proc/<pid>/environ` parse (`session_bus.rs`); -- the GTK4 `_GTK_FRAME_EXTENTS` parse + the `origin+inset+WINDOW` screen - reconstruction, anchored to the live `(132,375)` capture (`atspi/native.rs`, - `coord_tests`). - -The dispatch *contract* (every input tool advertises `delivery_mode`; -`bring_to_front` is a real EWMH activation) is asserted via -`rust/crates/cua-driver/tests/modality_dispatch_linux_test.rs`. Everything that -genuinely needs a live desktop + a11y bus — the dispatch ladder's *behaviour*, -the GTK4 coordinate invariant, recordings — is covered by the container lanes -(manual / `trycua/cua-xfce` rig) and the VM lanes (`derec.sh verify`), not CI. diff --git a/libs/cua-driver/tests/fixtures/linux-container/calc.sh b/libs/cua-driver/tests/fixtures/linux-container/calc.sh deleted file mode 100644 index 6285a14ebb..0000000000 --- a/libs/cua-driver/tests/fixtures/linux-container/calc.sh +++ /dev/null @@ -1,142 +0,0 @@ -#!/usr/bin/env bash -# cua-driver modality test harness for the XFCE-in-a-container lane. -# -# Drives galculator through the four parity modalities (background AX click, -# background pixel/vision click, foreground EWMH type, background type -# focus-limit) so each `delivery_mode` rung can be validated against a real WM -# (xfwm4 + EWMH) and a live AT-SPI tree. Invoke ONE action per call; state -# (pid/window) persists in /tmp/cstate.* between calls. -# -# Usage: bash calc.sh <action> [args...] -# See README.md in this directory for the full lane (image, install, gotchas). - -export DISPLAY=:1 -export HOME=/home/cua -export XDG_RUNTIME_DIR=/run/user/1000 - -# AT-SPI needs the desktop session bus. The cua-driver daemon now AUTO-DISCOVERS -# DBUS_SESSION_BUS_ADDRESS at startup (platform-linux/src/session_bus.rs), so the -# `serve` process self-recovers it even when started outside the session. We -# still export it here for the short-lived `cua-driver call` CLI invocations and -# for `gsettings`, discovering it from a running session process's environ the -# same way the daemon does (the VNC/XFCE session runs an ad-hoc bus). -for _p in xfce4-session xfsettingsd xfwm4 xfdesktop Thunar; do - _pid=$(pgrep -x "$_p" | head -1) - if [ -n "$_pid" ] && [ -r /proc/$_pid/environ ]; then - _a=$(tr '\0' '\n' < /proc/$_pid/environ | sed -n 's/^DBUS_SESSION_BUS_ADDRESS=//p' | head -1) - [ -n "$_a" ] && export DBUS_SESSION_BUS_ADDRESS="$_a" && break - fi -done -export XAUTHORITY=/home/cua/.Xauthority -CUA=/home/cua/.local/bin/cua-driver -S=/tmp/cstate -act="$1"; shift - -case "$act" in - daemon) - pgrep -f 'cua-driver serve' >/dev/null || (setsid "$CUA" serve >/tmp/cuad.log 2>&1 &) - sleep 2 - "$CUA" status 2>&1 | head -4 - ;; - doctor) - # Exercises the hardened AT-SPI probe (org.a11y.Bus name_has_owner + - # discovered DBUS_SESSION_BUS_ADDRESS). - "$CUA" call check_permissions '{}' 2>&1 | head -20 - ;; - a11y) - gsettings set org.gnome.desktop.interface toolkit-accessibility true 2>/dev/null - pgrep -f at-spi-bus-launcher >/dev/null || (setsid /usr/libexec/at-spi-bus-launcher --launch-immediately >/tmp/atspi.log 2>&1 &) - sleep 1 - pkill -x galculator 2>/dev/null; sleep 1 - GTK_MODULES=gail:atk-bridge NO_AT_BRIDGE=0 setsid galculator >/dev/null 2>&1 & - sleep 2 - pgrep -x galculator | head -1 > "$S.pid" - echo "a11y on; galculator pid=$(cat $S.pid) atspi=$(pgrep -f at-spi-bus-launcher | head -1)" - ;; - state) # snapshot the AT-SPI tree; prints element map (and degraded flag) - PID=$(cat "$S.pid") - [ -z "$PID" ] && PID=$(pgrep -x galculator | head -1) && echo "$PID" > "$S.pid" - WID=$("$CUA" call list_windows "{\"pid\":$PID}" 2>/dev/null | python3 -c "import sys,json;ws=json.load(sys.stdin).get('windows',[]);print(ws[0]['window_id'] if ws else '')" 2>/dev/null) - echo "$WID" > "$S.wid" - "$CUA" call get_window_state "{\"pid\":$PID,\"window_id\":$WID,\"capture_mode\":\"ax\"}" > /tmp/state.json 2>&1 - echo "pid=$PID wid=$WID" - python3 - <<'PY' -import json -try: - d=json.load(open('/tmp/state.json')) -except Exception as e: - print("STATE PARSE FAIL:", str(e)[:200]); print(open('/tmp/state.json').read()[:400]); raise SystemExit -if d.get('degraded'): - print("DEGRADED:", d.get('degraded_reason','')[:160]) -els=d.get('elements',[]) -print("total elements:", len(els)) -for e in els: - r=e.get('role',''); l=e.get('label') - if r and ('button' in r.lower() or 'push' in r.lower() or l): - print(e.get('element_index'), r, repr(l)[:22]) -PY - ;; - click) # click <element_index> <background|foreground> - PID=$(cat "$S.pid"); WID=$(cat "$S.wid") - "$CUA" call click "{\"pid\":$PID,\"window_id\":$WID,\"element_index\":$1,\"delivery_mode\":\"$2\"}" - ;; - pxclick) # pxclick <label> <background|foreground> — click a button by PIXEL coords (window-local) - PID=$(cat "$S.pid"); WID=$(cat "$S.wid") - read LX LY SX SY < <(python3 - "$1" <<'PY' -import json, sys -label = sys.argv[1] -st = json.load(open('/tmp/state.json')) -win = json.load(open('/tmp/win.json'))['windows'][0] -wx, wy = win['x'], win['y'] -for e in st.get('elements', []): - if str(e.get('label')) == label and e.get('frame'): - f = e['frame']; cx = f['x'] + f['w']/2; cy = f['y'] + f['h']/2 - print(int(cx - wx), int(cy - wy), int(cx), int(cy)); break -PY -) - echo "label=$1 screen=($SX,$SY) win-local=($LX,$LY)" - "$CUA" call click "{\"pid\":$PID,\"window_id\":$WID,\"x\":$LX,\"y\":$LY,\"delivery_mode\":\"$2\"}" - ;; - type) # type <text> <background|foreground> - PID=$(cat "$S.pid") - "$CUA" call type_text "{\"pid\":$PID,\"text\":\"$1\",\"delivery_mode\":\"$2\"}" - ;; - btf) - PID=$(cat "$S.pid"); WID=$(cat "$S.wid") - "$CUA" call bring_to_front "{\"pid\":$PID,\"window_id\":$WID}" - ;; - prep) # resolve window id into /tmp/win.json + cstate.wid (needed before pxclick) - PID=$(cat "$S.pid") - [ -z "$PID" ] && PID=$(pgrep -x galculator | head -1) && echo "$PID" > "$S.pid" - echo "pid=$PID" - "$CUA" call list_windows "{\"pid\":$PID}" | tee /tmp/win.json - WID=$(python3 -c "import json;ws=json.load(open('/tmp/win.json')).get('windows',[]);print(ws[0]['window_id'] if ws else '')" 2>/dev/null) - echo "$WID" > "$S.wid"; echo "wid=$WID" - ;; - fullreset) - # kill daemon (reaps zombie galculator children) + all galculator, enable - # a11y, restart, launch. Use when list_windows comes back empty (a zombie - # unreaped child of the daemon pollutes pgrep). - pkill -9 -f 'cua-driver serve' 2>/dev/null; pkill -9 -f galculator 2>/dev/null; sleep 3 - rm -f /home/cua/.cache/cua-driver/cua-driver.sock - gsettings set org.gnome.desktop.interface toolkit-accessibility true 2>/dev/null - pgrep -f at-spi-bus-launcher >/dev/null || (setsid /usr/libexec/at-spi-bus-launcher --launch-immediately >/tmp/atspi.log 2>&1 &) - sleep 1 - (setsid "$CUA" serve >/tmp/cuad.log 2>&1 &); sleep 2 - "$CUA" call launch_app '{"name":"galculator"}' > /tmp/launch.json 2>&1 - sleep 3 - python3 - <<'PY' -import json,re -t=open('/tmp/launch.json').read(); p=None -try: p=json.loads(t).get('pid') -except Exception: pass -if not p: - m=re.search(r'pid (\d+)',t); p=m.group(1) if m else '' -open('/tmp/cstate.pid','w').write(str(p or '')) -PY - echo "fullreset: galculator pid=$(cat $S.pid) remaining_galc=$(pgrep -x galculator|tr '\n' ' ')" - ;; - *) - echo "unknown action: $act"; echo "actions: daemon doctor a11y state click pxclick type btf prep fullreset" - ;; -esac diff --git a/libs/cua-driver/tests/fixtures/linux-container/derec.sh b/libs/cua-driver/tests/fixtures/linux-container/derec.sh deleted file mode 100755 index 6df8381098..0000000000 --- a/libs/cua-driver/tests/fixtures/linux-container/derec.sh +++ /dev/null @@ -1,190 +0,0 @@ -#!/bin/bash -# cua-driver desktop modality + recording harness — the SSH/VM lane. -# -# Companion to the container lane in this directory: where calc.sh / -# modality_matrix.sh drive ACI containers over the computer-server / `az exec` -# (XFCE, GTK3 galculator), derec.sh drives full Azure VMs over SSH and is the -# only lane that exercises the genuinely different desktops/toolkits: -# - KDE Plasma (X11) with kcalc (Qt) -# - GNOME Shell (X11, console Xorg) with gnome-calculator (GTK4) -# It auto-discovers the live session env (DISPLAY/XAUTHORITY/WAYLAND_DISPLAY/ -# dbus) from a running DE process, so the same script works on any of them — -# including the GNOME Mutter / KDE KWin *Wayland* lane (WAYLAND=1), where it -# drives the native Wayland backend off the release build. -# -# derec.sh setup # fresh daemon + launch $APP + snapshot AT-SPI tree -# derec.sh verify # assert the coordinate invariant (see below) -# derec.sh keytest # foreground-keyboard EFFECT check (3*4=12) -# derec.sh vclick 789 C 7 8 9 # vision pixel-click EFFECT check (no element_index) -# derec.sh record i.. i.. # record start_recording -> element clicks -> stop -# derec.sh env # print the discovered session env -# -# APP defaults to gnome-calculator; set APP=kcalc / galculator for the others. -# WAYLAND=1 selects the Wayland lane (GTK4 gnome-calculator on Mutter). -# -# COORDINATE INVARIANT (the GTK4 regression guard): -# GTK4's AT-SPI returns GetExtents(SCREEN) as (0,0) for every widget, which -# used to collapse every element `frame` to the window corner. The fix queries -# CoordType::Window + reconstructs screen = x11_origin + _GTK_FRAME_EXTENTS + -# WINDOW. `verify` asserts the observable invariant: per-button frames are -# DISTINCT (not collapsed) and every button center lies inside the window's -# X11 rect, with "7" left of "8" on the same row when both exist. -# Use the caller's $HOME; the fallback is the GNOME/KDE VM lane's user. Override -# by exporting HOME (or running as that user) on a differently-laid-out box. -export HOME="${HOME:-/home/fbonacci}" -W=$HOME/derec; mkdir -p "$W" -DE_PID="" -for p in gnome-shell plasmashell kwin_x11 metacity xfce4-session; do - DE_PID=$(pgrep -x "$p" | head -1); [ -n "$DE_PID" ] && break -done -if [ -n "$DE_PID" ]; then - while IFS= read -r kv; do export "$kv"; done < <(tr '\0' '\n' </proc/$DE_PID/environ 2>/dev/null | grep -E '^(DISPLAY|XAUTHORITY|DBUS_SESSION_BUS_ADDRESS|XDG_RUNTIME_DIR)=') -fi -# Lane selector (mirrors APP=): WAYLAND=1 engages the native Wayland backend. -# `is_wayland()` needs WAYLAND_DISPLAY set, DISPLAY UNSET, and the opt-in env; -# GTK apps need the wayland GDK backend; and the daemon must be the release -# build (the .local/bin copy can lag the Wayland fixes). gnome-shell's environ -# (discovered above) already supplied WAYLAND_DISPLAY/XDG_RUNTIME_DIR/DBUS. -if [ -n "$WAYLAND" ]; then - export CUA_DRIVER_RS_ENABLE_WAYLAND=1 - export WAYLAND_DISPLAY="${WAYLAND_DISPLAY:-wayland-0}" - unset DISPLAY - # Software-GL + wayland backend so GTK4 apps render on the GPU-less VM and - # are spawned as Wayland clients (children of the daemon inherit this env). - export GDK_BACKEND=wayland LIBGL_ALWAYS_SOFTWARE=1 GSK_RENDERER=cairo - CUA="${CUA_BIN:-$HOME/cua-rust/target/release/cua-driver}" -else - [ -n "$DISP" ] && export DISPLAY="$DISP" - export DISPLAY="${DISPLAY:-:0}" - CUA="${CUA_BIN:-$HOME/.local/bin/cua-driver}" -fi -APP="${APP:-gnome-calculator}"; SESS=demo - -resolve_pid(){ python3 -c "import json,re -t=open('$W/launch.json').read(); p=None -try: p=json.loads(t).get('pid') -except: pass -if not p: - m=re.search(r'pid (\d+)',t); p=m.group(1) if m else '' -print(p or '')"; } - -case "$1" in - env) echo "lane=$([ -n "$WAYLAND" ] && echo wayland || echo x11) DISPLAY=${DISPLAY:-<unset>} WAYLAND_DISPLAY=${WAYLAND_DISPLAY:-<unset>} XAUTHORITY=$XAUTHORITY dbus=${DBUS_SESSION_BUS_ADDRESS:+set} CUA=$CUA APP=$APP DE_PID=$DE_PID";; - setup) - pkill -f 'cua-driver serve' 2>/dev/null; pkill -f "$APP" 2>/dev/null; sleep 2 - rm -f ~/.cache/cua-driver/cua-driver.sock - (setsid "$CUA" serve >$W/cuad.log 2>&1 &); sleep 2 - "$CUA" call launch_app "{\"name\":\"$APP\"}" >$W/launch.json 2>&1; sleep 4 - PID=$(resolve_pid); echo "$PID" >$W/pid - "$CUA" call list_windows "{\"pid\":$PID}" >$W/win.json 2>&1 - WID=$(python3 -c "import json;ws=json.load(open('$W/win.json')).get('windows',[]);print(ws[0]['window_id'] if ws else '')" 2>/dev/null) - echo "$WID" >$W/wid - echo "app=$APP pid=$PID wid=$WID DISPLAY=$DISPLAY" - "$CUA" call get_window_state "{\"pid\":$PID,\"window_id\":$WID,\"capture_mode\":\"ax\"}" >$W/state.json 2>&1 - python3 -c "import json -d=json.load(open('$W/state.json')); els=d.get('elements',[]) -print('elements',len(els),'degraded',d.get('degraded')) -for e in els: - l=e.get('label') - if l and len(str(l))<=4: print(e.get('element_index'),repr(l))" 2>&1 | head -40 - ;; - verify) # assert the coordinate invariant against the X11 window geometry - WID=$(cat $W/wid) - read WX WY WW WH < <(xwininfo -id "$WID" 2>/dev/null | awk '/Absolute upper-left X/{x=$NF}/Absolute upper-left Y/{y=$NF}/Width:/{w=$NF}/Height:/{h=$NF}END{print x,y,w,h}') - FE=$(xprop -id "$WID" _GTK_FRAME_EXTENTS 2>/dev/null | grep -oE '= .*' | tr -d ' =') - echo "app=$APP window=($WX,$WY) ${WW}x${WH} _GTK_FRAME_EXTENTS=${FE:-absent}" - python3 - "$WX" "$WY" "$WW" "$WH" "$W/state.json" <<'PY' -import json,sys -wx,wy,ww,wh=map(int,sys.argv[1:5]); d=json.load(open(sys.argv[5])) -btns=[] -for e in d.get('elements',[]): - l=str(e.get('label')); r=str(e.get('role','')).lower(); f=e.get('frame') - if f and 'button' in r and len(l)<=3: - btns.append((l, f['x']+f['w']//2, f['y']+f['h']//2)) -fails=[] -xs={cx for _,cx,_ in btns} -if len(btns) >= 3 and len(xs) < 2: - fails.append(f'COLLAPSED: {len(btns)} buttons all at x={xs} (GTK4 (0,0) regression)') -for l,cx,cy in btns: - if not (wx-2 <= cx <= wx+ww+2 and wy-2 <= cy <= wy+wh+2): - fails.append(f'OUT-OF-WINDOW: {l!r} center=({cx},{cy}) outside [{wx},{wx+ww}]x[{wy},{wy+wh}]') -g={l:(cx,cy) for l,cx,cy in btns} -if '7' in g and '8' in g: - (x7,y7),(x8,y8)=g['7'],g['8'] - if not (x8 > x7 and abs(y8-y7) <= 6): - fails.append(f"ROW: 7={g['7']} 8={g['8']} not left-to-right on one row") -print(f'buttons checked: {len(btns)}') -print('COORD INVARIANT:', 'PASS' if not fails else 'FAIL') -for f in fails[:8]: print(' -', f) -sys.exit(0 if not fails else 1) -PY - ;; - keytest) # foreground-keyboard EFFECT check: AC, 3 * 4 = -> display reads 12 - # Guards the Xvnc XTEST keyboard fix (round-trip-before-close + shift-level - # auto-Shift so '*' multiplies instead of typing '8'). Effect-confirmed, not - # status-confirmed: scans the AT-SPI tree for the result "12" so it generalises - # across kcalc (Qt), gnome-calculator (GTK4) and galculator (GTK3), each of - # which exposes its LCD under a different role. - PID=$(cat $W/pid); WID=$(cat $W/wid) - "$CUA" call press_key "{\"pid\":$PID,\"window_id\":$WID,\"key\":\"Escape\",\"delivery_mode\":\"foreground\"}" >/dev/null 2>&1; sleep 0.4 - for k in 3 asterisk 4 equal; do - "$CUA" call press_key "{\"pid\":$PID,\"window_id\":$WID,\"key\":\"$k\",\"delivery_mode\":\"foreground\"}" >/dev/null 2>&1 - sleep 0.5 - done - sleep 0.4 - "$CUA" call get_window_state "{\"pid\":$PID,\"window_id\":$WID,\"capture_mode\":\"ax\"}" 2>/dev/null | python3 -c "import json,sys -d=json.load(sys.stdin) -hit=any(str(e.get('label')).strip()=='12' or str(e.get('value')).strip()=='12' for e in d.get('elements',[])) -print('FOREGROUND KEY EFFECT:', 'PASS (3*4=12 landed)' if hit else 'FAIL (display != 12)') -sys.exit(0 if hit else 1)" - ;; - vclick) # vclick <expected> <label>... VISION pixel-click EFFECT check. - # Clicks each labelled button by its SCREEN-frame CENTER via {x,y} (NO - # element_index), then confirms the calculator result — proving pixel/vision - # coordinates ACTUATE, not GTK4-false-succeed on the inner label. Works on - # X11 and Wayland; on Wayland the rung is `wayland_atspi` (screen pixel -> - # covering element -> element_index doAction), since Mutter drops synthetic - # pointer events. e.g. WAYLAND=1 derec.sh vclick 789 C 7 8 9 - PID=$(cat $W/pid); WID=$(cat $W/wid); shift - EXPECT="$1"; shift - "$CUA" call start_session "{\"session\":\"$SESS\"}" >/dev/null 2>&1 - for lbl in "$@"; do - read CX CY < <(python3 -c "import json -d=json.load(open('$W/state.json')) -for e in d['elements']: - if str(e.get('label'))=='$lbl' and 'button' in str(e.get('role','')).lower(): - f=e.get('frame') - if f: print(int(f['x']+f['w']/2), int(f['y']+f['h']/2)) - break") - if [ -z "$CX" ]; then echo " '$lbl' -> no frame (skipped)"; continue; fi - R=$("$CUA" call click "{\"pid\":$PID,\"window_id\":$WID,\"x\":$CX,\"y\":$CY,\"delivery_mode\":\"background\",\"session\":\"$SESS\"}" 2>&1) - PV=$(echo "$R" | python3 -c "import json,sys -try: print(json.load(sys.stdin).get('path','?')) -except: print('?')" 2>/dev/null) - echo " vision-click '$lbl' @($CX,$CY) -> path=$PV" - sleep 0.7 - done - sleep 0.5 - "$CUA" call get_window_state "{\"pid\":$PID,\"window_id\":$WID,\"capture_mode\":\"ax\"}" 2>/dev/null | python3 -c "import json,sys -d=json.load(sys.stdin); exp='$EXPECT' -hit=any(str(e.get('label')).strip()==exp or str(e.get('value')).strip()==exp for e in d.get('elements',[])) -print('VISION CLICK EFFECT:', f'PASS ({exp} via pixel coords)' if hit else f'FAIL (display != {exp})') -sys.exit(0 if hit else 1)" - ;; - record) # record <idx>... start_recording -> element clicks (in order) -> stop - PID=$(cat $W/pid); WID=$(cat $W/wid); shift - rm -rf $W/rec; mkdir -p $W/rec - "$CUA" call start_session "{\"session\":\"$SESS\"}" >/dev/null 2>&1 - "$CUA" call start_recording "{\"output_dir\":\"$W/rec\",\"record_video\":true}" 2>&1 | head -1 - sleep 2 - for idx in "$@"; do - "$CUA" call click "{\"pid\":$PID,\"window_id\":$WID,\"element_index\":$idx,\"delivery_mode\":\"background\",\"session\":\"$SESS\"}" >/dev/null 2>&1 - sleep 1.3 - done - sleep 1.5 - "$CUA" call stop_recording '{}' 2>&1 | head -1 - ls -la $W/rec/recording.mp4 2>&1 | tail -1 - ;; - *) echo "usage: derec.sh setup|verify|keytest|vclick <expected> <label>...|record <idx>...|env - APP=gnome-calculator|kcalc|galculator WAYLAND=1 for the Mutter/KWin Wayland lane";; -esac diff --git a/libs/cua-driver/tests/fixtures/linux-container/modality_matrix.sh b/libs/cua-driver/tests/fixtures/linux-container/modality_matrix.sh deleted file mode 100755 index a9eed91696..0000000000 --- a/libs/cua-driver/tests/fixtures/linux-container/modality_matrix.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/env bash -# Full modality matrix for the XFCE container, driven via `az container exec` -# (positional args only — the JSON is built inside the script). Proves the -# session-bus auto-discovery fix (the daemon is started with -# DBUS_SESSION_BUS_ADDRESS UNSET) and exercises every input tool across -# background/foreground × ax/vision. Screenshots land in /tmp/mm_*.png. -# -# modality_matrix.sh setup # fresh daemon (NO dbus env) + a11y + galculator + snapshot -# modality_matrix.sh matrix # run every tool × bg/fg, print path+result -# modality_matrix.sh shot <name> # screenshot -> /tmp/mm_<name>.png -# modality_matrix.sh b64 <name> # print a saved screenshot as base64 (host decodes) -set -uo pipefail -# Paths derive from $HOME so the same harness drives both container lanes: -# trycua/cua-xfce (user `cua`) and trycua/cua-ubuntu / Kasm (user `kasm-user`). -# Override CUA / XAUTHORITY / DISPLAY via env if your layout differs. -export DISPLAY=${DISPLAY:-:1} -export XAUTHORITY=${XAUTHORITY:-$HOME/.Xauthority} -CUA=${CUA:-$HOME/.local/bin/cua-driver} -S=/tmp/cstate - -pidf() { cat "$S.pid" 2>/dev/null; } -widf() { cat "$S.wid" 2>/dev/null; } - -snap() { # snapshot AT-SPI tree -> /tmp/state.json, print element count + degraded - local PID WID; PID=$(pidf); WID=$(widf) - "$CUA" call get_window_state "{\"pid\":$PID,\"window_id\":$WID,\"capture_mode\":\"ax\"}" >/tmp/state.json 2>&1 - python3 - <<'PY' -import json -try: d=json.load(open('/tmp/state.json')) -except Exception as e: print(" STATE PARSE FAIL:",str(e)[:120]); raise SystemExit -els=d.get('elements',[]) -print(f" AT-SPI elements={len(els)} degraded={d.get('degraded')}") -if d.get('degraded'): print(" degraded_reason:",str(d.get('degraded_reason'))[:120]) -PY -} - -call() { # call <label> <tool> <json> ; prints "<label> | path=.. ok=.." - local label="$1" tool="$2" body="$3" - "$CUA" call "$tool" "$body" >/tmp/last.json 2>&1 - python3 - "$label" <<'PY' -import json,sys -lbl=sys.argv[1] -try: d=json.load(open('/tmp/last.json')) -except Exception: - print(f" {lbl:30s} | RAW: "+open('/tmp/last.json').read()[:90].replace(chr(10),' ')); raise SystemExit -path=d.get('path') or d.get('delivery_path') or '-' -ok=d.get('ok', d.get('success', d.get('clicked', d.get('typed','?')))) -ver=d.get('verified') -extra=f" verified={ver}" if ver is not None else "" -print(f" {lbl:30s} | path={path:14s} ok={ok}{extra}") -PY -} - -case "${1:-help}" in - setup) - pkill -9 -f 'cua-driver serve' 2>/dev/null; pkill -9 -f galculator 2>/dev/null; sleep 2 - rm -f /home/cua/.cache/cua-driver/cua-driver.sock - gsettings set org.gnome.desktop.interface toolkit-accessibility true 2>/dev/null - pgrep -f at-spi-bus-launcher >/dev/null || (setsid /usr/libexec/at-spi-bus-launcher --launch-immediately >/tmp/atspi.log 2>&1 &) - sleep 1 - # THE TEST: start the daemon with DBUS_SESSION_BUS_ADDRESS removed from its - # environment. If AT-SPI still populates below, auto-discovery worked. - env -u DBUS_SESSION_BUS_ADDRESS setsid "$CUA" serve >/tmp/cuad.log 2>&1 & - sleep 3 - echo "=== daemon started with DBUS_SESSION_BUS_ADDRESS UNSET ===" - echo "daemon env DBUS set? $(env -u DBUS_SESSION_BUS_ADDRESS bash -c 'echo ${DBUS_SESSION_BUS_ADDRESS:-<unset>}')" - grep -iE "session bus|adopted the desktop|DBUS_SESSION" /tmp/cuad.log | head -3 || echo "(no discovery log line)" - "$CUA" call launch_app '{"name":"galculator"}' >/tmp/launch.json 2>&1 - sleep 3 - PID=$(python3 -c "import json,re;t=open('/tmp/launch.json').read() -try: p=json.loads(t).get('pid') -except Exception: p=None -if not p: - import re; m=re.search(r'pid (\d+)',t); p=m.group(1) if m else '' -print(p or '')") - echo "$PID" > "$S.pid" - "$CUA" call list_windows "{\"pid\":$PID}" >/tmp/win.json 2>&1 - WID=$(python3 -c "import json;ws=json.load(open('/tmp/win.json')).get('windows',[]);print(ws[0]['window_id'] if ws else '')" 2>/dev/null) - echo "$WID" > "$S.wid" - echo "galculator pid=$PID wid=$WID" - echo "=== AT-SPI tree (proves auto-discovery if elements>0) ===" - snap - ;; - - matrix) - PID=$(pidf); WID=$(widf) - echo "=== pid=$PID wid=$WID ===" - # resolve a couple of element indices + their pixel centers from the tree - read -r E7 PX7 PY7 E_PLUS < <(python3 - <<'PY' -import json -d=json.load(open('/tmp/state.json')); win=json.load(open('/tmp/win.json'))['windows'][0] -def find(lbl): - for e in d.get('elements',[]): - if str(e.get('label'))==lbl: return e - return None -e7=find('7'); ep=find('+') -def ctr(e): - f=e.get('frame'); - return (int(f['x']+f['w']/2), int(f['y']+f['h']/2)) if f else (-1,-1) -x7,y7=ctr(e7) if e7 else (-1,-1) -print(e7.get('element_index') if e7 else -1, x7, y7, ep.get('element_index') if ep else -1) -PY -) - echo "indices: 7->$E7 (px $PX7,$PY7) +->$E_PLUS" - echo "--- AX modality (element_index) ---" - call "click 7 [ax/bg]" click "{\"pid\":$PID,\"window_id\":$WID,\"element_index\":$E7,\"delivery_mode\":\"background\"}" - call "click + [ax/fg]" click "{\"pid\":$PID,\"window_id\":$WID,\"element_index\":$E_PLUS,\"delivery_mode\":\"foreground\"}" - call "double_click 7 [ax/bg]" double_click "{\"pid\":$PID,\"window_id\":$WID,\"element_index\":$E7,\"delivery_mode\":\"background\"}" - call "right_click 7 [ax/bg]" right_click "{\"pid\":$PID,\"window_id\":$WID,\"element_index\":$E7,\"delivery_mode\":\"background\"}" - echo "--- VISION modality (pixel coords) ---" - call "click px7 [vision/bg]" click "{\"pid\":$PID,\"window_id\":$WID,\"x\":$PX7,\"y\":$PY7,\"delivery_mode\":\"background\"}" - call "click px7 [vision/fg]" click "{\"pid\":$PID,\"window_id\":$WID,\"x\":$PX7,\"y\":$PY7,\"delivery_mode\":\"foreground\"}" - call "double_click px7 [v/bg]" double_click "{\"pid\":$PID,\"window_id\":$WID,\"x\":$PX7,\"y\":$PY7,\"delivery_mode\":\"background\"}" - echo "--- keyboard / scroll ---" - call "type 789 [bg]" type_text "{\"pid\":$PID,\"text\":\"789\",\"delivery_mode\":\"background\"}" - call "type 789 [fg]" type_text "{\"pid\":$PID,\"text\":\"789\",\"delivery_mode\":\"foreground\"}" - call "press_key Return [bg]" press_key "{\"pid\":$PID,\"window_id\":$WID,\"key\":\"Return\",\"delivery_mode\":\"background\"}" - call "press_key Return [fg]" press_key "{\"pid\":$PID,\"window_id\":$WID,\"key\":\"Return\",\"delivery_mode\":\"foreground\"}" - call "hotkey ctrl+c [bg]" hotkey "{\"pid\":$PID,\"window_id\":$WID,\"keys\":[\"ctrl\",\"c\"],\"delivery_mode\":\"background\"}" - call "scroll down [bg]" scroll "{\"pid\":$PID,\"window_id\":$WID,\"direction\":\"down\",\"amount\":3,\"delivery_mode\":\"background\"}" - call "bring_to_front (EWMH)" bring_to_front "{\"pid\":$PID,\"window_id\":$WID}" - ;; - - shot) - # Grab the X display directly (no `screenshot` tool; ffmpeg x11grab is present). - ffmpeg -y -loglevel error -f x11grab -i :1 -frames:v 1 "/tmp/mm_${2:-shot}.png" 2>/tmp/shot.err - ls -la "/tmp/mm_${2:-shot}.png" 2>&1 | tail -1 - [ -s /tmp/shot.err ] && head -c 200 /tmp/shot.err - ;; - - seq) # seq <label>... background element-click each labeled button (clean demo) - shift; PID=$(pidf); WID=$(widf) - for lbl in "$@"; do - IDX=$(python3 - "$lbl" <<'PY' -import json,sys -d=json.load(open('/tmp/state.json')) -for e in d.get('elements',[]): - if str(e.get('label'))==sys.argv[1]: print(e['element_index']); break -PY -) - [ -z "$IDX" ] && { echo " '$lbl' -> no element"; continue; } - "$CUA" call click "{\"pid\":$PID,\"window_id\":$WID,\"element_index\":$IDX,\"delivery_mode\":\"background\"}" >/dev/null 2>&1 - echo " '$lbl' -> element $IDX clicked (bg)" - sleep 0.4 - done - ;; - - b64) base64 -w0 "/tmp/mm_${2:-shot}.png" 2>/dev/null ;; - - *) echo "usage: modality_matrix.sh setup|matrix|shot <name>|b64 <name>" ;; -esac diff --git a/libs/cua-driver/tests/fixtures/linux-container/validate.sh b/libs/cua-driver/tests/fixtures/linux-container/validate.sh deleted file mode 100755 index 5c73c2793b..0000000000 --- a/libs/cua-driver/tests/fixtures/linux-container/validate.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -# Build + test cua-driver on the XFCE container, detached (survives `az -# container exec`). Output goes to /tmp/validate.log. -# validate.sh run # launch detached -# validate.sh log # print the log -set -uo pipefail -RUST=/opt/cua/libs/cua-driver/rust -LOG=/tmp/validate.log -export PATH=/home/cua/.cargo/bin:$PATH -export CARGO_TERM_COLOR=never - -_work() { - echo "=== validate start $(date -u +%H:%M:%S) ===" - cd "$RUST" || { echo "no rust dir"; exit 9; } - echo "HEAD: $(git -C /opt/cua rev-parse --short HEAD)" - echo "--- cargo build -p platform-linux ---" - cargo build -p platform-linux 2>&1 | tail -25; echo "build_rc=${PIPESTATUS[0]}" - echo "--- cargo build -p cua-driver (bin) ---" - cargo build -p cua-driver 2>&1 | tail -15; echo "bin_rc=${PIPESTATUS[0]}" - echo "--- cargo test -p platform-linux session_bus ---" - cargo test -p platform-linux session_bus 2>&1 | tail -18; echo "sbtest_rc=${PIPESTATUS[0]}" - echo "--- cargo test --test modality_dispatch_linux_test ---" - cargo test -p cua-driver --test modality_dispatch_linux_test 2>&1 | tail -18; echo "dispatch_rc=${PIPESTATUS[0]}" - echo "VALIDATE_DONE $(date -u +%H:%M:%S)" -} - -case "${1:-run}" in - run) : > "$LOG"; setsid bash "$0" _work >>"$LOG" 2>&1 </dev/null & echo "launched pid $! -> $LOG" ;; - _work) _work ;; - log) cat "$LOG" 2>/dev/null ;; - *) echo "usage: validate.sh run|log" ;; -esac diff --git a/libs/cua-driver/tests/fixtures/modality-recordings/FINDINGS.md b/libs/cua-driver/tests/fixtures/modality-recordings/FINDINGS.md deleted file mode 100644 index d1dbcd996b..0000000000 --- a/libs/cua-driver/tests/fixtures/modality-recordings/FINDINGS.md +++ /dev/null @@ -1,245 +0,0 @@ -# Modality recordings — findings (cua-driver behavior surfaced by the per-action verifier) - -These recordings run the 8-action matrix (click, double-click, right-click, drag, scroll, -set_value, type, press-key) against a controlled harness on Windows (WPF) and Linux (GTK3), in -5 modalities each (ax-fg, ax-bg, px-fg, px-bg, px-desktop). Each action now carries -**two independent measurements**: - -- **✓ worked / ✗ no-op** — did the action change the harness's own state (verified by reading the - harness status: `agreed=`, `slider_value=`, `last_action=`, `mirror=`, `menu_action=`, - `scroll_offset=`). Read via UIAutomation on Windows, via a harness state file on Linux. -- **held / STOLE FOCUS** — the no-foreground contract. - -Principle followed: the harness/recorder stays honest (no overfitting); real gaps are documented -here as cua-driver work, to be fixed in the driver first and then re-tested. - -**Historical note:** the Electron **7/8** and Linux **3/8** violation figures below -are recorder-baseline artifacts from an earlier pass and have since been addressed. -See the newer fix note later in this file and the canonical -`reference/cua-driver/modality-test-suite.mdx`. - -## What the verifier surfaced (to triage as driver work) - -### Windows / WPF (ax-bg, representative) — after fixes: 5/7 land -| action | effect | note | -|--------|--------|------| -| left-click checkbox | ✓ worked | UIA toggle | -| double-click button | ✓ worked | last_action=double_click | -| right-click | ✗ no-op (clean) | **Root cause: off-screen target, not a driver coordinate bug.** At 1024×768 the panel forces the harness to 556px wide, so the form reflows taller than the screen and the right-click button lands at y≈786 (below the screen). The synthetic tap previously clamped onto the **taskbar** (opened its menu). **DRIVER FIX:** `point_in_window_bounds()` now refuses a click whose resolved point is outside its window and returns a clear error — no more taskbar misfire (frame-verified). To actually *exercise* right-click needs a screen tall enough to show the control (this RDP session won't take a programmatic resize). | -| drag slider | ✓ worked | slider_value 0→48 | -| scroll | ✗ no-op (clean) | Same off-screen root cause (scroll-tall pane is lower still); now guarded, not misfiring. | -| set_value | ✓ worked | mirror=set-by-cua | -| type | ✓ worked | **Fixed:** the recorder now clicks the text box (focus) before `type_text`; type targets the focused control, and `set_value` doesn't focus, so without this it was a no-op. | -| press-key (Tab) | n/a | no asserted effect | - -**Driver fix shipped (this branch):** `platform-windows` — `point_in_window_bounds()` + guards in the -click / double_click / right_click element paths. Investigated and ruled out: UIA `GetClickablePoint` -for cursor-centering (returns the same rect center or fails), and SetFocus-to-scroll-into-view (would -steal foreground, breaking the very contract these bg runs measure). - -### Linux / GTK3 -- AT-SPI/element actions (single-click, set_value, type) **✓ work**. -- All **pixel-based** actions (double-click, right-click, drag, scroll, every coordinate click) - **✗ no-op**: cua-driver's Linux input is **XSendEvent** (synthetic, no focus steal — required for - the contract), but **GTK ignores synthetic XSendEvent** (`send_event` flag). Documented in - `platform-linux/src/input/mod.rs`. Real limitation; not a recorder bug. - -### Linux / Electron (Chromium, via AT-SPI) - -The cross-platform Electron harness runs on Linux straight from `node_modules` (no packaging), -launched `electron . --no-sandbox --disable-gpu --force-renderer-accessibility` under -Xvfb + a dbus session. `--force-renderer-accessibility` makes Chromium publish its full -**web-AX tree on AT-SPI**, which `get_window_state` returns cleanly alongside the -screenshot (check box "I agree", slider, entry "type here", the click-target section, and every -`agreed=`/`slider_value=`/`last_action=`/`mirror=` status label — 72 elements). Same verifier as -Windows, but reading AT-SPI text instead of UIA. Recorder: `linux/lin-rec-electron.py` -(+ `lin-run-electron.sh`, `lin-dashboard-electron.html`, `lin-all-electron.sh`). - -| mode | effects landed | landed actions | focus contract (stole) | -|------|----------------|----------------|------------------------| -| ax-fg | 3/6 | click, drag, type | foreground (8 actions) | -| ax-bg | 2/6 | click, type | **3/8 stole** (set_value, type, press-key) | -| px-fg | 1/5 | double-click | foreground (7 actions) | -| px-bg | 1/5 | double-click | **6/7 stole** | -| px-desktop | 1/2 | click | foreground (4 actions) | - -**Headline — Linux Electron breaks the no-foreground contract, but far less than Windows -Electron.** Windows Electron stole **7/8** in ax-bg (Chromium self-foregrounds on nearly every -action). Linux Electron steals **3/8 in ax-bg** — only the keyboard/focus actions (`set_value`, -`type`, `press-key`) pull the window frontmost via the X focus path; the AX pointer actions -(click/double/right/drag/scroll) all **held**. In **px-bg** it steals **6/7** (pixel dispatch -is coordinate-injection that foregrounds Chromium). Effect coverage mirrors the documented Linux -synthetic-input limit: AX `click`/`type` land; AX `double_click`/`right_click`/`drag`/`scroll` and -`set_value` are no-ops on the Chromium controls via AT-SPI, while **pixel** `double_click` *does* -fire on the click-target (`last_action=double_click`, frame-verified in px-fg/bg). All 5 modes -frame-verified; saved as `linux-electron-{ax-fg,ax-bg,px-fg,px-bg,px-desktop}.mp4`. - -## Agent-cursor centering (user-reported) -The agent cursor lands off-center on the checkbox. Root cause: the WPF checkbox's AX -BoundingRectangle is the **full-width row** (≈803px), so its center is mid-row. Verified that -UIA `GetClickablePoint` does **not** help — it either fails (slider) or returns the same rect -center (text box). So this is **not a driver defect**: the bounds are reported faithfully and the -element action works regardless of cursor position. The visual is a function of the control's -geometry (a stretched CheckBox). Most-honest fix would be the harness laying the checkbox out at -content width (how real apps render it); a driver left-bias heuristic would be unreliable. - -## Cross-toolkit coverage matrix (Windows, ax-bg) - -Same recorder (parameterized by `-Toolkit`), same 8-action plan, one ax-bg pass each. This is the -high-signal output: which cua-driver actions actually LAND per toolkit, and whether the no-foreground -contract holds. (right-click + scroll are ✗ across the board here: the harnesses either lack those -controls or they're off-screen at the 556px layout — see the off-screen guard above.) - -| Toolkit | effects landed | landed actions | focus contract (stole) | note | -|----------|---------------|------------------------------------------|------------------------|------| -| WPF | **5/7** | click, double, drag, set_value, type | 2/8 | fullest harness | -| WinUI3 | **3/7** | left-click(checkbox), scroll, set_value | 0/7 | re-recorded with parity controls; double/right/drag don't land (see WinUI3 dispatch below) | -| WebView2 | **4/7** | double, right, drag, set_value | 0/7 | re-recorded; left-click(checkbox) + scroll are honest no-ops (checkbox below the fold, web won't scroll in ax mode) | -| Electron | **5/7** | click, double, drag, set_value, type | **7/8** | **Electron/Chromium self-foregrounds → no-foreground contract VIOLATED** | - -**Headline:** the no-foreground contract holds on WPF / WinUI3 / WebView2, but **breaks on Electron** -(7/8 actions stole focus — Chromium foregrounds itself; consistent with the #1984 dispatch note in the -code). The verifier reads each toolkit's own status labels (WPF/WinUI3: UIAutomation; WebView2/Electron: -Chromium web-AX text via a substring window-title match because their titles carry a `[cdp=NNNN]` suffix). - -WinUI3 + WebView2 were **re-recorded** after the parity controls landed (frame-verified). Recorder fixes -that made WebView2 work at all: the web DOM only surfaces when `WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS= ---force-renderer-accessibility` is set (otherwise `get_window_state` returns only the chrome frame → -every web action resolves to nothing → the prior SIZE=0/empty-MP4 run); and the daemon's ffmpeg probe -missed the binary because the recorder runs as a different user than the one WinGet installed it under, so -ffmpeg must be on PATH before `serve`. Recording on the VM also requires Session 2 attached/active -(`tscon … /dest:console`) — a disconnected session blanks GPU content and fails `gdigrab`. - -**WinUI3 double/right-click — confirmed not driver-fixable via the WPF path.** AX `double_click`/ -`right_click` on the WinUI3 click-target produce `last_action=none, clicks=0` (no Click/DoubleTapped/ -RightTapped), while the *identical* actions land on WebView2 — so the harness wiring is correct; the gap -is WinUI3-specific dispatch. Matches the `platform-windows/src/input/dispatch.rs` note: routing WinUI3 -through the WPF synthetic-pen/coordinate path neither lands double/right NOR holds the contract (measured: -regressed ax-bg 0/8→8/8 stolen). Single left-click works via UIA Invoke. A real fix needs a WinUI3-specific -input path targeting the composition input-site (DirectComposition/ContentIsland InputSite), not -PostMessage-to-top-HWND; the same gap explains WinUI3 drag-slider being a no-op. - -Evidence: `matrix-{winui3,webview2,electron}-ax-bg.mp4` (+ the WPF set) on the Desktop. - -## Driver fixes shipped this round (cross-platform) - -- **Windows — cached UIA element use-after-free** (`platform-windows`): the element cache handed out a bare - COM pointer; under concurrent sessions a `get_window_state` snapshot-replace could `Release` it mid-action - (click/type/set_value) → daemon crash. Now a `RetainedElement` guard `AddRef`s under the cache lock and - `Release`s on drop — the Windows port of the macOS #1796 retain-under-lock fix. Compile-verified. -- **Linux — GTK left-click + value-only widgets** (`platform-linux`): pixel left-clicks now land via AT-SPI - hit-test + `doAction` (GTK drops synthetic X11 events; XTEST core events don't reach its XInput2 path) — - without stealing focus. Sliders/scroll bars (Value interface, no Action) now surface in `get_window_state` - (`is_indexable = actions || has_value`), so `set_value` drives them. Verified on the Linux VM. -- **macOS — numeric `set_value`** (`platform-macos`): CFNumber write for NSSlider, AXIncrement/AXDecrement - stepping fallback for SwiftUI sliders. Verified live (AppKit 0→50, SwiftUI 0→50). - -## Legacy modal popups (WPF) — driver CAN open + list them - -Confirmed: a **background `click` via UIA Invoke** fires the harness's popup buttons *even when they -are off-screen* (y≈876–1026 on the 768px display), and `list_windows` then enumerates the dialogs: -`Open MessageBox` → **"Harness MessageBox"**, `Open Owned Window` → **"Harness Owned Popup"**, -`Open Layered Popup` → **"Harness Layered Popup"**. - -This briefly regressed: the first cut of the off-screen guard (above) sat *before* the dispatch -branch and so blocked the UIA-Invoke path too (which needs no coordinates). **Fix:** the guard now -applies only to the coordinate-delivery paths (foreground SendInput tap + background coordinate -injection); UIA Invoke runs unguarded, so opening a modal from an off-screen button works again. - -## macOS (AppKit) — 5 modes, contract HOLDS - -Recorded on the macOS host (AppKit harness, ScreenCaptureKit). TCC already granted. Saved locally -only (`macos-appkit-*.mp4`) — these record a personal screen, not uploaded. - -| mode | effects | focus contract | -|------|---------|----------------| -| ax-bg | 5/7 | **0/8 stole** | -| px-bg | 4/6 | **0/7 stole** | -| ax-fg / px-fg / px-desktop | 5/7 / 4/6 / 2/3 | foreground | - -Headline: in **px-bg**, pixel left-click/double/drag landed on the harness while Chrome stayed -frontmost — **0 focus steals**; the no-foreground contract holds on macOS for both AX and pixel -dispatch. Honest macOS gaps surfaced: the NSView click-target isn't in the AX tree (needs pixels); -pixel right-click never fires `rightMouseDown`; pixel scroll doesn't move `NSScrollView` (element -scroll does); NSButton ignores synthetic pixel clicks (AX `element_index` targets it); and there's -no true window-less screen click on macOS (`click` requires `pid`; pixel dispatch is window-anchored). -Two driver gotchas to flag: **`end_session` poisons a reused session id** (subsequent actions -silently no-op), and AppKit **window height drifts between launches** (store targets as window-local -points, convert to live screenshot px). - -## Control-parity pass (all surfaces → WPF baseline) - -Brought every harness up to the WPF 6-control set (checkbox, click-target L/R/double, -slider, scroll-target, text-input, context-menu) so the same 8-action matrix is -exercisable everywhere; `shared/scenarios.json` + `shared/web/index.html` are the -single source of truth. Harness DOM/UI parity landed for GTK3, WinUI3, the shared web -(WebView2/Electron/WKWebView), and macOS AppKit/SwiftUI. - -**scroll_target — Linux Electron (re-recorded, frame-verified).** The shared web -`scroll-tall` clipped region now resolves on Linux Electron in all 5 modes via the -Chromium web-AX tree; the recorder aims the scroll at that section and checks a real -`scroll_offset=` delta. **Result: scroll is a no-op in every mode** (`scroll_offset` -stays 0) — the same AT-SPI synthetic-input limit as the other Linux pointer actions -(double/right/drag), for both the element_index and the pixel/vision path. Headline -intact: Linux Electron ax-bg still **stole 3/8** (scroll is now a counted failure, not -`na`). 5 mp4s re-encoded + overwritten (`linux-electron-*.mp4`). - -**WebView2 (Windows) — harness restored, full-parity recording deferred.** The shared -web DOM (with scroll-target) is deployed; a clean self-contained re-publish of the -harness was needed after an in-place publish corrupted the bundle (window stopped -resolving). With the clean bundle the window resolves again and `sld`/`txt` map, but the -new click-target/checkbox/scroll/context controls need the WebView2 recorder's resolver -extended (as was done for Linux Electron) before a full-parity recording is meaningful — -deferred as low marginal value: it would only re-confirm the captured headline (Windows -Chromium **steals 7/8**, the matrix-webview2-ax-bg.mp4 evidence). - -## macOS — 4 surfaces + numeric-set_value driver fix - -Surfaces recorded on the macOS host (local-only, personal screen — never uploaded): -**AppKit, SwiftUI, Electron, WKWebView**, 5 modes each. WKWebView (Apple WebKit, the -native analogue of WebView2) **holds the contract** — confirming the Windows Electron/ -WebView2 steal is specific to Windows-Chromium, not WebKit-on-macOS. - -**Driver fix shipped (this branch, `platform-macos`):** `set_value` now writes a -**CFNumber** for numeric `AXValue` controls (NSSlider/NSStepper reject a CFString); -falls back to CFString for text fields. - -### Live host verification (AppKit + SwiftUI + WKWebView, against the running driver) - -The CFNumber fix was verified live after a `install-local` reinstall (the new daemon is -the **Jun-27 post-fix** binary; the installer re-signed the `CuaDriver.app` bundle -ad-hoc and re-granted TCC — `permissions status` reports Accessibility + Screen -Recording both ✅). The daemon was driven directly via the `cua-driver call` CLI. - -- **set_value CFNumber fix — VERIFIED on AppKit NSSlider.** `set_value(AXSlider, "50")` - on the AppKit harness now **succeeds** (`✅ Set AXValue on [5] AXSlider`) and - `slider_value` goes **0 → 50** (frame-verified — thumb at midpoint). This is the control - the fix targets; the CFNumber write lands where a CFString write was rejected. -- **SwiftUI slider — NOT a value-type problem; `AXValue` is unsettable there.** The - SwiftUI `AXSlider` exposes only `actions=[increment,decrement]` and rejects *any* - `AXValue` write (CFNumber and the CFString fallback both return `-25200`). So the - CFNumber fix cannot help SwiftUI sliders — driving them needs repeated AXIncrement/ - AXDecrement (future driver work), not a value write. (Comment in `set_number_attr` - notes `-25200` as well as `-25201`.) -- **SwiftUI harness — full control parity confirmed.** `get_window_state` exposes all - six WPF-parity controls as actionable AX elements (text field, click-target, - `AXSlider sld-value`, `AXCheckBox "I agree"`, context-menu button, scroll region) and - the status labels (`counter=`, `mirror=`, `last_action=`, `clicks=`, `slider_value=`, - `agreed=`, `menu_action=`, `scroll_offset=`) render as text nodes in `tree_markdown` - for the verifier. An AX checkbox press flipped `agreed=false → true` (frame-verified). -- **WKWebView web-surface scroll — page works, nested overflow div is a no-op.** The - driver's keystroke `scroll` (PageDown on the `AXWebArea`) scrolls the whole page a full - page down (frame-verified). But the nested `scroll-tall` overflow div stays at - `scroll_offset=0`: keystroke-scroll only drives the focused/page scroller, and a CSS - `overflow:auto` div without `tabindex` never takes keyboard focus. Same root cause as the - Linux Electron inner-div no-op; a pixel-wheel scroll (not in the current `scroll` tool) - would be required to drive an arbitrary overflow container. - -The two macOS gotchas above (`end_session` reuse poisoning, AppKit window-height drift) -were also filed for the driver. - -## Deliverables -10 verified recordings (5 WPF + 5 GTK3) + index, published to the team demo blob -(URL tracked internally — not linked here to keep it out of the public repo). -Additional local-only sets (not uploaded): macOS AppKit/SwiftUI/Electron/WKWebView, -Linux Electron (with scroll-target), and the Windows cross-toolkit matrix. diff --git a/libs/cua-driver/tests/fixtures/modality-recordings/README.md b/libs/cua-driver/tests/fixtures/modality-recordings/README.md deleted file mode 100644 index acb9d04023..0000000000 --- a/libs/cua-driver/tests/fixtures/modality-recordings/README.md +++ /dev/null @@ -1,73 +0,0 @@ -# cua-driver — modality recordings (Windows WPF + Linux GTK3) - -Per-toolkit, per-modality recordings of cua-driver's no-foreground contract. -Each video: the **test harness on the LEFT**, the **`cua-driver-panel` dashboard on the RIGHT** -(always-on-top) showing the run's modality, a live **foreground/background indicator**, and a -**per-action measurement** of the contract (`held` vs `STOLE FOCUS`). The pink/cyan overlay is -cua-driver's per-session agent cursor (no real pointer move). - -Five single-modality runs per toolkit: -- **ax-fg** — accessibility-tree actions, app intentionally kept FOREGROUND -- **ax-bg** — accessibility-tree actions, app should stay BACKGROUND (contract measured) -- **px-fg** — pixel-only (screenshot + coordinates), kept FOREGROUND -- **px-bg** — pixel-only, should stay BACKGROUND (contract measured) -- **px-desktop** — whole-screen, window-less screen-pixel actions (no window targeted) - -Action set per run: left-click · double-click · right-click · drag · scroll · -set_value (AX-only) · type · press-key. - -Each action carries **two** independent measurements on the dashboard: -- **✓ worked / ✗ no-op** — did the action actually change the app's state (checkbox toggled, - slider moved, text changed)? Verified by reading the harness's own state after each action. -- **held / STOLE FOCUS** — did the action keep the app in the background (the no-foreground contract)? - -## Windows — WPF (`wpf-*.mp4`) -Recorded on the Windows VM (Session 2, 1024×768) via cua-driver `start_recording` (ffmpeg gdigrab). - -| File | Measured | -|------|----------| -| `wpf-ax-fg.mp4` | foreground-mode, 8 actions | -| `wpf-ax-bg.mp4` | **1/8 actions stole focus** (double-click) | -| `wpf-px-fg.mp4` | foreground-mode, 7 actions | -| `wpf-px-bg.mp4` | **2/7 actions stole focus** (left-click, double-click) | -| `wpf-px-desktop.mp4` | foreground-mode, 4 actions | - -## Linux — GTK3 (`gtk3-*.mp4`) -Recorded on the Ubuntu 24.04 VM under a headless **Xvfb + openbox + picom + AT-SPI** stack -(X11; picom composites the agent-cursor overlay), via cua-driver `start_recording` (x11grab). - -| File | Effects landed | Focus contract | -|------|----------------|----------------| -| `gtk3-ax-fg.mp4` | 3/7 actions changed the app | foreground-mode, 8 actions | -| `gtk3-ax-bg.mp4` | 3/7 actions changed the app | **3/8 stole focus** | -| `gtk3-px-fg.mp4` | 1/6 actions changed the app | foreground-mode, 7 actions | -| `gtk3-px-bg.mp4` | 1/6 actions changed the app | **0/7 stole focus** | -| `gtk3-px-desktop.mp4` | 1/3 actions changed the app | foreground-mode, 4 actions | - -> **Superseded:** GTK background pixel clicks now land via AT-SPI `doAction` at -> point. See `FINDINGS.md` and the canonical -> `reference/cua-driver/modality-test-suite.mdx` (§ Linux) for the current result. - -**Key Linux finding (visible in the ✓/✗ column):** cua-driver injects Linux input via **XSendEvent** -(synthetic events delivered to a window without stealing focus — required for the no-foreground -contract). **GTK ignores synthetic XSendEvent events** (the `send_event` flag, a security feature), -so cua-driver's *pixel-based* actions (double-click, right-click, drag, scroll, and all coordinate -clicks) **do not land** on the GTK harness — they're honestly marked ✗ no-op. The *AT-SPI / element* -actions (single-click, set_value, type) use accessibility actions, which GTK honors, so they ✓ work. -This is a real, documented driver behavior (see `platform-linux/src/input/mod.rs`), surfaced here by -the per-action verifier rather than hidden. On Windows/WPF the equivalent synthetic input *is* honored, -which is why those runs land all actions (e.g. the slider visibly moves). - -## Reading the result -In the background runs, a `STOLE FOCUS` row means that action brought the app to the foreground -(contract violated for that action); `held` means it stayed in the background. The dashboard -tally at the bottom is the live count. Results differ by toolkit and modality — that variance -is the point: the recordings *measure* the contract honestly rather than asserting it holds. - -### Notes / known limitations -- **Linux AT-SPI** exposes only buttons/text/checkbox roles (no sliders/scrollers) and no element - frames, so drag/scroll and all coordinate pixel actions are driven from harness-exported widget - geometry. The drag/scroll actions fire and are measured but may not visibly move those widgets. -- **px-desktop** uses screen-absolute (window-less) clicks; under Xvfb these are not always - delivered faithfully, so some clicks register as no-ops while still being measured. -- macOS (AppKit/SwiftUI) was out of scope for this pass (no recordable session on the host). diff --git a/libs/cua-driver/tests/fixtures/modality-recordings/index.html b/libs/cua-driver/tests/fixtures/modality-recordings/index.html deleted file mode 100644 index fc1f71d049..0000000000 --- a/libs/cua-driver/tests/fixtures/modality-recordings/index.html +++ /dev/null @@ -1,34 +0,0 @@ -<!doctype html><html lang="en"><head> -<meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"> -<title>cua-driver — modality recordings -

cua-driver — modality recordings

-

Per-modality recordings of the no-foreground contract. Each: test harness (left) + live dashboard (right) measuring, per action, whether it kept the app in the background. Tap a card to play or download.

- -

Windows — WPF

-
AX · foregroundplay / download ⤓
effects 5/7 landed · foreground, 8 actions
 -
AX · backgroundplay / download ⤓
effects 5/7 landed · 2/8 stole focus
 -
Vision · foregroundplay / download ⤓
effects 3/6 landed · foreground, 7 actions
 -
Vision · backgroundplay / download ⤓
effects 4/6 landed · 4/7 stole focus
 -
Vision · full desktopplay / download ⤓
effects 1/3 landed · 4 actions
 - -

Linux — GTK3

-
AX · foregroundplay / download ⤓
effects 3/7 landed · foreground, 8 actions
 -
AX · backgroundplay / download ⤓
effects 3/7 landed · 3/8 stole focus
 -
Vision · foregroundplay / download ⤓
effects 1/6 landed · foreground, 7 actions
 -
Vision · backgroundplay / download ⤓
effects 1/6 landed · 0/7 stole focus
 -
Vision · full desktopplay / download ⤓
effects 1/3 landed · 4 actions
 - -
READMEopen ⤓
full notes: action set, the two measurements, Linux XSendEvent finding
 -

Both Windows and Linux runs show a per-action ✓ worked / ✗ no-op verifier alongside the focus contract.

- diff --git a/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-all-electron.sh b/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-all-electron.sh deleted file mode 100755 index f98edbe921..0000000000 --- a/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-all-electron.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -# Run all 5 modality recordings against the Electron harness, save re-encoded -# h264 mp4s to ~/Desktop/cua-driver-modality-videos/linux-electron-.mp4. -rm -f /tmp/lin-electron-allmodes.log -OUT="$HOME/Desktop/cua-driver-modality-videos"; mkdir -p "$OUT" -for MODE in ax-fg ax-bg px-fg px-bg px-desktop; do - bash /tmp/lin-run-electron.sh "$MODE" >/dev/null 2>&1 - echo "DONE $MODE $(cat /tmp/cua-lin-electron-$MODE/metric.log 2>/dev/null)" | tee -a /tmp/lin-electron-allmodes.log - SRC="/tmp/cua-lin-electron-$MODE/rec/recording.mp4" - [ -f "$SRC" ] && ffmpeg -y -loglevel error -i "$SRC" -c:v libx264 -crf 26 -pix_fmt yuv420p "$OUT/linux-electron-$MODE.mp4" -done diff --git a/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-all.sh b/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-all.sh deleted file mode 100644 index 09d94e9ff2..0000000000 --- a/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-all.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -rm -f /tmp/lin-allmodes.log -for MODE in ax-fg ax-bg px-fg px-bg px-desktop; do - bash /tmp/lin-run.sh "$MODE" >/dev/null 2>&1 - echo "DONE $MODE $(cat /tmp/cua-lin-$MODE/metric.log 2>/dev/null)" | tee -a /tmp/lin-allmodes.log -done diff --git a/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-dash.py b/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-dash.py deleted file mode 100644 index 381d8ebe5e..0000000000 --- a/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-dash.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 -# Native dashboard panel: a GTK window hosting a WebKit2GTK webview that loads -# the SAME dashboard.html served over loopback (exact parity with the Windows -# chrome --app panel, but no browser process — works headless under Xvfb). -import sys -import gi -gi.require_version("Gtk", "3.0") -gi.require_version("WebKit2", "4.1") -from gi.repository import Gtk, WebKit2 - -URL = sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:8146/" -X = int(sys.argv[2]) if len(sys.argv) > 2 else 544 -Y = int(sys.argv[3]) if len(sys.argv) > 3 else 0 -W = int(sys.argv[4]) if len(sys.argv) > 4 else 480 -H = int(sys.argv[5]) if len(sys.argv) > 5 else 740 - -win = Gtk.Window(title="cua-driver-panel") -win.set_default_size(W, H) -win.set_resizable(False) -view = WebKit2.WebView() -win.add(view) -view.load_uri(URL) -win.connect("destroy", Gtk.main_quit) -win.show_all() -win.move(X, Y) -Gtk.main() diff --git a/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-dashboard-electron.html b/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-dashboard-electron.html deleted file mode 100644 index 4a71e896a0..0000000000 --- a/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-dashboard-electron.html +++ /dev/null @@ -1,31 +0,0 @@ -cua-driver-panel
-

cua-driver - single-modality run - Electron

...
-
...
-
✓ worked / ✗ no-op = did the action change the app · held / STOLE = focus contract
-
-
diff --git a/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-dashboard.html b/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-dashboard.html deleted file mode 100644 index d918f5e5d1..0000000000 --- a/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-dashboard.html +++ /dev/null @@ -1,31 +0,0 @@ -cua-driver-panel
-

cua-driver - single-modality run - GTK3

...
-
...
-
✓ worked / ✗ no-op = did the action change the app · held / STOLE = focus contract
-
-
diff --git a/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-harness.py b/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-harness.py deleted file mode 100644 index 30b9c08e30..0000000000 --- a/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-harness.py +++ /dev/null @@ -1,167 +0,0 @@ -#!/usr/bin/env python3 -# CuaTestHarness GTK3 (recording edition) — rich controls so the Windows/WPF -# 8-action matrix maps onto Linux. Each actionable control sets its AT-SPI -# accessible name. The harness also (a) exports each named widget's screen rect -# (cua-driver's Linux AT-SPI snapshot has no sliders/scrollers or frames) and -# (b) writes its live state to a file so the recorder can VERIFY each action's -# effect (slider moved? checkbox toggled? text changed?), not just focus-steal. -import json -import gi -gi.require_version("Gtk", "3.0") -from gi.repository import Gtk, Gdk, GLib - -STATE_FILE = "/tmp/cua-lin-state.json" -GEOM_FILE = "/tmp/cua-lin-geom.json" - - -def aid(widget, name): - widget.get_accessible().set_name(name) - return widget - - -class Harness(Gtk.Window): - def __init__(self): - super().__init__(title="CuaTestHarness GTK3") - self.set_default_size(536, 740) - self.set_resizable(False) - self.clicks = 0 - self._last_action = "none" - self._ctx = "none" - - scroller = Gtk.ScrolledWindow() - scroller.set_policy(Gtk.PolicyType.NEVER, Gtk.PolicyType.AUTOMATIC) - self.add(scroller) - root = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=12) - root.set_border_width(14) - scroller.add(root) - - def section(title): - root.pack_start(Gtk.Label(label=title, xalign=0), False, False, 0) - - section("text input") - self.entry = aid(Gtk.Entry(), "txt-input") - self.entry.set_placeholder_text("type here") - self.entry.connect("changed", self._on_entry) - root.pack_start(self.entry, False, False, 0) - self.mirror = Gtk.Label(label="mirror=", xalign=0) - root.pack_start(self.mirror, False, False, 0) - - section("click target") - self.btn = aid(Gtk.Button(label="Click target (left / right / double)"), "btn-clicktarget") - self.btn.connect("clicked", self._on_click) - self.btn.connect("button-press-event", self._on_btn_press) - root.pack_start(self.btn, False, False, 0) - self.btn_status = Gtk.Label(label="last_action=none clicks=0", xalign=0) - root.pack_start(self.btn_status, False, False, 0) - - section("slider") - adj = Gtk.Adjustment(value=0, lower=0, upper=100, step_increment=1, page_increment=10) - self.scale = aid(Gtk.Scale(orientation=Gtk.Orientation.HORIZONTAL, adjustment=adj), "sld-value") - self.scale.set_draw_value(False) - self.scale.connect("value-changed", self._on_scale) - root.pack_start(self.scale, False, False, 0) - self.scale_status = Gtk.Label(label="slider_value=0", xalign=0) - root.pack_start(self.scale_status, False, False, 0) - - section("checkable") - self.chk = aid(Gtk.CheckButton(label="I agree"), "chk-agree") - self.chk.connect("toggled", self._on_chk) - root.pack_start(self.chk, False, False, 0) - self.chk_status = Gtk.Label(label="agreed=False", xalign=0) - root.pack_start(self.chk_status, False, False, 0) - - section("context menu") - self.ctx = aid(Gtk.Button(label="Right-click for context menu"), "btn-context") - self.ctx.connect("button-press-event", self._on_ctx_press) - root.pack_start(self.ctx, False, False, 0) - self.ctx_status = Gtk.Label(label="ctx=none", xalign=0) - root.pack_start(self.ctx_status, False, False, 0) - self.menu = Gtk.Menu() - for lbl in ("Cut", "Copy", "Paste"): - mi = Gtk.MenuItem(label=lbl) - mi.connect("activate", self._on_ctx_item, lbl) - self.menu.append(mi) - self.menu.show_all() - - section("scroll-tall") - inner = aid(Gtk.ScrolledWindow(), "scroll-tall") - inner.set_policy(Gtk.PolicyType.NEVER, Gtk.PolicyType.AUTOMATIC) - inner.set_size_request(-1, 150) - tall = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=2) - for i in range(1, 61): - tall.pack_start(Gtk.Label(label=f"row {i}", xalign=0), False, False, 0) - inner.add(tall) - root.pack_start(inner, False, False, 0) - self.scroll_inner = inner - inner.get_vadjustment().connect("value-changed", lambda *_: self._save_state()) - - self.named = {"chk":self.chk,"btn":self.btn,"ctx":self.ctx, - "sld":self.scale,"scr":self.scroll_inner,"txt":self.entry} - self.connect("destroy", Gtk.main_quit) - self._save_state() - - # ---- state file (for the recorder's per-action effect verifier) ---- - def _save_state(self): - try: sc = int(self.scroll_inner.get_vadjustment().get_value()) - except Exception: sc = 0 - st = {"slider": int(self.scale.get_value()), "agreed": self.chk.get_active(), - "mirror": self.entry.get_text(), "last_action": self._last_action, - "ctx": self._ctx, "clicks": self.clicks, "scroll": sc} - try: json.dump(st, open(STATE_FILE, "w")) - except Exception: pass - - def export_geom(self): - out = {} - for k, wdg in self.named.items(): - try: - res = wdg.translate_coordinates(self, 0, 0) - if not res: continue - lx, ly = res - ok, gx, gy = self.get_window().get_origin() - a = wdg.get_allocation() - out[k] = {"x":gx+lx, "y":gy+ly, "w":a.width, "h":a.height} - except Exception: - pass - json.dump(out, open(GEOM_FILE, "w")) - self._save_state() - return False - - def _on_entry(self, e): - self.mirror.set_text(f"mirror={e.get_text()}"); self._save_state() - - def _on_click(self, *_): - self.clicks += 1; self._last_action = "click" - self.btn_status.set_text(f"last_action=click clicks={self.clicks}"); self._save_state() - - def _on_btn_press(self, _w, ev): - if ev.type == Gdk.EventType.DOUBLE_BUTTON_PRESS: - self._last_action = "double_click" - self.btn_status.set_text(f"last_action=double_click clicks={self.clicks}"); self._save_state() - - def _on_scale(self, s): - self.scale_status.set_text(f"slider_value={int(s.get_value())}"); self._save_state() - - def _on_chk(self, c): - self.chk_status.set_text(f"agreed={c.get_active()}"); self._save_state() - - def _on_ctx_press(self, _w, ev): - if ev.button == 3: - self._ctx = "opened"; self.ctx_status.set_text("ctx=opened") - self.menu.popup_at_pointer(ev); self._save_state() - - def _on_ctx_item(self, _w, lbl): - self._ctx = lbl; self.ctx_status.set_text(f"ctx={lbl}"); self._save_state() - - -if __name__ == "__main__": - import sys, traceback - try: - w = Harness() - print("INIT_DONE", flush=True) - w.move(0, 0) - w.show_all() - print("SHOWN", flush=True) - GLib.timeout_add(1200, w.export_geom) - except Exception: - traceback.print_exc(); sys.stdout.flush(); raise - Gtk.main() diff --git a/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-rec-electron.py b/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-rec-electron.py deleted file mode 100644 index 7721e779e1..0000000000 --- a/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-rec-electron.py +++ /dev/null @@ -1,353 +0,0 @@ -#!/usr/bin/env python3 -# Linux modality recorder for the ELECTRON harness — parity with the Windows -# Electron lane in wpf-recorder.ps1 and the Linux GTK3 lin-rec.py. -# Runs under: xvfb-run -a --server-args="-screen 0 1024x768x24" dbus-run-session -- python3 lin-rec-electron.py MODE -# Electron LEFT, WebKit dashboard RIGHT (per-action held/STOLE + ✓worked/✗no-op verifier). -# -# Differences vs lin-rec.py (GTK): -# - launches the Electron app (Chromium) instead of lin-harness.py -# - resolves the harness window by substring "CuaTestHarness Electron" (cdp-suffixed title) -# - reads BOTH control geometry AND harness state from the web-AX tree -# (get_window_state capture_mode=ax), the way the Windows recorder reads UIA Text. -import json, os, subprocess, sys, time, glob, re - -MODE = sys.argv[1] if len(sys.argv) > 1 else "ax-bg" -HOME = os.path.expanduser("~") -DRV = f"{HOME}/cua/libs/cua-driver/rust/target/release/cua-driver" -ELECTRON_DIR = f"{HOME}/cua/libs/cua-driver/tests/fixtures/apps/cross-platform/electron" -ELECTRON_BIN = f"{ELECTRON_DIR}/node_modules/electron/dist/electron" -DASH = "/tmp/lin-dash.py" -DASH_HTML = "/tmp/lin-dashboard-electron.html" -PORT = 8146 -WORK = f"/tmp/cua-lin-electron-{MODE}" -REC = f"{WORK}/rec" -HARW, HARH = 536, 740 -PANX, PANW, PANH = 544, 480, 740 -WIN_TITLE = "CuaTestHarness Electron" - -META = { - "ax-fg": {"title":"AX - FOREGROUND","scope":"window","see":"accessibility tree (element-level)","fg":True, "expect":"App kept in FRONT on purpose. Each action runs via the accessibility tree; we measure the foreground."}, - "ax-bg": {"title":"AX - BACKGROUND","scope":"window","see":"accessibility tree (element-level)","fg":False,"expect":"App should stay in the BACKGROUND. Each action runs via the accessibility tree; we measure which actions steal focus."}, - "px-fg": {"title":"VISION - FOREGROUND","scope":"window","see":"screenshot only (pixels)","fg":True, "expect":"Pure pixel-driven, app kept in FRONT. We measure the foreground."}, - "px-bg": {"title":"VISION - BACKGROUND","scope":"window","see":"screenshot only (pixels)","fg":False,"expect":"Pure pixel-driven, app should stay in the BACKGROUND. We measure which pixel actions steal focus."}, - "px-desktop":{"title":"VISION - FULL DESKTOP","scope":"desktop","see":"full-screen screenshot","fg":True,"expect":"Whole-screen, window-less screen-pixel actions (no window targeted)."}, -} -m = META[MODE] -VISION = m["see"].startswith("screenshot") or m["scope"] == "desktop" -DESKTOP = m["scope"] == "desktop" - -def sh(args, t=10): - try: return subprocess.run(args, capture_output=True, text=True, timeout=t).stdout - except Exception: return "" - -def D(tool, payload): - try: - p = subprocess.run([DRV, "call", tool], input=json.dumps(payload), capture_output=True, text=True, timeout=25) - return p.stdout - except Exception: - return "" - -def DJ(tool, payload): - try: return json.loads(D(tool, payload) or "{}") - except Exception: return {} - -def active_name(): - return sh(["xdotool", "getactivewindow", "getwindowname"]).strip() - -def active_id(): - return sh(["xdotool", "getactivewindow"]).strip() - -# ---------- plan ---------- -PLAN = [ - ("click","chk","left-click a checkbox"), - ("double","btn","double-click a button"), - ("right","ctx","right-click (context menu)"), - ("drag","sld","drag the slider"), - ("scroll","scr","scroll the panel"), - ("setval","txt","set_value on the text box"), - ("type","txt","type into the text box"), - ("key","txt","press a key (Tab)"), -] -if VISION: PLAN = [p for p in PLAN if p[0] != "setval"] -if DESKTOP: PLAN = [p for p in PLAN if p[0] in ("click","scroll","type","key")] - -steps = [{"label":l,"state":"pending","result":"","verified":""} for (_,_,l) in PLAN] -state = {"steals":0,"actions":0} - -# ---------- per-action EFFECT verifier: read the web harness's own status labels via web-AX ---------- -def hstate(): - E = els() - joined = " || ".join(str(e.get("label", e.get("name",""))) for e in E) - h = {} - mm = re.search(r'agreed=(\w+)', joined); h["agreed"] = mm.group(1) if mm else None - mm = re.search(r'slider_value=(\d+)', joined); h["slider"] = int(mm.group(1)) if mm else 0 - mm = re.search(r'last_action=(\w+)', joined); h["last_action"] = mm.group(1) if mm else None - mm = re.search(r'mirror=([^|]*)', joined); h["mirror"] = (mm.group(1).strip() if mm else "") - mm = re.search(r'counter=(\d+)', joined); h["counter"] = int(mm.group(1)) if mm else 0 - mm = re.search(r'scroll_offset=(\d+)', joined); h["scroll"] = int(mm.group(1)) if mm else 0 - h["_joined"] = joined - return h - -def verify(t, before, after): - if t == "click": return "ok" if after.get("agreed") != before.get("agreed") else "fail" - if t == "double": return "ok" if after.get("last_action") == "double_click" else "fail" - if t == "right": return "ok" if after.get("last_action") == "right_click" else "fail" - if t == "drag": return "ok" if after.get("slider", 0) > before.get("slider", 0) else "fail" - if t == "scroll": return "ok" if after.get("scroll", 0) > before.get("scroll", 0) else "fail" # scroll_offset= now exposed by web-AX - if t == "setval": return "ok" if "set-by-cua" in str(after.get("mirror", "")) else "fail" - if t == "type": return "ok" if "typed-by-cua" in str(after.get("mirror", "")) else "fail" - return "na" # press-key (Tab): no observable harness-state effect - -def flush(): - fg = active_name() - af = WIN_TITLE in fg - st = {"run":m["title"],"expect":m["expect"],"fgmode":m["fg"],"foreground":fg,"appFront":af, - "steals":state["steals"],"actions":state["actions"],"steps":steps} - with open(f"{WORK}/status.json","w") as f: json.dump(st, f) - -def pulse(sec): - end = time.time()+sec - while time.time() < end: flush(); time.sleep(0.15) - -# ---------- setup ---------- -os.makedirs(REC, exist_ok=True) -subprocess.run("pkill -x electron; pkill -f lin-dash.py", shell=True) -time.sleep(1) -import shutil; shutil.copy(DASH_HTML, f"{WORK}/dashboard.html") -with open(f"{WORK}/status.json","w") as f: json.dump({"run":"","steps":[]}, f) - -def dbgcnt(tag): - try: - w = json.loads(D("list_windows", {}) or "{}").get("windows", []) - line = "%s: %d %s\n" % (tag, len(w), [x.get("title") for x in w]) - except Exception as e: - line = "%s: ERR %s\n" % (tag, e) - open(f"{WORK}/setup.log","a").write(line) - -open(f"{WORK}/setup.log","a").write("DISPLAY=%s XAUTH=%s\n" % (os.environ.get("DISPLAY"), os.environ.get("XAUTHORITY"))) -dbgcnt("at start") -D("set_agent_cursor_enabled", {"enabled":True,"session":"d1"}) -D("set_agent_cursor_motion", {"session":"d1","cursor_color":"#FF2D2D","cursor_label":"cua-driver","glide_duration_ms":600,"dwell_after_click_ms":700,"idle_hide_ms":120000}) -dbgcnt("after cursor") - -subprocess.Popen(["python3","-m","http.server",str(PORT),"--directory",WORK], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - -# launch Electron — Chromium needs --no-sandbox under Xvfb, --disable-gpu for the -# software path, --force-renderer-accessibility so the web-AX tree populates via AT-SPI. -eenv = dict(os.environ) -eenv["ELECTRON_DISABLE_SECURITY_WARNINGS"] = "1" -elog = open(f"{WORK}/electron.log","w") -subprocess.Popen([ELECTRON_BIN, ELECTRON_DIR, "--no-sandbox", "--disable-gpu", - "--force-renderer-accessibility"], - cwd=ELECTRON_DIR, env=eenv, stdout=elog, stderr=elog) -time.sleep(10) -dbgcnt("after electron") -subprocess.Popen(["python3", DASH, f"http://127.0.0.1:{PORT}/dashboard.html", str(PANX),"0",str(PANW),str(PANH)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL); time.sleep(4) -dbgcnt("after dashboard") -# position windows: harness LEFT, dashboard RIGHT + above -sh(["wmctrl","-r",WIN_TITLE,"-e","0,0,0,%d,%d"%(HARW,HARH)]) -sh(["wmctrl","-r","cua-driver-panel","-e","0,%d,0,%d,%d"%(PANX,PANW,PANH)]) -sh(["wmctrl","-r","cua-driver-panel","-b","add,above"]) -time.sleep(1) - -# ---------- foreground-baseline ANCHOR (background modes only) ---------- -# The no-foreground contract is "did this action steal foreground". Measuring that needs a -# GENUINE foreground baseline before each action: a real, ACTIVATED, non-harness window. -# Re-asserting the dashboard panel with `wmctrl -b add,above` is z-order / _NET_WM_STATE_ABOVE -# ONLY (no activation), so it never holds a true active-window baseline — once the first inject -# action click-activates the target, the harness silently stays the active window and every later -# step false-positives as a "steal". Anchor on a real xterm: park it under the (above) panel rect -# so it stays invisible in the recording but remains a valid activatable non-harness foreground -# window. (macOS mac-rec.py already does a real `activate` — no flaw; Windows fix was 49bdb41b.) -ANCHOR_ID = None -ANCHOR_PROC = None -if not m["fg"]: - ANCHOR_PROC = subprocess.Popen( - ["xterm","-class","cua-anchor","-T","cua-anchor","-geometry","18x3", - "-e","bash","-c","while true; do sleep 3600; done"], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - for _ in range(20): - ids = sh(["xdotool","search","--class","cua-anchor"]).split() - if ids: ANCHOR_ID = ids[0]; break - time.sleep(0.4) - if ANCHOR_ID: - sh(["wmctrl","-i","-r",ANCHOR_ID,"-e","0,%d,0,%d,%d"%(PANX,PANW,PANH)]) - sh(["wmctrl","-r","cua-driver-panel","-b","add,above"]) - open(f"{WORK}/baseline.log","w").write("ANCHOR_ID=%s launched=%s\n" % (ANCHOR_ID, ANCHOR_PROC is not None)) - -def anchor_front(): - # GENUINELY activate (not z-order) the anchor and confirm it actually became the active window, - # so each action is measured against a true non-harness foreground baseline. Then re-assert the - # dashboard panel ABOVE for the recording (z-order only — does not change the active window). - if not ANCHOR_ID: return False - held = False - for _ in range(8): - sh(["xdotool","windowactivate","--sync",ANCHOR_ID]) - time.sleep(0.18) - if active_id() == ANCHOR_ID: held = True; break - sh(["wmctrl","-r","cua-driver-panel","-b","add,above"]) - return held - -# resolve harness window -w = None -for _ in range(30): - wins = DJ("list_windows", {}).get("windows", []) - w = next((x for x in wins if WIN_TITLE in (x.get("title","") or "")), None) - if w: break - time.sleep(0.5) -if not w: - dbg = "DISPLAY=%s\n" % os.environ.get("DISPLAY") - dbg += "raw_list_windows=%s\n" % (D("list_windows", {})[:2000]) - dbg += "xwininfo=%s\n" % sh(["bash","-c","xwininfo -root -tree 2>/dev/null | grep -iE 'Cua|panel|Electron|harness'"]) - dbg += "electron_log=%s\n" % (open(f"{WORK}/electron.log").read()[-2000:] if os.path.exists(f"{WORK}/electron.log") else "none") - open(f"{WORK}/debug.log","w").write(dbg) - open(f"{WORK}/metric.log","w").write("FATAL: no harness window"); sys.exit(1) -WP, WD = w["pid"], w["window_id"] -WB = w.get("bounds") or w.get("frame") or {"x":0,"y":0,"w":HARW,"h":HARH} - -def els(): - return DJ("get_window_state", {"pid":WP,"window_id":WD,"capture_mode":"ax"}).get("elements", []) - -def rect(el): - r = el.get("frame") or el.get("bounds") or {} - return (int(r.get("x",0)), int(r.get("y",0)), int(r.get("w",0)), int(r.get("h",0))) - -def find(E, name=None, role=None, anyname=None): - for e in E: - nm = str(e.get("label", e.get("name",""))) - rl = str(e.get("role","")) - if name and name.lower() not in nm.lower(): continue - if role and role.lower() not in rl.lower(): continue - if anyname and not any(a.lower() in nm.lower() for a in anyname): continue - return e - return None - -def find_scroll(E): - # scroll-tall is the CLIPPED viewport: a 'section' ~260w x ~120h (border/pad → ~270x131 - # in web-AX), distinct from the inner lines container (h~880) and the 23px line rows. - for e in E: - if "section" not in str(e.get("role","")).lower(): continue - _, _, w, h = rect(e) - if 250 <= w <= 290 and 110 <= h <= 155: - return e - return None - -E = [] -resolve = {} -for _ in range(16): - E = els() - resolve = { - "chk": find(E, role="check") or find(E, name="agree"), - "btn": find(E, name="Click target") or find(E, name="click target"), - "ctx": find(E, name="Click target") or find(E, name="click target"), - "sld": find(E, role="slider"), - "scr": find_scroll(E), - "txt": find(E, role="text") or find(E, role="entry") or find(E, name="type here"), - } - if resolve["chk"] or resolve["txt"]: break - time.sleep(0.8) -dump = "\n".join("[%s] role=%r name=%r frame=%r" % (e.get("element_index"), e.get("role"), e.get("label", e.get("name","")), e.get("frame") or e.get("bounds")) for e in E) -open(f"{WORK}/resolve.log","w").write("WIN bounds=%r\nRESOLVE "%WB+" ".join("%s=%s"%(k,bool(v)) for k,v in resolve.items())+" count=%d\n--- elements ---\n%s"%(len(E),dump)) - -def gcenter(sel): - el = resolve.get(sel) - if not el: return None - x,y,ww,hh = rect(el); return (x+ww//2, y+hh//2) -def gwinlocal(sel): - el = resolve.get(sel) - if not el: return None - x,y,ww,hh = rect(el); return (x-int(WB["x"])+ww//2, y-int(WB["y"])+hh//2) - -# seed agent cursor overlay BEFORE recording -D("move_cursor", {"x":HARW-30,"y":HARH-30,"session":"d1"}) -time.sleep(0.4) -if DESKTOP: D("set_config", {"key":"capture_scope","value":"desktop"}) -D("start_recording", {"output_dir":REC,"record_video":True}) -pulse(2) - -def do(t, sel): - el = resolve.get(sel) - eidx = el.get("element_index") if el else None - use_ax = (eidx is not None) and (not VISION) and (not DESKTOP) - c = gcenter(sel); wl = gwinlocal(sel) or (HARW//2, HARH//2) - if c: D("move_cursor", {"x":c[0],"y":c[1],"session":"d1"}); time.sleep(0.55) - if t == "click": - if DESKTOP and c: D("click", {"x":c[0],"y":c[1],"session":"d1"}) - elif use_ax: D("click", {"pid":WP,"window_id":WD,"element_index":eidx,"session":"d1"}) - elif c: D("click", {"pid":WP,"window_id":WD,"x":wl[0],"y":wl[1],"delivery_mode":("foreground" if m["fg"] else "background"),"session":"d1"}) - elif t == "double": - if use_ax: D("double_click", {"pid":WP,"window_id":WD,"element_index":eidx,"session":"d1"}) - elif c: D("double_click", {"pid":WP,"window_id":WD,"x":wl[0],"y":wl[1],"session":"d1"}) - elif t == "right": - if use_ax: D("right_click", {"pid":WP,"window_id":WD,"element_index":eidx,"session":"d1"}) - elif c: D("right_click", {"pid":WP,"window_id":WD,"x":wl[0],"y":wl[1],"session":"d1"}) - time.sleep(0.5); D("press_key", {"pid":WP,"key":"escape","session":"d1"}) - elif t == "drag": - if el: - x,y,ww,hh = rect(el); fx=x-int(WB["x"])+8; fy=y-int(WB["y"])+hh//2 - D("drag", {"pid":WP,"window_id":WD,"from_x":fx,"from_y":fy,"to_x":fx+150,"to_y":fy,"session":"d1"}) - elif t == "scroll": - # aim at the resolved scroll-tall viewport; fall back to window center if unresolved - sa = c or (HARW//2, HARH//2) # screen/abs center of scroll-tall - sl = wl # window-local center of scroll-tall - if DESKTOP: D("scroll", {"x":sa[0],"y":sa[1],"direction":"down","session":"d1"}) - elif VISION: D("scroll", {"pid":WP,"window_id":WD,"x":sl[0],"y":sl[1],"direction":"down","session":"d1"}) - elif use_ax: D("scroll", {"pid":WP,"window_id":WD,"element_index":eidx,"direction":"down","session":"d1"}) - else: D("scroll", {"pid":WP,"window_id":WD,"x":sl[0],"y":sl[1],"direction":"down","session":"d1"}) - elif t == "setval": - if eidx is not None: D("set_value", {"pid":WP,"window_id":WD,"element_index":eidx,"value":"set-by-cua","session":"d1"}) - elif t == "type": - if eidx is not None and not VISION: - D("click", {"pid":WP,"window_id":WD,"element_index":eidx,"session":"d1"}); time.sleep(0.35) - elif c: - D("click", {"pid":WP,"window_id":WD,"x":wl[0],"y":wl[1],"delivery_mode":("foreground" if m["fg"] else "background"),"session":"d1"}); time.sleep(0.35) - D("type_text", {"pid":WP,"text":"typed-by-cua","session":"d1"}) - elif t == "key": - D("press_key", {"pid":WP,"key":"tab","session":"d1"}) - -# ---------- run ---------- -try: - for i,(t,sel,label) in enumerate(PLAN): - steps[i]["state"]="active"; flush() - # establish the genuine foreground baseline: ACTIVATE the anchor and confirm it actually took - # the active window BEFORE the action runs. A "steal" is then the active window moving OFF the - # anchor ONTO the harness; "held" is the anchor staying active. (fg modes keep the harness in - # front by design, so no baseline anchor there.) - if not m["fg"] and ANCHOR_ID: - held = anchor_front() - open(f"{WORK}/baseline.log","a").write( - "step %d '%s': baseline active='%s' anchor_held=%s\n" % (i, t, active_name(), held)) - before = hstate() - do(t, sel) - # measure: steal = active window moved OFF the anchor baseline onto the harness after the action. - stole=False; end=time.time()+1.5 - while time.time() 1 else "ax-bg" -HOME = os.path.expanduser("~") -DRV = f"{HOME}/cua/libs/cua-driver/rust/target/release/cua-driver" -HARNESS = "/tmp/lin-harness.py" -DASH = "/tmp/lin-dash.py" -DASH_HTML = "/tmp/lin-dashboard.html" -PORT = 8146 -WORK = f"/tmp/cua-lin-{MODE}" -REC = f"{WORK}/rec" -HARW, HARH = 536, 740 -PANX, PANW, PANH = 544, 480, 740 - -META = { - "ax-fg": {"title":"AX - FOREGROUND","scope":"window","see":"accessibility tree (element-level)","fg":True, "expect":"App kept in FRONT on purpose. Each action runs via the accessibility tree; we measure the foreground."}, - "ax-bg": {"title":"AX - BACKGROUND","scope":"window","see":"accessibility tree (element-level)","fg":False,"expect":"App should stay in the BACKGROUND. Each action runs via the accessibility tree; we measure which actions steal focus."}, - "px-fg": {"title":"VISION - FOREGROUND","scope":"window","see":"screenshot only (pixels)","fg":True, "expect":"Pure pixel-driven, app kept in FRONT. We measure the foreground."}, - "px-bg": {"title":"VISION - BACKGROUND","scope":"window","see":"screenshot only (pixels)","fg":False,"expect":"Pure pixel-driven, app should stay in the BACKGROUND. We measure which pixel actions steal focus."}, - "px-desktop":{"title":"VISION - FULL DESKTOP","scope":"desktop","see":"full-screen screenshot","fg":True,"expect":"Whole-screen, window-less screen-pixel actions (no window targeted)."}, -} -m = META[MODE] -VISION = m["see"].startswith("screenshot") or m["scope"] == "desktop" -DESKTOP = m["scope"] == "desktop" - -def sh(args, t=10): - try: return subprocess.run(args, capture_output=True, text=True, timeout=t).stdout - except Exception: return "" - -def D(tool, payload): - try: - p = subprocess.run([DRV, "call", tool], input=json.dumps(payload), capture_output=True, text=True, timeout=25) - return p.stdout - except Exception as e: - return "" - -def DJ(tool, payload): - try: return json.loads(D(tool, payload) or "{}") - except Exception: return {} - -def active_name(): - return sh(["xdotool", "getactivewindow", "getwindowname"]).strip() - -def active_id(): - return sh(["xdotool", "getactivewindow"]).strip() - -# ---------- plan ---------- -PLAN = [ - ("click","chk","left-click a checkbox"), - ("double","btn","double-click a button"), - ("right","ctx","right-click (context menu)"), - ("drag","sld","drag the slider"), - ("scroll","scr","scroll the panel"), - ("setval","txt","set_value on the text box"), - ("type","txt","type into the text box"), - ("key","txt","press a key (Tab)"), -] -if VISION: PLAN = [p for p in PLAN if p[0] != "setval"] -if DESKTOP: PLAN = [p for p in PLAN if p[0] in ("click","scroll","type","key")] - -steps = [{"label":l,"state":"pending","result":"","verified":""} for (_,_,l) in PLAN] -state = {"steals":0,"actions":0} - -def hstate(): - try: return json.load(open("/tmp/cua-lin-state.json")) - except Exception: return {} - -def verify(t, before, after): - # did the action actually change the harness's state? - if t == "click": return "ok" if after.get("agreed") != before.get("agreed") else "fail" - if t == "double": return "ok" if after.get("last_action") == "double_click" else "fail" - if t == "right": return "ok" if after.get("ctx", "none") != "none" else "fail" - if t == "drag": return "ok" if after.get("slider", 0) > before.get("slider", 0) else "fail" - if t == "scroll": return "ok" if after.get("scroll", 0) > before.get("scroll", 0) else "fail" - if t == "setval": return "ok" if after.get("mirror", "") == "set-by-cua" else "fail" - if t == "type": return "ok" if "typed-by-cua" in str(after.get("mirror", "")) else "fail" - return "na" # press-key (Tab): no observable harness-state effect to assert - -def flush(): - fg = active_name() - af = "CuaTestHarness" in fg - st = {"run":m["title"],"expect":m["expect"],"fgmode":m["fg"],"foreground":fg,"appFront":af, - "steals":state["steals"],"actions":state["actions"],"steps":steps} - with open(f"{WORK}/status.json","w") as f: json.dump(st, f) - -def pulse(sec): - end = time.time()+sec - while time.time() < end: flush(); time.sleep(0.15) - -# ---------- setup ---------- -os.makedirs(REC, exist_ok=True) -# NB: do NOT pkill cua-driver here — the wrapper owns the daemon. Port is freed by the wrapper. -subprocess.run("pkill -f lin-harness.py; pkill -f lin-dash.py", shell=True) -time.sleep(1) -import shutil; shutil.copy(DASH_HTML, f"{WORK}/dashboard.html") -with open(f"{WORK}/status.json","w") as f: json.dump({"run":"","steps":[]}, f) - -def dbgcnt(tag): - try: - w = json.loads(D("list_windows", {}) or "{}").get("windows", []) - line = "%s: %d %s\n" % (tag, len(w), [x.get("title") for x in w]) - except Exception as e: - line = "%s: ERR %s\n" % (tag, e) - open(f"{WORK}/setup.log","a").write(line) - -# openbox + `cua-driver serve` are launched by the bash wrapper (so the daemon -# reliably inherits DISPLAY/XAUTHORITY — a python-launched daemon saw 0 windows). -open(f"{WORK}/setup.log","a").write("DISPLAY=%s XAUTH=%s\n" % (os.environ.get("DISPLAY"), os.environ.get("XAUTHORITY"))) -dbgcnt("at start") -D("set_agent_cursor_enabled", {"enabled":True,"session":"d1"}) -D("set_agent_cursor_motion", {"session":"d1","cursor_color":"#FF2D2D","cursor_label":"cua-driver","glide_duration_ms":600,"dwell_after_click_ms":700,"idle_hide_ms":120000}) -dbgcnt("after cursor") -subprocess.Popen(["python3","-m","http.server",str(PORT),"--directory",WORK], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) -subprocess.Popen(["python3", HARNESS], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL); time.sleep(3) -dbgcnt("after harness") -subprocess.Popen(["python3", DASH, f"http://127.0.0.1:{PORT}/dashboard.html", str(PANX),"0",str(PANW),str(PANH)], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL); time.sleep(4) -dbgcnt("after dashboard") -# position windows: harness LEFT, dashboard RIGHT + above -sh(["wmctrl","-r","CuaTestHarness GTK3","-e","0,0,0,%d,%d"%(HARW,HARH)]) -sh(["wmctrl","-r","cua-driver-panel","-e","0,%d,0,%d,%d"%(PANX,PANW,PANH)]) -sh(["wmctrl","-r","cua-driver-panel","-b","add,above"]) -time.sleep(1) - -# ---------- foreground-baseline ANCHOR (background modes only) ---------- -# The no-foreground contract is "did this action steal foreground". Measuring that needs a -# GENUINE foreground baseline before each action: a real, ACTIVATED, non-harness window. -# Re-asserting the dashboard panel with `wmctrl -b add,above` is z-order / _NET_WM_STATE_ABOVE -# ONLY (no activation), so it never holds a true active-window baseline — once the first inject -# action click-activates the target, the harness silently stays the active window and every later -# step false-positives as a "steal". Anchor on a real xterm: park it under the (above) panel rect -# so it stays invisible in the recording but remains a valid activatable non-harness foreground -# window. (macOS mac-rec.py already does a real `activate` — no flaw; Windows fix was 49bdb41b.) -ANCHOR_ID = None -ANCHOR_PROC = None -if not m["fg"]: - ANCHOR_PROC = subprocess.Popen( - ["xterm","-class","cua-anchor","-T","cua-anchor","-geometry","18x3", - "-e","bash","-c","while true; do sleep 3600; done"], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - for _ in range(20): - ids = sh(["xdotool","search","--class","cua-anchor"]).split() - if ids: ANCHOR_ID = ids[0]; break - time.sleep(0.4) - if ANCHOR_ID: - sh(["wmctrl","-i","-r",ANCHOR_ID,"-e","0,%d,0,%d,%d"%(PANX,PANW,PANH)]) - sh(["wmctrl","-r","cua-driver-panel","-b","add,above"]) - open(f"{WORK}/baseline.log","w").write("ANCHOR_ID=%s launched=%s\n" % (ANCHOR_ID, ANCHOR_PROC is not None)) - -def anchor_front(): - # GENUINELY activate (not z-order) the anchor and confirm it actually became the active window, - # so each action is measured against a true non-harness foreground baseline. Then re-assert the - # dashboard panel ABOVE for the recording (z-order only — does not change the active window). - if not ANCHOR_ID: return False - held = False - for _ in range(8): - sh(["xdotool","windowactivate","--sync",ANCHOR_ID]) - time.sleep(0.18) - if active_id() == ANCHOR_ID: held = True; break - sh(["wmctrl","-r","cua-driver-panel","-b","add,above"]) - return held - -# resolve harness window -w = None -for _ in range(20): - wins = DJ("list_windows", {}).get("windows", []) - w = next((x for x in wins if "CuaTestHarness" in (x.get("title","") or "")), None) - if w: break - time.sleep(0.5) -if not w: - dbg = "DISPLAY=%s\n" % os.environ.get("DISPLAY") - try: xalive = subprocess.run(["xdpyinfo"], capture_output=True, timeout=8).returncode == 0 - except Exception: xalive = False - dbg += "X_alive=%s\n" % xalive - dbg += "raw_list_windows=%s\n" % (D("list_windows", {})[:1800]) - dbg += "xwininfo=%s\n" % sh(["bash","-c","xwininfo -root -tree 2>/dev/null | grep -iE 'Cua|panel|GTK'"]) - open(f"{WORK}/debug.log","w").write(dbg) - open(f"{WORK}/metric.log","w").write("FATAL: no harness window"); sys.exit(1) -WP, WD = w["pid"], w["window_id"] -WB = w.get("bounds") or w.get("frame") or {"x":0,"y":0,"w":HARW,"h":HARH} - -def els(): - return DJ("get_window_state", {"pid":WP,"window_id":WD,"capture_mode":"ax"}).get("elements", []) - -def find(E, **kw): - for e in E: - nm = str(e.get("label", e.get("name",""))) - role = str(e.get("role","")) - if "name" in kw and kw["name"] not in nm: continue - if "role" in kw and kw["role"].lower() not in role.lower(): continue - return e - return None - -E = [] -resolve = {} -for _ in range(14): - E = els() - resolve = { - "chk": find(E, name="chk-agree") or find(E, role="check"), - "btn": find(E, name="btn-clicktarget"), - "ctx": find(E, name="btn-context"), - "sld": find(E, name="sld-value") or find(E, role="slider"), - "scr": find(E, name="scroll-tall"), - "txt": find(E, name="txt-input") or find(E, role="text"), - } - if resolve["chk"] or resolve["txt"]: break - time.sleep(0.7) -dump = "\n".join("[%s] role=%r name=%r" % (e.get("element_index"), e.get("role"), e.get("label", e.get("name",""))) for e in E) -open(f"{WORK}/resolve.log","w").write("RESOLVE "+" ".join("%s=%s"%(k,bool(v)) for k,v in resolve.items())+" count=%d\n--- elements ---\n%s"%(len(E),dump)) - -try: GEOM = json.load(open("/tmp/cua-lin-geom.json")) -except Exception: GEOM = {} -open(f"{WORK}/resolve.log","a").write("\n--- geom ---\n"+json.dumps(GEOM)) -def gcenter(sel): - g = GEOM.get(sel) - return (g["x"]+g["w"]//2, g["y"]+g["h"]//2) if g else None -def gwinlocal(sel): - g = GEOM.get(sel) - return (g["x"]-int(WB["x"])+g["w"]//2, g["y"]-int(WB["y"])+g["h"]//2) if g else None - -# seed agent cursor overlay BEFORE recording -D("move_cursor", {"x":HARW-30,"y":HARH-30,"session":"d1"}) -time.sleep(0.4) -if DESKTOP: D("set_config", {"key":"capture_scope","value":"desktop"}) -D("start_recording", {"output_dir":REC,"record_video":True}) -pulse(2) - -def do(t, sel): - el = resolve.get(sel) - g = GEOM.get(sel) - eidx = el.get("element_index") if el else None - use_ax = (eidx is not None) and (not VISION) and (not DESKTOP) - c = gcenter(sel); wl = gwinlocal(sel) or (0,0) - if c: D("move_cursor", {"x":c[0],"y":c[1],"session":"d1"}); time.sleep(0.55) - if t == "click": - if DESKTOP and c: D("click", {"x":c[0],"y":c[1],"session":"d1"}) - elif use_ax: D("click", {"pid":WP,"window_id":WD,"element_index":eidx,"session":"d1"}) - elif c: D("click", {"pid":WP,"window_id":WD,"x":wl[0],"y":wl[1],"delivery_mode":("foreground" if m["fg"] else "background"),"session":"d1"}) - elif t == "double": - if use_ax: D("double_click", {"pid":WP,"window_id":WD,"element_index":eidx,"session":"d1"}) - elif c: D("double_click", {"pid":WP,"window_id":WD,"x":wl[0],"y":wl[1],"session":"d1"}) - elif t == "right": - if use_ax: D("right_click", {"pid":WP,"window_id":WD,"element_index":eidx,"session":"d1"}) - elif c: D("right_click", {"pid":WP,"window_id":WD,"x":wl[0],"y":wl[1],"session":"d1"}) - time.sleep(0.5); D("press_key", {"pid":WP,"key":"escape","session":"d1"}) - elif t == "drag": - if g: fx=g["x"]-int(WB["x"])+8; fy=g["y"]-int(WB["y"])+g["h"]//2; D("drag", {"pid":WP,"from_x":fx,"from_y":fy,"to_x":fx+150,"to_y":fy,"session":"d1"}) - elif t == "scroll": - if DESKTOP and c: D("scroll", {"x":c[0],"y":c[1],"direction":"down","session":"d1"}) - elif use_ax: D("scroll", {"pid":WP,"window_id":WD,"element_index":eidx,"direction":"down","session":"d1"}) - elif c: D("scroll", {"pid":WP,"window_id":WD,"x":wl[0],"y":wl[1],"direction":"down","session":"d1"}) - elif t == "setval": - if eidx is not None: D("set_value", {"pid":WP,"window_id":WD,"element_index":eidx,"value":"set-by-cua","session":"d1"}) - elif t == "type": - D("type_text", {"pid":WP,"text":"typed-by-cua","session":"d1"}) - elif t == "key": - D("press_key", {"pid":WP,"key":"tab","session":"d1"}) - -# ---------- run ---------- -try: - for i,(t,sel,label) in enumerate(PLAN): - steps[i]["state"]="active"; flush() - # establish the genuine foreground baseline: ACTIVATE the anchor and confirm it actually took - # the active window BEFORE the action runs. A "steal" is then the active window moving OFF the - # anchor ONTO the harness; "held" is the anchor staying active. (fg modes keep the harness in - # front by design, so no baseline anchor there.) - if not m["fg"] and ANCHOR_ID: - held = anchor_front() - open(f"{WORK}/baseline.log","a").write( - "step %d '%s': baseline active='%s' anchor_held=%s\n" % (i, t, active_name(), held)) - before = hstate() - do(t, sel) - # measure: steal = active window moved OFF the anchor baseline onto the harness after the action. - stole=False; end=time.time()+1.5 - while time.time()/dev/null; pkill cua-driver 2>/dev/null; pkill picom 2>/dev/null -pkill -f 'http.server' 2>/dev/null; fuser -k 8146/tcp 2>/dev/null -pkill -x electron 2>/dev/null; pkill -f lin-dash 2>/dev/null -sleep 1.5 -rm -f "/tmp/cua-lin-electron-$MODE/setup.log" -xvfb-run -a --server-args="-screen 0 1024x768x24" dbus-run-session -- bash -c ' - export GDK_BACKEND=x11 CUA_DRIVER_RS_DRAW_SYSTEM_CURSOR=0 - DRV="'"$DRV"'" - # bring up the AT-SPI a11y bus so Chromium can register its web-AX tree - for p in /usr/libexec/at-spi-bus-launcher /usr/lib/at-spi2-core/at-spi-bus-launcher; do - [ -x "$p" ] && "$p" --launch-immediately & break - done - sleep 1 - openbox & sleep 1 - picom --backend xrender --config /dev/null >/tmp/picom.log 2>&1 & sleep 1 - xdotool mousemove 1010 758 - "$DRV" serve >"/tmp/cua-lin-electron-'"$MODE"'-drv.log" 2>&1 & sleep 3 - python3 /tmp/lin-rec-electron.py "'"$MODE"'" - pkill -x electron; pkill cua-driver; pkill picom; pkill openbox -' > "/tmp/cua-lin-electron-$MODE-run.log" 2>&1 -echo "exit=$? mode=$MODE" diff --git a/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-run.sh b/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-run.sh deleted file mode 100644 index 8139c51c45..0000000000 --- a/libs/cua-driver/tests/fixtures/modality-recordings/linux/lin-run.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -MODE="${1:-ax-bg}" -DRV="$HOME/cua/libs/cua-driver/rust/target/release/cua-driver" -export NO_AT_BRIDGE=0 GTK_A11Y=1 GDK_BACKEND=x11 CUA_DRIVER_RS_DRAW_SYSTEM_CURSOR=0 -pkill Xvfb 2>/dev/null; pkill cua-driver 2>/dev/null; pkill picom 2>/dev/null -pkill -f 'http.server' 2>/dev/null; fuser -k 8146/tcp 2>/dev/null -pkill -f lin-harness 2>/dev/null; pkill -f lin-dash 2>/dev/null -sleep 1.5 -rm -f "/tmp/cua-lin-$MODE/setup.log" /tmp/cua-lin-state.json -xvfb-run -a --server-args="-screen 0 1024x768x24" dbus-run-session -- bash -c ' - export GDK_BACKEND=x11 CUA_DRIVER_RS_DRAW_SYSTEM_CURSOR=0 - DRV="'"$DRV"'" - openbox & sleep 1 - picom --backend xrender --config /dev/null >/tmp/picom.log 2>&1 & sleep 1 - xdotool mousemove 1010 758 # park the real X pointer in the corner (only the agent cursor should show) - "$DRV" serve >"/tmp/cua-lin-'"$MODE"'-drv.log" 2>&1 & sleep 3 - python3 /tmp/lin-rec.py "'"$MODE"'" - pkill cua-driver; pkill picom; pkill openbox -' > "/tmp/cua-lin-$MODE-run.log" 2>&1 -echo "exit=$? mode=$MODE" diff --git a/libs/cua-driver/tests/fixtures/modality-recordings/macos/mac-rec.py b/libs/cua-driver/tests/fixtures/modality-recordings/macos/mac-rec.py deleted file mode 100644 index f3446b3acc..0000000000 --- a/libs/cua-driver/tests/fixtures/modality-recordings/macos/mac-rec.py +++ /dev/null @@ -1,546 +0,0 @@ -#!/usr/bin/env python3 -# macOS AppKit modality recorder — parity with the Linux GTK3 / Windows WPF golden recorders. -# Harness LEFT (CuaTestHarness.AppKit), Chrome --app dashboard RIGHT. -# Per-action: no-foreground contract (held / STOLE) + effect verifier (worked / no-op). -import json, os, subprocess, sys, time, re, glob, signal, atexit - -MODE = sys.argv[1] if len(sys.argv) > 1 else "ax-bg" -SURFACE = sys.argv[2] if len(sys.argv) > 2 else "appkit" -HOME = os.path.expanduser("~") -DRV = f"{HOME}/cua/libs/cua-driver/rust/target/release/cua-driver" -DASH_HTML = "/tmp/mac-dashboard.html" -CHROME = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" -PORT = 8147 - -# ---------- per-surface configuration ---------- -SURF = { - "appkit": { - "bundle":"com.trycua.harness.appkit", "title":"CuaTestHarness AppKit", - "dash":"AppKit", "appmatch":"CuaTestHarness", - # window-local POINTS (measured live against the rebuilt parity harness, window 700x1112). - "coords":{"click_target":(129.0,304.0),"slider":(180.0,372.0),"scroll":(279.0,791.0), - "txt":(140.0,232.0),"increment":(62.0,96.0)}, - }, - "swiftui": { - "bundle":"com.trycua.harness.swiftui", "title":"CuaTestHarness SwiftUI", - "dash":"SwiftUI", "appmatch":"CuaTestHarness", - # window-local POINTS (measured live against the rebuilt parity harness, window 700x820). - "coords":{"click_target":(179.0,348.0),"slider":(230.0,424.0),"scroll":(210.0,690.0), - "txt":(190.0,268.0),"increment":(112.0,100.0),"popover":(125.0,904.0)}, - }, - "electron": { - "bundle":None, "launch_match":"CuaTestHarness Electron", "title":"CuaTestHarness Electron", - "dash":"Electron", "appmatch":"Electron", - "coords":{"click_target":(167.0,459.0),"slider":(45.0,367.0),"txt":(106.0,282.0), - "scroll":(340.0,400.0),"increment":(70.0,197.0)}, - }, - "wkwebview": { # Apple WebKit (NOT Chromium); same shared web DOM as Electron - "bundle":"com.trycua.harness.wkwebview", "title":"CuaTestHarness WKWebView", - "dash":"WKWebView", "appmatch":"CuaTestHarness", - "coords":{"click_target":(167.0,459.0),"slider":(45.0,367.0),"txt":(106.0,282.0), - "scroll":(340.0,400.0),"increment":(70.0,197.0)}, - }, -} -WEBISH = ("electron", "wkwebview") # web-AX harnesses sharing the same DOM/verifier -S = SURF[SURFACE] -BUNDLE = S["bundle"]; TITLE = S["title"]; APPMATCH = S["appmatch"] -WORK = f"/tmp/cua-mac-{SURFACE}-{MODE}" -REC = f"{WORK}/rec" - -# window layout (points). screen ~1512x982, menu bar ~33. -HX, HY = 0, 40 # harness top-left -DASH_X, DASH_Y = 712, 40 # dashboard top-left -DASH_W, DASH_H = 792, 900 - -META = { - "ax-fg": {"title":"AX - FOREGROUND","see":"accessibility tree (element-level)","fg":True, "expect":"App kept in FRONT on purpose. Each action runs via the accessibility tree / element_index; we measure the foreground."}, - "ax-bg": {"title":"AX - BACKGROUND","see":"accessibility tree (element-level)","fg":False,"expect":"App should stay in the BACKGROUND. Each action runs via the accessibility tree / element_index; we measure which actions steal focus."}, - "px-fg": {"title":"VISION - FOREGROUND","see":"screenshot only (pixels)","fg":True, "expect":"Pure pixel-driven, app kept in FRONT. We measure the foreground."}, - "px-bg": {"title":"VISION - BACKGROUND","see":"screenshot only (pixels)","fg":False,"expect":"Pure pixel-driven, app should stay in the BACKGROUND. We measure which pixel actions steal focus."}, - "px-desktop":{"title":"VISION - FULL DESKTOP","see":"full-screen screenshot","fg":True,"expect":"capture_scope=desktop (full-display capture). Pure pixel-driven, app in FRONT. On macOS pixel dispatch is window-anchored, so actions route to the on-screen window."}, -} -m = META[MODE] -VISION = m["see"].startswith("screenshot") -DESKTOP = MODE == "px-desktop" -SESS = f"macd-{MODE}-{int(time.time())}" - -def D(tool, payload, t=25): - try: - p = subprocess.run([DRV, "call", tool], input=json.dumps(payload), - capture_output=True, text=True, timeout=t) - return p.stdout - except Exception: - return "" - -def DJ(tool, payload, t=25): - try: return json.loads(D(tool, payload, t) or "{}") - except Exception: return {} - -# Safety net: dispose the agent-cursor session (remove its overlay cursor) even if -# the run raises before the explicit end_session below. This script does NOT kill -# the daemon on exit, so the daemon is still alive at interpreter shutdown and the -# atexit call reaches it. end_session is idempotent, so the happy-path call is a -# no-op here. -atexit.register(lambda: D("end_session", {"session": SESS})) - -def osa(script, t=6): - try: return subprocess.run(["osascript","-e",script], capture_output=True, text=True, timeout=t).stdout.strip() - except Exception: return "" - -def frontmost(): - return osa('tell application "System Events" to get name of first application process whose frontmost is true') - -def app_is_front(): - return APPMATCH in frontmost() - -def activate_harness(): - osa(f'tell application "System Events" to set frontmost of (first process whose unix id is {PID}) to true') - -def activate_dash(): - osa('tell application "Google Chrome" to activate') - -def position_harness(): - # pin position AND size: window height otherwise varies (832/858) between launches, - # which shifts the screenshot dims and breaks pixel-coord calibration. - osa(f'''tell application "System Events" - set p to first process whose unix id is {PID} - set w to first window of p whose title contains "{TITLE}" - set position of w to {{{HX}, {HY}}} - set size of w to {{700, 820}} - end tell''') - -# ---------- coords (window-local screenshot-px; window 700x858 -> shot 1279x1568) ---------- -# Targets stored as WINDOW-LOCAL POINTS (stable across window-height drift, top-anchored -# layout). Converted to live screenshot-pixels at runtime: px = pt * shot_dim / win_dim. -# Calibrated against a pinned 700x820 window (shot 1339x1568). -WIN_W = 700.0 -COORDS_PT = S["coords"] -# live dims, measured right before recording -SHOT_W, SHOT_H, WIN_H = 1339.0, 1568.0, 820.0 -def coord(sel): - px, py = COORDS_PT[sel] - return (px * SHOT_W / WIN_W, py * SHOT_H / WIN_H) - -# ---------- plan (per surface) ---------- -PLANS = { - "appkit": [ - ("click","click_target","left-click the click-target"), - ("double","click_target","double-click the click-target"), - ("right","click_target","right-click the click-target"), - ("drag","click_target","drag across the click-target"), - ("scroll","scroll","scroll the list"), - ("setval","txt","set_value on the text box"), - ("type","txt","type into the text box"), - ("key","txt","press a key (Tab)"), - ], - "swiftui": [ - ("click","click_target","left-click the click-target"), - ("double","click_target","double-click the click-target"), - ("right","click_target","right-click the click-target"), - ("drag","slider","drag the slider"), - ("scroll","scroll","scroll the list"), - ("setval","txt","set_value on the text box"), - ("type","txt","type into the text box"), - ("key","txt","press a key (Tab)"), - ], - "electron": [ - ("click","click_target","left-click the click-target"), - ("double","click_target","double-click the click-target"), - ("right","click_target","right-click the click-target"), - ("drag","slider","drag the slider"), - ("scroll","scroll","scroll the page"), - ("setval","txt","set_value on the text box"), - ("type","txt","type into the text box"), - ("key","txt","press a key (Tab)"), - ], -} -PLANS["wkwebview"] = PLANS["electron"] -PLAN = list(PLANS[SURFACE]) -if VISION: PLAN = [p for p in PLAN if p[0] != "setval"] -if DESKTOP: PLAN = [p for p in PLAN if p[0] in ("click","scroll","type","key")] - -steps = [{"label":l,"state":"pending","result":"","verified":""} for (_,_,l) in PLAN] -state = {"steals":0,"actions":0} - -# ---------- harness state reader (parse AX tree_markdown) ---------- -def hstate(): - d = DJ("get_window_state", {"pid":PID,"window_id":WID,"capture_mode":"ax"}) - tm = d.get("tree_markdown","") or "" - if SURFACE in WEBISH: - # web harness exposes plain-text status labels (counter=N, mirror=..., etc.) - st = {"last":"none","clicks":0,"slider":0,"counter":0,"txt":""} - m1 = re.search(r"last_action=(\w+)", tm); st["last"] = m1.group(1) if m1 else "none" - m2 = re.search(r"clicks=(\d+)", tm); st["clicks"] = int(m2.group(1)) if m2 else 0 - m3 = re.search(r"slider_value=(\d+)", tm); st["slider"] = int(m3.group(1)) if m3 else 0 - m4 = re.search(r"counter=(\d+)", tm); st["counter"] = int(m4.group(1)) if m4 else 0 - m5 = re.search(r"mirror=([^\"]*)\"", tm); st["txt"] = m5.group(1) if m5 else "" - return st - # NATIVE (appkit/swiftui): rebuilt parity harness exposes the SAME web-style status - # labels in the AX tree_markdown — last_action=, clicks=, slider_value=, scroll_offset=, - # agreed=, menu_action=. Text mirror is the AXTextField value (id=txt-input). - st = {"last":"none","clicks":0,"slider":0,"scroll":0,"menu":"none","agreed":"false","txt":""} - m1 = re.search(r"last_action=(\w+)", tm); st["last"] = m1.group(1) if m1 else "none" - m2 = re.search(r"clicks=(\d+)", tm); st["clicks"] = int(m2.group(1)) if m2 else 0 - m3 = re.search(r"slider_value=(\d+)", tm); st["slider"] = int(m3.group(1)) if m3 else 0 - m4 = re.search(r"scroll_offset=(\d+)", tm); st["scroll"] = int(m4.group(1)) if m4 else 0 - m5 = re.search(r"menu_action=([^\"]+)\"", tm); st["menu"] = m5.group(1) if m5 else "none" - m6 = re.search(r"agreed=(\w+)", tm); st["agreed"] = m6.group(1) if m6 else "false" - # txt-input value (placeholder "Type here…" when empty — substring checks below won't match it) - tv = re.search(r'AXTextField = "([^"]*)" \[id=txt-input', tm) or re.search(r'AXTextField = "([^"]*)"', tm) - if tv: st["txt"] = tv.group(1) - return st - -AXMODE = (not VISION) and (not DESKTOP) -def verify(t, b, a): - if SURFACE in WEBISH: - if t == "click": return "ok" if ("left_click" in a["last"] and a["clicks"]>b["clicks"]) else "fail" - if t == "double": return "ok" if "double_click" in a["last"] else "fail" - if t == "right": return "ok" if "right_click" in a["last"] else "fail" - if t == "drag": return "ok" if a["slider"]>b["slider"] else "fail" - if t == "setval": return "ok" if "set-by-cua" in a["txt"] else "fail" - if t == "type": return "ok" if "typed-by-cua" in a["txt"] else "fail" - return "na" # scroll/key: no verifiable readout - # NATIVE appkit/swiftui — real parity signals from the click-target / slider / - # scroll-target / text box (matches the WEBISH verifier; only the readouts differ). - if t == "click": return "ok" if (a["clicks"]>b["clicks"] or a["last"] in ("click","left_click")) else "fail" - # AppKit's click-target reports last_action=double_click for a pixel double; SwiftUI's - # does not distinguish it, so a landed double shows up as clicks incrementing (+2). - if t == "double": return "ok" if ("double" in a["last"] or a["clicks"]>b["clicks"]) else "fail" - # right-click: last_action=right_click (AppKit) or a context menu_action change. - if t == "right": return "ok" if ("right" in a["last"] or a["menu"]!=b["menu"]) else "fail" - if t == "drag": return "ok" if a["slider"]>b["slider"] else "fail" - if t == "scroll": return "ok" if a["scroll"]>b["scroll"] else "fail" - if t == "setval": return "ok" if "set-by-cua" in a["txt"] else "fail" - if t == "type": return "ok" if "typed-by-cua" in a["txt"] else "fail" - return "na" # key: no verifiable readout - -# ---------- status.json writer for dashboard ---------- -_front_cache = {"v":"", "t":0} -def flush(refresh_front=True): - if refresh_front: - _front_cache["v"] = frontmost() - fg = _front_cache["v"] - af = APPMATCH in fg - st = {"run":m["title"],"expect":m["expect"],"fgmode":m["fg"],"foreground":fg,"appFront":af, - "steals":state["steals"],"actions":state["actions"],"steps":steps} - with open(f"{WORK}/status.json","w") as f: json.dump(st, f) - -def pulse(sec): - end = time.time()+sec - while time.time() < end: - flush(); time.sleep(0.18) - -# ---------- setup ---------- -os.makedirs(REC, exist_ok=True) -subprocess.run("pkill -f mac-dashboard; pkill -f 'http.server 8147'", shell=True) -subprocess.run(f"pkill -f CuaTestHarness", shell=True) -if SURFACE == "electron": - subprocess.run("pkill -f 'electron/dist/Electron'; pkill -f 'Electron.app/Contents/MacOS/Electron'", shell=True) -time.sleep(2.0) -# dashboard.html with per-surface title suffix (h1 "... - AppKit" -> "... - ") -_html = open(DASH_HTML).read().replace("single-modality run - AppKit", f"single-modality run - {S['dash']}") -open(f"{WORK}/dashboard.html","w").write(_html) -with open(f"{WORK}/status.json","w") as f: json.dump({"run":"","steps":[]}, f) -log = open(f"{WORK}/setup.log","w") -def L(s): log.write(s+"\n"); log.flush() - -# http server for dashboard -subprocess.Popen(["python3","-m","http.server",str(PORT),"--directory",WORK], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) -time.sleep(1) - -# launch harness -if BUNDLE: - la = DJ("launch_app", {"bundle_id":BUNDLE}) - PID = la.get("pid") - wins = la.get("windows", []) - main = next((w for w in wins if TITLE in (w.get("title") or "")), None) - WID = main["window_id"] if main else (wins[0]["window_id"] if wins else None) -else: - # electron: launched out-of-band (see ELECTRON_CMD); poll list_windows by title substring - if os.environ.get("ELECTRON_CMD"): - subprocess.Popen(os.environ["ELECTRON_CMD"], shell=True, - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - PID = WID = None - for _ in range(40): - for w in DJ("list_windows", {}).get("windows", []): - if S["launch_match"] in (w.get("title") or ""): - PID, WID = w["pid"], w["window_id"]; break - if PID: break - time.sleep(0.5) -L(f"launched pid={PID} wid={WID}") -if not PID or not WID: - L("FATAL no harness"); open(f"{WORK}/metric.log","w").write("FATAL no harness"); sys.exit(1) - -# make harness visible (recording carve-out) + position LEFT -activate_harness(); time.sleep(0.6) -position_harness(); time.sleep(0.4) -# refresh window id (re-list, pick on-screen main window) -wl = DJ("list_windows", {"pid":PID}).get("windows", []) -mm = next((w for w in wl if TITLE in (w.get("title") or "") and w.get("is_on_screen")), None) -if mm: WID = mm["window_id"]; WIN_ORIGIN = (mm["bounds"]["x"], mm["bounds"]["y"]) -else: WIN_ORIGIN = (HX, HY) -L(f"after position wid={WID} origin={WIN_ORIGIN} onscreen_main={bool(mm)}") - -# launch Chrome --app dashboard, positioned RIGHT -prof = f"/tmp/cua-dash-profile-{SURFACE}-{MODE}" -subprocess.Popen([CHROME, f"--app=http://127.0.0.1:{PORT}/dashboard.html", - f"--user-data-dir={prof}", "--no-first-run", "--no-default-browser-check", - f"--window-position={DASH_X},{DASH_Y}", f"--window-size={DASH_W},{DASH_H}"], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) -time.sleep(4) -# re-position dashboard via System Events as a backstop -osa(f'''tell application "System Events" - set cp to first process whose name is "Google Chrome" - try - set position of front window of cp to {{{DASH_X}, {DASH_Y}}} - set size of front window of cp to {{{DASH_W}, {DASH_H}}} - end try -end tell''') -time.sleep(1) -L(f"frontmost after chrome: {frontmost()}") - -# resolve element indices FRESH from a snapshot (first occurrence = primary window subtree). -# Re-resolved right before each AX action to honour the snapshot-before-action invariant. -def axidx(): - d = DJ("get_window_state", {"pid":PID,"window_id":WID,"capture_mode":"ax"}) - tm = d.get("tree_markdown","") or "" - idx = {} - for mt in re.finditer(r'\[(\d+)\] (AX\w+)[^\n]*?\[id=([\w-]+)', tm): - if mt.group(3) not in idx: idx[mt.group(3)] = int(mt.group(1)) - ta = re.search(r'\[(\d+)\] AXTextArea', tm) - if ta: idx["scroll-area"] = int(ta.group(1)) - # role/label fallbacks (Electron web-AX exposes no [id=...]) - if "txt-input" not in idx: - mt = re.search(r'\[(\d+)\] AXTextField', tm) - if mt: idx["txt-input"] = int(mt.group(1)) - if "slider" not in idx: - mt = re.search(r'\[(\d+)\] AXSlider', tm) - if mt: idx["slider"] = int(mt.group(1)) - if "btn-increment" not in idx: - mt = re.search(r'\[(\d+)\] AXButton "Increment"', tm) - if mt: idx["btn-increment"] = int(mt.group(1)) - return idx - -IDX0 = axidx() -open(f"{WORK}/resolve.log","w").write("IDX="+json.dumps(IDX0)) -L(f"resolved idx={IDX0}") - -# agent cursor -D("start_session", {"session":SESS}) -D("set_agent_cursor_enabled", {"enabled":True,"session":SESS}) -D("set_agent_cursor_motion", {"session":SESS,"cursor_color":"#FF2D2D","cursor_label":"cua-driver", - "glide_duration_ms":600,"dwell_after_click_ms":650,"idle_hide_ms":120000}) - -# re-pin size and measure LIVE screenshot dims + window height (drift-proof coords) -position_harness(); time.sleep(0.6) -ws = DJ("get_window_state", {"pid":PID,"window_id":WID,"capture_mode":"som"}) -if ws.get("screenshot_width"): SHOT_W = float(ws["screenshot_width"]) -if ws.get("screenshot_height"): SHOT_H = float(ws["screenshot_height"]) -wl2 = DJ("list_windows", {"pid":PID}).get("windows", []) -mm2 = next((w for w in wl2 if w.get("window_id")==WID), None) -if mm2: - WIN_H = float(mm2["bounds"]["height"]); WIN_ORIGIN = (mm2["bounds"]["x"], mm2["bounds"]["y"]) -L(f"live dims shot={SHOT_W}x{SHOT_H} win_h={WIN_H} origin={WIN_ORIGIN}") - -# foreground/background precondition -if m["fg"]: activate_harness() -else: activate_dash() -time.sleep(0.8) - -# seed cursor on-screen before recording (so AX actions glide visibly) -D("move_cursor", {"pid":PID,"window_id":WID,"x":480,"y":760,"session":SESS}) -time.sleep(0.4) - - -if DESKTOP: - D("set_config", {"key":"capture_scope","value":"desktop"}) -D("start_recording", {"output_dir":REC,"record_video":True}) -time.sleep(0.5) -pulse(2.0) - -# ---------- action dispatch ---------- -def to_screen(px, py): - # screenshot-px -> screen points - return (WIN_ORIGIN[0] + px*WIN_W/SHOT_W, WIN_ORIGIN[1] + py*WIN_H/SHOT_H) - -def glide(sel): - px, py = coord(sel) - if DESKTOP: - sx, sy = to_screen(px, py) - D("move_cursor", {"x":sx,"y":sy,"session":SESS}) - else: - D("move_cursor", {"pid":PID,"window_id":WID,"x":px,"y":py,"session":SESS}) - time.sleep(0.5) - -def web_snapshot(): - d = DJ("get_window_state", {"pid":PID,"window_id":WID,"capture_mode":"ax"}) - return d.get("elements", []) or [] - -def web_el(els, sel): - # Resolve a web control to (element_index, screenshot_px_x, screenshot_px_y) from its - # ACTUAL live frame — never the per-surface COORDS_PT, which drifted ~185px off WKWebView's - # real web layout so every pixel action missed. WebKit duplicates the web subtree; the - # lower-index copy is the live/hittable one, so pick the smallest matching element_index. - def first(pred): - cand = [e for e in els if pred(e)] - return min(cand, key=lambda e: e.get("element_index", 9999)) if cand else None - role = lambda e: str(e.get("role", "")) - lbl = lambda e: str(e.get("label") or e.get("name") or "") - if sel == "click_target": e = first(lambda e: "Click target" in lbl(e)) - elif sel == "slider": e = first(lambda e: "AXSlider" in role(e)) - elif sel == "txt": e = first(lambda e: "AXTextField" in role(e)) - elif sel == "scroll": e = first(lambda e: "scroll_target" in lbl(e)) or first(lambda e: "AXTextArea" in role(e)) - else: e = None - if not e: return (None, None, None) - f = e["frame"]; cx = f["x"] + f["w"]/2.0; cy = f["y"] + f["h"]/2.0 - px = (cx - WIN_ORIGIN[0]) * SHOT_W / WIN_W - py = (cy - WIN_ORIGIN[1]) * SHOT_H / WIN_H - return (e.get("element_index"), px, py) - -def glide_to(px, py): - if px is None: return - if DESKTOP: - sx, sy = to_screen(px, py); D("move_cursor", {"x":sx,"y":sy,"session":SESS}) - else: - D("move_cursor", {"pid":PID,"window_id":WID,"x":px,"y":py,"session":SESS}) - time.sleep(0.5) - -def do_electron(t, sel): - # Web harness (WKWebView / Electron). AX mode dispatches by element_index — AXPress works on - # the web spans (verified live: click/double/right/type all land). Vision mode uses pixel - # coords derived from the live element frame. Both replace the stale hardcoded COORDS_PT that - # made every WKWebView pixel action miss. set_value on a web input sets AXValue but doesn't - # fire the DOM input event (mirror stays empty) — an honest web limitation; type_text works. - els = web_snapshot() - ct_idx, cpx, cpy = web_el(els, "click_target") - sl_idx, spx, spy = web_el(els, "slider") - tx_idx, tpx, tpy = web_el(els, "txt") - sc_idx, rpx, rpy = web_el(els, "scroll") - if t == "click": - glide_to(cpx, cpy) - if AXMODE and ct_idx is not None: D("click", {"pid":PID,"window_id":WID,"element_index":ct_idx,"session":SESS}) - else: D("click", {"pid":PID,"window_id":WID,"x":cpx,"y":cpy,"session":SESS}) - elif t == "double": - glide_to(cpx, cpy) - if AXMODE and ct_idx is not None: D("double_click", {"pid":PID,"window_id":WID,"element_index":ct_idx,"session":SESS}) - else: D("double_click", {"pid":PID,"window_id":WID,"x":cpx,"y":cpy,"count":2,"session":SESS}) - elif t == "right": - glide_to(cpx, cpy) - if AXMODE and ct_idx is not None: D("right_click", {"pid":PID,"window_id":WID,"element_index":ct_idx,"session":SESS}) - else: D("right_click", {"pid":PID,"window_id":WID,"x":cpx,"y":cpy,"session":SESS}) - elif t == "drag": - # slider thumb: a real synthetic pixel drag fires the range input's event (AXValue-set - # does not). Correct coords come from the live slider frame. - glide_to(spx, spy) - if spx is not None: D("drag", {"pid":PID,"window_id":WID,"from_x":spx,"from_y":spy,"to_x":spx+220,"to_y":spy,"session":SESS}) - elif t == "scroll": - glide_to(rpx, rpy) - if rpx is not None: D("scroll", {"pid":PID,"window_id":WID,"x":rpx,"y":rpy,"direction":"down","amount":5,"session":SESS}) - elif t == "setval": - if tx_idx is not None: D("set_value", {"pid":PID,"window_id":WID,"element_index":tx_idx,"value":"set-by-cua","session":SESS}) - elif t == "type": - if AXMODE and tx_idx is not None: - glide_to(tpx, tpy) - D("set_value", {"pid":PID,"window_id":WID,"element_index":tx_idx,"value":"","session":SESS}); time.sleep(0.4) - D("click", {"pid":PID,"window_id":WID,"element_index":tx_idx,"session":SESS}); time.sleep(0.3) - D("type_text", {"pid":PID,"window_id":WID,"element_index":tx_idx,"text":"typed-by-cua","session":SESS}) - else: - glide_to(tpx, tpy); D("click", {"pid":PID,"window_id":WID,"x":tpx,"y":tpy,"session":SESS}); time.sleep(0.3) - D("type_text", {"pid":PID,"text":"typed-by-cua","session":SESS}) - elif t == "key": - D("press_key", {"pid":PID,"key":"tab","session":SESS}) - -def do_native(t, sel): - # appkit / swiftui — rebuilt parity controls (click-target / slider / scroll-target / - # text box), mirroring the electron plan. AX press lands the click-target's primary - # action (last_action=click, works backgrounded). double/right/drag are real - # mouse-button gestures the view records via pixel events. scroll: AppKit exposes an - # AXTextArea (id=scroll-tall) addressable by element_index; SwiftUI's nested ScrollView - # is pixel-only. type: a PIXEL click grabs first responder (AX-press on a textfield does - # not), then keyboard type_text — set_value clears first in AX modes. - IDX = axidx() if AXMODE else {} - ct = IDX.get("btn-clicktarget"); txt = IDX.get("txt-input") - sa = IDX.get("scroll-tall") or IDX.get("scroll-area") # AppKit AXTextArea (multiline value hides its [id=]) - cpx, cpy = coord("click_target"); spx, spy = coord("slider"); tpx, tpy = coord("txt") - if t == "click": - glide("click_target") - if AXMODE and ct is not None: D("click", {"pid":PID,"window_id":WID,"element_index":ct,"session":SESS}) - else: D("click", {"pid":PID,"window_id":WID,"x":cpx,"y":cpy,"session":SESS}) - elif t == "double": - glide("click_target"); D("double_click", {"pid":PID,"window_id":WID,"x":cpx,"y":cpy,"count":2,"session":SESS}) - elif t == "right": - glide("click_target"); D("right_click", {"pid":PID,"window_id":WID,"x":cpx,"y":cpy,"session":SESS}) - D("press_key", {"pid":PID,"key":"escape","session":SESS}) # dismiss any context menu - elif t == "drag": - glide("slider"); D("drag", {"pid":PID,"window_id":WID,"from_x":spx,"from_y":spy,"to_x":spx+220,"to_y":spy,"session":SESS}) - elif t == "scroll": - glide("scroll"); rpx, rpy = coord("scroll") - if AXMODE and sa is not None: - D("scroll", {"pid":PID,"window_id":WID,"element_index":sa,"direction":"down","amount":5,"session":SESS}) - else: - D("scroll", {"pid":PID,"window_id":WID,"x":rpx,"y":rpy,"direction":"down","amount":8,"session":SESS}) - elif t == "setval": - glide("txt") - if AXMODE and txt is not None: - D("set_value", {"pid":PID,"window_id":WID,"element_index":txt,"value":"set-by-cua","session":SESS}) - elif t == "type": - glide("txt") - if AXMODE and txt is not None: - D("set_value", {"pid":PID,"window_id":WID,"element_index":txt,"value":"","session":SESS}); time.sleep(0.4) - D("click", {"pid":PID,"window_id":WID,"x":tpx,"y":tpy,"session":SESS}); time.sleep(0.4) - D("type_text", {"pid":PID,"text":"typed-by-cua","session":SESS}) - elif t == "key": - D("press_key", {"pid":PID,"key":"tab","session":SESS}) - -def do(t, sel): - if SURFACE in WEBISH: - return do_electron(t, sel) - return do_native(t, sel) - -# ---------- run ---------- -for i,(t,sel,label) in enumerate(PLAN): - steps[i]["state"]="active"; flush() - if not m["fg"]: - activate_dash(); time.sleep(0.4) # re-assert background precondition - before = hstate() - do(t, sel) - stole=False; end=time.time()+1.6 - while time.time()1.5s, esp. backgrounded; - # re-poll so a landed-but-late effect isn't mis-scored as a no-op. - for _ in range(7): - time.sleep(0.4); after = hstate() - if verify(t, before, after) != "fail": break - steps[i]["verified"] = verify(t, before, after) - if t == "popover": # dismiss so it doesn't poison later snapshots - D("press_key", {"pid":PID,"key":"escape","session":SESS}); time.sleep(0.3) - state["actions"]+=1 - if m["fg"]: steps[i]["result"]="front" - elif stole: state["steals"]+=1; steps[i]["result"]="stole" - else: steps[i]["result"]="held" - steps[i]["state"]="done"; flush() - L(f"step {t}: result={steps[i]['result']} verified={steps[i]['verified']} before={before} after={after}") - -pulse(2.5) -if DESKTOP: - D("set_config", {"key":"capture_scope","value":"window"}) -D("stop_recording", {}); time.sleep(3) -D("end_session", {"session":SESS}) -subprocess.run("pkill -f mac-dashboard; pkill -f 'http.server 8147'", shell=True) -subprocess.run(f"pkill -f 'user-data-dir=/tmp/cua-dash-profile-{SURFACE}-{MODE}'", shell=True) -if SURFACE == "electron": - subprocess.run("pkill -f 'Electron.app/Contents/MacOS/Electron'", shell=True) - -mp4 = (glob.glob(f"{REC}/**/*.mp4", recursive=True) or [None])[0] -size = os.path.getsize(mp4) if mp4 else 0 -worked = sum(1 for s in steps if s.get("verified")=="ok") -ver = sum(1 for s in steps if s.get("verified") in ("ok","fail")) -verdict = ("foreground-mode, %d actions"%state["actions"]) if m["fg"] else ("%d/%d stole focus"%(state["steals"],state["actions"])) -line = "MODE=%s MEASURE=%s EFFECTS=%d/%d_landed MP4=%s SIZE=%d" % (MODE, verdict, worked, ver, mp4, size) -open(f"{WORK}/metric.log","w").write(line); print(line) diff --git a/libs/cua-driver/tests/fixtures/modality-recordings/windows/run-one.ps1 b/libs/cua-driver/tests/fixtures/modality-recordings/windows/run-one.ps1 deleted file mode 100644 index 09a84543c8..0000000000 --- a/libs/cua-driver/tests/fixtures/modality-recordings/windows/run-one.ps1 +++ /dev/null @@ -1,4 +0,0 @@ -param([string]$Mode="ax-bg") -Get-Process cua-driver,CuaTestHarness.Wpf,chrome -EA SilentlyContinue | Stop-Process -Force -EA SilentlyContinue -Start-Sleep 2 -& powershell -NoProfile -ExecutionPolicy Bypass -File "$PSScriptRoot\wpf-recorder.ps1" -Mode $Mode *> "C:\Users\Public\cua-$Mode-run.log" diff --git a/libs/cua-driver/tests/fixtures/modality-recordings/windows/wpf-recorder.ps1 b/libs/cua-driver/tests/fixtures/modality-recordings/windows/wpf-recorder.ps1 deleted file mode 100644 index 5d847e19f2..0000000000 --- a/libs/cua-driver/tests/fixtures/modality-recordings/windows/wpf-recorder.ps1 +++ /dev/null @@ -1,365 +0,0 @@ -param([string]$Mode="ax-bg",[string]$Toolkit="wpf") # Mode: ax-fg|ax-bg|px-fg|px-bg|px-desktop ; Toolkit: wpf|winui3|webview2|electron -$TK=@{ - wpf =@{exe="C:\Users\cuademo\cua\libs\cua-driver\rust\test-apps\harness-wpf\CuaTestHarness.Wpf.exe"; title="CuaTestHarness WPF"; label="WPF"} - winui3 =@{exe="C:\Users\cuademo\cua\libs\cua-driver\rust\test-apps\harness-winui3\CuaTestHarness.WinUI3.exe"; title="CuaTestHarness WinUI3"; label="WinUI3"} - webview2=@{exe="C:\Users\cuademo\cua\libs\cua-driver\rust\test-apps\harness-webview\CuaTestHarness.WebView.exe"; title="CuaTestHarness WebView"; label="WebView2"} - electron=@{exe="C:\Users\cuademo\cua\libs\cua-driver\rust\test-apps\harness-electron\CuaTestHarness.Electron.exe";title="CuaTestHarness Electron"; label="Electron"} -} -$tk=$TK[$Toolkit]; if(-not $tk){ Write-Output "unknown toolkit $Toolkit"; exit 3 } -$ErrorActionPreference="Continue" -# screen is 1024x768 -> harness LEFT, dashboard panel RIGHT, both fully on-screen. -# Harness LEFT, dashboard panel RIGHT (1024x768). At this width the WPF form reflows -# taller than the screen, so the right-click button + scroll area sit off-screen; the -# cua-driver off-screen guard now reports those as a clean no-op (no taskbar misfire) -# instead of clicking the wrong target. Full-width avoids that but makes the harness -# the foreground window (breaks the no-foreground measurement) — so we keep this layout. -$HARX=0;$HARY=0;$HARW=556;$HARH=742 -$PANX=560;$PANY=0;$PANW=462;$PANH=742 -$META=@{ - "ax-fg" =@{title="AX - FOREGROUND"; scope="window"; see="accessibility tree (element-level)"; fg=$true; expect="App kept in FRONT on purpose. Each action runs via the accessibility tree; we measure the foreground."} - "ax-bg" =@{title="AX - BACKGROUND"; scope="window"; see="accessibility tree (element-level)"; fg=$false; expect="App should stay in the BACKGROUND. Each action runs via the accessibility tree; we measure which actions steal focus."} - "px-fg" =@{title="VISION - FOREGROUND"; scope="window"; see="screenshot only (pixels)"; fg=$true; expect="Pure pixel-driven, app kept in FRONT. We measure the foreground."} - "px-bg" =@{title="VISION - BACKGROUND"; scope="window"; see="screenshot only (pixels)"; fg=$false; expect="Pure pixel-driven, app should stay in the BACKGROUND. We measure which pixel actions steal focus."} - "px-desktop"=@{title="VISION - FULL DESKTOP"; scope="desktop"; see="full-screen screenshot"; fg=$true; expect="Whole-screen, window-less screen-pixel actions (no window targeted)."} -} -$m=$META[$Mode]; if(-not $m){ Write-Output "unknown mode $Mode"; exit 2 } -$vision=($m.see -like 'screenshot*'); $desktop=($m.scope -eq 'desktop') -$dir="C:\Users\Public\cua-$Toolkit-$Mode"; $rec="$dir\rec" -Remove-Item $dir -Recurse -Force -EA SilentlyContinue; New-Item -ItemType Directory -Force $rec | Out-Null -# ---------- dashboard (compact for 462px) ---------- -$html=@' -cua-driver-panel
-

cua-driver - single-modality run - TKLABEL

...
-
...
-
✓ worked / ✗ no-op = did the action change the app · held / STOLE = focus contract
-
-
-'@ -$html=$html -replace 'TKLABEL', $tk.label -Set-Content "$dir\dashboard.html" $html -Encoding UTF8 -# ---------- loopback server ---------- -$srv=Start-Job -ScriptBlock { param($d,$port) - $l=[System.Net.Sockets.TcpListener]::new([System.Net.IPAddress]::Loopback,$port);$l.Start() - while($true){try{$c=$l.AcceptTcpClient();$st=$c.GetStream();$rd=[IO.StreamReader]::new($st);$ln=$rd.ReadLine() - if($ln -match 'GET\s+(\S+)'){$pp=($matches[1].TrimStart('/') -split '\?')[0];if($pp -eq ''){$pp='dashboard.html'} - $fp=Join-Path $d $pp; if(Test-Path $fp){$b=[IO.File]::ReadAllBytes($fp);$ct=if($fp -like '*.html'){'text/html; charset=utf-8'}else{'application/json; charset=utf-8'} - $h="HTTP/1.1 200 OK`r`nContent-Type: $ct`r`nContent-Length: $($b.Length)`r`nCache-Control: no-store`r`nConnection: close`r`n`r`n";$hb=[Text.Encoding]::ASCII.GetBytes($h);$st.Write($hb,0,$hb.Length);$st.Write($b,0,$b.Length)}} - $st.Flush();$c.Close()}catch{}} -} -ArgumentList $dir,8146 -Add-Type @" -using System;using System.Runtime.InteropServices;using System.Text; -public class W{[DllImport("user32.dll")]public static extern bool MoveWindow(IntPtr h,int x,int y,int w,int ht,bool r); - [DllImport("user32.dll")]public static extern bool SetForegroundWindow(IntPtr h); - [DllImport("user32.dll")]public static extern IntPtr GetForegroundWindow(); - [DllImport("user32.dll")]public static extern int GetWindowText(IntPtr h,StringBuilder s,int n); - [DllImport("user32.dll")]public static extern bool SetWindowPos(IntPtr h,IntPtr after,int x,int y,int cx,int cy,uint flags); - [DllImport("user32.dll")]public static extern bool ShowWindow(IntPtr h,int cmd); - [DllImport("user32.dll")]static extern void keybd_event(byte vk,byte scan,uint flags,UIntPtr extra); - public static void Restore(IntPtr h){ ShowWindow(h,9); } - public static void Topmost(IntPtr h,int x,int y,int cx,int cy){ SetWindowPos(h,(IntPtr)(-1),x,y,cx,cy,0x40); } - // Genuinely ACTIVATE (not just z-order) a real anchor window so each action is measured - // against a true foreground baseline. The synthetic ALT tap clears the foreground lock so - // SetForegroundWindow is honoured even when our thread is not already the foreground one. - public static bool Front(IntPtr h){ keybd_event(0x12,0,0,UIntPtr.Zero); keybd_event(0x12,0,2,UIntPtr.Zero); ShowWindow(h,9); return SetForegroundWindow(h); } - public static string Title(IntPtr h){var sb=new StringBuilder(256);GetWindowText(h,sb,256);return sb.ToString();}} -"@ -$drv="C:\Users\cuademo\cua\libs\cua-driver\rust\target\release\cua-driver.exe"; if(-not(Test-Path $drv)){$drv=$drv -replace 'release','debug'} -$wpf=$tk.exe -# WebView2 renderer builds its UIA tree only when an AT requests it. Force it so the -# web DOM (click-target/checkbox/scroll-tall/etc.) surfaces in get_window_state — the -# Windows analog of the Electron recorder's --force-renderer-accessibility. Without this -# get_window_state returns only the chrome frame (TitleBar/Min/Max/Close) and every -# web action resolves to nothing (the prior empty-MP4 / SIZE=0 run). -if($Toolkit -eq 'webview2'){ $env:WEBVIEW2_ADDITIONAL_BROWSER_ARGUMENTS="--force-renderer-accessibility" } -$chrome="C:\Program Files\Google\Chrome\Application\chrome.exe" -$recfwd=$rec -replace '\\','/' -function D($t,$j){ ($j | & $drv call $t 2>&1 | Out-String) } -function Els(){ ((D "get_window_state" ('{{"pid":{0},"window_id":{1},"capture_mode":"ax"}}' -f $script:wp,$script:wd))|ConvertFrom-Json).elements } -# ---------- per-action EFFECT verifier: read the WPF harness's own status labels via UIAutomation ---------- -Add-Type -AssemblyName UIAutomationClient,UIAutomationTypes -$script:AE=[System.Windows.Automation.AutomationElement]; $script:UTS=[System.Windows.Automation.TreeScope]; $script:UCT=[System.Windows.Automation.ControlType] -function ReadState(){ - try{ - $win=$null # match by substring so cdp-suffixed web-harness titles ("... [cdp=9222]") resolve - foreach($c in $script:AE::RootElement.FindAll($script:UTS::Children,[System.Windows.Automation.Condition]::TrueCondition)){ if("$($c.Current.Name)" -like "*$($tk.title)*"){ $win=$c; break } } - if(-not $win){return @{}} - $tc=New-Object System.Windows.Automation.PropertyCondition($script:AE::ControlTypeProperty,$script:UCT::Text) - $all=(($win.FindAll($script:UTS::Descendants,$tc)|%{ $_.Current.Name }) -join " || ") - $h=@{} - if($all -match 'agreed=(\w+)'){$h.agreed=$matches[1]} - if($all -match 'slider_value=(\d+)'){$h.slider=[int]$matches[1]} - if($all -match 'last_action=(\w+)'){$h.last_action=$matches[1]} - if($all -match 'mirror=([^|]*)'){$h.mirror=$matches[1].Trim()} - if($all -match 'menu_action=(\w+)'){$h.menu=$matches[1]} - if($all -match 'scroll_offset=(\d+)'){$h.scroll=[int]$matches[1]} - if($all -match 'counter=(\d+)'){$h.counter=[int]$matches[1]} - return $h - }catch{ return @{} } -} -function Verify($t,$b,$a){ - switch($t){ - 'click' { if("$($a.agreed)" -ne "$($b.agreed)"){'ok'}else{'fail'} } - 'double' { if("$($a.last_action)" -eq 'double_click'){'ok'}else{'fail'} } - 'right' { if("$($a.last_action)" -eq 'right_click'){'ok'} # WinUI3/web: click-target records last_action=right_click - elseif("$($a.menu)" -ne "$($b.menu)" -and "$($a.menu)" -ne 'none' -and "$($a.menu)" -ne ''){'ok'} # WPF: dedicated context-menu sets menu_action= - else{'fail'} } - 'drag' { if([int]$a.slider -gt [int]$b.slider){'ok'}else{'fail'} } - 'scroll' { if([int]$a.scroll -gt [int]$b.scroll){'ok'}else{'fail'} } - 'setval' { if("$($a.mirror)" -match 'set-by-cua'){'ok'}else{'fail'} } - 'type' { if("$($a.mirror)" -match 'typed-by-cua'){'ok'}else{'fail'} } - 'key' { if("$($a.mirror)" -ne "$($b.mirror)"){'ok'}else{'fail'} } # Backspace edits the focused textbox -> mirror changes - default { 'na' } - } -} -# bring an off-screen control into the viewport (UIA ScrollItemPattern) so coordinate actions -# land and the control is actually visible in the recording. $ctype is a UIA ControlType. -function ScrollIntoView($ctype){ - try{ - $win=$null - foreach($c in $script:AE::RootElement.FindAll($script:UTS::Children,[System.Windows.Automation.Condition]::TrueCondition)){ if("$($c.Current.Name)" -like "*$($tk.title)*"){ $win=$c; break } } - if(-not $win){return} - $cond=New-Object System.Windows.Automation.PropertyCondition($script:AE::ControlTypeProperty,$ctype) - $el=$win.FindFirst($script:UTS::Descendants,$cond) - if($el){ $pat=$null; if($el.TryGetCurrentPattern([System.Windows.Automation.ScrollItemPattern]::Pattern,[ref]$pat)){ $pat.ScrollIntoView() } } - }catch{} -} -# re-resolve a single plan selector against a FRESH ax snapshot (frames change after a scroll). -function Pick($sel){ - $EE=Els - switch($sel){ - 'chk'{ $EE|?{ "$($_.role)" -match 'Check' -or "$($_.label)" -match 'agree' }|Select -First 1 } - 'btn'{ $EE|?{ "$($_.label)" -match 'lick target' -or "$($_.name)" -match 'lick target' }|Select -First 1 } - 'sld'{ $EE|?{ "$($_.role)" -match 'Slider' }|Select -First 1 } - 'txt'{ $EE|?{ "$($_.role)" -match 'Edit' }|Select -First 1 } - 'ctx'{ $cc=$EE|?{ "$($_.label)" -match 'context menu' }|Select -First 1; if(-not $cc){ $cc=$EE|?{ "$($_.label)" -match 'lick target' -or "$($_.name)" -match 'lick target' }|Select -First 1 }; $cc } - 'scr'{ $ss=$EE|?{ "$($_.role)" -match 'Pane|Group' -and "$($_.label)" -match 'scroll' }|Select -First 1; if(-not $ss){ $ss=$EE|?{ "$($_.role)" -match 'Pane|Group' }|Select -Last 1 }; $ss } - default{ $null } - } -} -# ---------- action plan (filtered per mode) ---------- -$plan=@( - @{t='click'; sel='chk'; label='left-click a checkbox'} - @{t='double'; sel='btn'; label='double-click a button'} - @{t='right'; sel='ctx'; label='right-click (context menu)'} - @{t='drag'; sel='sld'; label='drag the slider'} - @{t='scroll'; sel='scr'; label='scroll the panel'} - @{t='setval'; sel='txt'; label='set_value on the text box'} - @{t='type'; sel='txt'; label='type into the text box'} - @{t='key'; sel='txt'; label='press a key (Backspace)'} -) -if($vision){ $plan=@($plan | ? { $_.t -ne 'setval' }) } # set_value is AX-only -if($desktop){ $plan=@($plan | ? { $_.t -in @('click','scroll','type','key') }) } # window-less supports click+scroll+global -$script:steals=0;$script:actions=0;$script:cur="" -$script:steps=@(); foreach($p in $plan){ $script:steps+=@{label=$p.label;state='pending';result='';verified=''} } -function Flush(){ $fgt=[W]::Title([W]::GetForegroundWindow());$af=($fgt -like '*CuaTestHarness*') - (@{run=$m.title;expect=$m.expect;fgmode=$m.fg;foreground=$fgt;appFront=$af;steals=$script:steals;actions=$script:actions;steps=$script:steps}|ConvertTo-Json -Depth 6 -Compress)|Set-Content "$dir\status.json" -Encoding UTF8 } -function Pulse($sec){ $e=(Get-Date).AddSeconds($sec); while((Get-Date)-lt $e){ Flush; Start-Sleep -Milliseconds 150 } } -function Ctr($el){ ,@([int]($el.frame.x+$el.frame.w/2),[int]($el.frame.y+$el.frame.h/2)) } # screen center -function Win0($el){ ,@([int]($el.frame.x-$w.bounds.x+$el.frame.w/2),[int]($el.frame.y-$w.bounds.y+$el.frame.h/2)) } # window-local center -'{"run":"","expect":"","fgmode":true,"foreground":"","appFront":false,"steals":0,"actions":0,"steps":[]}' | Set-Content "$dir\status.json" -Encoding UTF8 -# ---------- launch ---------- -# Put ffmpeg on PATH for the daemon BEFORE serve. The video backend's find_ffmpeg() -# checks `ffmpeg` on PATH first, then %LOCALAPPDATA%\Microsoft\WinGet\Packages — but this -# task runs as fbonacci (schtasks /ru fbonacci) while ffmpeg is installed under cuademo's -# WinGet, so the daemon's LOCALAPPDATA probe misses it and video silently degrades to -# present:false (the empty-MP4 / SIZE=0 failures). Find the Gyan.FFmpeg bin under any user -# profile and prepend it so the PATH check succeeds for whichever account serves. -$ffbin=Get-ChildItem "C:\Users\*\AppData\Local\Microsoft\WinGet\Packages\Gyan.FFmpeg*\*\bin\ffmpeg.exe" -EA SilentlyContinue | Select -First 1 -if($ffbin){ $env:PATH=$ffbin.DirectoryName+";"+$env:PATH; "FFMPEG on PATH: $($ffbin.FullName)"|Set-Content "$dir\ffmpeg.log" } else { "FFMPEG NOT FOUND"|Set-Content "$dir\ffmpeg.log" } -Start-Process $drv -ArgumentList "serve" -WindowStyle Hidden; Start-Sleep 4 -D "set_agent_cursor_enabled" '{"enabled":true,"session":"d1"}'|Out-Null -D "set_agent_cursor_motion" '{"session":"d1","cursor_color":"#FF2D2D","cursor_label":"cua-driver","glide_duration_ms":600,"dwell_after_click_ms":700,"idle_hide_ms":120000}'|Out-Null -$startWait = if($Toolkit -in @('electron','webview2')){18}else{5} # web harnesses are slow to start; WebView2 a11y tree needs extra settle -Start-Process $wpf; Start-Sleep $startWait -Start-Process $chrome -ArgumentList "--app=http://localhost:8146/","--user-data-dir=C:\Users\Public\cdp-$Mode","--no-first-run","--window-position=$PANX,$PANY","--window-size=$PANW,$PANH","--new-window"; Start-Sleep 4 -# resolve harness window -$w=$null -for($i=0;$i -lt 20;$i++){ $w=(D "list_windows" "{}"|ConvertFrom-Json).windows | ? { $_.title -like "*$($tk.title)*" } | Select -First 1; if($w){break}; Start-Sleep -Milliseconds 500 } -if(-not $w){ "FATAL: harness window never appeared" | Set-Content "$dir\metric.log"; exit 1 } -$script:wp=$w.pid;$script:wd=$w.window_id;$hHar=[IntPtr][int64]$w.window_id -# layout: harness LEFT (un-maximize first), panel RIGHT + topmost. retry panel handle. -[W]::Restore($hHar)|Out-Null; Start-Sleep -Milliseconds 400; [W]::MoveWindow($hHar,$HARX,$HARY,$HARW,$HARH,$true)|Out-Null; Start-Sleep -Milliseconds 300; [W]::MoveWindow($hHar,$HARX,$HARY,$HARW,$HARH,$true)|Out-Null -$hPanel=[IntPtr]::Zero -for($i=0;$i -lt 16;$i++){ $hPanel=(Get-Process chrome -EA SilentlyContinue|?{$_.MainWindowTitle -like "*cua-driver-panel*"}|Select -First 1).MainWindowHandle; if($hPanel -and $hPanel -ne [IntPtr]::Zero){break}; Start-Sleep -Milliseconds 400 } -if($hPanel -and $hPanel -ne [IntPtr]::Zero){ [W]::MoveWindow($hPanel,$PANX,$PANY,$PANW,$PANH,$true)|Out-Null; [W]::Topmost($hPanel,$PANX,$PANY,$PANW,$PANH) } -# ---------- foreground-baseline ANCHOR (background modes only) ---------- -# The no-foreground contract is "did this action steal foreground". To measure that we must -# hold a GENUINE foreground baseline before each action: a real, ACTIVATED, non-harness window. -# Re-asserting the dashboard panel with SetWindowPos is z-order only (no activation), so it never -# defines a true foreground holder — after the first inject-based action click-activates the -# target, the harness silently stays frontmost and every later step false-positives as a "steal". -# Anchor on mspaint (Win11 Notepad exposes no MainWindowHandle, so it can never take foreground -# and would fabricate an all-steal). Park it under the topmost panel rect so it is invisible in -# the recording but still a valid activatable foreground window off to the side of the harness. -$hAnchor=[IntPtr]::Zero;$anchorPid=0 -if(-not $m.fg){ - Start-Process mspaint | Out-Null - for($i=0;$i -lt 20;$i++){ $ap=(Get-Process -EA SilentlyContinue|?{ $_.MainWindowTitle -like '*Paint*' -and $_.MainWindowHandle -ne [IntPtr]::Zero }|Select -First 1); if($ap){ $hAnchor=$ap.MainWindowHandle;$anchorPid=$ap.Id; break }; Start-Sleep -Milliseconds 500 } - if($hAnchor -ne [IntPtr]::Zero){ [W]::Restore($hAnchor)|Out-Null; [W]::MoveWindow($hAnchor,$PANX,$PANY,$PANW,$PANH,$true)|Out-Null; [W]::Topmost($hPanel,$PANX,$PANY,$PANW,$PANH); Start-Sleep -Milliseconds 300 } -} -"HANDLES hHar=$hHar hPanel=$hPanel hAnchor=$hAnchor anchorPid=$anchorPid screen=1024x768"|Set-Content "$dir\handles.log" -Start-Sleep 1 -# re-read window bounds after the move (window-local coords need post-move origin) -$w2=(D "list_windows" "{}"|ConvertFrom-Json).windows | ? { $_.window_id -eq $script:wd } | Select -First 1; if($w2){ $w=$w2 } -# resolve control targets (post-move snapshot) with settle-retry. -# Resolver is shared across WPF / WinUI3 / WebView2: roles differ per toolkit -# click-target : label "Click target (left / right / double)" (button on WPF/WinUI3, span on web) -# scroll-tall : role Pane (WinUI3 ScrollViewer) | Group (WebView2 div) , label "scroll-tall" -# checkbox : role Check* | label "I agree" -# context-menu : WPF has a dedicated control (label "context menu"); WinUI3/web have none, -# so right-click targets the click-target and records last_action=right_click. -$E=$null;$resolve=$null -$web=($Toolkit -in @('webview2','electron')) -for($i=0;$i -lt 16;$i++){ - $E=Els - $resolve=@{ - chk=($E|?{ "$($_.role)" -match 'Check' -or "$($_.label)" -match 'agree' }|Select -First 1) - btn=($E|?{ "$($_.label)" -match 'lick target' -or "$($_.name)" -match 'lick target' }|Select -First 1) - sld=($E|?{ "$($_.role)" -match 'Slider' }|Select -First 1) - scr=($E|?{ "$($_.role)" -match 'Pane|Group' -and "$($_.label)" -match 'scroll' }|Select -First 1) - txt=($E|?{ "$($_.role)" -match 'Edit' }|Select -First 1) - } - if(-not $resolve.scr){ $resolve.scr=($E|?{ "$($_.role)" -match 'Pane|Group' }|Select -Last 1) } - $resolve.ctx=($E|?{ "$($_.label)" -match 'context menu' }|Select -First 1) # WPF dedicated control - if(-not $resolve.ctx){ $resolve.ctx=$resolve.btn } # WinUI3/web: right-click the click-target - # WinUI3 realizes the checkbox once the window is sized (break on chk). On the WebView2 web - # surface the click-target IS in the viewport but the checkbox sits below the fold, and the - # web content cannot be scrolled by the driver in ax mode (verified: AX-scroll on the - # Document/scroll-tall Group, coordinate WM_MOUSEWHEEL, and keyboard PageDown are all no-ops - # — the WebView2 host HWND does not route scroll to the Chromium renderer). So we can't bring - # the checkbox into the tree; break as soon as the click-target resolves and leave chk unset - # (its click step then honestly reports a no-op). - if($resolve.chk -or ($web -and $resolve.btn)){break} - Start-Sleep -Milliseconds 700 -} -"RESOLVE "+(($resolve.GetEnumerator()|%{"$($_.Key)=$([bool]$_.Value)"}) -join ' ')+" count=$(@($E).Count)"|Set-Content "$dir\resolve.log" -("picked: "+(($resolve.GetEnumerator()|%{"$($_.Key)=[idx $($_.Value.element_index) role '$($_.Value.role)' lbl '$($_.Value.label)']"}) -join ' '))|Add-Content "$dir\resolve.log" -"--- elements ---"|Add-Content "$dir\resolve.log" -$E|%{ "[$($_.element_index)] role='$($_.role)' name='$($_.name)' label='$($_.label)' frame=$($_.frame.x),$($_.frame.y),$($_.frame.w),$($_.frame.h)" }|Add-Content "$dir\resolve.log" -# seed the agent-cursor overlay BEFORE recording (off the controls) -D "move_cursor" ('{{"x":{0},"y":{1},"session":"d1"}}' -f ($HARW-30),($HARH-30))|Out-Null; Start-Sleep -Milliseconds 400 -if($desktop){ D "set_config" '{"key":"capture_scope","value":"desktop"}'|Out-Null } -D "start_recording" ('{{"output_dir":"{0}","record_video":true}}' -f $recfwd)|Out-Null -Pulse 2 -# one action, dispatched per mode (mirrors v1's proven Click: glide cursor, then act) -function DoAct($t,$el){ - if((-not $el) -and $t -notin @('type','key')){return} - if($el){ $c=Ctr $el; D "move_cursor" ('{{"x":{0},"y":{1},"session":"d1"}}' -f $c[0],$c[1])|Out-Null; Start-Sleep -Milliseconds 550 } - $wl= if($el){Win0 $el}else{$null} - switch($t){ - 'click' { if($desktop){D "click" ('{{"x":{0},"y":{1},"session":"d1"}}' -f $c[0],$c[1])|Out-Null} - elseif($vision){$disp=if($m.fg){'foreground'}else{'background'};D "click" ('{{"pid":{0},"window_id":{1},"x":{2},"y":{3},"delivery_mode":"{4}","session":"d1"}}' -f $wp,$wd,$wl[0],$wl[1],$disp)|Out-Null} - else{D "click" ('{{"pid":{0},"window_id":{1},"element_index":{2},"session":"d1"}}' -f $wp,$wd,$el.element_index)|Out-Null} } - 'double' { if($vision){D "double_click" ('{{"pid":{0},"window_id":{1},"x":{2},"y":{3},"session":"d1"}}' -f $wp,$wd,$wl[0],$wl[1])|Out-Null} - else{D "double_click" ('{{"pid":{0},"window_id":{1},"element_index":{2},"session":"d1"}}' -f $wp,$wd,$el.element_index)|Out-Null} } - 'right' { if($vision){D "right_click" ('{{"pid":{0},"window_id":{1},"x":{2},"y":{3},"session":"d1"}}' -f $wp,$wd,$wl[0],$wl[1])|Out-Null} - else{D "right_click" ('{{"pid":{0},"window_id":{1},"element_index":{2},"session":"d1"}}' -f $wp,$wd,$el.element_index)|Out-Null} - if($Toolkit -eq 'wpf' -and $m.fg){ Start-Sleep -Milliseconds 800 # let the context menu render (visible in the recording), then pick the first item so menu_action=ctx_* -> the right-click is scored as landed instead of a false no-op - D "press_key" ('{{"pid":{0},"key":"down","session":"d1"}}' -f $wp)|Out-Null; Start-Sleep -Milliseconds 350 - D "press_key" ('{{"pid":{0},"key":"enter","session":"d1"}}' -f $wp)|Out-Null } - else{ Start-Sleep -Milliseconds 500; D "press_key" ('{{"pid":{0},"key":"escape","session":"d1"}}' -f $wp)|Out-Null } } - 'drag' { if(-not $vision -and -not $desktop){ - # AX mode: drive the slider through its RangeValue pattern (set_value) — the ax-path - # way to move a slider, reliable and FOREGROUND-INDEPENDENT. A coordinate thumb-drag - # only tracks under forced harness-foreground, which re-activates the window and eats - # the element-path double/right-click. Vision mode below does the real pixel drag. - D "set_value" ('{{"pid":{0},"window_id":{1},"element_index":{2},"value":"48","session":"d1"}}' -f $wp,$wd,$el.element_index)|Out-Null - } else { - $fx=[int]($el.frame.x-$w.bounds.x+8);$fy=[int]($el.frame.y-$w.bounds.y+$el.frame.h/2);$tx=$fx+150 - D "drag" ('{{"pid":{0},"from_x":{1},"from_y":{2},"to_x":{3},"to_y":{4},"delivery_mode":"foreground","session":"d1"}}' -f $wp,$fx,$fy,$tx,$fy)|Out-Null - } } - 'scroll' { if($desktop){D "scroll" ('{{"x":{0},"y":{1},"direction":"down","session":"d1"}}' -f $c[0],$c[1])|Out-Null} - elseif($vision){D "scroll" ('{{"pid":{0},"window_id":{1},"x":{2},"y":{3},"direction":"down","session":"d1"}}' -f $wp,$wd,$wl[0],$wl[1])|Out-Null} - else{D "scroll" ('{{"pid":{0},"window_id":{1},"element_index":{2},"direction":"down","session":"d1"}}' -f $wp,$wd,$el.element_index)|Out-Null} } - 'setval' { D "set_value" ('{{"pid":{0},"window_id":{1},"element_index":{2},"value":"set-by-cua","session":"d1"}}' -f $wp,$wd,$el.element_index)|Out-Null } - 'type' { if($el){ D "click" ('{{"pid":{0},"window_id":{1},"element_index":{2},"session":"d1"}}' -f $wp,$wd,$el.element_index)|Out-Null; Start-Sleep -Milliseconds 350 }; D "type_text" ('{{"pid":{0},"text":"typed-by-cua","session":"d1"}}' -f $wp)|Out-Null } - 'key' { if($el){ D "click" ('{{"pid":{0},"window_id":{1},"element_index":{2},"session":"d1"}}' -f $wp,$wd,$el.element_index)|Out-Null; Start-Sleep -Milliseconds 300 } - D "press_key" ('{{"pid":{0},"key":"backspace","session":"d1"}}' -f $wp)|Out-Null } - } -} -# ---------- run ---------- -try { -for($i=0;$i -lt $plan.Count;$i++){ - $p=$plan[$i]; $el=$resolve[$p.sel] - # the 556px reflow pushes the slider / textbox / context-menu button below the fold; bring the - # target into view (so coordinate dispatch lands and the step is visible) then re-read its frame. - if($p.sel -in @('sld','txt','ctx')){ - $ct=switch($p.sel){ 'sld'{$script:UCT::Slider} 'txt'{$script:UCT::Edit} 'ctx'{$script:UCT::Button} } - ScrollIntoView $ct; Start-Sleep -Milliseconds 450 - $re=Pick $p.sel; if($re){ $el=$re } - } - $script:steps[$i].state='active'; $script:cur=$p.label; Flush - # establish the genuine foreground baseline: ACTIVATE the anchor and confirm it actually took - # foreground BEFORE the action runs. A "steal" is then foreground moving OFF the anchor TO the - # harness; "held" is the anchor staying frontmost. (fg modes keep the harness in front by design.) - if((-not $m.fg) -and $hAnchor -ne [IntPtr]::Zero){ - $anchorHeld=$false - for($a=0;$a -lt 8;$a++){ [W]::Front($hAnchor)|Out-Null; Start-Sleep -Milliseconds 180; if([W]::Title([W]::GetForegroundWindow()) -like '*Paint*'){ $anchorHeld=$true; break } } - if($hPanel -and $hPanel -ne [IntPtr]::Zero){ [W]::Topmost($hPanel,$PANX,$PANY,$PANW,$PANH) } - "step $i '$($p.t)': anchor-baseline foreground='$([W]::Title([W]::GetForegroundWindow()))' held=$anchorHeld"|Add-Content "$dir\baseline.log" - } - elseif(($vision -or $desktop) -and $hHar -ne [IntPtr]::Zero){ - # VISION/DESKTOP fg modes only: every step dispatches by pixel coordinate, which needs the - # harness to be the real foreground window. Assert it before each step. (Not done in AX-fg: - # ax steps use element_index — foreground-independent — and forcing foreground there re- - # activates the window and eats the element double/right-click. The ax slider 'drag' is - # element-driven via set_value, so no ax step needs a forced foreground.) - $harFront=$false - for($a=0;$a -lt 8;$a++){ [W]::Front($hHar)|Out-Null; Start-Sleep -Milliseconds 150; if([W]::Title([W]::GetForegroundWindow()) -like '*CuaTestHarness*'){ $harFront=$true; break } } - if($hPanel -and $hPanel -ne [IntPtr]::Zero){ [W]::Topmost($hPanel,$PANX,$PANY,$PANW,$PANH) } - "step $i '$($p.t)': fg-mode harness foreground='$([W]::Title([W]::GetForegroundWindow()))' held=$harFront"|Add-Content "$dir\baseline.log" - } - $before=ReadState - DoAct $p.t $el - # measure: sample foreground ~1.5s while the dashboard updates live. steal = foreground moved - # off the anchor onto the harness after the action (with a real activated baseline now held). - $stole=$false; $e=(Get-Date).AddSeconds(1.5); while((Get-Date)-lt $e){ Flush; if([W]::Title([W]::GetForegroundWindow()) -like '*CuaTestHarness*'){$stole=$true}; Start-Sleep -Milliseconds 110 } - $script:steps[$i].verified=(Verify $p.t $before (ReadState)) - $script:actions++ - if($m.fg){ $script:steps[$i].result='front' } elseif($stole){ $script:steals++;$script:steps[$i].result='stole' } else { $script:steps[$i].result='held' } - $script:steps[$i].state='done'; Flush - if($hPanel -and $hPanel -ne [IntPtr]::Zero){ [W]::Topmost($hPanel,$PANX,$PANY,$PANW,$PANH) } -} -$script:cur="done"; Pulse 2.5 -if($desktop){ D "set_config" '{"key":"capture_scope","value":"window"}'|Out-Null } -D "stop_recording" "{}"|Out-Null; Start-Sleep 3 -} -finally { - # Dispose the agent-cursor session (remove its overlay cursor) BEFORE cua-driver is - # killed below. Runs even if the action loop threw, so the session is never leaked to - # the idle-TTL reaper. end_session is idempotent. - D "end_session" '{"session":"d1"}' | Out-Null -} -Stop-Job $srv -EA SilentlyContinue; Remove-Job $srv -Force -EA SilentlyContinue -if($anchorPid){ Stop-Process -Id $anchorPid -Force -EA SilentlyContinue } -Get-Process cua-driver,CuaTestHarness.Wpf,chrome,mspaint -EA SilentlyContinue | Stop-Process -Force -EA SilentlyContinue -Get-Process -EA SilentlyContinue | ?{ $_.MainWindowTitle -like '*Paint*' } | Stop-Process -Force -EA SilentlyContinue -$mp4=Get-ChildItem $rec -Recurse -Filter *.mp4 -EA SilentlyContinue | Select -First 1 -$verdict= if($m.fg){"foreground-mode, $($script:actions) actions"}else{"$($script:steals)/$($script:actions) actions stole focus"} -$worked=@($script:steps|?{$_.verified -eq 'ok'}).Count; $ver=@($script:steps|?{$_.verified -in @('ok','fail')}).Count -"MODE=$Mode MEASURE=$verdict EFFECTS=$worked/$ver`_landed MP4=$($mp4.FullName) SIZE=$($mp4.Length)" | Tee-Object "$dir\metric.log" diff --git a/libs/cua-driver/tests/fixtures/shared/scenarios.json b/libs/cua-driver/tests/fixtures/shared/scenarios.json index 564f7ba896..f1f4276c87 100644 --- a/libs/cua-driver/tests/fixtures/shared/scenarios.json +++ b/libs/cua-driver/tests/fixtures/shared/scenarios.json @@ -187,12 +187,6 @@ "btn-increment", "btn-reset", "lbl-counter", - "calc-1", - "calc-2", - "calc-4", - "calc-plus", - "calc-equals", - "calc-display", "txt-input", "lbl-input-mirror", "editor-document", diff --git a/libs/cua-driver/tests/fixtures/shared/web/index.html b/libs/cua-driver/tests/fixtures/shared/web/index.html index b1f93c6944..f44e725425 100644 --- a/libs/cua-driver/tests/fixtures/shared/web/index.html +++ b/libs/cua-driver/tests/fixtures/shared/web/index.html @@ -7,16 +7,38 @@ :root { color-scheme: light dark; } body { font-family: 'Segoe UI', system-ui, sans-serif; - max-width: 860px; + max-width: 980px; margin: 0 auto; - padding: 16px; + padding: 10px; } - fieldset { margin-bottom: 12px; padding: 10px 14px; border-radius: 6px; border: 1px solid #ccc; } + h2 { margin: 0 0 2px; } + body > p { margin: 0 0 8px; } + .harness-grid { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 8px; + align-items: start; + } + .harness-grid h3 { margin: 8px 0; font-size: 14px; overflow-wrap: anywhere; } + fieldset { min-width: 0; margin: 0; padding: 6px 8px; border-radius: 6px; border: 1px solid #ccc; } legend { font-weight: 600; padding: 0 6px; } - label { display: inline-block; min-width: 120px; } - .row { display: flex; align-items: center; gap: 10px; margin: 6px 0; flex-wrap: wrap; } + label { display: inline-block; min-width: 90px; } + .row { display: flex; align-items: center; gap: 8px; margin: 3px 0; flex-wrap: wrap; } .mirror, .state { font-family: Consolas, monospace; color: #555; } - input[type="range"] { width: 280px; } + input[type="range"] { width: 180px; } + textarea { max-width: 100%; } + #drag-source, #drop-target { + width: 110px; + height: 48px; + display: flex; + align-items: center; + justify-content: center; + color: white; + user-select: none; + touch-action: none; + } + #drag-source { background: #1268d6; } + #drop-target { background: #178a38; } #click-target { display: inline-block; padding: 8px 18px; @@ -33,6 +55,7 @@

cua-driver Web Harness

WEB_HARNESS_MARKER_v1

+
counter
@@ -42,22 +65,11 @@

cua-driver Web Harness

-
- calculator_task -
- - - - - - display=0 -
-
-
text_input
- + mirror=
@@ -92,8 +104,8 @@

cua-driver Web Harness

drag_task
-
Drag source
-
Drop target
+
Drag source
+
Drop target
drag_status=idle
@@ -115,7 +127,8 @@

cua-driver Web Harness

click_target
- Click target (left / right / double) + Click target (left / right / double)
last_action=none @@ -157,7 +170,7 @@

cua-driver Web Harness

Jump to section hash=
-
+

SECTION_TARGET_MARKER_v1

@@ -168,12 +181,12 @@

SECTION_TARGET_MARKER_v1child_windows=0 +

diff --git a/libs/cua-driver/tests/fixtures/smoke/macos.sh b/libs/cua-driver/tests/fixtures/smoke/macos.sh index 5537558b32..8df92254fc 100755 --- a/libs/cua-driver/tests/fixtures/smoke/macos.sh +++ b/libs/cua-driver/tests/fixtures/smoke/macos.sh @@ -158,7 +158,7 @@ else run_tool click "{\"pid\":$HARNESS_PID,\"window_id\":$WIN_ID,\"x\":120,\"y\":80}" run_tool double_click "{\"pid\":$HARNESS_PID,\"window_id\":$WIN_ID,\"x\":120,\"y\":80}" run_tool right_click "{\"pid\":$HARNESS_PID,\"window_id\":$WIN_ID,\"x\":120,\"y\":80}" - run_tool drag "{\"pid\":$HARNESS_PID,\"window_id\":$WIN_ID,\"from_x\":120,\"from_y\":80,\"to_x\":180,\"to_y\":120}" + run_tool drag "{\"pid\":$HARNESS_PID,\"window_id\":$WIN_ID,\"from_x\":120,\"from_y\":80,\"to_x\":180,\"to_y\":120,\"delivery_mode\":\"foreground\"}" run_tool scroll "{\"pid\":$HARNESS_PID,\"window_id\":$WIN_ID,\"x\":200,\"y\":400,\"direction\":\"down\"}" run_tool type_text "{\"pid\":$HARNESS_PID,\"window_id\":$WIN_ID,\"text\":\"hi\"}" run_tool press_key "{\"pid\":$HARNESS_PID,\"window_id\":$WIN_ID,\"key\":\"a\"}" diff --git a/libs/cua-driver/tests/fixtures/vision-agent-test/README.md b/libs/cua-driver/tests/fixtures/vision-agent-test/README.md deleted file mode 100644 index b146155e9a..0000000000 --- a/libs/cua-driver/tests/fixtures/vision-agent-test/README.md +++ /dev/null @@ -1,52 +0,0 @@ -# Vision-agent coordinate-invariant test - -Tests cua-driver the way a **vision agent** actually hits it — and without the -overfit the modality recorder has (hand-tuned window-local points run through a -private ratio, which never exercises the driver's image→screen mapping). - -## The invariant under test -**The pixel an agent reads off the returned screenshot is the pixel that gets -clicked** — verified by the target's own instrumented state changing. - -## The loop (no cheating in locate/click) -1. **capture** — `get_window_state` (window; returns the screenshot alongside the - tree by default) / `get_desktop_state` (desktop, true pixels): the exact image - an agent receives. -2. **locate** — a deterministic pixel **in the returned-image coordinate space** - (`PixelRegistryLocator`: a pre-measured pixel read off the real PNG, with a - dims-guard that fails loud if the pinned geometry drifts). No AX `element_index`, - no hand-converted window-local points. Pluggable `locate(image, target, dims)->(x,y)`. -3. **act** — `click`/`right_click`/`scroll` at that pixel (scope set to match capture). -4. **verify** — the harness oracle (`last_action=`, `clicks=`, …) — reliable pass/fail. - A coordinate mis-map leaves the oracle unchanged → FAIL. - -Run: `python3 vision_agent_test.py {wkwebview-click-window|wkwebview-click-desktop|appkit-click-window|safari-learnmore-desktop|all}` - -## Two separate axes (don't conflate) -- **Driver coordinate invariant** (this test): deterministic locate → objective - pass/fail → the regression guard. This is what turns the 2× Retina escape into - a permanently-guarded one-liner. -- **Agent locator quality** (future, LLM): same `locate()` signature, send the PNG - to a model, score localization hit-rate separately — never gates the coordinate - regression. -- **Agent self-judged success with no oracle** (future, separate track): re-capture, - ask the model "did it work", score its self-judgment vs the oracle. Kept isolated - so a bad self-judge can't mask a driver regression. - -## What the deterministic version caught (that the modality suite couldn't) -- **2× Retina desktop path now correct + guarded** — a read pixel (340,1358) in the - 3024×1964 desktop PNG converts to screen-point (170,679) and lands (oracle + a real - Safari navigation confirm it). -- **Vision pixel-click is a no-op on AppKit `NSButton`** even frontmost — lands - pixel-perfect (crosshair) but `NSButton`'s modal mouseDown loop reads the - window-server queue, not the per-pid `CGEvent.postToPid` queue. The suite drives - this via AXPress, so it never saw a vision agent's pixel click do nothing here. -- **Pixel path requires the target app frontmost** (the AX path doesn't). -- **AppKit harness window AX returns only the menu bar** — its `clicks=` oracle is - unreachable that way; WKWebView exposes it fine. - -## Full harness (scope) -Cross-product, each a one-line registry entry: -`{appkit, swiftui, wkwebview, electron, real-app} × {window, desktop(, secondary-display)} × {left, right, double, scroll, drag, type}`. -A color-fiducial locator (harness renders a unique-color dot per control) would make -the registry robust to window moves without OCR. diff --git a/libs/cua-driver/tests/fixtures/vision-agent-test/vision_agent_test.py b/libs/cua-driver/tests/fixtures/vision-agent-test/vision_agent_test.py deleted file mode 100644 index 1f6812e770..0000000000 --- a/libs/cua-driver/tests/fixtures/vision-agent-test/vision_agent_test.py +++ /dev/null @@ -1,381 +0,0 @@ -#!/usr/bin/env python3 -""" -vision_agent_test.py — driver coordinate-invariant test for cua-driver's -vision/pixel action path on macOS. - -WHY THIS EXISTS ---------------- -The modality suite overfits. It drives actions with hand-tuned window-local -POINTS run through a SHOT_W/ratio conversion, and it confirms success by -reading harness-injected AX labels via element_index. A real agent in vision -mode has NEITHER: it receives a screenshot, picks a PIXEL off THAT image, and -calls click(x,y). The AX/element_index path (what the modality suite uses) maps -coordinates through a totally different code path than the pixel path, so the -suite never exercised the driver's image->screen mapping. A real desktop -coordinate bug (clicks off by the 2x Retina factor) hid behind it. - -WHAT THIS TESTS (and what it explicitly does NOT) -------------------------------------------------- -Under test -> the DRIVER COORDINATE INVARIANT: a pixel read off the returned - screenshot is the pixel that gets acted on. Deterministic. - Pass/fail is objective. THIS is the regression guard. -NOT under test here -> agent LOCATOR QUALITY (can a model find the button in - the image?). That is fuzzy, model-dependent, future work. We - keep the locate step behind a pluggable interface so an - LLM/OCR locator can be swapped in, but we do NOT couple the - coordinate-mapping regression to a flaky locator: a flaky - locator would conflate "locator missed" with "driver mis-mapped", - which defeats the test. - -THE LOOP (per target) - 1. CAPTURE the exact image an agent receives: - window scope -> get_window_state(capture_mode=vision) (window-local PNG) - desktop scope -> get_desktop_state (full display, TRUE pixels) - 2. LOCATE the target's pixel IN THE RETURNED-IMAGE SPACE via a - DETERMINISTIC locator (a pinned, pre-measured pixel — see - PixelRegistryLocator). NOT a window-local POINT + ratio - conversion (that is the overfit we are replacing); the pixel - lives in the same coordinate space the agent reads off the PNG. - 3. ACT click(x,y) that pixel. Window scope passes window_id; desktop - scope omits it. (Pixel path needs the app frontmost — see - activate_pid — because CGEvent.postToPid mouse events are only - consumed by a control when its app/window is key. The AX path - works backgrounded; the pixel path does not.) - 4. VERIFY did the action LAND ON THE TARGET? Re-read the target control's - OWN state (AX ground truth for the harness; visible page change - for a real app) and assert it moved. A coordinate mis-map (e.g. - the 2x desktop bug) lands the click on empty space / the wrong - control, the target's state does NOT change, and the test FAILS. - We verify by RESULT, never by "we clicked where we intended". - -The deterministic pixel is the FLOOR of agent capability, not a stand-in for an -agent: if the driver cannot honour a pixel a human/agent can plainly read off -the image, no locator on earth fixes it. -""" - -import argparse -import json -import subprocess -import sys -import time - -# -------------------------------------------------------------------------- -# driver plumbing -# -------------------------------------------------------------------------- -def driver(tool: str, payload: dict) -> dict: - """Invoke `cua-driver call `; return parsed structuredContent.""" - proc = subprocess.run( - ["cua-driver", "call", tool, json.dumps(payload)], - capture_output=True, text=True, - ) - if proc.returncode != 0: - raise RuntimeError(f"{tool} failed: {proc.stderr.strip() or proc.stdout.strip()}") - txt = proc.stdout.strip() - try: - doc = json.loads(txt) - return doc.get("structuredContent", doc) - except json.JSONDecodeError: - return {"_raw": txt} - - -def resolve_window(app_substr: str, title_substr: str = "") -> tuple[int, int]: - """Find (pid, window_id) of the largest matching window. pids/window_ids - change every launch, so the test resolves them at runtime — the only thing - we hardcode is the image-space pixel (the deterministic locator).""" - wins = driver("list_windows", {}).get("windows", []) - cands = [ - w for w in wins - if app_substr.lower() in w.get("app_name", "").lower() - and title_substr.lower() in (w.get("title") or "").lower() - and w["bounds"]["width"] > 200 and w["bounds"]["height"] > 200 - ] - if not cands: - raise RuntimeError(f"no window matching app~{app_substr!r} title~{title_substr!r}") - best = max(cands, key=lambda w: w["bounds"]["width"] * w["bounds"]["height"]) - return best["pid"], best["window_id"] - - -def activate_pid(pid: int): - """Bring the target app frontmost. The driver intentionally refuses to do - this (no-foreground contract). But the PIXEL path needs the target key: - CGEvent.postToPid mouse events are only consumed by a control when its app - is frontmost (the AX/element_index path has no such requirement). So the - TEST harness — not the driver — does the activation. This is a real - constraint a vision agent must satisfy, surfaced here on purpose.""" - subprocess.run( - ["osascript", "-e", - f'tell application "System Events" to set frontmost of ' - f'(first process whose unix id is {pid}) to true'], - capture_output=True, - ) - time.sleep(1.2) - - -# -------------------------------------------------------------------------- -# Step 1: CAPTURE -# -------------------------------------------------------------------------- -def set_scope(scope: str): - """The driver gates desktop-scope clicks (no pid/window_id, screen-absolute - pixels) behind capture_scope=desktop, and window-scope clicks behind - capture_scope=window. Set it to match the target so the click path matches - the capture path the agent used.""" - driver("set_config", {"capture_scope": scope}) - - -def window_origin(pid: int, window_id: int) -> tuple[int, int]: - for w in driver("list_windows", {}).get("windows", []): - if w["pid"] == pid and w["window_id"] == window_id: - return int(w["bounds"]["x"]), int(w["bounds"]["y"]) - return (-1, -1) - - -def capture(scope: str, pid: int, window_id: int, out: str) -> tuple[int, int]: - """Write the agent-visible PNG; return its (width, height) in pixels.""" - if scope == "window": - r = driver("get_window_state", { - "pid": pid, "window_id": window_id, - "capture_mode": "vision", "screenshot_out_file": out, - }) - return r["screenshot_width"], r["screenshot_height"] - elif scope == "desktop": - r = driver("get_desktop_state", {"screenshot_out_file": out}) - return r["screenshot_width"], r["screenshot_height"] - raise ValueError(scope) - - -# -------------------------------------------------------------------------- -# Step 2: LOCATE — pluggable. Contract: (image, expected_dims) -> pixel. -# -------------------------------------------------------------------------- -class PixelRegistryLocator: - """DETERMINISTIC locator. Returns a pre-measured pixel in the returned-image - coordinate space. The pixel was read ONCE off a real capture of the actual - get_window_state / get_desktop_state PNG (see scripts/derive_pixel.md notes - in the report) — it is NOT a window-local point fed through a ratio. - - Guard: it asserts the live capture's dimensions match the dimensions the - pixel was measured at. The harness window is pinned (non-resizable, fixed - content size) so window-scope captures are byte-stable; if the dims drift, - the pixel is stale and we FAIL LOUD rather than click blind. - - Drop-in seam for the future fuzzy locator: an LLMLocator/OCRLocator would - implement the same locate(image_path, key, dims) -> (x, y) signature, taking - the PNG + an intent string instead of a registry key. The loop below is - identical. That locator's *quality* is a separate axis from this coordinate - invariant and must be scored separately (see report).""" - name = "deterministic-pixel-registry" - - def locate(self, image_path: str, target, live_dims: tuple[int, int]): - exp = target["image_dims"] - if tuple(live_dims) != tuple(exp): - raise RuntimeError( - f"image dims {live_dims} != measured {exp} for {target['key']!r}: " - f"pinned geometry drifted, registry pixel is stale") - return target["pixel"] - - -# -------------------------------------------------------------------------- -# Step 3: ACT -# -------------------------------------------------------------------------- -def act(action: str, scope: str, pid: int, window_id: int, x: int, y: int, crosshair=None): - tool = {"left": "click", "right": "right_click", "double": "double_click"}[action] - payload = {"x": x, "y": y} - if action == "left": - payload["button"] = "left" - if scope == "window": - payload["pid"] = pid - payload["window_id"] = window_id - if crosshair and tool == "click": - payload["debug_image_out"] = crosshair # crosshair only on pixel `click` + window_id - # desktop scope: no pid/window_id — driver hit-tests the screen point itself - return driver(tool, payload) - - -# -------------------------------------------------------------------------- -# Step 4: VERIFY — pluggable oracle. Reads the TARGET's own resulting state. -# -------------------------------------------------------------------------- -def ax_label(pid: int, window_id: int, prefix: str): - """Reliable ground-truth read of a harness mirror label via AX (e.g. - 'clicks=' -> 'clicks=12'). Used to answer 'did the click LAND', not to - drive the click. A coordinate miss leaves this unchanged -> FAIL.""" - r = driver("get_window_state", { - "pid": pid, "window_id": window_id, "capture_mode": "ax", "query": prefix, - }) - md = r.get("tree_markdown", "") - vals = [] - for line in md.splitlines(): - if prefix in line and '"' in line: - seg = line.split('"')[1] - if seg.startswith(prefix): - vals.append(seg) - # dedupe, keep order - return list(dict.fromkeys(vals)) - - -# -------------------------------------------------------------------------- -# Target registry — the demonstrated cases. Pixels measured off real captures. -# pid/window_id are resolved at runtime; ONLY the image-space pixel is fixed. -# -------------------------------------------------------------------------- -TARGETS = { - # WKWebView harness click-target, WINDOW scope. Web consumes - # synthetic clicks when frontmost. Pixel measured on the 1288x1568 - # window-local vision PNG. - "wkwebview-click-window": { - "key": "wkwebview-click-window", - "scope": "window", - "app": "WKWebView", "title": "CuaTestHarness", - "image_dims": (1288, 1568), - "pixel": (305, 1186), - "action": "left", - "verify_prefix": "clicks=", # AX ground truth must increment - }, - # AppKit harness click-target, WINDOW scope. NSButton. Coordinate maps - # pixel-perfect (crosshair) but the control does NOT consume synthetic - # CGEvent clicks -> verify is EXPECTED to show no change. Documents the - # pixel-vs-AX gap the modality suite (AXPress only) masks. - "appkit-click-window": { - "key": "appkit-click-window", - "scope": "window", - "app": "CuaTestHarness.AppKit", "title": "CuaTestHarness AppKit", - "image_dims": (987, 1568), - "pixel": (181, 431), - "action": "left", - "verify_prefix": "clicks=", - "expect_no_change": True, # known NSButton synthetic-click swallow - }, - # WKWebView harness click-target, DESKTOP scope — THE 2x RETINA PATH, the - # one the modality suite never exercised and where the real bug hid. Pixel - # measured on the 3024x1964 TRUE-pixel desktop PNG. Verified by the harness - # oracle (clicks=/last_action via AX) — reliable. Requires the window pinned - # to a known origin (the desktop pixel is position-dependent); the test - # asserts the origin before clicking and FAILS LOUD if it drifted. - "wkwebview-click-desktop": { - "key": "wkwebview-click-desktop", - "scope": "desktop", - "app": "WKWebView", "title": "CuaTestHarness", - "image_dims": (3024, 1964), - "pixel": (340, 1358), - "require_window_origin": (0, 33), - "action": "left", - "verify_prefix": "clicks=", - }, - # Safari "Learn more" on example.com, DESKTOP scope — a GENUINELY REAL app - # (no harness instrumentation). Pixel measured on the 3024x1964 desktop PNG. - # Verified by the visible navigation example.com -> iana.org (no oracle). - # Position-dependent: Safari must show example.com maximized-ish; re-measure - # the pixel if Safari's window moved. Kept as a real-app cross-check. - "safari-learnmore-desktop": { - "key": "safari-learnmore-desktop", - "scope": "desktop", - "app": "Safari", "title": "", - "image_dims": (3024, 1964), - "pixel": (797, 592), - "action": "left", - "verify_prefix": None, # real app: verify visibly (see report) - }, -} - - -def run(target_key: str, scratch: str): - t = TARGETS[target_key] - scope = t["scope"] - loc = PixelRegistryLocator() - print(f"=== {target_key} scope={scope} locator={loc.name} ===") - - pid, window_id = resolve_window(t["app"], t["title"]) - print(f"[resolve] pid={pid} window_id={window_id}") - - # pixel path requires the target frontmost - activate_pid(pid) - - # desktop-scope pixels are position-dependent: assert the pinned origin so a - # moved window FAILS LOUD instead of clicking a stale pixel. - if t.get("require_window_origin"): - org = window_origin(pid, window_id) - if tuple(org) != tuple(t["require_window_origin"]): - raise RuntimeError( - f"window origin {org} != required {t['require_window_origin']}: " - f"pin the window there (the desktop pixel was measured at that origin)") - print(f"[resolve] window origin {org} OK") - - # match the click path to the capture path - set_scope(scope) - - # 1. CAPTURE (before) - before_png = f"{scratch}/{target_key}_before.png" - dims = capture(scope, pid, window_id, before_png) - print(f"[capture] {before_png} dims={dims}") - - # ground-truth BEFORE (window-scope harness only) - before_state = None - if t.get("verify_prefix"): - before_state = ax_label(pid, window_id, t["verify_prefix"]) - print(f"[verify ] before: {before_state}") - - # 2. LOCATE — deterministic pixel in returned-image space - x, y = loc.locate(before_png, t, dims) - print(f"[locate ] pixel ({x},{y}) [measured @ {t['image_dims']}]") - - # 3. ACT — click that exact pixel - crosshair = f"{scratch}/{target_key}_crosshair.png" - res = act(t["action"], scope, pid, window_id, x, y, crosshair) - msg = res.get("_raw") or json.dumps(res) - print(f"[act ] {t['action']} @ ({x},{y}): {msg[:120]}") - if scope == "window": - print(f"[act ] crosshair: {crosshair}") - time.sleep(0.8) - - # 4. VERIFY — did it LAND? (re-read target's own state) - landed = None - if t.get("verify_prefix"): - after_state = ax_label(pid, window_id, t["verify_prefix"]) - print(f"[verify ] after : {after_state}") - if not before_state and not after_state: - # AX exposes no such label for this toolkit (AppKit mirror labels - # are not in the AX tree). The AX oracle is UNAVAILABLE — do NOT - # report a fake PASS from empty==empty. Fall back to the visible - # screenshot: the crosshair proves the pixel landed on the control; - # read the rendered label region to confirm the count. - print(f"[verify ] AX oracle UNAVAILABLE for this toolkit " - f"(no '{t['verify_prefix']}' AXStaticText). " - f"Confirm visibly: crosshair {scratch}/{target_key}_crosshair.png " - f"lands on the control; rendered label in " - f"{scratch}/{target_key}_before.png unchanged after the click.") - landed = None if t.get("expect_no_change") else None - else: - changed = after_state != before_state - if t.get("expect_no_change"): - landed = not changed - note = "(EXPECTED no change — known NSButton synthetic-click swallow)" - else: - landed = changed - note = "" - print(f"[verify ] state changed: {changed} -> coordinate-invariant " - f"{'PASS' if landed else 'FAIL'} {note}") - else: - after_png = f"{scratch}/{target_key}_after.png" - capture(scope, pid, window_id, after_png) - print(f"[verify ] desktop re-capture: {after_png}") - print(f"[verify ] real-app target: confirm visible change in {after_png} " - f"(e.g. address bar / page navigated). Driver echoes the screen-point " - f"it converted to in [act]; a 2x mis-map lands off-target.") - landed = None # human/visible confirm for real-app desktop demo - - return 0 if landed in (True, None) else 2 - - -def main(): - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("target", choices=list(TARGETS) + ["all"]) - p.add_argument("--scratch", default="/tmp/vision-agent-test") - a = p.parse_args() - subprocess.run(["mkdir", "-p", a.scratch]) - keys = list(TARGETS) if a.target == "all" else [a.target] - rc = 0 - for k in keys: - rc |= run(k, a.scratch) - print() - sys.exit(rc) - - -if __name__ == "__main__": - main() diff --git a/libs/cua-driver/tests/runners/windows-sandbox/run-tests-in-sandbox.ps1 b/libs/cua-driver/tests/runners/windows-sandbox/run-tests-in-sandbox.ps1 index b4ac011867..0f52b7abca 100644 --- a/libs/cua-driver/tests/runners/windows-sandbox/run-tests-in-sandbox.ps1 +++ b/libs/cua-driver/tests/runners/windows-sandbox/run-tests-in-sandbox.ps1 @@ -70,10 +70,6 @@ try { cargo test -p cua-driver --test protocol_handshake_test --no-run if ($LASTEXITCODE -ne 0) { throw "cargo test --no-run (protocol_handshake_test) failed" } - Write-Host "`n[BUILD] cargo test --no-run (guard_ux_test)..." -ForegroundColor Yellow - cargo test -p cua-driver --test guard_ux_test --no-run - if ($LASTEXITCODE -ne 0) { throw "cargo test --no-run (guard_ux_test) failed" } - Write-Host "`n[BUILD] cargo test --no-run (harness_wpf_test)..." -ForegroundColor Yellow cargo test -p cua-driver --test harness_wpf_test --no-run if ($LASTEXITCODE -ne 0) { throw "cargo test --no-run (harness_wpf_test) failed" } @@ -86,9 +82,14 @@ try { cargo test -p cua-driver --test harness_web_test --no-run if ($LASTEXITCODE -ne 0) { throw "cargo test --no-run (harness_web_test) failed" } - Write-Host "`n[BUILD] cargo test --no-run (modality_input_e2e_test)..." -ForegroundColor Yellow - cargo test -p cua-driver --test modality_input_e2e_test --no-run - if ($LASTEXITCODE -ne 0) { throw "cargo test --no-run (modality_input_e2e_test) failed" } + Write-Host "`n[BUILD] cargo test --no-run (launch_windows_test)..." -ForegroundColor Yellow + cargo test -p cua-driver --test launch_windows_test --no-run + if ($LASTEXITCODE -ne 0) { throw "cargo test --no-run (launch_windows_test) failed" } + + Write-Host "`n[BUILD] cargo test --no-run (agent_cursor_windows_test)..." -ForegroundColor Yellow + cargo test -p cua-driver --test agent_cursor_windows_test --no-run + if ($LASTEXITCODE -ne 0) { throw "cargo test --no-run (agent_cursor_windows_test) failed" } + } finally { Pop-Location } # -- 1.5. Build the test fixtures if dependencies are on PATH ------------------ @@ -123,10 +124,6 @@ $protocolBin = Get-ChildItem "$rustRoot\target\debug\deps\protocol_handshake_tes if (-not $protocolBin) { throw "protocol_handshake_test-*.exe not found" } Write-Host "protocol_handshake_test: $($protocolBin.Name)" -$guardBin = Get-ChildItem "$rustRoot\target\debug\deps\guard_ux_test-*.exe" | - Sort-Object LastWriteTime -Descending | Select-Object -First 1 -if ($guardBin) { Write-Host "guard_ux_test : $($guardBin.Name)" } else { Write-Host "guard_ux_test : (not found, will skip)" } - # -- 2. Prepare shared output folder ------------------------------------------ $outputDir = "$env:TEMP\cua-sandbox-output" New-Item -ItemType Directory -Force $outputDir | Out-Null diff --git a/libs/cua-driver/tests/runners/windows-sandbox/sandbox-runner.ps1 b/libs/cua-driver/tests/runners/windows-sandbox/sandbox-runner.ps1 index 30dc7f2593..3e27c7e21d 100644 --- a/libs/cua-driver/tests/runners/windows-sandbox/sandbox-runner.ps1 +++ b/libs/cua-driver/tests/runners/windows-sandbox/sandbox-runner.ps1 @@ -32,15 +32,14 @@ if (-not (Test-Path $driverExe)) { Log "cua-driver : $driverExe" # -- find test binaries ------------------------------------------------------- -# Run protocol_handshake_test first, then guard_ux_test (UX guard needs a real -# desktop session and spawns visible windows, so it runs second). +# Run the protocol test first, followed by the typed interactive harnesses. $testSuites = @( @{ Pattern = "protocol_handshake_test-*.exe"; Label = "protocol_handshake_test" }, - @{ Pattern = "guard_ux_test-*.exe"; Label = "guard_ux_test" }, @{ Pattern = "harness_wpf_test-*.exe"; Label = "harness_wpf_test"; Extra = @("--ignored") }, @{ Pattern = "harness_winui3_test-*.exe"; Label = "harness_winui3_test"; Extra = @("--ignored") }, @{ Pattern = "harness_web_test-*.exe"; Label = "harness_web_test"; Extra = @("--ignored") }, - @{ Pattern = "modality_input_e2e_test-*.exe"; Label = "modality_input_e2e_test"; Extra = @("--ignored") } + @{ Pattern = "launch_windows_test-*.exe"; Label = "launch_windows_test"; Extra = @("--ignored") }, + @{ Pattern = "agent_cursor_windows_test-*.exe"; Label = "agent_cursor_windows_test"; Extra = @("--ignored") } ) # -- stage harness binaries to %TEMP% (same Zone-3 ShellExecute workaround) -- diff --git a/libs/cua-driver/tests/runners/windows/README.md b/libs/cua-driver/tests/runners/windows/README.md index ffb2e409c0..a64ba79cf0 100644 --- a/libs/cua-driver/tests/runners/windows/README.md +++ b/libs/cua-driver/tests/runners/windows/README.md @@ -9,7 +9,6 @@ Run from `libs/cua-driver` in an RDP or console session: .\tests\runners\windows\run-all.ps1 -RequireGui ``` -The runner builds repo-local Windows fixtures and runs the Rust default, -guard, harness, and modality suites. It intentionally skips optional -external-app suites such as LibreOffice because those require extra software -on the VM image. +The runner builds repo-local Windows fixtures and runs the Rust unit and typed +harness matrix. It intentionally skips optional external-app suites such as +LibreOffice because those require extra software on the VM image. diff --git a/libs/cua-driver/tests/runners/windows/run-all.ps1 b/libs/cua-driver/tests/runners/windows/run-all.ps1 index 4661ec5494..2df101f53a 100644 --- a/libs/cua-driver/tests/runners/windows/run-all.ps1 +++ b/libs/cua-driver/tests/runners/windows/run-all.ps1 @@ -12,95 +12,12 @@ param( Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" -$runnerDir = Split-Path -Parent $MyInvocation.MyCommand.Definition -$runnersDir = Split-Path -Parent $runnerDir -$testsDir = Split-Path -Parent $runnersDir -$cuaDriverRoot = Split-Path -Parent $testsDir -$rustRoot = Join-Path $cuaDriverRoot "rust" -$fixtureBuild = Join-Path $cuaDriverRoot "tests\fixtures\build\windows.ps1" +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..\..\..\..") +$canonicalRunner = Join-Path $repoRoot "scripts\ci\windows\run-rust-e2e.ps1" -Write-Host "=== cua-driver Windows Rust run-all ===" -ForegroundColor Cyan -Write-Host "cua-driver root: $cuaDriverRoot" -Write-Host "Rust workspace : $rustRoot" - -if ($RequireGui) { - $env:CUA_REQUIRE_GUI = "1" - Write-Host "CUA_REQUIRE_GUI=1" -ForegroundColor Yellow -} - -$results = New-Object System.Collections.Generic.List[object] - -function Add-Result { - param([string]$Name, [int]$ExitCode) - $status = if ($ExitCode -eq 0) { "PASS" } else { "FAIL" } - $results.Add([pscustomobject]@{ - Name = $Name - Status = $status - ExitCode = $ExitCode - }) | Out-Null -} - -function Run-Step { - param( - [string]$Name, - [string]$WorkingDirectory, - [string[]]$CommandArgs - ) - - Write-Host "`n[RUN] $Name" -ForegroundColor Yellow - Push-Location $WorkingDirectory - try { - & cargo @CommandArgs - $code = if ($null -eq $LASTEXITCODE) { 0 } else { $LASTEXITCODE } - } finally { - Pop-Location - } - Add-Result $Name $code - if ($code -ne 0) { - Write-Host "[FAIL] $Name exited $code" -ForegroundColor Red - } +if (-not (Test-Path $canonicalRunner)) { + throw "Canonical Windows E2E runner not found: $canonicalRunner" } -if (-not $NoBuild) { - if (-not (Test-Path $fixtureBuild)) { - throw "Windows fixture build script not found: $fixtureBuild" - } - Write-Host "`n[BUILD] Windows fixtures" -ForegroundColor Yellow - & $fixtureBuild - if ($LASTEXITCODE -ne 0) { - Add-Result "windows fixtures" $LASTEXITCODE - throw "Windows fixture build failed with exit $LASTEXITCODE" - } - Add-Result "windows fixtures" 0 -} - -Run-Step "default Rust tests" $rustRoot @( - "test", "-p", "cua-driver", "-p", "platform-windows", "--", "--nocapture" -) -Run-Step "guard UX" $rustRoot @( - "test", "-p", "cua-driver", "--test", "guard_ux_test", "--", "--nocapture", "--test-threads=1" -) -Run-Step "WPF harness" $rustRoot @( - "test", "-p", "cua-driver", "--test", "harness_wpf_test", "--", "--ignored", "--nocapture", "--test-threads=1" -) -Run-Step "WinUI3 harness" $rustRoot @( - "test", "-p", "cua-driver", "--test", "harness_winui3_test", "--", "--ignored", "--nocapture", "--test-threads=1" -) -Run-Step "WebView2/Electron harness" $rustRoot @( - "test", "-p", "cua-driver", "--test", "harness_web_test", "--", "--ignored", "--nocapture", "--test-threads=1" -) -Run-Step "Windows modality input e2e" $rustRoot @( - "test", "-p", "cua-driver", "--test", "modality_input_e2e_test", "--", "--ignored", "--nocapture", "--test-threads=1" -) - -Write-Host "`n=== Summary ===" -ForegroundColor Cyan -$failed = 0 -foreach ($r in $results) { - $color = if ($r.Status -eq "PASS") { "Green" } else { "Red" } - Write-Host ("{0,-32} {1} ({2})" -f $r.Name, $r.Status, $r.ExitCode) -ForegroundColor $color - if ($r.ExitCode -ne 0) { $failed++ } -} - -if ($failed -gt 0) { - exit 1 -} +& $canonicalRunner -NoBuild:$NoBuild -RequireGui:$RequireGui +exit $LASTEXITCODE diff --git a/libs/cua-driver/wayland-helper/README.md b/libs/cua-driver/wayland-helper/README.md index e74921de5b..6bbb679cb2 100644 --- a/libs/cua-driver/wayland-helper/README.md +++ b/libs/cua-driver/wayland-helper/README.md @@ -1,16 +1,24 @@ # cua WinRects — GNOME Shell helper extension (Wayland) -A ~40-line GNOME Shell extension that lets cua-driver get **pixel coordinates** -and draw the **agent cursor** on GNOME Mutter Wayland — two things a normal -Wayland client cannot do (no global coordinates; no `zwlr_layer_shell_v1`). +A small GNOME Shell extension that lets cua-driver get **pixel coordinates**, +activate an exact target window, capture the compositor stage, and draw the +**agent cursor** on GNOME Mutter Wayland. A normal Wayland client cannot do +these things globally. It exposes `org.cua.WinRects` on the session bus: -- `GetRects() -> json` — every window's `meta_window.get_frame_rect()` (screen - geometry). cua-driver combines the window origin with AT-SPI +- `GetRects() -> json` — every window's frame geometry and surface-buffer + origin. cua-driver combines the buffer origin with AT-SPI `CoordType::Window` per-widget coords: `screen = origin + window_xy`. This is the GNOME analogue of the X11 `_GTK_FRAME_EXTENTS` reconstruction (AT-SPI's - `CoordType::Screen` is `(0,0)` for every widget on Mutter). + `CoordType::Screen` is `(0,0)` for every widget on Mutter). Keeping the frame + and buffer origins separate accounts for GTK client-side shadows. +- `Activate(id) -> bool` — activate one Shell stable-sequence window and report + whether the request was accepted. cua-driver verifies focus through a second + `GetRects` snapshot before sending focus-bound portal/libei input, preventing + input from leaking into whichever application happened to be focused. +- `Capture() -> png_base64` — capture the compositor stage through Shell's + screenshot API. cua-driver crops it with the same authoritative geometry. - `MoveCursor(x,y)` / `ClickPulse(x,y)` / `HideCursor()` — render the agent cursor as a Clutter actor on the compositor stage. @@ -25,9 +33,14 @@ needed (unlike libei/RemoteDesktop). gnome-extensions info winrects@cua # -> State: ACTIVE ``` -cua-driver auto-detects it at runtime (`wayland::shell_helper`); everything is -best-effort, so the driver still runs (without screen coords / Wayland cursor) -when the extension is absent. wlroots compositors (sway/labwc/KWin) don't need -it — cua-driver uses `zwlr_layer_shell` + foreign-toplevel there. +cua-driver auto-detects it at runtime (`wayland::shell_helper`). AX operations +still work when it is absent, but pixel geometry, the Shell cursor, and safe +foreground portal input are unavailable. cua-driver refuses focus-bound input +instead of injecting into an unverified target. -KDE Plasma Wayland (KWin) would need an equivalent KWin script; not yet provided. +wlroots compositors such as Sway and labwc do not need it: cua-driver uses +foreign-toplevel activation, virtual-pointer input, and layer-shell there. + +KDE Plasma Wayland needs an equivalent target-addressable KWin activation +adapter; it is not yet provided. Portal reachability alone is insufficient +because RemoteDesktop/libei input is global to the compositor focus. diff --git a/libs/cua-driver/wayland-helper/winrects@cua/extension.js b/libs/cua-driver/wayland-helper/winrects@cua/extension.js index 9668f5fdc3..817a44a9ab 100644 --- a/libs/cua-driver/wayland-helper/winrects@cua/extension.js +++ b/libs/cua-driver/wayland-helper/winrects@cua/extension.js @@ -1,12 +1,19 @@ import Gio from 'gi://Gio'; +import GLib from 'gi://GLib'; +import Shell from 'gi://Shell'; import St from 'gi://St'; import Clutter from 'gi://Clutter'; import Cairo from 'cairo'; import * as Main from 'resource:///org/gnome/shell/ui/main.js'; import {Extension} from 'resource:///org/gnome/shell/extensions/extension.js'; +Gio._promisify(Shell.Screenshot.prototype, 'screenshot_stage_to_content'); +Gio._promisify(Shell.Screenshot, 'composite_to_stream'); + const IFACE = ` + + @@ -62,15 +69,80 @@ export default class WinRectsExtension extends Extension { if (this._nameId) { Gio.bus_unown_name(this._nameId); this._nameId = 0; } } GetRects() { + const actors = global.get_window_actors(); + const actorByWindow = new Map(); + for (const actor of actors) { + if (actor.meta_window) + actorByWindow.set(actor.meta_window, actor); + } + const windows = global.display.sort_windows_by_stacking([...actorByWindow.keys()]); + const focusedWindow = global.display.focus_window; const out = []; - for (const a of global.get_window_actors()) { - const w = a.meta_window; - if (!w) continue; + for (let stacking = 0; stacking < windows.length; stacking++) { + const w = windows[stacking]; + const actor = actorByWindow.get(w); const r = w.get_frame_rect(); - out.push({pid: w.get_pid(), title: w.get_title(), x: r.x, y: r.y, w: r.width, h: r.height}); + let buffer = r; + try { + buffer = w.get_buffer_rect(); + } catch (_error) { + // Older Shell releases may not expose the buffer rectangle. + } + const minimized = Boolean(w.minimized); + out.push({ + id: w.get_stable_sequence(), + pid: w.get_pid(), + title: w.get_title() ?? '', + x: r.x, + y: r.y, + w: r.width, + h: r.height, + buffer_x: buffer.x, + buffer_y: buffer.y, + focused: focusedWindow === w, + minimized, + visible: Boolean(actor?.visible) && !minimized, + stacking, + }); } return JSON.stringify(out); } + async CaptureAsync(_params, invocation) { + try { + const shooter = new Shell.Screenshot(); + const [content, scale] = await shooter.screenshot_stage_to_content(); + const stream = Gio.MemoryOutputStream.new_resizable(); + await Shell.Screenshot.composite_to_stream( + content.get_texture(), + 0, 0, -1, -1, + scale, + null, 0, 0, 1, + stream + ); + stream.close(null); + const encoded = GLib.base64_encode(stream.steal_as_bytes().get_data()); + invocation.return_value(new GLib.Variant('(s)', [encoded])); + } catch (error) { + invocation.return_dbus_error('org.cua.WinRects.CaptureFailed', String(error)); + } + } + ActivateAsync([id], invocation) { + const target = global.get_window_actors() + .map(actor => actor.meta_window) + .find(window => window?.get_stable_sequence() === id); + if (!target) { + invocation.return_value(new GLib.Variant('(b)', [false])); + return; + } + target.activate(global.get_current_time()); + GLib.timeout_add(GLib.PRIORITY_DEFAULT, 100, () => { + invocation.return_value(new GLib.Variant( + '(b)', + [global.display.focus_window === target] + )); + return GLib.SOURCE_REMOVE; + }); + } MoveCursor(x, y) { if (!this._cursor) return; this._cursor.show(); diff --git a/libs/cua-driver/wayland-helper/winrects@cua/metadata.json b/libs/cua-driver/wayland-helper/winrects@cua/metadata.json index efc503bf18..89f708d423 100644 --- a/libs/cua-driver/wayland-helper/winrects@cua/metadata.json +++ b/libs/cua-driver/wayland-helper/winrects@cua/metadata.json @@ -1 +1 @@ -{"uuid":"winrects@cua","name":"WinRects","description":"Expose window frame rects over D-Bus for cua-driver coordinate reconstruction on Wayland.","shell-version":["46","45","47"],"version":1} +{"uuid":"winrects@cua","name":"WinRects","description":"Expose window geometry, capture, cursor, and verified activation for cua-driver on Wayland.","shell-version":["45","46","47","48"],"version":3} diff --git a/libs/typescript/.prettierignore b/libs/typescript/.prettierignore new file mode 100644 index 0000000000..a695b1a918 --- /dev/null +++ b/libs/typescript/.prettierignore @@ -0,0 +1,3 @@ +node_modules/ +**/dist/ +**/coverage/ diff --git a/libs/typescript/package.json b/libs/typescript/package.json index ebc0f82fd8..86cd628fff 100644 --- a/libs/typescript/package.json +++ b/libs/typescript/package.json @@ -14,7 +14,7 @@ "test:core": "pnpm --filter @trycua/core test", "test:computer": "pnpm --filter @trycua/computer test", "test": "pnpm -r test", - "typecheck": "pnpm -r typecheck", + "typecheck": "pnpm build:core && pnpm -r typecheck", "format": "prettier --write .", "format:check": "prettier --check ." }, diff --git a/nix/cua-driver/README.md b/nix/cua-driver/README.md index 649a5a7490..9daf552323 100644 --- a/nix/cua-driver/README.md +++ b/nix/cua-driver/README.md @@ -1,37 +1,27 @@ # Linux driver checks -Nix is the Linux test environment. It is used for reproducible builds and for -containerized or compositor-backed Linux checks; Windows and macOS checks do -not belong here. - -## Check tiers - -| Tier | Check family | Trigger | Evidence | -| --- | --- | --- | --- | -| Source | `cua-driver-linux-rust-unit` | Linux Rust/Nix changes | Cargo unit tests and compilation | -| Contract | `cua-driver-integration`, `cua-driver-set-config`, `cua-driver-screenshot` | Maintainer or Linux CI | service, config, and capture assertions | -| Behavioral | `e2e-rust-linux` and the shared Rust matrix | Manual maintainer dispatch | external app state, AX trees, logs, failure screenshots | -| Supporting | legacy background GUI and compatibility checks | Explicitly selected | toolkit-specific diagnostics | - -The behavioral matrix is owned by -`libs/cua-driver/rust/crates/cua-driver/tests/cross_platform_behavior_test.rs`. -Nix checks must not create a second scenario list in Nix expressions. - -The current automatic Nix lane builds the driver and Rust workspace from source. -The manual Linux desktop runner builds Electron/Tauri from the repo-local -fixture sources in the desktop session. Moving those app builds fully inside -Nix requires committing and hashing their dependency lockfiles, especially the -Electron npm graph, and is intentionally a follow-up rather than an opaque -prebuilt download. - -## Naming and artifacts - -Use `cua-driver-linux--` for new checks. Do not add `gif` -to new check names. A failing behavioral check should retain structured output, -the driver log, an AX/UIA tree where available, and a PNG when the scenario is -pixel-based. GIF recorders remain legacy supporting diagnostics until their -behavior is covered by the Rust matrix. - -The old `linux-background-gui.nix` and compositor matrix are not deleted in one -step. They contain useful toolkit and regression evidence, but read-only -skeleton entries must not be reported as equivalent to a passing user workflow. +Nix has two jobs in the cua-driver test stack: + +| Attribute | Purpose | CI | +| --- | --- | --- | +| `cua-driver-build` | Build the shipped Linux package from the locked Rust source | `ci-nix-linux.yml` | +| `cua-driver-linux-rust-unit` | Compile and run the source-owned headless Rust tests | `ci-nix-linux.yml` | +| `cua-driver-wayland-e2e` | Provide the Sway, GTK, WebKit, Electron, capture, and Rust toolchain used by native Wayland E2E | `e2e-rust-linux-wayland.yml` | +| `cua-driver-inject-e2e` | Provide the same typed harness toolchain plus the nested `cua-compositor` package | Experimental nested-injection workflow | +| `cua-compositor-build` | Build the optional compositor-owned injection backend against pinned wlroots | Flake check | + +The Rust tests own all protocol and desktop behavior. The old NixOS Python +clients, GIF scenarios, real-app smoke rows, and compositor matrix were removed +after equivalent or stronger checks moved into the repo-owned Rust harnesses. +This avoids a second scenario catalog with different assertions. + +Run the native Wayland matrix from the repository root: + +```bash +nix develop .#cua-driver-wayland-e2e -c \ + scripts/ci/linux/run-rust-e2e-wayland.sh +``` + +The wrapper starts a pure Wayland Sway session with Xwayland disabled, then +calls the same canonical Rust runner used by X11. Results use the common typed +JSONL schema and retain MP4 trajectories under `artifacts/cua-driver/linux/`. diff --git a/nix/cua-driver/compositor/cua_compositor_patch.py b/nix/cua-driver/compositor/cua_compositor_patch.py index e6abdaabd5..80a9d38b44 100644 --- a/nix/cua-driver/compositor/cua_compositor_patch.py +++ b/nix/cua-driver/compositor/cua_compositor_patch.py @@ -7,17 +7,19 @@ # * focus-FREE per-surface KEYBOARD injection (type into an unfocused window), # * MULTI-cursor pointer injection (N independent cursors on one window), # -# both routed to a target window by its xdg app_id, driven over a tiny line +# both routed by stable process identity (with app_id fallback), driven over a tiny line # protocol on a unix control socket ($CUA_INJECT_SOCKET). It also exposes # foreign-toplevel-management (so cua-driver's list_windows enumerates windows) # and screencopy (so grim captures the output) — the same protocols labwc gives -# the non-nested cells — so the EIS cells keep working end to end. +# the non-nested cells — so the nested-injection cells keep working end to end. # # The injection primitives (cua_motion/cua_button/cua_kbd_*) deliver wl_pointer/ # wl_keyboard straight to a target client's resources, bypassing seat focus — -# routed by app_id (not positional device index) and transported over a plain -# socket instead of libei/EIS, since cua owns both ends and the portal/libei -# layer buys nothing here. +# routed by process identity rather than a connection-local object id and transported over a plain +# socket rather than libei/EIS, since cua owns both ends and the portal/libei +# layer buys nothing here. The socket speaks a versioned v1 line protocol: the +# client sends the `cua-inject v1` banner (echoed back on match), then every +# command line is answered by exactly one `ok` / `err ` acknowledgement. # # Usage: cua_compositor_patch.py import sys, io @@ -54,9 +56,13 @@ static int g_keymap_fd = -1; static size_t g_keymap_size = 0; static xkb_mod_mask_t g_shift_mask = 1; +static xkb_mod_mask_t g_ctrl_mask = 0; +static xkb_mod_mask_t g_alt_mask = 0; +static xkb_mod_mask_t g_logo_mask = 0; struct cua_keyent { uint32_t keycode; int shift; int valid; }; static struct cua_keyent g_chartab[128]; static struct wlr_foreign_toplevel_manager_v1 *g_ftl_mgr = NULL; +static void cua_ftl_request_activate(struct wl_listener *listener, void *data); """ @@ -64,26 +70,137 @@ STRUCT_FIELD = ( "\tstruct wlr_xdg_toplevel *xdg_toplevel;\n" "\tstruct wlr_foreign_toplevel_handle_v1 *ftl;\n" + "\tstruct wl_listener ftl_request_activate;\n" ) FUNCS = r""" +/* v1 control-protocol banner: the client sends this line, the compositor echoes + * it to confirm both speak v1. Any other first line is refused. */ +#define CUA_PROTO_HELLO "cua-inject v1" +static void cua_ftl_request_activate(struct wl_listener *listener, void *data) { + (void)data; + struct tinywl_toplevel *t = wl_container_of(listener, t, ftl_request_activate); + focus_toplevel(t); +} static uint32_t cua_now_ms(void) { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (uint32_t)(ts.tv_sec * 1000 + ts.tv_nsec / 1000000); } +/* Write a single acknowledgement line back to the control client. Best-effort: + * a dead peer is torn down by the read side on the next loop iteration. */ +static void cua_reply(int fd, const char *line) { + char buf[256]; + int n = snprintf(buf, sizeof buf, "%s\n", line); + if (n < 0) return; + if (n >= (int)sizeof buf) n = (int)sizeof buf - 1; + ssize_t off = 0; + while (off < n) { + ssize_t w = write(fd, buf + off, (size_t)n - (size_t)off); + if (w <= 0) break; + off += w; + } +} static void cua_pframe(struct wl_resource *res) { if (wl_resource_get_version(res) >= WL_POINTER_FRAME_SINCE_VERSION) wl_pointer_send_frame(res); } -/* Resolve a target window by its xdg app_id. */ -static struct tinywl_toplevel *cua_find_appid(struct tinywl_server *server, const char *app_id) { - struct tinywl_toplevel *t; +static pid_t cua_toplevel_pid(struct tinywl_toplevel *t); +/* Resolve a target window by its xdg app_id, refusing missing and ambiguous + * matches so a command never silently drives the wrong window. In v1 duplicate + * app_ids are simply not addressable. On failure returns NULL and points *err + * at a stable reason token; on success *err is left untouched. */ +static struct tinywl_toplevel *cua_resolve_target(struct tinywl_server *server, const char *app_id, const char **err) { + struct tinywl_toplevel *t, *found = NULL; + int matches = 0; + if (!strncmp(app_id, "pid:", 4)) { + char *end = NULL; + long pid = strtol(app_id + 4, &end, 10); + if (pid <= 0 || !end || *end) { *err = "bad-pid"; return NULL; } + wl_list_for_each(t, &server->toplevels, link) { + if (cua_toplevel_pid(t) == (pid_t)pid) { found = t; matches++; } + } + if (matches == 0) { *err = "unknown-pid"; return NULL; } + if (matches > 1) { *err = "ambiguous-pid"; return NULL; } + return found; + } wl_list_for_each(t, &server->toplevels, link) { const char *a = t->xdg_toplevel ? t->xdg_toplevel->app_id : NULL; - if (a && strcmp(a, app_id) == 0) return t; + if (a && strcmp(a, app_id) == 0) { found = t; matches++; } + } + if (matches == 0) { *err = "unknown-app-id"; return NULL; } + if (matches > 1) { *err = "ambiguous-app-id"; return NULL; } + return found; +} +static pid_t cua_toplevel_pid(struct tinywl_toplevel *t) { + if (!t || !t->xdg_toplevel || !t->xdg_toplevel->base->surface) return 0; + struct wl_client *client = wl_resource_get_client(t->xdg_toplevel->base->surface->resource); + pid_t pid = 0; uid_t uid = 0; gid_t gid = 0; + wl_client_get_credentials(client, &pid, &uid, &gid); + return pid; +} +/* Focus exactly one mapped toplevel owned by `target_pid`. Refuse ambiguity: + * process-scoped activation is only safe when the process owns one window. */ +static const char *cua_activate_pid(struct tinywl_server *server, pid_t target_pid) { + struct tinywl_toplevel *t, *found = NULL; + int matches = 0; + wl_list_for_each(t, &server->toplevels, link) { + if (target_pid > 0 && cua_toplevel_pid(t) == target_pid) { + found = t; + matches++; + } + } + if (matches == 0) return "unknown-pid"; + if (matches > 1) return "ambiguous-pid"; + focus_toplevel(found); + /* tinywl only notifies seat keyboard focus when a physical wlr_keyboard is + * attached. Headless CI has none, so establish the logical focus explicitly + * for observer truth and client activation semantics. */ + struct wlr_surface *surface = found->xdg_toplevel->base->surface; + if (server->seat->keyboard_state.focused_surface != surface) { + struct wlr_keyboard_modifiers modifiers = {0}; + wlr_seat_keyboard_notify_enter(server->seat, surface, NULL, 0, &modifiers); } return NULL; } +/* Independent observer query used only by the Rust E2E testkit. The target is + * selected by the Wayland client's process credentials, not by driver-owned + * object ids. In this minimal compositor every mapped toplevel shares origin; + * a non-focused target beneath another focused surface is therefore occluded. */ +static void cua_query_state(struct tinywl_server *server, pid_t target_pid, char *out, size_t out_len) { + struct wlr_surface *focused = server->seat->keyboard_state.focused_surface; + struct wlr_surface *focused_root = focused ? wlr_surface_get_root_surface(focused) : NULL; + struct tinywl_toplevel *t, *target = NULL, *focused_toplevel = NULL; + wl_list_for_each(t, &server->toplevels, link) { + struct wlr_surface *surface = t->xdg_toplevel ? t->xdg_toplevel->base->surface : NULL; + if (surface == focused_root) focused_toplevel = t; + if (target_pid > 0 && cua_toplevel_pid(t) == target_pid) target = t; + } + pid_t focused_pid = cua_toplevel_pid(focused_toplevel); + const char *state = !target ? "not_found" : + (target == focused_toplevel ? "foreground" : + (focused_toplevel ? "background_occluded" : "background_visible")); + snprintf(out, out_len, "state %d %s", (int)focused_pid, state); +} +static const char *cua_query_geometry(struct tinywl_server *server, pid_t target_pid, char *out, size_t out_len) { + struct tinywl_toplevel *t, *target = NULL; + int matches = 0; + wl_list_for_each(t, &server->toplevels, link) { + if (cua_toplevel_pid(t) == target_pid) { target = t; matches++; } + } + if (!matches) return "target-not-found"; + if (matches > 1) return "ambiguous-pid"; + int x = 0, y = 0; + if (!wlr_scene_node_coords(&target->scene_tree->node, &x, &y)) return "unmapped-target"; + /* AT-SPI Window coordinates for native GTK are rooted at the xdg window + * geometry, while scene coordinates and screencopy include the complete root + * surface (including client-side decorations). Rebase into that coordinate + * system so `origin + accessible_window_xy` lands on the captured pixel. */ + int scene_x = x, scene_y = y; + struct wlr_box geo = target->xdg_toplevel->base->geometry; + x -= geo.x; y -= geo.y; + snprintf(out, out_len, "geometry %d %d %d %d", x, y, scene_x, scene_y); + return NULL; +} static void cua_ptr_leave(struct wlr_seat *seat, struct wlr_surface *surf) { if (!surf) return; struct wlr_seat_client *sc = wlr_seat_client_for_wl_client(seat, wl_resource_get_client(surf->resource)); @@ -97,13 +214,34 @@ * independent cursors against the same or different surfaces. Focus-free: we * write straight to the client's wl_pointer resources, never touching the seat * focus or any real cursor. */ -static void cua_motion(struct tinywl_server *server, struct tinywl_toplevel *t, int idx, double x, double y) { - if (!t || idx < 0 || idx >= CUA_MAXDEV) return; - struct wlr_surface *surface = t->xdg_toplevel->base->surface; - struct wlr_box geo = t->xdg_toplevel->base->geometry; - wl_fixed_t sx = wl_fixed_from_double(geo.x + x), sy = wl_fixed_from_double(geo.y + y); +static bool cua_motion(struct tinywl_server *server, struct tinywl_toplevel *t, int idx, double x, double y) { + if (!t || idx < 0 || idx >= CUA_MAXDEV) return false; + /* Public PX coordinates come from the cropped root-surface screenshot. Hit + * test that point through the scene so Chromium/WebKit child surfaces receive + * enter/motion in their own local coordinates instead of the top-level root. */ + int scene_x = 0, scene_y = 0; + if (!wlr_scene_node_coords(&t->scene_tree->node, &scene_x, &scene_y)) return false; + double local_x = 0, local_y = 0; + /* Search only the requested toplevel's scene subtree. A global hit test + * would select the foreground sentinel when this target is occluded. */ + struct wlr_scene_node *node = wlr_scene_node_at(&t->scene_tree->node, + scene_x + x, scene_y + y, &local_x, &local_y); + if (!node || node->type != WLR_SCENE_NODE_BUFFER) return false; + struct wlr_scene_buffer *buffer = wlr_scene_buffer_from_node(node); + struct wlr_scene_surface *scene_surface = wlr_scene_surface_try_from_buffer(buffer); + if (!scene_surface) return false; + struct wlr_surface *surface = scene_surface->surface; struct wlr_seat_client *sc = wlr_seat_client_for_wl_client(server->seat, wl_resource_get_client(surface->resource)); - if (!sc || wl_list_empty(&sc->pointers)) return; + if (!sc || wl_list_empty(&sc->pointers)) { + /* Chromium can compose a renderer-owned child surface whose wl_client + * never bound wl_pointer while the owning toplevel client did. Use that + * target root while retaining the caller's screenshot-local point. */ + surface = t->xdg_toplevel->base->surface; + local_x = x; local_y = y; + sc = wlr_seat_client_for_wl_client(server->seat, wl_resource_get_client(surface->resource)); + if (!sc || wl_list_empty(&sc->pointers)) return false; + } + wl_fixed_t sx = wl_fixed_from_double(local_x), sy = wl_fixed_from_double(local_y); struct wl_resource *res; if (cua_ptr[idx].entered != surface) { if (cua_ptr[idx].entered) cua_ptr_leave(server->seat, cua_ptr[idx].entered); @@ -113,18 +251,68 @@ } uint32_t tm = cua_now_ms(); wl_resource_for_each(res, &sc->pointers) { wl_pointer_send_motion(res, tm, sx, sy); cua_pframe(res); } + return true; } -static void cua_button(struct tinywl_server *server, struct tinywl_toplevel *t, int idx, uint32_t button, bool pressed) { - if (!t || idx < 0 || idx >= CUA_MAXDEV) return; - struct wlr_surface *surface = t->xdg_toplevel->base->surface; +/* Resolve an output-layout point through the compositor scene, preserving + * subsurface offsets and output scaling. This is the desktop-scope path. */ +static struct tinywl_toplevel *cua_desktop_motion(struct tinywl_server *server, double x, double y) { + double sx = 0, sy = 0; + struct wlr_surface *surface = NULL; + struct tinywl_toplevel *t = desktop_toplevel_at(server, x, y, &surface, &sx, &sy); + if (!t || !surface) return NULL; struct wlr_seat_client *sc = wlr_seat_client_for_wl_client(server->seat, wl_resource_get_client(surface->resource)); - if (!sc || wl_list_empty(&sc->pointers)) return; + if (!sc || wl_list_empty(&sc->pointers)) return NULL; + struct wl_resource *res; + if (cua_ptr[0].entered != surface) { + if (cua_ptr[0].entered) cua_ptr_leave(server->seat, cua_ptr[0].entered); + uint32_t serial = wlr_seat_client_next_serial(sc); + wl_resource_for_each(res, &sc->pointers) { + wl_pointer_send_enter(res, serial, surface->resource, wl_fixed_from_double(sx), wl_fixed_from_double(sy)); + cua_pframe(res); + } + cua_ptr[0].entered = surface; + } + uint32_t tm = cua_now_ms(); + wl_resource_for_each(res, &sc->pointers) { + wl_pointer_send_motion(res, tm, wl_fixed_from_double(sx), wl_fixed_from_double(sy)); + cua_pframe(res); + } + return t; +} +static bool cua_button(struct tinywl_server *server, struct tinywl_toplevel *t, int idx, uint32_t button, bool pressed) { + if (!t || idx < 0 || idx >= CUA_MAXDEV) return false; + /* `cua_motion` establishes the exact child or root surface for this logical + * pointer. Button and axis events must use that same wl_pointer resource. */ + struct wlr_surface *surface = cua_ptr[idx].entered ? cua_ptr[idx].entered : t->xdg_toplevel->base->surface; + struct wlr_seat_client *sc = wlr_seat_client_for_wl_client(server->seat, wl_resource_get_client(surface->resource)); + if (!sc || wl_list_empty(&sc->pointers)) return false; uint32_t tm = cua_now_ms(), bs = wlr_seat_client_next_serial(sc); struct wl_resource *res; wl_resource_for_each(res, &sc->pointers) { wl_pointer_send_button(res, bs, tm, button, pressed ? WL_POINTER_BUTTON_STATE_PRESSED : WL_POINTER_BUTTON_STATE_RELEASED); cua_pframe(res); } + return true; +} +static bool cua_axis(struct tinywl_server *server, struct tinywl_toplevel *t, int idx, uint32_t axis, double value) { + if (!t || idx < 0 || idx >= CUA_MAXDEV) return false; + struct wlr_surface *surface = cua_ptr[idx].entered ? cua_ptr[idx].entered : t->xdg_toplevel->base->surface; + struct wlr_seat_client *sc = wlr_seat_client_for_wl_client(server->seat, wl_resource_get_client(surface->resource)); + if (!sc || wl_list_empty(&sc->pointers)) return false; + uint32_t tm = cua_now_ms(); + struct wl_resource *res; + wl_resource_for_each(res, &sc->pointers) { + int32_t step = value < 0 ? -1 : 1; + if (wl_resource_get_version(res) >= WL_POINTER_AXIS_SOURCE_SINCE_VERSION) + wl_pointer_send_axis_source(res, WL_POINTER_AXIS_SOURCE_WHEEL); + if (wl_resource_get_version(res) >= WL_POINTER_AXIS_VALUE120_SINCE_VERSION) + wl_pointer_send_axis_value120(res, axis, step * 120); + else if (wl_resource_get_version(res) >= WL_POINTER_AXIS_DISCRETE_SINCE_VERSION) + wl_pointer_send_axis_discrete(res, axis, step); + wl_pointer_send_axis(res, tm, axis, wl_fixed_from_double(value)); + cua_pframe(res); + } + return true; } static void cua_init_keymap(void) { struct xkb_context *ctx = xkb_context_new(XKB_CONTEXT_NO_FLAGS); @@ -143,6 +331,12 @@ * evdev+8; wl_keyboard.key wants the evdev code. */ xkb_mod_index_t shift = xkb_keymap_mod_get_index(km, XKB_MOD_NAME_SHIFT); if (shift != XKB_MOD_INVALID) g_shift_mask = (xkb_mod_mask_t)1 << shift; + xkb_mod_index_t ctrl = xkb_keymap_mod_get_index(km, XKB_MOD_NAME_CTRL); + if (ctrl != XKB_MOD_INVALID) g_ctrl_mask = (xkb_mod_mask_t)1 << ctrl; + xkb_mod_index_t alt = xkb_keymap_mod_get_index(km, XKB_MOD_NAME_ALT); + if (alt != XKB_MOD_INVALID) g_alt_mask = (xkb_mod_mask_t)1 << alt; + xkb_mod_index_t logo = xkb_keymap_mod_get_index(km, XKB_MOD_NAME_LOGO); + if (logo != XKB_MOD_INVALID) g_logo_mask = (xkb_mod_mask_t)1 << logo; for (xkb_keycode_t kc = 9; kc < 256; kc++) { for (int lvl = 0; lvl < 2; lvl++) { const xkb_keysym_t *syms; @@ -192,27 +386,28 @@ wl_keyboard_send_key(res, wlr_seat_client_next_serial(sc), tm, keycode, pressed ? WL_KEYBOARD_KEY_STATE_PRESSED : WL_KEYBOARD_KEY_STATE_RELEASED); } -static void cua_type_cp(struct tinywl_server *server, struct tinywl_toplevel *t, uint32_t cp) { - if (cp >= 128 || !g_chartab[cp].valid) return; +static bool cua_type_cp(struct tinywl_server *server, struct tinywl_toplevel *t, uint32_t cp) { + if (cp >= 128 || !g_chartab[cp].valid) return false; struct wlr_seat_client *sc = cua_kbd_enter(server, t); - if (!sc) return; + if (!sc) return false; struct cua_keyent e = g_chartab[cp]; if (e.shift) cua_kbd_mods(sc, g_shift_mask); cua_kbd_key(sc, e.keycode, true); cua_kbd_key(sc, e.keycode, false); if (e.shift) cua_kbd_mods(sc, 0); + return true; } /* Decode a hex-encoded ASCII string and type it focus-free into `t`. */ -static void cua_type_hex(struct tinywl_server *server, struct tinywl_toplevel *t, const char *hex) { - if (!t) return; +static bool cua_type_hex(struct tinywl_server *server, struct tinywl_toplevel *t, const char *hex) { + if (!t) return false; for (const char *p = hex; p[0] && p[1]; p += 2) { int hi = (p[0] <= '9') ? p[0] - '0' : (p[0] | 0x20) - 'a' + 10; int lo = (p[1] <= '9') ? p[1] - '0' : (p[1] | 0x20) - 'a' + 10; - cua_type_cp(server, t, (uint32_t)((hi << 4) | lo)); + if (!cua_type_cp(server, t, (uint32_t)((hi << 4) | lo))) return false; } + return true; } -static void cua_key_named(struct tinywl_server *server, struct tinywl_toplevel *t, const char *name) { - if (!t) return; +static uint32_t cua_named_keycode(const char *name) { uint32_t kc = 0; if (!strcasecmp(name, "enter") || !strcasecmp(name, "return")) kc = KEY_ENTER; else if (!strcasecmp(name, "tab")) kc = KEY_TAB; @@ -223,35 +418,114 @@ else if (!strcasecmp(name, "down")) kc = KEY_DOWN; else if (!strcasecmp(name, "left")) kc = KEY_LEFT; else if (!strcasecmp(name, "right")) kc = KEY_RIGHT; - if (!kc) return; + else if (!strncasecmp(name, "f", 1)) { + char *end = NULL; long fn = strtol(name + 1, &end, 10); + if (end && !*end && fn >= 1 && fn <= 10) kc = KEY_F1 + (uint32_t)fn - 1; + else if (end && !*end && fn == 11) kc = KEY_F11; + else if (end && !*end && fn == 12) kc = KEY_F12; + } + return kc; +} +/* Returns 1 when `name` is a recognised key (delivered if the target had a + * keyboard bound), 0 when it is outside the whitelist so the caller can NAK. */ +static int cua_key_named(struct tinywl_server *server, struct tinywl_toplevel *t, const char *name) { + if (!t) return 0; + uint32_t kc = cua_named_keycode(name); + if (!kc) return 0; struct wlr_seat_client *sc = cua_kbd_enter(server, t); - if (!sc) return; + if (!sc) return -1; cua_kbd_key(sc, kc, true); cua_kbd_key(sc, kc, false); + return 1; +} +static int cua_hotkey(struct tinywl_server *server, struct tinywl_toplevel *t, const char *mods, const char *key) { + if (!t) return 0; + uint32_t kc = cua_named_keycode(key); + if (!kc && key[0] && !key[1]) { + unsigned char cp = (unsigned char)key[0]; + if (cp < 128 && g_chartab[cp].valid) kc = g_chartab[cp].keycode; + } + if (!kc) return 0; + xkb_mod_mask_t mask = 0; + char copy[128]; snprintf(copy, sizeof copy, "%s", mods); + char *save = NULL; + for (char *mod = strtok_r(copy, ",", &save); mod; mod = strtok_r(NULL, ",", &save)) { + if (!strcasecmp(mod, "ctrl") || !strcasecmp(mod, "control")) mask |= g_ctrl_mask; + else if (!strcasecmp(mod, "shift")) mask |= g_shift_mask; + else if (!strcasecmp(mod, "alt") || !strcasecmp(mod, "option")) mask |= g_alt_mask; + else if (!strcasecmp(mod, "meta") || !strcasecmp(mod, "super") || !strcasecmp(mod, "win") || !strcasecmp(mod, "cmd")) mask |= g_logo_mask; + else return 0; + } + struct wlr_seat_client *sc = cua_kbd_enter(server, t); + if (!sc) return -1; + cua_kbd_mods(sc, mask); + cua_kbd_key(sc, kc, true); + cua_kbd_key(sc, kc, false); + cua_kbd_mods(sc, 0); + return 1; } /* ── control socket: one line per command, routed by app_id ───────────────── */ -static void cua_handle_cmd(struct tinywl_server *server, char *line) { +/* Process one command line. Returns NULL on success, else a stable error token + * the caller sends back as `err `. The command is only acknowledged + * after it has been resolved and applied — never before. */ +static const char *cua_handle_cmd(struct tinywl_server *server, char *line) { char cmd[8], app[128]; - if (sscanf(line, "%7s", cmd) != 1) return; - if (!strcmp(cmd, "m")) { + if (sscanf(line, "%7s", cmd) != 1) return "empty"; + const char *err = NULL; + struct tinywl_toplevel *t; + if (!strcmp(cmd, "d")) { + double x, y; unsigned count, btn; + if (sscanf(line, "d %lf %lf %u %u", &x, &y, &count, &btn) != 4) return "bad-args"; + if (!(t = cua_desktop_motion(server, x, y))) return "no-surface-at-point"; + for (unsigned i = 0; i < (count ? count : 1); i++) { + if (!cua_button(server, t, 0, btn, true)) return "no-pointer-resource"; + if (!cua_button(server, t, 0, btn, false)) return "no-pointer-resource"; + } + return NULL; + } else if (!strcmp(cmd, "m")) { int idx; double x, y; - if (sscanf(line, "m %127s %d %lf %lf", app, &idx, &x, &y) == 4) - cua_motion(server, cua_find_appid(server, app), idx, x, y); + if (sscanf(line, "m %127s %d %lf %lf", app, &idx, &x, &y) != 4) return "bad-args"; + if (!(t = cua_resolve_target(server, app, &err))) return err; + if (!cua_motion(server, t, idx, x, y)) return "no-pointer-resource"; + return NULL; } else if (!strcmp(cmd, "b")) { int idx; unsigned btn, pr; - if (sscanf(line, "b %127s %d %u %u", app, &idx, &btn, &pr) == 4) - cua_button(server, cua_find_appid(server, app), idx, btn, pr != 0); + if (sscanf(line, "b %127s %d %u %u", app, &idx, &btn, &pr) != 4) return "bad-args"; + if (!(t = cua_resolve_target(server, app, &err))) return err; + if (!cua_button(server, t, idx, btn, pr != 0)) return "no-pointer-resource"; + return NULL; } else if (!strcmp(cmd, "t")) { char hex[8192]; - if (sscanf(line, "t %127s %8191s", app, hex) == 2) - cua_type_hex(server, cua_find_appid(server, app), hex); + if (sscanf(line, "t %127s %8191s", app, hex) != 2) return "bad-args"; + if (!(t = cua_resolve_target(server, app, &err))) return err; + if (!cua_type_hex(server, t, hex)) return "no-keyboard-resource"; + return NULL; } else if (!strcmp(cmd, "k")) { char key[32]; - if (sscanf(line, "k %127s %31s", app, key) == 2) - cua_key_named(server, cua_find_appid(server, app), key); + if (sscanf(line, "k %127s %31s", app, key) != 2) return "bad-args"; + if (!(t = cua_resolve_target(server, app, &err))) return err; + int result = cua_key_named(server, t, key); + if (result < 0) return "no-keyboard-resource"; + if (!result) return "unknown-key"; + return NULL; + } else if (!strcmp(cmd, "h")) { + char mods[128], key[32]; + if (sscanf(line, "h %127s %127s %31s", app, mods, key) != 3) return "bad-args"; + if (!(t = cua_resolve_target(server, app, &err))) return err; + int result = cua_hotkey(server, t, mods, key); + if (result < 0) return "no-keyboard-resource"; + if (!result) return "unknown-hotkey"; + return NULL; + } else if (!strcmp(cmd, "a")) { + int idx; unsigned axis; double value; + if (sscanf(line, "a %127s %d %u %lf", app, &idx, &axis, &value) != 4) return "bad-args"; + if (!(t = cua_resolve_target(server, app, &err))) return err; + if (!cua_axis(server, t, idx, axis, value)) return "no-pointer-resource"; + return NULL; } + return "unknown-command"; } -struct cua_conn { struct tinywl_server *server; struct wl_event_source *src; char buf[16384]; size_t len; }; +struct cua_conn { struct tinywl_server *server; struct wl_event_source *src; int hello; char buf[16384]; size_t len; }; /* Tear a connection down once: remove its event source (else the loop fires it * again on freed data -> double free), close the fd, free the state. */ static int cua_conn_drop(struct cua_conn *c, int fd) { @@ -267,7 +541,54 @@ c->len += (size_t)n; c->buf[c->len] = 0; char *p = c->buf, *nl; while ((nl = memchr(p, '\n', (size_t)(c->buf + c->len - p)))) { - *nl = 0; cua_handle_cmd(c->server, p); p = nl + 1; + *nl = 0; + /* Tolerate CRLF clients by trimming a trailing carriage return. */ + if (nl > p && nl[-1] == '\r') nl[-1] = 0; + if (!c->hello) { + /* The first line must be the versioned v1 handshake. */ + if (!strcmp(p, CUA_PROTO_HELLO)) { + c->hello = 1; + cua_reply(fd, CUA_PROTO_HELLO); + } else { + cua_reply(fd, "err unsupported-version"); + return cua_conn_drop(c, fd); + } + } else { + int query_pid, geometry_pid, activate_pid; + if (sscanf(p, "q %d", &query_pid) == 1) { + char msg[128]; cua_query_state(c->server, (pid_t)query_pid, msg, sizeof msg); + cua_reply(fd, msg); + } else if (sscanf(p, "g %d", &geometry_pid) == 1) { + char msg[128]; + const char *err = cua_query_geometry(c->server, (pid_t)geometry_pid, msg, sizeof msg); + if (err) { + char reply[128]; snprintf(reply, sizeof reply, "err %s", err); cua_reply(fd, reply); + } else { + cua_reply(fd, msg); + } + } else if (sscanf(p, "f %d", &activate_pid) == 1) { + const char *err = cua_activate_pid(c->server, (pid_t)activate_pid); + wl_display_flush_clients(c->server->wl_display); + if (err) { + char msg[128]; snprintf(msg, sizeof msg, "err %s", err); cua_reply(fd, msg); + } else { + cua_reply(fd, "ok"); + } + } else { + const char *err = cua_handle_cmd(c->server, p); + /* Deliver injected events before acking so `ok` means "processed", + * never merely "parsed". */ + wl_display_flush_clients(c->server->wl_display); + if (err) { + char msg[128]; + snprintf(msg, sizeof msg, "err %s", err); + cua_reply(fd, msg); + } else { + cua_reply(fd, "ok"); + } + } + } + p = nl + 1; } size_t rem = (size_t)(c->buf + c->len - p); memmove(c->buf, p, rem); c->len = rem; @@ -327,13 +648,20 @@ def repl(s, old, new, label, count=1): "\t\t\twlr_foreign_toplevel_handle_v1_set_title(toplevel->ftl, toplevel->xdg_toplevel->title);\n" "\t\tif (toplevel->xdg_toplevel->app_id)\n" "\t\t\twlr_foreign_toplevel_handle_v1_set_app_id(toplevel->ftl, toplevel->xdg_toplevel->app_id);\n" + "\t\ttoplevel->ftl_request_activate.notify = cua_ftl_request_activate;\n" + "\t\twl_signal_add(&toplevel->ftl->events.request_activate, &toplevel->ftl_request_activate);\n" "\t}\n\n\tfocus_toplevel(toplevel);", "ftl-on-map") # 4) On unmap: drop the foreign-toplevel handle. src = repl(src, "\twl_list_remove(&toplevel->link);\n}", - "\tif (toplevel->ftl) { wlr_foreign_toplevel_handle_v1_destroy(toplevel->ftl); toplevel->ftl = NULL; }\n" + "\tstruct wlr_surface *cua_surface = toplevel->xdg_toplevel->base->surface;\n" + "\tfor (int i = 0; i < CUA_MAXDEV; i++) {\n" + "\t\tif (cua_ptr[i].entered && wlr_surface_get_root_surface(cua_ptr[i].entered) == cua_surface) cua_ptr[i].entered = NULL;\n" + "\t\tif (cua_kbd_state[i].entered && wlr_surface_get_root_surface(cua_kbd_state[i].entered) == cua_surface) cua_kbd_state[i].entered = NULL;\n" + "\t}\n" + "\tif (toplevel->ftl) { wl_list_remove(&toplevel->ftl_request_activate.link); wlr_foreign_toplevel_handle_v1_destroy(toplevel->ftl); toplevel->ftl = NULL; }\n" "\twl_list_remove(&toplevel->link);\n}", "ftl-on-unmap") diff --git a/nix/cua-driver/package.nix b/nix/cua-driver/package.nix index 4a59f798ac..df27036613 100644 --- a/nix/cua-driver/package.nix +++ b/nix/cua-driver/package.nix @@ -80,6 +80,8 @@ pkgs.rustPlatform.buildRustPackage { pipewire # libei via reis — same as the ashpd RemoteDesktop+EIS flow. libei + # reis links its keyboard support directly against libxkbcommon. + libxkbcommon ]; # Skip tests that require a running X11 display or AT-SPI bus diff --git a/nix/cua-driver/tests/README.md b/nix/cua-driver/tests/README.md index f1b8a59c35..3b699d7616 100644 --- a/nix/cua-driver/tests/README.md +++ b/nix/cua-driver/tests/README.md @@ -1,17 +1,15 @@ -# Linux Nix test layout +# Linux Nix check layout Nix expressions are grouped by what they prove: | Path | Role | | --- | --- | | `rust-unit.nix` | Source-built Rust workspace checks without a desktop session | -| `integration.nix` | Driver service starts and answers protocol requests | -| `set-config.nix` | Configuration persistence contract | -| `screenshot.nix` | Capture contract in a managed display | -| `wayland/` | Compositor-specific supporting and red/green checks | -| `linux-background-gui.nix` | Legacy toolkit matrix, supporting coverage only | -| `linux-*-gif.nix` | Legacy visual diagnostics, never the canonical Rust matrix | + +This directory intentionally contains no desktop behavior catalog. Nix builds +the driver, unit checks, session dependencies, and optional compositor package. +The canonical GUI scenarios and assertions live in the typed Rust harnesses. New user-behavior coverage belongs in the Rust test harness first. A Nix check -may provide the session and package environment, but it should invoke the -shared Rust scenario or clearly identify itself as supporting coverage. +may provide the session and package environment, but it must invoke the shared +Rust catalog instead of defining a second set of behavioral assertions. diff --git a/nix/cua-driver/tests/integration.nix b/nix/cua-driver/tests/integration.nix deleted file mode 100644 index c391385147..0000000000 --- a/nix/cua-driver/tests/integration.nix +++ /dev/null @@ -1,198 +0,0 @@ -# CUA Driver Integration Test -# -# Tests the cua-driver MCP server end-to-end in a NixOS container with Xvfb. -# Verifies CLI subcommands, MCP protocol handshake, and tool invocation. -# -# To run: nix build .#checks.x86_64-linux.cua-driver-integration -# -{ - pkgs, - lib ? pkgs.lib, - cuaDriverModule, - ... -}: - -let - # Python MCP client that drives cua-driver over stdio. - # Sends JSON-RPC requests, reads responses, validates structure. - mcpClientTest = pkgs.writeText "mcp-client-test.py" '' - import subprocess - import json - import sys - import os - import threading - import time - - DRIVER_BIN = os.environ.get("CUA_DRIVER_BIN", "cua-driver") - - def main(): - print("=== CUA Driver MCP Integration Test ===", flush=True) - - proc = subprocess.Popen( - [DRIVER_BIN, "mcp", "--no-daemon-relaunch"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env={**os.environ}, - ) - - # Drain stderr in background to prevent blocking - def drain_stderr(): - for line in proc.stderr: - sys.stderr.buffer.write(line) - sys.stderr.buffer.flush() - t = threading.Thread(target=drain_stderr, daemon=True) - t.start() - - def send_request(method, params=None, req_id=None): - msg = {"jsonrpc": "2.0", "method": method} - if params is not None: - msg["params"] = params - if req_id is not None: - msg["id"] = req_id - line = json.dumps(msg) + "\n" - print(f"[send] {line.strip()}", flush=True) - proc.stdin.write(line.encode()) - proc.stdin.flush() - - def read_response(timeout=30): - # Simple blocking read with timeout via thread - result = [None] - def reader(): - result[0] = proc.stdout.readline() - rt = threading.Thread(target=reader) - rt.start() - rt.join(timeout) - if rt.is_alive(): - raise TimeoutError("No response within timeout") - line = result[0].decode().strip() - print(f"[recv] {line}", flush=True) - return json.loads(line) - - try: - # Test 1: Initialize - print("\n--- Test 1: Initialize ---", flush=True) - send_request("initialize", { - "protocolVersion": "2024-11-05", - "capabilities": {}, - "clientInfo": {"name": "nixos-test", "version": "1.0.0"}, - }, req_id=1) - resp = read_response() - assert resp.get("id") == 1, f"Expected id=1, got {resp.get('id')}" - assert "result" in resp, f"Expected result in response: {resp}" - assert "serverInfo" in resp["result"], f"Expected serverInfo: {resp['result']}" - print("PASS: initialize", flush=True) - - # Send initialized notification (no response expected) - send_request("notifications/initialized", {}) - time.sleep(0.5) - - # Test 2: List tools - print("\n--- Test 2: tools/list ---", flush=True) - send_request("tools/list", {}, req_id=2) - resp = read_response() - assert resp.get("id") == 2, f"Expected id=2, got {resp.get('id')}" - tools = resp.get("result", {}).get("tools", []) - tool_names = [t["name"] for t in tools] - print(f"Found {len(tools)} tools: {tool_names[:10]}...", flush=True) - assert "click" in tool_names, f"click not in tools: {tool_names}" - assert "type_text" in tool_names, f"type_text not in tools: {tool_names}" - assert "get_screen_size" in tool_names, f"get_screen_size not in tools: {tool_names}" - assert "get_window_state" in tool_names, f"get_window_state not in tools: {tool_names}" - print("PASS: tools/list", flush=True) - - # Test 3: Call get_screen_size (works without any windows) - print("\n--- Test 3: tools/call get_screen_size ---", flush=True) - send_request("tools/call", { - "name": "get_screen_size", - "arguments": {}, - }, req_id=3) - resp = read_response(timeout=15) - assert resp.get("id") == 3, f"Expected id=3, got {resp.get('id')}" - # get_screen_size should return display dimensions from Xvfb - if "result" in resp: - content = resp["result"].get("content", []) - text = content[0].get("text", "") if content else "" - print(f"PASS: get_screen_size returned: {text[:200]}", flush=True) - elif "error" in resp: - err_msg = resp.get("error", {}).get("message", "unknown") - print(f"PASS: get_screen_size returned error (acceptable in headless): {err_msg}", flush=True) - else: - raise AssertionError(f"Unexpected response: {resp}") - - print("\n=== All MCP tests passed! ===", flush=True) - - finally: - proc.stdin.close() - proc.terminate() - proc.wait(timeout=5) - - if __name__ == "__main__": - main() - ''; - -in - -pkgs.testers.nixosTest { - name = "cua-driver-integration-test"; - meta = { - maintainers = [ ]; - }; - - containers.machine = - { - config, - pkgs, - lib, - ... - }: - { - imports = [ cuaDriverModule ]; - services.cua-driver.enable = true; - environment.systemPackages = with pkgs; [ - xorg.xorgserver # Xvfb for headless X11 - python3 # MCP client test script - jq - procps # pgrep/pkill - ]; - }; - - testScript = '' - # cache-bust 2026-06-08.1: this comment is part of the test derivation, so - # bumping it changes the output path and forces the test to actually re-run - # instead of being substituted from the binary cache. Bump again to re-run. - machine.start() - machine.wait_for_unit("multi-user.target") - - with subtest("Binary exists and runs"): - machine.succeed("cua-driver --help") - - with subtest("list-tools prints available tools"): - result = machine.succeed("cua-driver list-tools") - assert "click" in result, f"click not in: {result}" - assert "type_text" in result, f"type_text not in: {result}" - assert "get_screen_size" in result, f"get_screen_size not in: {result}" - - with subtest("describe tool outputs schema"): - result = machine.succeed("cua-driver describe get_screen_size") - assert "input_schema" in result, f"Unexpected describe output: {result[:200]}" - - with subtest("Start Xvfb"): - machine.execute("Xvfb :99 -screen 0 1280x1024x24 >/dev/null 2>&1 &") - machine.wait_until_succeeds("test -e /tmp/.X11-unix/X99", timeout=10) - - with subtest("doctor with X11 display"): - # Timeout doctor to 15s — AT-SPI/gdbus probes can hang without a session bus - result = machine.succeed("timeout 15 env DISPLAY=:99 cua-driver doctor 2>&1 || true") - machine.log(result) - - with subtest("MCP protocol handshake and tool listing"): - machine.copy_from_host("${mcpClientTest}", "/tmp/mcp-client-test.py") - result = machine.succeed( - "timeout 60 env DISPLAY=:99 " - "python3 /tmp/mcp-client-test.py 2>&1" - ) - machine.log(result) - assert "All MCP tests passed" in result, f"MCP tests failed: {result}" - ''; -} diff --git a/nix/cua-driver/tests/linux-background-gui.nix b/nix/cua-driver/tests/linux-background-gui.nix deleted file mode 100644 index 7af374d0a6..0000000000 --- a/nix/cua-driver/tests/linux-background-gui.nix +++ /dev/null @@ -1,1062 +0,0 @@ -# Linux background GUI test — matrix over REAL desktop apps, via AT-SPI. -# -# X11 only routes keystrokes to the focused toplevel's focused widget, so the -# driver reads/types into GUI apps focus-free through AT-SPI instead. This test -# stands up an AT-SPI accessibility bus, launches a real app in the background -# (a separate control terminal stays focused), then drives cua-driver against -# the inactive app window and asserts the accessibility tree is reachable. -# -# Two classes of entry live in `apps`: -# -# 1. SKELETON entries (skeleton = true) — the real-app matrix. These run a -# LENIENT, READ-ONLY smoke test: find the app window, drive cua-driver -# `page get_text` (read) against it, assert it returned a non-error -# accessibility response, assert focus stayed on the control terminal, and -# produce a GIF. Focus-free WRITE / typed-text assertions are intentionally -# OUT OF SCOPE here — they are added later per-app via trajectories. -# -# 2. Full entries (skeleton unset) — chromium and tk. These keep their -# original full behaviour: chromium exercises the approved CDP focus-free -# *write* override (Input.insertText into the background window); tk -# exercises the Tk `send` focus-free write override. Both also type via the -# native AT-SPI path and read it back. -# -# Each run screen-records X11 display :99 into a per-app animated GIF -# (/tmp/cua-driver-linux-background-gui-.gif) and copies it into the test -# derivation's $out, so the Nix workflow's `visual: true` artifact upload finds -# a `.gif` for every matrix job. The GIF is stopped and copied out before any -# toolkit assertion can fail, so even failing jobs still upload one. -# -# `app` selects one entry from `apps`, so flake.nix wires one matrix job per app. -# To run: nix build .#checks.x86_64-linux.cua-driver-linux-background-gui- -{ - pkgs, - lib ? pkgs.lib, - cuaDriverModule, - app, - ... -}: - -let - typed = "cuatyped1234"; - - # Records an animated GIF of X11 display :99 while the driver interacts with - # the background window, so every matrix job (not just the dedicated GIF - # tests) produces a `.gif` artifact for the Nix workflow's `visual: true` - # upload. Shared with the linux-cursor-click / linux-background-terminal GIF - # tests. Needs `pkgs.imagemagick` in environment.systemPackages (below). - recordGifScript = import ./record-x11-gif.nix { inherit pkgs; }; - - # openbox config with focusNew=no so background apps can't steal X focus when - # their window maps mid-test (see openbox-rc.nix). - openboxRc = import ./openbox-rc.nix { inherit pkgs; }; - - # Distinct per-app GIF name so concurrent matrix jobs and their artifacts - # never collide; the workflow's `find -L "/" -name '*.gif'` picks it - # up once it has been copied into the test derivation's $out. - outputGif = "/tmp/cua-driver-linux-background-gui-${app}.gif"; - - # Per-app still screenshot + AT-SPI overlay PNGs. The raw PNG is a full-screen - # capture whose pixel coordinates align 1:1 with the AT-SPI screen-coordinate - # bounds (no translation), so the overlay can draw element boxes directly. - rawPng = "/tmp/cua-driver-linux-background-gui-${app}.png"; - atspiPng = "/tmp/cua-driver-linux-background-gui-${app}-atspi.png"; - # Per-app copy of the AT-SPI element bounds JSON ({element_index, role, name, - # x, y, width, height} in screen coords) — emitted as a CI artifact so the - # coordinates are inspectable alongside the annotated screenshots. - elementsJson = "/tmp/cua-driver-linux-background-gui-${app}-elements.json"; - - # Reads /tmp/cua-elements.json and draws each element's screen-coordinate box - # + label onto a copy of the raw PNG via a single ImageMagick `convert`. If - # the element list is empty (or anything goes wrong) it just copies the raw - # PNG through, so an overlay hiccup never fails the job. - # stdlib-python overlay: reads the raw PNG + /tmp/cua-elements.json and shells - # out to ImageMagick `convert` to draw a red box + label per element, in a - # single invocation. Tolerant: on an empty list or ANY error it copies the raw - # PNG through, so the *-atspi.png artifact is always produced. - atspiOverlayPy = pkgs.writeText "cua-atspi-overlay.py" '' - import json, os, shutil, subprocess, sys - - CONVERT = "${pkgs.imagemagick}/bin/convert" - # -annotate renders TEXT and needs an explicit font: the minimal test container has - # no fontconfig-discoverable fonts, so convert exits 1 without this. - FONT = "${pkgs.dejavu_fonts}/share/fonts/truetype/DejaVuSans.ttf" - - def main(): - raw, out = sys.argv[1], sys.argv[2] - elems_path = sys.argv[3] if len(sys.argv) > 3 else "/tmp/cua-elements.json" - if not (os.path.exists(raw) and os.path.getsize(raw) > 0): - return - try: - with open(elems_path) as f: - elems = json.load(f) - except Exception: - elems = [] - if not isinstance(elems, list): - elems = [] - argv = [CONVERT, raw] - drew = False - for e in elems: - try: - x = int(e["x"]); y = int(e["y"]) - w = int(e["width"]); h = int(e["height"]) - idx = e.get("element_index") - except Exception: - continue - # Skip AT-SPI's "no extents" sentinel (i32::MIN) and degenerate - # 1x1 boxes from unrealized widgets (items inside closed menus) — - # ImageMagick errors out on those coordinates. The driver filters - # these too; this is belt-and-braces for older driver builds. - if x < 0 or y < 0 or x > 16384 or y > 16384 or w <= 1 or h <= 1: - continue - label = "%s (%d,%d %dx%d)" % (idx, x, y, w, h) - argv += ["-stroke", "red", "-fill", "none", - "-draw", "rectangle %d,%d %d,%d" % (x, y, x + w, y + h)] - argv += ["-stroke", "none", "-fill", "red", "-font", FONT, - "-pointsize", "12", - "-annotate", "+%d+%d" % (x + 2, max(y + 12, 12)), label] - drew = True - if not drew: - shutil.copyfile(raw, out) - print("ATSPI_OVERLAY: no elements, copied raw -> " + out, flush=True) - return - argv.append(out) - try: - r = subprocess.run(argv, capture_output=True, text=True) - if r.returncode != 0: - raise RuntimeError("convert rc=%d stderr=%s" % (r.returncode, r.stderr[-500:])) - print("ATSPI_OVERLAY: drew overlay -> " + out, flush=True) - except Exception as ex: - print("ATSPI_OVERLAY_ERROR: " + repr(ex), flush=True) - shutil.copyfile(raw, out) - - if __name__ == "__main__": - try: - main() - except Exception as ex: - print("ATSPI_OVERLAY_FATAL: " + repr(ex), flush=True) - ''; - - # Plain python3 (stdlib only) to drive the MCP JSON-RPC handshake in the test. - # The driver now speaks AT-SPI natively over D-Bus (no pyatspi / GI typelibs), - # so no accessibility Python packages are needed anywhere. - testPython = pkgs.python3; - - # Shared environment: same session D-Bus + a11y settings for the apps and the - # driver, so the driver's native AT-SPI client reaches the same registry the - # apps register with. Fixed bus path so every `machine.*` shell can opt in. - a11yEnv = lib.concatStringsSep " " [ - "DISPLAY=:99" - "DBUS_SESSION_BUS_ADDRESS=unix:path=/tmp/cua-session-bus" - "XDG_RUNTIME_DIR=/run/user/0" - "XDG_DATA_DIRS=/run/current-system/sw/share" - # A GTK3 app dlopens libatk-bridge-2.0.so by soname to join the AT-SPI bus; - # in this hand-rolled session it isn't on the loader path, so expose it. - # (Chromium ships its own AT-SPI implementation and doesn't need this.) - "LD_LIBRARY_PATH=${pkgs.at-spi2-atk}/lib" - # GTK3 only exports its accessible tree when assistive tech is enabled. That - # flag is the GSettings key org.gnome.desktop.interface toolkit-accessibility. - # Use the keyfile backend with a shared config dir so a one-shot `gsettings - # set` is visible to every app + the bus launcher, without poking org.a11y.Bus - # at runtime (which D-Bus-activates a second launcher and breaks the bus). - "GSETTINGS_BACKEND=keyfile" - "XDG_CONFIG_HOME=/tmp/cua-cfg" - "GSETTINGS_SCHEMA_DIR=${pkgs.gsettings-desktop-schemas}/share/gsettings-schemas/${pkgs.gsettings-desktop-schemas.name}/glib-2.0/schemas" - "GTK_MODULES=gail:atk-bridge" - "GNOME_ACCESSIBILITY=1" - "QT_ACCESSIBILITY=1" - "NO_AT_BRIDGE=0" - ]; - - # A page whose input is autofocused; its title is fixed so the window can be - # found by name. Readback is via AT-SPI, so no JS mirroring is needed. Used by - # the chromium full entry. - htmlFile = pkgs.writeText "cua-input.html" '' - cua-initial - - ''; - - # ── Per-toolkit launch-environment helpers (shared across the real-app sets) ── - - # GTK4 talks AT-SPI directly (not via atk-bridge), but only exports its - # accessible tree when it selects the AT-SPI backend at startup; GTK_A11Y=atspi - # forces it on. x11 backend + cairo renderer keep it headless-safe. - gtk4EnvExports = '' - export GTK_A11Y=atspi - export GDK_BACKEND=x11 - export GSK_RENDERER=cairo - ''; - - # Qt5: point Qt at qtbase's xcb platform plugin (bare apps don't always inherit - # it) and force the AT-SPI bridge on regardless of the bus enabled-handshake. - qt5EnvExports = '' - export QT_QPA_PLATFORM=xcb - export QT_PLUGIN_PATH=${pkgs.qt5.qtbase}/${pkgs.qt5.qtbase.qtPluginPrefix} - export QT_QPA_PLATFORM_PLUGIN_PATH=${pkgs.qt5.qtbase}/${pkgs.qt5.qtbase.qtPluginPrefix}/platforms - export QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1 - export QT_ACCESSIBILITY=1 - ''; - - # Qt6: Qt 6.5+ aborts loading the xcb plugin unless libxcb-cursor is present; - # put it on LD_LIBRARY_PATH and force the AT-SPI bridge on. - qt6EnvExports = '' - export QT_QPA_PLATFORM=xcb - export LD_LIBRARY_PATH=${pkgs.xcb-util-cursor}/lib:''${LD_LIBRARY_PATH:-} - export QT_LINUX_ACCESSIBILITY_ALWAYS_ON=1 - export QT_ACCESSIBILITY=1 - ''; - - # Electron: heavy Chromium embed; standard headless-safe flags. - electronCommonFlags = "--no-sandbox --disable-gpu --disable-dev-shm-usage"; - - # ── CDP (Chrome DevTools Protocol) focus-free write — approved override ────── - # AT-SPI exposes Chromium read-only, so the driver can't write into a - # *background* browser window through it. CDP talks to the renderer over the - # debug socket instead: Input.insertText lands in the page's focused DOM - # element (document.activeElement) regardless of whether the OS window holds X - # focus. This is a Chromium-specific override, not the generic path. - cdpPort = 9222; - cdpMarker = "cdptyped5678"; - - # Self-contained CDP client (stdlib only: HTTP target discovery + a minimal - # RFC-6455 WebSocket client), so it runs under plain python3 with no extra deps. - cdpWriteScript = pkgs.writeText "cdp-write.py" '' - import json, sys, time, socket, base64, os, struct, urllib.request - from urllib.parse import urlparse - - PORT = ${toString cdpPort} - MARKER = "${cdpMarker}" - - def http_json(path): - url = "http://127.0.0.1:%d%s" % (PORT, path) - with urllib.request.urlopen(url, timeout=10) as r: - return json.load(r) - - def pick_page(): - cands = [t for t in http_json("/json") - if t.get("type") == "page" and t.get("webSocketDebuggerUrl")] - for t in cands: - if t.get("title") == "cua-initial" or "cua-input" in t.get("url", ""): - return t["webSocketDebuggerUrl"] - return cands[0]["webSocketDebuggerUrl"] if cands else None - - class WS: - def __init__(self, url): - u = urlparse(url) - self.sock = socket.create_connection((u.hostname, u.port), timeout=15) - key = base64.b64encode(os.urandom(16)).decode() - path = (u.path or "/") + (("?" + u.query) if u.query else "") - req = ( - "GET %s HTTP/1.1\r\nHost: %s:%d\r\nUpgrade: websocket\r\n" - "Connection: Upgrade\r\nSec-WebSocket-Key: %s\r\n" - "Sec-WebSocket-Version: 13\r\n\r\n" - ) % (path, u.hostname, u.port, key) - self.sock.sendall(req.encode()) - self._buf = b"" - while b"\r\n\r\n" not in self._buf: - chunk = self.sock.recv(4096) - if not chunk: - raise RuntimeError("ws handshake closed early") - self._buf += chunk - head, self._buf = self._buf.split(b"\r\n\r\n", 1) - if b"101" not in head.split(b"\r\n")[0]: - raise RuntimeError("ws handshake failed: " + head.decode("latin1")) - - def _exact(self, n): - while len(self._buf) < n: - chunk = self.sock.recv(4096) - if not chunk: - raise RuntimeError("ws closed") - self._buf += chunk - out, self._buf = self._buf[:n], self._buf[n:] - return out - - def send_text(self, text): - payload = text.encode() - n = len(payload) - header = bytearray([0x81]) - if n < 126: - header.append(0x80 | n) - elif n < 65536: - header.append(0x80 | 126); header += struct.pack(">H", n) - else: - header.append(0x80 | 127); header += struct.pack(">Q", n) - mask = os.urandom(4) - header += mask - self.sock.sendall(bytes(header) + bytes(b ^ mask[i % 4] for i, b in enumerate(payload))) - - def recv_text(self): - data = b"" - while True: - b0, b1 = self._exact(2) - fin, opcode, masked, length = b0 & 0x80, b0 & 0x0F, b1 & 0x80, b1 & 0x7F - if length == 126: - length = struct.unpack(">H", self._exact(2))[0] - elif length == 127: - length = struct.unpack(">Q", self._exact(8))[0] - mask = self._exact(4) if masked else None - payload = self._exact(length) - if mask: - payload = bytes(b ^ mask[i % 4] for i, b in enumerate(payload)) - if opcode == 0x8: - raise RuntimeError("ws closed by server") - if opcode in (0x9, 0xA): - continue - data += payload - if fin: - return data.decode() - - def close(self): - try: - self.sock.close() - except Exception: - pass - - ws_url = None - for _ in range(30): - try: - ws_url = pick_page() - except Exception: - ws_url = None - if ws_url: - break - time.sleep(1) - if not ws_url: - print("NO_CDP_PAGE_TARGET", flush=True); sys.exit(1) - - ws = WS(ws_url) - _id = [0] - def cmd(method, params=None): - _id[0] += 1 - mid = _id[0] - ws.send_text(json.dumps({"id": mid, "method": method, "params": params or {}})) - while True: - msg = json.loads(ws.recv_text()) - if msg.get("id") == mid: - return msg - - cmd("Runtime.enable") - cmd("DOM.enable") - # Focus + clear the page input through the DOM (not OS window focus). - cmd("Runtime.evaluate", {"expression": 'var i=document.querySelector("input"); i.focus(); i.value=""; "ok"'}) - # The override: CDP injects into the renderer's focused element, no OS focus. - cmd("Input.insertText", {"text": MARKER}) - time.sleep(0.3) - res = cmd("Runtime.evaluate", {"expression": "document.querySelector('input').value", "returnByValue": True}) - val = res.get("result", {}).get("result", {}).get("value", "") - print("CDP_VALUE: " + repr(val), flush=True) - ws.close() - if val == MARKER: - print("CDP_READBACK_OK", flush=True); sys.exit(0) - print("CDP_READBACK_MISMATCH", flush=True); sys.exit(1) - ''; - - # Tk (tkinter) app — Tk has no AT-SPI bridge, so focus-free writes use Tk's - # `send` command instead: the app registers itself with a known name, and the - # driver injects text by invoking `wish` to send Tcl commands over X11 IPC. - # This is the Tk-specific override (like CDP for Chromium), proving that - # non-accessible toolkits can still support background input with bespoke paths. - tkEnv = pkgs.python3.withPackages (ps: [ ps.tkinter ]); - tkScript = pkgs.writeText "cua-tk.py" '' - import tkinter as tk - root = tk.Tk() - root.title("cua-initial") - # Register the app with a known name so `send` commands can reach it. - tk._default_root.tk.call('tk', 'appname', 'cua-tk-target') - entry = tk.Entry(root, width=40, name='entry') - entry.pack(padx=20, pady=20) - entry.focus_set() - root.geometry("400x120+700+150") - root.mainloop() - ''; - - # ── Helpers to build real-app skeleton entries tersely ────────────────────── - # mkSkeleton builds a "skeleton = true" app entry: a launch script (the given - # `cmd` run with the per-toolkit env exports) plus a `windowMatch` xdotool - # search expression used to find the background window. - mkSkeleton = - { - packages, - envExports ? "", - cmd, - windowMatch, - }: - { - inherit packages windowMatch; - skeleton = true; - launch = pkgs.writeShellScript "cua-launch-${app}.sh" '' - ${envExports} - exec ${cmd} - ''; - }; - - apps = { - # ── GTK3 real apps (READ-ONLY SKELETON) ───────────────────────────────── - gtk3-gedit = mkSkeleton { - packages = [ pkgs.gedit ]; - cmd = "${pkgs.gedit}/bin/gedit --new-window"; - windowMatch = "--class gedit"; - }; - gtk3-mousepad = mkSkeleton { - packages = [ pkgs.mousepad ]; - cmd = "${pkgs.mousepad}/bin/mousepad --disable-server"; - windowMatch = "--class mousepad"; - }; - gtk3-geany = mkSkeleton { - packages = [ pkgs.geany ]; - cmd = "${pkgs.geany}/bin/geany"; - windowMatch = "--class geany"; - }; - gtk3-scite = mkSkeleton { - packages = [ pkgs.scite ]; - cmd = "${pkgs.scite}/bin/SciTE"; - windowMatch = "--class scite"; - }; - gtk3-abiword = mkSkeleton { - packages = [ pkgs.abiword ]; - cmd = "${pkgs.abiword}/bin/abiword"; - windowMatch = "--class abiword"; - }; - - # ── GTK4 real apps (READ-ONLY SKELETON) ───────────────────────────────── - # Only gnome-characters reliably surfaces a window headless; the other GNOME - # GTK4 apps (text-editor/console/contacts/calendar) never mapped a window - # within 120s in CI (missing portals/EDS/VTE runtime), so they were dropped. - gtk4-characters = mkSkeleton { - packages = [ pkgs.gnome-characters ]; - envExports = gtk4EnvExports; - cmd = "${pkgs.gnome-characters}/bin/gnome-characters"; - windowMatch = "--class org.gnome.Characters"; - }; - - # ── Qt5 real apps (READ-ONLY SKELETON) ────────────────────────────────── - # manuskript is PyQt5; klog/openambit are Qt5 (qtbase 5.15.x). wsjtx (no - # window headless) and qsstv (steals focus on launch) were dropped. - qt5-manuskript = mkSkeleton { - packages = [ pkgs.manuskript ]; - envExports = qt5EnvExports; - cmd = "${pkgs.manuskript}/bin/manuskript"; - windowMatch = "--class manuskript"; - }; - qt5-klog = mkSkeleton { - packages = [ pkgs.klog ]; - envExports = qt5EnvExports; - cmd = "${pkgs.klog}/bin/klog"; - windowMatch = "--class klog"; - }; - qt5-openambit = mkSkeleton { - packages = [ pkgs.openambit ]; - envExports = qt5EnvExports; - cmd = "${pkgs.openambit}/bin/openambit"; - windowMatch = "--class openambit"; - }; - - # ── Qt6 real apps (READ-ONLY SKELETON) ────────────────────────────────── - # All from the kdePackages (Qt6) scope, plus qownnotes (Qt6). ghostwriter - # was dropped (no window surfaced headless within 120s). - qt6-kate = mkSkeleton { - packages = [ pkgs.kdePackages.kate ]; - envExports = qt6EnvExports; - cmd = "${pkgs.kdePackages.kate}/bin/kate --new"; - windowMatch = "--class kate"; - }; - qt6-kcalc = mkSkeleton { - packages = [ pkgs.kdePackages.kcalc ]; - envExports = qt6EnvExports; - cmd = "${pkgs.kdePackages.kcalc}/bin/kcalc"; - windowMatch = "--class kcalc"; - }; - qt6-okular = mkSkeleton { - packages = [ pkgs.kdePackages.okular ]; - envExports = qt6EnvExports; - cmd = "${pkgs.kdePackages.okular}/bin/okular"; - windowMatch = "--class okular"; - }; - qt6-qownnotes = mkSkeleton { - packages = [ pkgs.qownnotes ]; - envExports = qt6EnvExports; - cmd = "${pkgs.qownnotes}/bin/QOwnNotes"; - windowMatch = "--class qownnotes"; - }; - - # ── Electron real apps (READ-ONLY SKELETON) ───────────────────────────── - # Heavy Chromium embeds; given a longer CI timeout. Read-only - # skeleton (CDP write is exercised by the chromium full entry instead). - # marktext (steals focus on launch) and vscodium (no window headless within - # 120s) were dropped. - electron-zettlr = mkSkeleton { - packages = [ pkgs.zettlr ]; - cmd = "${pkgs.zettlr}/bin/zettlr ${electronCommonFlags}"; - windowMatch = "--class zettlr"; - }; - electron-joplin = mkSkeleton { - packages = [ pkgs.joplin-desktop ]; - cmd = "${pkgs.joplin-desktop}/bin/joplin-desktop ${electronCommonFlags}"; - windowMatch = "--class joplin"; - }; - electron-logseq = mkSkeleton { - packages = [ pkgs.logseq ]; - cmd = "${pkgs.logseq}/bin/logseq ${electronCommonFlags}"; - windowMatch = "--class logseq"; - }; - - # ── Full entries (unchanged behaviour): chromium (CDP) + tk (send) ─────── - chromium = { - packages = [ pkgs.chromium ]; - cdp = true; - windowMatch = "--name cua-initial"; - launch = pkgs.writeShellScript "cua-launch-chromium.sh" '' - exec ${pkgs.chromium}/bin/chromium \ - --no-sandbox --no-first-run --no-default-browser-check --disable-gpu \ - --force-renderer-accessibility \ - --remote-debugging-port=${toString cdpPort} --remote-allow-origins=* \ - --disable-backgrounding-occluded-windows --disable-renderer-backgrounding \ - --user-data-dir=/tmp/cua-chromium --window-position=700,150 --window-size=480,360 \ - --new-window file://${htmlFile} - ''; - }; - tk = { - packages = [ tkEnv pkgs.tk ]; - tksend = true; - windowMatch = "--name cua-initial"; - launch = pkgs.writeShellScript "cua-launch-tk.sh" '' - exec ${tkEnv}/bin/python3 ${tkScript} - ''; - }; - }; - - selected = apps.${app}; - isSkeleton = selected.skeleton or false; - - # CDP focus-free write subtest — only for Chromium-backed full entries. Asserts - # the approved override writes into the *background* window while the control - # terminal keeps focus. (Skeleton entries skip this.) - cdpSubtest = lib.optionalString (selected.cdp or false) '' - with subtest("CDP focus-free write into the background window (approved override)"): - # CDP reaches the renderer over the debug socket, so Input.insertText lands - # in the page's focused DOM element while the OS window stays in the - # background. This is the one path that writes into an unfocused browser - # window; AT-SPI exposes Chromium read-only. - machine.copy_from_host("${cdpWriteScript}", "/tmp/cdp-write.py") - cdp_out = machine.succeed("${a11yEnv} timeout 120 python3 /tmp/cdp-write.py 2>&1") - machine.log(cdp_out) - assert "CDP_READBACK_OK" in cdp_out, cdp_out - # The override must remain focus-free: control terminal still active. - cdp_control = machine.succeed("head -1 /tmp/control-xid.txt").strip() - cdp_active = machine.succeed("DISPLAY=:99 xdotool getactivewindow").strip() - assert cdp_control == cdp_active, "focus moved during CDP write: got " + cdp_active - ''; - - # Tk send focus-free write subtest — only for the Tk full entry. Reads the - # entry value back over Tk `send`, guarded by a Tcl `after` timer + `timeout` - # backstop so wish always terminates promptly. - tkGetScript = pkgs.writeText "tk-get-value.tcl" '' - set ::rc 1 - after 20000 { - puts "TK_SEND_READBACK_TIMEOUT" - flush stdout - exit 1 - } - if {[catch {send cua-tk-target {.entry get}} val]} { - puts "TK_SEND_READBACK_ERROR: $val" - flush stdout - exit 1 - } - puts $val - flush stdout - exit 0 - ''; - tkSubtest = lib.optionalString (selected.tksend or false) '' - with subtest("Tk send focus-free write into the background window (Tk override)"): - # The driver already typed via inject_tk_send in the main test. Now read - # the entry widget's value back via Tk send to prove the write landed. - machine.copy_from_host("${tkGetScript}", "/tmp/tk-get-value.tcl") - status, tk_readback = machine.execute("${a11yEnv} timeout 30 ${pkgs.tk}/bin/wish /tmp/tk-get-value.tcl 2>&1") - tk_readback = tk_readback.strip() - machine.log("Tk send readback (exit=" + str(status) + "): " + repr(tk_readback)) - assert "${typed}" in tk_readback, f"Expected '${typed}' in Tk entry, got (exit={status}): {tk_readback}" - # The override must remain focus-free: control terminal still active. - tk_control = machine.succeed("head -1 /tmp/control-xid.txt").strip() - tk_active = machine.succeed("DISPLAY=:99 xdotool getactivewindow").strip() - assert tk_control == tk_active, "focus moved during Tk write: got " + tk_active - ''; - - # Full-entry MCP driver script: type via native AT-SPI then read back. - mcpTest = pkgs.writeText "mcp-background-gui-test.py" '' - import json, os, sys, threading, time - - DRIVER_BIN = os.environ.get("CUA_DRIVER_BIN", "cua-driver") - - def start_driver(): - import subprocess - env = {**os.environ, "CUA_ATSPI_DEBUG": "1"} - proc = subprocess.Popen( - [DRIVER_BIN, "mcp", "--no-daemon-relaunch"], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=env, - ) - def drain(): - for line in proc.stderr: - sys.stderr.buffer.write(line); sys.stderr.buffer.flush() - threading.Thread(target=drain, daemon=True).start() - return proc - - def send(proc, method, params=None, req_id=None): - msg = {"jsonrpc": "2.0", "method": method} - if params is not None: - msg["params"] = params - if req_id is not None: - msg["id"] = req_id - proc.stdin.write((json.dumps(msg) + "\n").encode()); proc.stdin.flush() - - def recv(proc, timeout=45): - result = [None] - def reader(): - result[0] = proc.stdout.readline() - t = threading.Thread(target=reader); t.start(); t.join(timeout) - if t.is_alive(): - raise TimeoutError("No response within timeout") - line = result[0].decode().strip() - if not line: - raise RuntimeError("Driver returned an empty response") - return json.loads(line) - - def call_tool(proc, req_id, name, arguments): - send(proc, "tools/call", {"name": name, "arguments": arguments}, req_id=req_id) - resp = recv(proc) - if resp.get("error"): - raise RuntimeError(f"{name} failed: {resp}") - if resp.get("result", {}).get("isError"): - raise RuntimeError(f"{name} returned isError: {resp}") - return resp - - def main(): - with open("/tmp/target-xid.txt") as f: - target_xid = int(f.read().strip()) - with open("/tmp/target-pid.txt") as f: - target_pid = int(f.read().strip()) - - proc = start_driver() - try: - send(proc, "initialize", { - "protocolVersion": "2024-11-05", - "capabilities": {}, - "clientInfo": {"name": "nixos-background-gui-test", "version": "1.0.0"}, - }, req_id=1) - recv(proc) - send(proc, "notifications/initialized", {}) - time.sleep(0.3) - # Type into the *inactive* app window — focus-free, via native AT-SPI. - call_tool(proc, 2, "type_text", { - "pid": target_pid, - "window_id": target_xid, - "text": "${typed}", - }) - time.sleep(1.5) - print("background GUI test typed", flush=True) - - # Read it back through the driver's *own* native AT-SPI client. - readback = "" - last_resp = None - for _ in range(8): - resp = call_tool(proc, 3, "page", { - "action": "get_text", - "pid": target_pid, - "window_id": target_xid, - }) - last_resp = resp - content = resp.get("result", {}).get("content", []) - readback = " ".join( - c.get("text", "") for c in content if c.get("type") == "text" - ) - if "${typed}" in readback: - break - time.sleep(1.0) - print("RAW_GET_TEXT_RESPONSE: " + json.dumps(last_resp), flush=True) - print("READBACK_BEGIN", flush=True) - print(readback, flush=True) - print("READBACK_END", flush=True) - finally: - proc.stdin.close(); proc.terminate(); proc.wait(timeout=5) - - if __name__ == "__main__": - main() - ''; - - # ── SKELETON: read-only smoke + GIF; focus-free WRITE / typed-text assertions - # are added later via trajectories. ───────────────────────────────────────── - # Skeleton MCP driver script: ONLY drives `page get_text` (read) against the - # found window. It does NOT call type_text and does NOT assert any typed text. - # It prints the raw get_text response so the testScript can check it returned a - # non-error accessibility payload. - skeletonMcpTest = pkgs.writeText "mcp-background-gui-skeleton.py" '' - import json, os, sys, threading, time - - DRIVER_BIN = os.environ.get("CUA_DRIVER_BIN", "cua-driver") - - def start_driver(): - import subprocess - env = {**os.environ, "CUA_ATSPI_DEBUG": "1"} - proc = subprocess.Popen( - [DRIVER_BIN, "mcp", "--no-daemon-relaunch"], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=env, - ) - def drain(): - for line in proc.stderr: - sys.stderr.buffer.write(line); sys.stderr.buffer.flush() - threading.Thread(target=drain, daemon=True).start() - return proc - - def send(proc, method, params=None, req_id=None): - msg = {"jsonrpc": "2.0", "method": method} - if params is not None: - msg["params"] = params - if req_id is not None: - msg["id"] = req_id - proc.stdin.write((json.dumps(msg) + "\n").encode()); proc.stdin.flush() - - def recv(proc, timeout=45): - result = [None] - def reader(): - result[0] = proc.stdout.readline() - t = threading.Thread(target=reader); t.start(); t.join(timeout) - if t.is_alive(): - raise TimeoutError("No response within timeout") - line = result[0].decode().strip() - if not line: - raise RuntimeError("Driver returned an empty response") - return json.loads(line) - - def main(): - with open("/tmp/target-xid.txt") as f: - target_xid = int(f.read().strip()) - with open("/tmp/target-pid.txt") as f: - target_pid = int(f.read().strip()) - - proc = start_driver() - try: - send(proc, "initialize", { - "protocolVersion": "2024-11-05", - "capabilities": {}, - "clientInfo": {"name": "nixos-background-gui-skeleton", "version": "1.0.0"}, - }, req_id=1) - recv(proc) - send(proc, "notifications/initialized", {}) - time.sleep(0.3) - - # READ-ONLY: drive page/get_text against the inactive window. Retry a - # few times — real apps (Electron/KDE) take a moment to build their - # accessibility tree after first paint. - readback = "" - last_resp = None - is_error = True - for _ in range(10): - send(proc, "tools/call", { - "name": "page", - "arguments": { - "action": "get_text", - "pid": target_pid, - "window_id": target_xid, - }, - }, req_id=3) - resp = recv(proc) - last_resp = resp - # A transport-level error or isError=true is a non-result; keep - # retrying. Any structured content counts as a non-error response. - if resp.get("error") or resp.get("result", {}).get("isError"): - time.sleep(1.0) - continue - content = resp.get("result", {}).get("content", []) - readback = " ".join( - c.get("text", "") for c in content if c.get("type") == "text" - ) - is_error = False - if readback.strip(): - break - time.sleep(1.0) - - print("RAW_GET_TEXT_RESPONSE: " + json.dumps(last_resp), flush=True) - print("GET_TEXT_IS_ERROR: " + ("yes" if is_error else "no"), flush=True) - if not is_error: - print("GET_TEXT_OK", flush=True) - print("READBACK_BEGIN", flush=True) - print(readback, flush=True) - print("READBACK_END", flush=True) - - # READ-ONLY: pull the AT-SPI element bounds via get_window_state so - # we can draw an overlay PNG as a CI artifact. Tolerant — any - # failure here just yields an empty element list; it never affects - # the read-only assertions above. - elements = [] - try: - send(proc, "tools/call", { - "name": "get_window_state", - "arguments": { - "pid": target_pid, - "window_id": target_xid, - }, - }, req_id=4) - # Bounds collection does one D-Bus GetExtents round-trip per - # action node; big trees (geany walked 787 nodes) need well over - # the default 45s, so give this call a longer window. - ws = recv(proc, timeout=90) - result_obj = ws.get("result", {}) if isinstance(ws, dict) else {} - # The structured `elements` array lives in structuredContent; - # fall back to scanning any text content that carries JSON. - structured = result_obj.get("structuredContent") or {} - if isinstance(structured, dict) and isinstance(structured.get("elements"), list): - elements = structured["elements"] - else: - for c in result_obj.get("content", []): - if c.get("type") == "text": - try: - obj = json.loads(c.get("text", "")) - except Exception: - continue - if isinstance(obj, dict) and isinstance(obj.get("elements"), list): - elements = obj["elements"] - break - except Exception as e: - print("GET_WINDOW_STATE_ERROR: " + repr(e), flush=True) - elements = [] - if not isinstance(elements, list): - elements = [] - with open("/tmp/cua-elements.json", "w") as f: - json.dump(elements, f) - print("ELEMENTS_JSON: " + json.dumps(elements), flush=True) - print("ELEMENTS_COUNT: " + str(len(elements)), flush=True) - finally: - proc.stdin.close(); proc.terminate(); proc.wait(timeout=5) - - if __name__ == "__main__": - main() - ''; - - # The window-find shell command, parameterised on the app's windowMatch. Tries - # the toolkit class/name match first, then falls back to the launched PID's - # window, then the newest visible window — so real apps that don't expose the - # expected class still surface a window for the read-only drive. - # A store-path script (not an inline multi-line string): it is interpolated - # into the Python testScript as a single `machine.wait_until_succeeds("...")` - # argument. A multi-line shell snippet with embedded quotes/newlines would - # break that Python string literal (it did — every GUI job failed the - # testScript type-check). As a script path it is one safe token. - windowFindCmd = pkgs.writeShellScript "cua-window-find.sh" '' - export DISPLAY=:99 - xid=$(${pkgs.xdotool}/bin/xdotool search --sync --onlyvisible ${selected.windowMatch} 2>/dev/null | head -1) - if [ -z "$xid" ]; then - xid=$(${pkgs.xdotool}/bin/xdotool search --all --pid "$(cat /tmp/target-pid.txt)" 2>/dev/null | head -1) - fi - if [ -z "$xid" ]; then - xid=$(${pkgs.xdotool}/bin/xdotool search --onlyvisible "" 2>/dev/null | tail -1) - fi - test -n "$xid" && printf "%s" "$xid" >/tmp/target-xid.txt && test -s /tmp/target-xid.txt - ''; - - # ── Skeleton (read-only) drive + assertions ───────────────────────────────── - skeletonDrive = '' - with subtest("SKELETON read-only: drive cua-driver page/get_text against the inactive window"): - # SKELETON: read-only smoke + GIF; focus-free WRITE / typed-text - # assertions are added later via trajectories. This path only proves the - # app window appeared and the driver can READ its accessibility tree. - machine.copy_from_host("${skeletonMcpTest}", "/tmp/mcp-background-gui-skeleton.py") - machine.execute( - "sh -lc '${recordGifScript} :99 /tmp/gui-frames ${outputGif} " - "/tmp/stop-gui-recorder /tmp/record-gui.log 10 0.2 >/dev/null 2>&1 & echo $! >/tmp/record-gui.pid'" - ) - # 300s: get_text retries + the bounded get_window_state bounds walk - # (recv timeout 150s) must both fit. - status, result = machine.execute("${a11yEnv} timeout 300 python3 /tmp/mcp-background-gui-skeleton.py 2>&1") - machine.log(result) - # Stop the recorder and copy the GIF out *now*, before any assertion can - # fail, so every matrix job uploads a GIF of the interaction. - machine.execute("touch /tmp/stop-gui-recorder") - machine.execute("timeout 60 sh -lc 'while kill -0 $(cat /tmp/record-gui.pid) 2>/dev/null; do sleep 0.2; done'") - # If the recorder (or its convert) is still grinding past the wait, - # kill it hard so it can't thrash the container and wedge later commands. - machine.execute("pkill -9 -f record-x11-gif >/dev/null 2>&1; pkill -9 -x convert >/dev/null 2>&1; pkill -9 -x import >/dev/null 2>&1; true") - machine.log(machine.execute("sh -lc 'cat /tmp/record-gui.log || true'")[1]) - # Best-effort: a missing GIF (recorder/convert hiccup under load) must - # not fail the read-only job — copy only when the file exists. - if machine.execute("test -s ${outputGif}")[0] == 0: - machine.copy_from_machine("${outputGif}", "") - else: - machine.log("WARN: ${outputGif} missing; skipping GIF copy") - - # Annotated screenshots (CI artifacts), produced BEFORE any assertion can - # fail and wrapped in execute() so a capture/drawing hiccup never fails - # the read-only job: - # ${rawPng} full-screen still (screen coords == AT-SPI bounds) - # ${atspiPng} raw + AT-SPI element boxes/labels overlay - # The downstream `som-annotate` job consumes the raw PNG to emit *-som.png. - # `import` can block indefinitely if another client wedges the X server - # grab (it hung a job to the GH 15-min cap once) — bound it hard. - machine.execute("${a11yEnv} timeout 30 ${pkgs.imagemagick}/bin/import -window root ${rawPng}") - machine.copy_from_host("${atspiOverlayPy}", "/tmp/cua-atspi-overlay.py") - st_ov, out_ov = machine.execute( - "${a11yEnv} timeout 60 ${pkgs.python3}/bin/python3 /tmp/cua-atspi-overlay.py " - "${rawPng} ${atspiPng} /tmp/cua-elements.json 2>&1" - ) - machine.log(out_ov) - # Always emit both PNGs; fall back to copying the raw PNG if the overlay - # step produced nothing. Copies are best-effort — a capture hiccup must - # not fail the read-only job. - machine.execute("test -s ${atspiPng} || cp ${rawPng} ${atspiPng} 2>/dev/null") - if machine.execute("test -s ${rawPng}")[0] == 0: - machine.copy_from_machine("${rawPng}", "") - else: - machine.log("WARN: ${rawPng} missing; skipping raw screenshot copy") - if machine.execute("test -s ${atspiPng}")[0] == 0: - machine.copy_from_machine("${atspiPng}", "") - else: - machine.log("WARN: ${atspiPng} missing; skipping atspi overlay copy") - # Emit the element-bounds JSON (the coordinates) as an artifact too. - machine.execute("cp /tmp/cua-elements.json ${elementsJson} 2>/dev/null; true") - if machine.execute("test -s ${elementsJson}")[0] == 0: - machine.copy_from_machine("${elementsJson}", "") - else: - machine.log("WARN: ${elementsJson} missing; skipping elements JSON copy") - - # Lenient assertion: get_text returned a NON-error accessibility response. - # Do NOT require any specific role (entry/text/...) — any content is fine. - assert "GET_TEXT_OK" in result, ( - "driver page/get_text did not return a non-error response for the " - "background app:\n" + result - ) - - with subtest("Focus stayed on the control terminal"): - control = machine.succeed("head -1 /tmp/control-xid.txt").strip() - active = machine.succeed("DISPLAY=:99 xdotool getactivewindow").strip() - assert control == active, "expected active window " + control + ", got " + active - ''; - - # ── Full-entry (chromium/tk) drive + assertions (original behaviour) ───────── - fullDrive = '' - with subtest("Drive cua-driver against the inactive window (AT-SPI)"): - machine.copy_from_host("${mcpTest}", "/tmp/mcp-background-gui-test.py") - machine.execute( - "sh -lc '${recordGifScript} :99 /tmp/gui-frames ${outputGif} " - "/tmp/stop-gui-recorder /tmp/record-gui.log 10 0.2 >/dev/null 2>&1 & echo $! >/tmp/record-gui.pid'" - ) - status, result = machine.execute("${a11yEnv} timeout 200 python3 /tmp/mcp-background-gui-test.py 2>&1") - machine.log(result) - machine.execute("touch /tmp/stop-gui-recorder") - machine.execute("timeout 60 sh -lc 'while kill -0 $(cat /tmp/record-gui.pid) 2>/dev/null; do sleep 0.2; done'") - # If the recorder (or its convert) is still grinding past the wait, - # kill it hard so it can't thrash the container and wedge later commands. - machine.execute("pkill -9 -f record-x11-gif >/dev/null 2>&1; pkill -9 -x convert >/dev/null 2>&1; pkill -9 -x import >/dev/null 2>&1; true") - machine.log(machine.execute("sh -lc 'cat /tmp/record-gui.log || true'")[1]) - # Best-effort GIF copy (see skeleton path): a recorder hiccup must not - # fail the job before the real assertions run. - if machine.execute("test -s ${outputGif}")[0] == 0: - machine.copy_from_machine("${outputGif}", "") - else: - machine.log("WARN: ${outputGif} missing; skipping GIF copy") - assert "background GUI test typed" in result, result - - with subtest("Input landed: driver's native AT-SPI reads the window back"): - assert any(tok in result for tok in ('frame "', 'window "', 'document', 'text "', 'entry "')), ( - "driver get_text did not return an accessibility node for the background app:\n" - + result - ) - - with subtest("Focus stayed on the control terminal"): - control = machine.succeed("head -1 /tmp/control-xid.txt").strip() - active = machine.succeed("DISPLAY=:99 xdotool getactivewindow").strip() - assert control == active, "expected active window " + control + ", got " + active - - ${cdpSubtest} - ${tkSubtest} - ''; - - driveBody = if isSkeleton then skeletonDrive else fullDrive; -in - -pkgs.testers.nixosTest { - name = "cua-driver-linux-background-gui-${app}-test"; - meta.maintainers = [ ]; - - containers.machine = - { pkgs, ... }: - { - imports = [ cuaDriverModule ]; - services.cua-driver.enable = true; - services.dbus.enable = true; - environment.systemPackages = with pkgs; [ - xorg.xorgserver - xterm - openbox - picom - xdotool - imagemagick # `import` + `convert` for the screen-recorded GIF - dbus - at-spi2-core - testPython - jq - procps - glib # `gsettings` to flip toolkit-accessibility - gsettings-desktop-schemas # provides org.gnome.desktop.interface schema - ] ++ selected.packages; - }; - - testScript = '' - # cache-bust 2026-06-08.1: this comment is part of the test derivation, so - # bumping it changes the output path and forces the test to actually re-run - # instead of being substituted from the binary cache. Bump again to re-run. - machine.start() - machine.wait_for_unit("multi-user.target") - - with subtest("Start X11 + session D-Bus + AT-SPI bus"): - machine.execute("Xvfb :99 -screen 0 1280x1024x24 >/tmp/xvfb.log 2>&1 &") - machine.wait_until_succeeds("test -e /tmp/.X11-unix/X99", timeout=10) - machine.execute("DISPLAY=:99 openbox --config-file ${openboxRc} >/tmp/openbox.log 2>&1 &") - machine.execute("DISPLAY=:99 picom --backend xrender >/tmp/picom.log 2>&1 &") - machine.succeed("mkdir -p /run/user/0 && chmod 700 /run/user/0") - machine.execute("dbus-daemon --session --address=unix:path=/tmp/cua-session-bus --fork >/tmp/dbus.log 2>&1") - machine.wait_until_succeeds("test -S /tmp/cua-session-bus", timeout=10) - machine.succeed("mkdir -p /tmp/cua-cfg") - machine.execute("${a11yEnv} ${pkgs.at-spi2-core}/libexec/at-spi-bus-launcher --launch-immediately >/tmp/atspi-launcher.log 2>&1 &") - machine.wait_until_succeeds( - "${a11yEnv} dbus-send --session --print-reply " - "--dest=org.freedesktop.DBus / org.freedesktop.DBus.NameHasOwner " - "string:org.a11y.Bus | grep -q 'boolean true'", - timeout=15, - ) - machine.execute( - "${a11yEnv} dbus-send --session --print-reply --dest=org.a11y.Bus " - "/org/a11y/bus org.freedesktop.DBus.Properties.Set " - "string:org.a11y.Status string:IsEnabled variant:boolean:true 2>&1 | tee /tmp/a11y-enable.log" - ) - machine.log("a11y IsEnabled set: " + machine.execute("cat /tmp/a11y-enable.log")[1]) - machine.execute( - "${a11yEnv} dbus-send --session --print-reply --dest=org.a11y.Bus " - "/org/a11y/bus org.freedesktop.DBus.Properties.Get " - "string:org.a11y.Status string:IsEnabled 2>&1 | tee /tmp/a11y-get.log" - ) - machine.log("a11y IsEnabled get: " + machine.execute("cat /tmp/a11y-get.log")[1]) - machine.log("atspi-launcher.log: " + machine.execute("cat /tmp/atspi-launcher.log")[1]) - - with subtest("Focused control terminal"): - machine.execute("sh -lc 'DISPLAY=:99 xterm -T Control -geometry 60x20+40+120 >/tmp/control.log 2>&1 & echo $! >/tmp/control-pid.txt'") - machine.wait_until_succeeds("DISPLAY=:99 xdotool search --sync --pid $(cat /tmp/control-pid.txt) >/tmp/control-xid.txt", timeout=20) - machine.succeed("DISPLAY=:99 xdotool windowactivate --sync $(head -1 /tmp/control-xid.txt)") - - with subtest("Launch target app (${app}) in the background"): - machine.execute("sh -lc '${a11yEnv} ${selected.launch} >/tmp/target.log 2>&1 & echo $! >/tmp/target-pid.txt'") - # Surface the app's own stdout/stderr early so launch failures (e.g. a Qt - # platform-plugin error) are visible instead of just a window-find timeout. - machine.sleep(5) - machine.log("target.log after launch: " + machine.execute("cat /tmp/target.log")[1]) - # Find the background window via the per-app matcher, with a PID / newest - # -window fallback. Generous timeout: Electron/KDE are slow to first paint. - machine.wait_until_succeeds("${windowFindCmd}", timeout=120) - machine.log("target-xid: " + machine.execute("cat /tmp/target-xid.txt")[1]) - # Keep focus on the control terminal — launching the app in the - # background must not steal focus. - machine.succeed("DISPLAY=:99 xdotool windowactivate --sync $(head -1 /tmp/control-xid.txt)") - machine.succeed("DISPLAY=:99 xdotool windowfocus --sync $(head -1 /tmp/control-xid.txt)") - - ${driveBody} - ''; -} diff --git a/nix/cua-driver/tests/linux-background-terminal-gif.nix b/nix/cua-driver/tests/linux-background-terminal-gif.nix deleted file mode 100644 index 949c129d8d..0000000000 --- a/nix/cua-driver/tests/linux-background-terminal-gif.nix +++ /dev/null @@ -1,203 +0,0 @@ -# Linux background terminal GIF test -# -# Records a GIF while cua-driver types into an inactive xterm window and -# executes a shell command without stealing focus. -# -# To run: nix build .#checks.x86_64-linux.cua-driver-linux-background-terminal-gif -# -{ - pkgs, - lib ? pkgs.lib, - cuaDriverModule, - ... -}: - -let - mcpBackgroundTerminalTest = pkgs.writeText "mcp-background-terminal-gif-test.py" '' - import json - import os - import subprocess - import sys - import threading - import time - - DRIVER_BIN = os.environ.get("CUA_DRIVER_BIN", "cua-driver") - - def start_driver(): - proc = subprocess.Popen( - [DRIVER_BIN, "mcp", "--no-daemon-relaunch"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env={**os.environ}, - ) - - def drain_stderr(): - for line in proc.stderr: - sys.stderr.buffer.write(line) - sys.stderr.buffer.flush() - - threading.Thread(target=drain_stderr, daemon=True).start() - return proc - - def send(proc, method, params=None, req_id=None): - msg = {"jsonrpc": "2.0", "method": method} - if params is not None: - msg["params"] = params - if req_id is not None: - msg["id"] = req_id - proc.stdin.write((json.dumps(msg) + "\n").encode()) - proc.stdin.flush() - - def recv(proc, timeout=30): - result = [None] - - def reader(): - result[0] = proc.stdout.readline() - - thread = threading.Thread(target=reader) - thread.start() - thread.join(timeout) - if thread.is_alive(): - raise TimeoutError("No response within timeout") - line = result[0].decode().strip() - if not line: - raise RuntimeError("Driver returned an empty response") - return json.loads(line) - - def call_tool(proc, req_id, name, arguments): - send(proc, "tools/call", {"name": name, "arguments": arguments}, req_id=req_id) - resp = recv(proc, timeout=45) - if "error" in resp and resp["error"] is not None: - raise RuntimeError(f"{name} failed: {resp}") - if resp.get("result", {}).get("isError"): - raise RuntimeError(f"{name} returned isError: {resp}") - return resp - - def main(): - with open("/tmp/background-target-xid.txt", "r", encoding="utf-8") as f: - target_window_id = int(f.read().strip()) - with open("/tmp/background-target-pid.txt", "r", encoding="utf-8") as f: - target_pid = int(f.read().strip()) - with open("/tmp/background-control-xid.txt", "r", encoding="utf-8") as f: - control_window_id = int(f.read().strip()) - with open("/tmp/background-control-pid.txt", "r", encoding="utf-8") as f: - control_pid = int(f.read().strip()) - - proc = start_driver() - try: - send(proc, "initialize", { - "protocolVersion": "2024-11-05", - "capabilities": {}, - "clientInfo": {"name": "nixos-background-terminal-gif-test", "version": "1.0.0"}, - }, req_id=1) - recv(proc) - send(proc, "notifications/initialized", {}) - time.sleep(0.3) - - call_tool(proc, 2, "set_agent_cursor_enabled", {"enabled": True}) - call_tool(proc, 3, "move_cursor", {"x": 50.0, "y": 900.0}) - time.sleep(0.5) - call_tool(proc, 4, "click", { - "pid": control_pid, - "window_id": control_window_id, - "x": 120.0, - "y": 120.0, - }) - time.sleep(0.6) - call_tool(proc, 5, "type_text", { - "pid": target_pid, - "window_id": target_window_id, - "text": "echo hello | tee /tmp/background-hello.txt", - }) - call_tool(proc, 6, "press_key", { - "pid": target_pid, - "window_id": target_window_id, - "key": "enter", - }) - time.sleep(1.8) - print("background terminal GIF test complete", flush=True) - finally: - proc.stdin.close() - proc.terminate() - proc.wait(timeout=5) - - if __name__ == "__main__": - main() - ''; - - recordGifScript = import ./record-x11-gif.nix { inherit pkgs; }; - - # openbox config with focusNew=no (see openbox-rc.nix) so focus stays on the - # control terminal when the target xterm maps. - openboxRc = import ./openbox-rc.nix { inherit pkgs; }; -in - -pkgs.testers.nixosTest { - name = "cua-driver-linux-background-terminal-gif-test"; - meta.maintainers = [ ]; - - containers.machine = - { - pkgs, - ... - }: - { - imports = [ cuaDriverModule ]; - services.cua-driver.enable = true; - environment.systemPackages = with pkgs; [ - xorg.xorgserver - xterm - openbox - picom - xdotool - imagemagick - python3 - jq - procps - ]; - }; - - testScript = '' - # cache-bust 2026-06-08.1: this comment is part of the test derivation, so - # bumping it changes the output path and forces the test to actually re-run - # instead of being substituted from the binary cache. Bump again to re-run. - machine.start() - machine.wait_for_unit("multi-user.target") - - with subtest("Start X11 desktop and target/control xterms"): - machine.execute("Xvfb :99 -screen 0 1280x1024x24 >/tmp/xvfb.log 2>&1 &") - machine.wait_until_succeeds("test -e /tmp/.X11-unix/X99", timeout=10) - machine.execute("DISPLAY=:99 openbox --config-file ${openboxRc} >/tmp/openbox.log 2>&1 &") - machine.execute("DISPLAY=:99 picom --backend xrender >/tmp/picom.log 2>&1 &") - machine.execute("sh -lc \"DISPLAY=:99 xterm -T 'Background Target' -fa Monospace -fs 14 -geometry 70x24+40+120 >/tmp/background-target.log 2>&1 & echo \\$! >/tmp/background-target-pid.txt\"") - machine.execute("sh -lc \"DISPLAY=:99 xterm -T 'Background Control' -fa Monospace -fs 14 -geometry 70x24+690+120 >/tmp/background-control.log 2>&1 & echo \\$! >/tmp/background-control-pid.txt\"") - machine.wait_until_succeeds("DISPLAY=:99 xdotool search --sync --pid $(cat /tmp/background-target-pid.txt) >/tmp/background-target-xid.txt", timeout=20) - machine.wait_until_succeeds("DISPLAY=:99 xdotool search --sync --pid $(cat /tmp/background-control-pid.txt) >/tmp/background-control-xid.txt", timeout=20) - machine.succeed("DISPLAY=:99 xdotool windowactivate --sync $(head -1 /tmp/background-control-xid.txt)") - machine.succeed("DISPLAY=:99 xdotool windowfocus --sync $(head -1 /tmp/background-control-xid.txt)") - - with subtest("Record GIF and execute command in inactive terminal"): - machine.copy_from_host("${mcpBackgroundTerminalTest}", "/tmp/mcp-background-terminal-gif-test.py") - machine.execute( - "sh -lc '${recordGifScript} :99 /tmp/background-frames /tmp/cua-driver-linux-background-terminal.gif " - "/tmp/stop-background-recorder /tmp/ffmpeg-background.log 10 0.15 >/dev/null 2>&1 & echo $! >/tmp/ffmpeg-background.pid'" - ) - result = machine.succeed("timeout 60 env DISPLAY=:99 python3 /tmp/mcp-background-terminal-gif-test.py 2>&1") - machine.log(result) - assert "background terminal GIF test complete" in result, result - machine.succeed("touch /tmp/stop-background-recorder") - machine.wait_until_succeeds("! kill -0 $(cat /tmp/ffmpeg-background.pid) 2>/dev/null", timeout=60) - machine.log(machine.succeed("sh -lc 'cat /tmp/ffmpeg-background.log || true'")) - machine.succeed("test -s /tmp/cua-driver-linux-background-terminal.gif") - machine.wait_until_succeeds("grep -Fx 'hello' /tmp/background-hello.txt", timeout=20) - - with subtest("Verify focus stayed on control terminal"): - control = machine.succeed("head -1 /tmp/background-control-xid.txt").strip() - active = machine.succeed("DISPLAY=:99 xdotool getactivewindow").strip() - assert control == active, f"expected active window {control}, got {active}" - - with subtest("Copy GIF out of the container"): - machine.copy_from_machine("/tmp/cua-driver-linux-background-terminal.gif", "") - ''; -} diff --git a/nix/cua-driver/tests/linux-cursor-click-gif.nix b/nix/cua-driver/tests/linux-cursor-click-gif.nix deleted file mode 100644 index d847d538fb..0000000000 --- a/nix/cua-driver/tests/linux-cursor-click-gif.nix +++ /dev/null @@ -1,186 +0,0 @@ -# Linux cursor click GIF test -# -# Records a GIF of the Linux overlay cursor moving as part of a click action -# and proves the click focused the target xterm and allowed a shell command to -# execute. -# -# To run: nix build .#checks.x86_64-linux.cua-driver-linux-cursor-click-gif -# -{ - pkgs, - lib ? pkgs.lib, - cuaDriverModule, - ... -}: - -let - mcpClickTest = pkgs.writeText "mcp-click-gif-test.py" '' - import json - import os - import subprocess - import sys - import threading - import time - - DRIVER_BIN = os.environ.get("CUA_DRIVER_BIN", "cua-driver") - - def start_driver(): - proc = subprocess.Popen( - [DRIVER_BIN, "mcp", "--no-daemon-relaunch"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env={**os.environ}, - ) - - def drain_stderr(): - for line in proc.stderr: - sys.stderr.buffer.write(line) - sys.stderr.buffer.flush() - - threading.Thread(target=drain_stderr, daemon=True).start() - return proc - - def send(proc, method, params=None, req_id=None): - msg = {"jsonrpc": "2.0", "method": method} - if params is not None: - msg["params"] = params - if req_id is not None: - msg["id"] = req_id - proc.stdin.write((json.dumps(msg) + "\n").encode()) - proc.stdin.flush() - - def recv(proc, timeout=30): - result = [None] - - def reader(): - result[0] = proc.stdout.readline() - - thread = threading.Thread(target=reader) - thread.start() - thread.join(timeout) - if thread.is_alive(): - raise TimeoutError("No response within timeout") - line = result[0].decode().strip() - if not line: - raise RuntimeError("Driver returned an empty response") - return json.loads(line) - - def call_tool(proc, req_id, name, arguments): - send(proc, "tools/call", {"name": name, "arguments": arguments}, req_id=req_id) - resp = recv(proc, timeout=45) - if "error" in resp and resp["error"] is not None: - raise RuntimeError(f"{name} failed: {resp}") - if resp.get("result", {}).get("isError"): - raise RuntimeError(f"{name} returned isError: {resp}") - return resp - - def main(): - with open("/tmp/target-click-xid.txt", "r", encoding="utf-8") as f: - window_id = int(f.read().strip()) - with open("/tmp/target-click-pid.txt", "r", encoding="utf-8") as f: - pid = int(f.read().strip()) - - proc = start_driver() - try: - send(proc, "initialize", { - "protocolVersion": "2024-11-05", - "capabilities": {}, - "clientInfo": {"name": "nixos-click-gif-test", "version": "1.0.0"}, - }, req_id=1) - recv(proc) - send(proc, "notifications/initialized", {}) - time.sleep(0.3) - - call_tool(proc, 2, "set_agent_cursor_enabled", {"enabled": True}) - call_tool(proc, 3, "move_cursor", {"x": 1100.0, "y": 900.0}) - time.sleep(0.8) - call_tool(proc, 4, "click", {"pid": pid, "window_id": window_id, "x": 120.0, "y": 120.0}) - time.sleep(0.5) - call_tool(proc, 5, "type_text", { - "pid": pid, - "window_id": window_id, - "text": "echo click-focus > /tmp/click-focus.txt", - }) - call_tool(proc, 6, "press_key", {"pid": pid, "window_id": window_id, "key": "enter"}) - time.sleep(1.5) - print("click GIF test complete", flush=True) - finally: - proc.stdin.close() - proc.terminate() - proc.wait(timeout=5) - - if __name__ == "__main__": - main() - ''; - - recordGifScript = import ./record-x11-gif.nix { inherit pkgs; }; - - # openbox config with focusNew=no (see openbox-rc.nix) so the control terminal - # keeps X focus when the second xterm maps. - openboxRc = import ./openbox-rc.nix { inherit pkgs; }; -in - -pkgs.testers.nixosTest { - name = "cua-driver-linux-cursor-click-gif-test"; - meta.maintainers = [ ]; - - containers.machine = - { - pkgs, - ... - }: - { - imports = [ cuaDriverModule ]; - services.cua-driver.enable = true; - environment.systemPackages = with pkgs; [ - xorg.xorgserver - xterm - openbox - picom - xdotool - imagemagick - python3 - jq - procps - ]; - }; - - testScript = '' - # cache-bust 2026-06-08.1: this comment is part of the test derivation, so - # bumping it changes the output path and forces the test to actually re-run - # instead of being substituted from the binary cache. Bump again to re-run. - machine.start() - machine.wait_for_unit("multi-user.target") - - with subtest("Start X11 desktop and two xterms"): - machine.execute("Xvfb :99 -screen 0 1280x1024x24 >/tmp/xvfb.log 2>&1 &") - machine.wait_until_succeeds("test -e /tmp/.X11-unix/X99", timeout=10) - machine.execute("DISPLAY=:99 openbox --config-file ${openboxRc} >/tmp/openbox.log 2>&1 &") - machine.execute("DISPLAY=:99 picom --backend xrender >/tmp/picom.log 2>&1 &") - machine.execute("sh -lc \"DISPLAY=:99 xterm -T 'Target Click' -fa Monospace -fs 14 -geometry 70x24+80+120 >/tmp/target-click.log 2>&1 & echo \\$! >/tmp/target-click-pid.txt\"") - machine.execute("sh -lc \"DISPLAY=:99 xterm -T 'Control Click' -fa Monospace -fs 14 -geometry 70x24+700+120 >/tmp/control-click.log 2>&1 & echo \\$! >/tmp/control-click-pid.txt\"") - machine.wait_until_succeeds("DISPLAY=:99 xdotool search --sync --pid $(cat /tmp/target-click-pid.txt) >/tmp/target-click-xid.txt", timeout=20) - machine.wait_until_succeeds("DISPLAY=:99 xdotool search --sync --pid $(cat /tmp/control-click-pid.txt) >/tmp/control-click-xid.txt", timeout=20) - machine.succeed("DISPLAY=:99 xdotool windowactivate --sync $(head -1 /tmp/control-click-xid.txt)") - machine.succeed("DISPLAY=:99 xdotool windowfocus --sync $(head -1 /tmp/control-click-xid.txt)") - - with subtest("Record click GIF and run driver actions"): - machine.copy_from_host("${mcpClickTest}", "/tmp/mcp-click-gif-test.py") - machine.execute( - "sh -lc '${recordGifScript} :99 /tmp/click-frames /tmp/cua-driver-linux-cursor-click.gif " - "/tmp/stop-click-recorder /tmp/ffmpeg-click.log 10 0.15 >/dev/null 2>&1 & echo $! >/tmp/ffmpeg-click.pid'" - ) - result = machine.succeed("timeout 60 env DISPLAY=:99 python3 /tmp/mcp-click-gif-test.py 2>&1") - machine.log(result) - assert "click GIF test complete" in result, result - machine.succeed("touch /tmp/stop-click-recorder") - machine.wait_until_succeeds("! kill -0 $(cat /tmp/ffmpeg-click.pid) 2>/dev/null", timeout=60) - machine.log(machine.succeed("sh -lc 'cat /tmp/ffmpeg-click.log || true'")) - machine.succeed("test -s /tmp/cua-driver-linux-cursor-click.gif") - machine.wait_until_succeeds("test -f /tmp/click-focus.txt", timeout=20) - - with subtest("Copy GIF out of the container"): - machine.copy_from_machine("/tmp/cua-driver-linux-cursor-click.gif", "") - ''; -} diff --git a/nix/cua-driver/tests/linux-parallel-drag-gif.nix b/nix/cua-driver/tests/linux-parallel-drag-gif.nix deleted file mode 100644 index dc2562211f..0000000000 --- a/nix/cua-driver/tests/linux-parallel-drag-gif.nix +++ /dev/null @@ -1,387 +0,0 @@ -# Linux parallel multi-cursor drag GIF test -# -# NOT WIRED INTO CI (flake checks / nix-build.yml). It needs a real Xorg -# (dummy video + libinput) so the MPX path's uinput slaves enumerate as X -# input devices, and that server does not start reliably in the emulated -# GHA nixos-test VM (the hand-launched Xorg times out before the socket -# appears). The feature's logic is covered by unit tests in -# platform-linux/src/input/mod.rs (arc-length path glide + fn sampling); -# this scenario is kept for local / real-X manual runs: -# nix build .#checks.x86_64-linux.cua-driver-linux-parallel-drag-gif -# (re-add the flake check + matrix entry to run it where a real Xorg works). -# -# Pilots cua-driver through its Linux MPX `parallel_mouse_drag` path: two -# per-session master pointers drawing concurrent strokes into the SAME window, -# while a separate control window keeps the input focus. Proves the three -# guarantees the feature is built on: -# -# 1. Concurrent delivery — both masters' presses/motions/releases reach the -# target window as cooked, window-targeted XI2 events (an XI2 paint app -# logs every event with its device id; we assert two distinct devices). -# 2. No focus steal — the "shield grab" keeps the window manager blind to -# the presses, so the control window stays active throughout. -# 3. Pixel-exact endpoints — the recorded GIF shows two crossing strokes. -# -# Unlike the other Linux visual tests, this one needs a REAL Xorg server, not -# Xvfb: the MPX path attaches uinput slave devices to per-session master -# pointers, and only a real Xorg with the libinput input driver enumerates -# uinput devices as X input devices (Xvfb — like the old Xtigervnc setup — -# does not). So we launch Xorg with the `dummy` video driver + libinput, with -# the `uinput` kernel module loaded. -# -# To run: nix build .#checks.x86_64-linux.cua-driver-linux-parallel-drag-gif -# -{ - pkgs, - lib ? pkgs.lib, - cuaDriverModule, - ... -}: - -let - # XI2 paint + event logger. Selects XI2 events for ALL master devices on its - # own window and, for each ButtonPress/Motion/ButtonRelease, appends a line - # " dev= x=<> y=<>" to a log file and paints a square (per-device - # colour). The log is what proves cooked, window-delivered, multi-device - # input — not just raw motion. Prints "READY 0x" on stdout at startup. - xi2paintSrc = pkgs.writeText "xi2paint.c" '' - #include - #include - #include - #include - #include - - int main(int argc, char **argv) { - const char *logpath = argc > 1 ? argv[1] : "/tmp/xi2paint-events.log"; - Display *dpy = XOpenDisplay(NULL); - if (!dpy) { fprintf(stderr, "no display\n"); return 1; } - int xi_opcode, ev, err; - if (!XQueryExtension(dpy, "XInputExtension", &xi_opcode, &ev, &err)) { - fprintf(stderr, "no XInputExtension\n"); return 1; - } - int major = 2, minor = 3; - XIQueryVersion(dpy, &major, &minor); - - int scr = DefaultScreen(dpy); - Window win = XCreateSimpleWindow(dpy, RootWindow(dpy, scr), 40, 40, 800, 600, - 1, BlackPixel(dpy, scr), WhitePixel(dpy, scr)); - XStoreName(dpy, win, "XI2 MPX Paint"); - XSelectInput(dpy, win, ExposureMask); - - unsigned char mask[XIMaskLen(XI_LASTEVENT)]; - memset(mask, 0, sizeof mask); - XISetMask(mask, XI_ButtonPress); - XISetMask(mask, XI_Motion); - XISetMask(mask, XI_ButtonRelease); - XISetMask(mask, XI_Enter); - XISetMask(mask, XI_FocusIn); - XIEventMask em; - em.deviceid = XIAllMasterDevices; - em.mask_len = sizeof mask; - em.mask = mask; - XISelectEvents(dpy, win, &em, 1); - XMapWindow(dpy, win); - XFlush(dpy); - GC gc = XCreateGC(dpy, win, 0, NULL); - - FILE *logf = fopen(logpath, "w"); - if (!logf) { fprintf(stderr, "cannot open log\n"); return 1; } - setvbuf(logf, NULL, _IOLBF, 0); - printf("READY 0x%lx\n", win); fflush(stdout); - fprintf(logf, "READY 0x%lx\n", win); - - int down[256]; memset(down, 0, sizeof down); - unsigned long colors[] = { 0xd00000, 0x0040d0, 0x00a000, 0xc08000 }; - for (;;) { - XEvent e; - XNextEvent(dpy, &e); - if (e.type == GenericEvent && e.xcookie.extension == xi_opcode && - XGetEventData(dpy, &e.xcookie)) { - int t = e.xcookie.evtype; - if (t == XI_ButtonPress || t == XI_Motion || t == XI_ButtonRelease) { - XIDeviceEvent *de = e.xcookie.data; - const char *n = t == XI_ButtonPress ? "PRESS" - : t == XI_Motion ? "MOTION" : "RELEASE"; - fprintf(logf, "%s dev=%d x=%.0f y=%.0f\n", - n, de->deviceid, de->event_x, de->event_y); - int d = de->deviceid & 255; - if (t == XI_ButtonPress) down[d] = 1; - if (t == XI_ButtonRelease) down[d] = 0; - if ((t == XI_Motion && down[d]) || t == XI_ButtonPress) { - XSetForeground(dpy, gc, colors[de->deviceid % 4]); - XFillRectangle(dpy, win, gc, (int)de->event_x - 3, - (int)de->event_y - 3, 6, 6); - XFlush(dpy); - } - } else if (t == XI_Enter || t == XI_FocusIn) { - XIEnterEvent *ee = e.xcookie.data; - fprintf(logf, "%s dev=%d\n", t == XI_Enter ? "ENTER" : "FOCUSIN", - ee->deviceid); - } - XFreeEventData(dpy, &e.xcookie); - } - } - return 0; - } - ''; - - xi2paint = pkgs.runCommandCC "xi2paint" { - buildInputs = [ pkgs.xorg.libX11 pkgs.xorg.libXi ]; - } '' - mkdir -p $out/bin - cc -O2 -o $out/bin/xi2paint ${xi2paintSrc} -lX11 -lXi - ''; - - # Real Xorg config: dummy video driver (software framebuffer) + libinput - # input hotplug so cua-driver's uinput slaves get enumerated as X devices. - xorgConf = pkgs.writeText "xorg-dummy.conf" '' - Section "ServerFlags" - Option "AutoAddDevices" "true" - Option "AutoEnableDevices" "true" - Option "DontVTSwitch" "true" - EndSection - Section "Device" - Identifier "dummy" - Driver "dummy" - VideoRam 256000 - EndSection - Section "Monitor" - Identifier "mon" - HorizSync 30.0 - 1000.0 - VertRefresh 30.0 - 200.0 - Modeline "1280x1024" 109.00 1280 1368 1496 1712 1024 1027 1034 1063 -hsync +vsync - EndSection - Section "Screen" - Identifier "screen" - Device "dummy" - Monitor "mon" - DefaultDepth 24 - SubSection "Display" - Depth 24 - Modes "1280x1024" - EndSubSection - EndSection - ''; - - # Manually-launched Xorg needs an explicit module path covering the server's - # own modules plus the separately-packaged dummy video + libinput drivers. - xorgModulePath = lib.concatStringsSep "," [ - "${pkgs.xorg.xorgserver}/lib/xorg/modules" - "${pkgs.xorg.xf86videodummy}/lib/xorg/modules/drivers" - "${pkgs.xorg.xf86inputlibinput}/lib/xorg/modules/input" - ]; - - mcpDragTest = pkgs.writeText "mcp-parallel-drag-test.py" '' - import json - import os - import subprocess - import sys - import threading - import time - - DRIVER_BIN = os.environ.get("CUA_DRIVER_BIN", "cua-driver") - - def start_driver(): - proc = subprocess.Popen( - [DRIVER_BIN, "mcp", "--no-daemon-relaunch"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env={**os.environ}, - ) - - def drain_stderr(): - for line in proc.stderr: - sys.stderr.buffer.write(line) - sys.stderr.buffer.flush() - - threading.Thread(target=drain_stderr, daemon=True).start() - return proc - - def send(proc, method, params=None, req_id=None): - msg = {"jsonrpc": "2.0", "method": method} - if params is not None: - msg["params"] = params - if req_id is not None: - msg["id"] = req_id - proc.stdin.write((json.dumps(msg) + "\n").encode()) - proc.stdin.flush() - - def recv(proc, timeout=30): - result = [None] - - def reader(): - result[0] = proc.stdout.readline() - - thread = threading.Thread(target=reader) - thread.start() - thread.join(timeout) - if thread.is_alive(): - raise TimeoutError("No response within timeout") - line = result[0].decode().strip() - if not line: - raise RuntimeError("Driver returned an empty response") - return json.loads(line) - - def call_tool(proc, req_id, name, arguments, timeout=60): - send(proc, "tools/call", {"name": name, "arguments": arguments}, req_id=req_id) - resp = recv(proc, timeout=timeout) - if "error" in resp and resp["error"] is not None: - raise RuntimeError(f"{name} failed: {resp}") - if resp.get("result", {}).get("isError"): - raise RuntimeError(f"{name} returned isError: {resp}") - return resp - - def main(): - with open("/tmp/paint-xid.txt", "r", encoding="utf-8") as f: - window_id = int(f.readline().strip()) - - proc = start_driver() - try: - send(proc, "initialize", { - "protocolVersion": "2024-11-05", - "capabilities": {}, - "clientInfo": {"name": "nixos-parallel-drag-test", "version": "1.0.0"}, - }, req_id=1) - recv(proc) - send(proc, "notifications/initialized", {}) - time.sleep(0.3) - - # Two concurrent strokes forming the left half of an "X" with two - # cursors, into a window that does NOT hold focus. - call_tool(proc, 2, "parallel_mouse_drag", {"drags": [ - {"session": "agent-1", "window_id": window_id, - "from_x": 100.0, "from_y": 100.0, "to_x": 380.0, "to_y": 420.0, - "duration_ms": 2500, "steps": 80}, - {"session": "agent-2", "window_id": window_id, - "from_x": 700.0, "from_y": 100.0, "to_x": 420.0, "to_y": 420.0, - "duration_ms": 2500, "steps": 80}, - ]}) - time.sleep(0.6) - - # Second pass completes both "X" shapes. - call_tool(proc, 3, "parallel_mouse_drag", {"drags": [ - {"session": "agent-1", "window_id": window_id, - "from_x": 100.0, "from_y": 420.0, "to_x": 380.0, "to_y": 120.0, - "duration_ms": 2500, "steps": 80}, - {"session": "agent-2", "window_id": window_id, - "from_x": 700.0, "from_y": 420.0, "to_x": 420.0, "to_y": 120.0, - "duration_ms": 2500, "steps": 80}, - ]}) - time.sleep(0.6) - print("parallel drag test complete", flush=True) - finally: - proc.stdin.close() - proc.terminate() - proc.wait(timeout=5) - - if __name__ == "__main__": - main() - ''; - - recordGifScript = import ./record-x11-gif.nix { inherit pkgs; }; -in - -pkgs.testers.nixosTest { - name = "cua-driver-linux-parallel-drag-gif-test"; - meta.maintainers = [ ]; - - nodes.machine = - { - pkgs, - ... - }: - { - imports = [ cuaDriverModule ]; - virtualisation = { - cores = 2; - memorySize = 2048; - }; - services.cua-driver.enable = true; - # The MPX path opens /dev/uinput and relies on Xorg+libinput enumerating - # the resulting devices, so the uinput module must be present. - boot.kernelModules = [ "uinput" ]; - environment.systemPackages = with pkgs; [ - xorg.xorgserver - xorg.xf86videodummy - xorg.xf86inputlibinput - xorg.xinput - xi2paint - xterm - openbox - picom - xdotool - imagemagick - python3 - jq - procps - ]; - }; - - testScript = '' - machine.start() - machine.wait_for_unit("multi-user.target") - machine.succeed("modprobe uinput && test -e /dev/uinput") - - with subtest("Start a real Xorg (dummy video + libinput) and a WM"): - machine.execute( - "Xorg :99 -ac -noreset -keeptty -nolisten tcp " - "-config ${xorgConf} -modulepath ${xorgModulePath} " - "-logfile /tmp/xorg.log >/tmp/xorg.out 2>&1 &" - ) - machine.wait_until_succeeds("test -e /tmp/.X11-unix/X99", timeout=30) - machine.wait_until_succeeds("DISPLAY=:99 xdpyinfo >/dev/null 2>&1", timeout=30) - machine.execute("DISPLAY=:99 openbox >/tmp/openbox.log 2>&1 &") - machine.execute("DISPLAY=:99 picom --backend xrender >/tmp/picom.log 2>&1 &") - - with subtest("Launch the XI2 paint target and a control window that holds focus"): - machine.execute( - "sh -lc \"DISPLAY=:99 xi2paint /tmp/xi2paint-events.log >/tmp/paint.log 2>&1 & echo \\$! >/tmp/paint-pid.txt\"" - ) - machine.execute( - "sh -lc \"DISPLAY=:99 xterm -T 'Control' -fa Monospace -fs 14 -geometry 50x12+980+120 >/tmp/control.log 2>&1 & echo \\$! >/tmp/control-pid.txt\"" - ) - machine.wait_until_succeeds("DISPLAY=:99 xdotool search --sync --name 'XI2 MPX Paint' >/tmp/paint-xid.txt", timeout=20) - machine.wait_until_succeeds("DISPLAY=:99 xdotool search --sync --pid $(cat /tmp/control-pid.txt) >/tmp/control-xid.txt", timeout=20) - # Give input focus to the CONTROL window — the drags target the paint - # window, which must never steal it. - machine.succeed("DISPLAY=:99 xdotool windowactivate --sync $(head -1 /tmp/control-xid.txt)") - machine.succeed("DISPLAY=:99 xdotool windowfocus --sync $(head -1 /tmp/control-xid.txt)") - - with subtest("Record GIF and pilot cua-driver through parallel_mouse_drag"): - machine.copy_from_host("${mcpDragTest}", "/tmp/mcp-parallel-drag-test.py") - machine.execute( - "sh -lc '${recordGifScript} :99 /tmp/drag-frames /tmp/cua-driver-linux-parallel-drag.gif " - "/tmp/stop-drag-recorder /tmp/ffmpeg-drag.log 8 0.12 >/dev/null 2>&1 & echo $! >/tmp/ffmpeg-drag.pid'" - ) - result = machine.succeed("timeout 90 env DISPLAY=:99 python3 /tmp/mcp-parallel-drag-test.py 2>&1") - machine.log(result) - assert "parallel drag test complete" in result, result - machine.succeed("touch /tmp/stop-drag-recorder") - machine.wait_until_succeeds("! kill -0 $(cat /tmp/ffmpeg-drag.pid) 2>/dev/null", timeout=60) - machine.log(machine.succeed("sh -lc 'cat /tmp/ffmpeg-drag.log || true'")) - - with subtest("Both cursors delivered cooked, window-targeted events"): - machine.log(machine.succeed("cat /tmp/xi2paint-events.log")) - # Four presses total (two strokes x two cursors). - machine.succeed("test \"$(grep -c '^PRESS ' /tmp/xi2paint-events.log)\" -eq 4") - # From two DISTINCT master pointer devices — i.e. genuinely concurrent - # multi-cursor input, not one pointer reused. - distinct = machine.succeed( - "grep '^PRESS ' /tmp/xi2paint-events.log | sed -n 's/.*dev=\\([0-9]*\\).*/\\1/p' | sort -u | wc -l" - ).strip() - assert int(distinct) == 2, f"expected 2 distinct devices, got {distinct}" - machine.succeed("test \"$(grep -c '^MOTION ' /tmp/xi2paint-events.log)\" -ge 4") - - with subtest("The drags did not steal focus from the control window"): - active = machine.succeed("DISPLAY=:99 xdotool getactivewindow").strip() - control = machine.succeed("head -1 /tmp/control-xid.txt").strip() - assert active == control, f"focus stolen: active={active} control={control}" - - with subtest("GIF artifact exists"): - machine.succeed("test -s /tmp/cua-driver-linux-parallel-drag.gif") - - with subtest("Copy GIF out of the VM"): - machine.copy_from_machine("/tmp/cua-driver-linux-parallel-drag.gif", "") - ''; -} diff --git a/nix/cua-driver/tests/linux-parallel-drag-xserver.nix b/nix/cua-driver/tests/linux-parallel-drag-xserver.nix deleted file mode 100644 index f812ca5482..0000000000 --- a/nix/cua-driver/tests/linux-parallel-drag-xserver.nix +++ /dev/null @@ -1,314 +0,0 @@ -# Linux parallel multi-cursor drag test — real Xorg via services.xserver -# -# A CI-viable rewrite of linux-parallel-drag-gif.nix. The MPX path needs a -# REAL Xorg with the libinput input backend so cua-driver's uinput slaves -# enumerate as X input devices (Xvfb can't; a hand-launched Xorg couldn't get -# a VT/seat in the emulated nixos-test VM and timed out). Here NixOS's -# services.xserver brings up Xorg properly on a seat via a display manager, -# with the `dummy` video driver (software framebuffer, headless) and libinput. -# A normal user is auto-logged-in to an icewm session; the session runs -# `xhost +local:` so the root-run test driver / cua-driver can connect to :0. -# -# Proves: two per-session master pointers draw concurrent cooked, window- -# targeted XI2 events into one window (assert 2 distinct devices), the shield -# grab keeps focus on a separate control window, and a GIF is produced. -# -# To run: nix build .#checks.x86_64-linux.cua-driver-linux-parallel-drag-xserver -{ - pkgs, - lib ? pkgs.lib, - cuaDriverModule, - ... -}: - -let - # XI2 paint + event logger (selects XI2 for all master devices; logs each - # press/motion/release with its device id and paints per-device squares). - xi2paintSrc = pkgs.writeText "xi2paint.c" '' - #include - #include - #include - #include - - int main(int argc, char **argv) { - const char *logpath = argc > 1 ? argv[1] : "/tmp/xi2paint-events.log"; - Display *dpy = XOpenDisplay(NULL); - if (!dpy) { fprintf(stderr, "no display\n"); return 1; } - int xi_opcode, ev, err; - if (!XQueryExtension(dpy, "XInputExtension", &xi_opcode, &ev, &err)) { - fprintf(stderr, "no XInputExtension\n"); return 1; - } - int major = 2, minor = 3; - XIQueryVersion(dpy, &major, &minor); - int scr = DefaultScreen(dpy); - Window win = XCreateSimpleWindow(dpy, RootWindow(dpy, scr), 40, 40, 800, 600, - 1, BlackPixel(dpy, scr), WhitePixel(dpy, scr)); - XStoreName(dpy, win, "XI2 MPX Paint"); - XSelectInput(dpy, win, ExposureMask); - unsigned char mask[XIMaskLen(XI_LASTEVENT)]; - memset(mask, 0, sizeof mask); - XISetMask(mask, XI_ButtonPress); - XISetMask(mask, XI_Motion); - XISetMask(mask, XI_ButtonRelease); - XIEventMask em; - em.deviceid = XIAllMasterDevices; - em.mask_len = sizeof mask; - em.mask = mask; - XISelectEvents(dpy, win, &em, 1); - XMapWindow(dpy, win); - XFlush(dpy); - GC gc = XCreateGC(dpy, win, 0, NULL); - FILE *logf = fopen(logpath, "w"); - if (!logf) { fprintf(stderr, "cannot open log\n"); return 1; } - setvbuf(logf, NULL, _IOLBF, 0); - printf("READY 0x%lx\n", win); fflush(stdout); - int down[256]; memset(down, 0, sizeof down); - unsigned long colors[] = { 0xd00000, 0x0040d0, 0x00a000, 0xc08000 }; - for (;;) { - XEvent e; - XNextEvent(dpy, &e); - if (e.type == GenericEvent && e.xcookie.extension == xi_opcode && - XGetEventData(dpy, &e.xcookie)) { - int t = e.xcookie.evtype; - if (t == XI_ButtonPress || t == XI_Motion || t == XI_ButtonRelease) { - XIDeviceEvent *de = e.xcookie.data; - const char *n = t == XI_ButtonPress ? "PRESS" - : t == XI_Motion ? "MOTION" : "RELEASE"; - fprintf(logf, "%s dev=%d x=%.0f y=%.0f\n", - n, de->deviceid, de->event_x, de->event_y); - int d = de->deviceid & 255; - if (t == XI_ButtonPress) down[d] = 1; - if (t == XI_ButtonRelease) down[d] = 0; - if ((t == XI_Motion && down[d]) || t == XI_ButtonPress) { - XSetForeground(dpy, gc, colors[de->deviceid % 4]); - XFillRectangle(dpy, win, gc, (int)de->event_x - 3, - (int)de->event_y - 3, 6, 6); - XFlush(dpy); - } - } - XFreeEventData(dpy, &e.xcookie); - } - } - return 0; - } - ''; - - xi2paint = pkgs.runCommandCC "xi2paint" { - buildInputs = [ pkgs.xorg.libX11 pkgs.xorg.libXi ]; - } '' - mkdir -p $out/bin - cc -O2 -o $out/bin/xi2paint ${xi2paintSrc} -lX11 -lXi - ''; - - mcpDragTest = pkgs.writeText "mcp-parallel-drag-test.py" '' - import json, os, subprocess, sys, threading, time - DRIVER_BIN = os.environ.get("CUA_DRIVER_BIN", "cua-driver") - - def start_driver(): - proc = subprocess.Popen([DRIVER_BIN, "mcp", "--no-daemon-relaunch"], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env={**os.environ}) - def drain(): - for line in proc.stderr: - sys.stderr.buffer.write(line); sys.stderr.buffer.flush() - threading.Thread(target=drain, daemon=True).start() - return proc - - def send(proc, method, params=None, req_id=None): - msg = {"jsonrpc": "2.0", "method": method} - if params is not None: msg["params"] = params - if req_id is not None: msg["id"] = req_id - proc.stdin.write((json.dumps(msg) + "\n").encode()); proc.stdin.flush() - - def recv(proc, timeout=30): - result = [None] - def reader(): result[0] = proc.stdout.readline() - th = threading.Thread(target=reader); th.start(); th.join(timeout) - if th.is_alive(): raise TimeoutError("no response") - line = result[0].decode().strip() - if not line: raise RuntimeError("empty response") - return json.loads(line) - - def call_tool(proc, rid, name, args, timeout=60): - send(proc, "tools/call", {"name": name, "arguments": args}, req_id=rid) - resp = recv(proc, timeout=timeout) - if resp.get("error"): raise RuntimeError(f"{name} failed: {resp}") - if resp.get("result", {}).get("isError"): raise RuntimeError(f"{name} isError: {resp}") - return resp - - def main(): - with open("/tmp/paint-xid.txt") as f: - wid = int(f.readline().strip()) - proc = start_driver() - try: - send(proc, "initialize", {"protocolVersion": "2024-11-05", "capabilities": {}, - "clientInfo": {"name": "nixos-parallel-drag-xserver", "version": "1.0.0"}}, req_id=1) - recv(proc); send(proc, "notifications/initialized", {}); time.sleep(0.3) - # Two concurrent strokes (one per cursor) into an UNFOCUSED window; - # each is a single held-path drag (press once, glide, release once). - call_tool(proc, 2, "parallel_mouse_drag", {"drags": [ - {"session": "agent-1", "window_id": wid, "from_x": 100.0, "from_y": 100.0, "to_x": 380.0, "to_y": 420.0, "duration_ms": 2200, "steps": 80}, - {"session": "agent-2", "window_id": wid, "from_x": 700.0, "from_y": 100.0, "to_x": 420.0, "to_y": 420.0, "duration_ms": 2200, "steps": 80}, - ]}) - time.sleep(0.6) - call_tool(proc, 3, "parallel_mouse_drag", {"drags": [ - {"session": "agent-1", "window_id": wid, "from_x": 100.0, "from_y": 420.0, "to_x": 380.0, "to_y": 120.0, "duration_ms": 2200, "steps": 80}, - {"session": "agent-2", "window_id": wid, "from_x": 700.0, "from_y": 420.0, "to_x": 420.0, "to_y": 120.0, "duration_ms": 2200, "steps": 80}, - ]}) - time.sleep(0.6) - print("parallel drag test complete", flush=True) - finally: - proc.stdin.close(); proc.terminate(); proc.wait(timeout=5) - - if __name__ == "__main__": - main() - ''; - - recordGifScript = import ./record-x11-gif.nix { inherit pkgs; }; -in - -pkgs.testers.nixosTest { - name = "cua-driver-linux-parallel-drag-xserver-test"; - meta.maintainers = [ ]; - - nodes.machine = - { pkgs, ... }: - { - imports = [ cuaDriverModule ]; - virtualisation = { - cores = 2; - memorySize = 2048; - }; - services.cua-driver.enable = true; - boot.kernelModules = [ "uinput" ]; - # Root opens /dev/uinput directly; the group rule is belt-and-suspenders. - services.udev.extraRules = '' - KERNEL=="uinput", MODE="0660", GROUP="input", OPTIONS+="static_node=uinput" - ''; - - # Real Xorg on a seat (DM handles the VT), dummy video, libinput input. - services.xserver = { - enable = true; - videoDrivers = [ "dummy" ]; - deviceSection = ''VideoRam 256000''; - monitorSection = '' - HorizSync 30.0 - 1000.0 - VertRefresh 30.0 - 200.0 - Modeline "1280x1024" 109.00 1280 1368 1496 1712 1024 1027 1034 1063 -hsync +vsync - ''; - screenSection = '' - DefaultDepth 24 - SubSection "Display" - Depth 24 - Modes "1280x1024" - Virtual 1280 1024 - EndSubSection - ''; - windowManager.icewm.enable = true; - # A display manager is what actually starts Xorg on a seat/VT — the bit - # the hand-launched Xorg couldn't arrange in the emulated VM. lightdm is - # the lightest. (lightdm.enable still lives under xserver.displayManager; - # autoLogin/defaultSession moved to the top-level services.displayManager.) - displayManager.lightdm.enable = true; - # Disable X access control outright: this is a throwaway single-user - # test VM, and the root-run cua-driver / clients must reach :0. Relying - # on the session's `xhost +local:` proved unreliable across uids (the - # server's auth ACL only lists the autologin user), so `-ac` is the - # bulletproof grant. `xhost +local:` is kept as belt-and-suspenders. - displayManager.xserverArgs = [ "-ac" ]; - displayManager.sessionCommands = '' - ${pkgs.xorg.xhost}/bin/xhost +local: || true - ''; - }; - services.libinput.enable = true; - services.displayManager = { - defaultSession = "none+icewm"; - autoLogin = { - enable = true; - user = "cua"; - }; - }; - users.users.cua = { - isNormalUser = true; - extraGroups = [ "input" ]; - }; - - environment.systemPackages = with pkgs; [ - xorg.xinput - xorg.xhost - xorg.xdpyinfo # the :0 connectivity probe in the test script - xi2paint - xterm - xdotool - imagemagick - python3 - jq - procps - ]; - }; - - testScript = '' - machine.start() - machine.wait_for_unit("multi-user.target") - machine.succeed("modprobe uinput && test -e /dev/uinput") - - with subtest("Real Xorg up via the display manager"): - try: - machine.wait_for_x() - # With `-ac` the root-run clients can connect to :0 immediately. - machine.wait_until_succeeds("DISPLAY=:0 xdpyinfo >/dev/null 2>&1", timeout=30) - except Exception: - # Diagnostics: surface why X/lightdm did not come up or why root - # can't reach :0, so we don't burn a CI round-trip guessing. - machine.log(machine.execute("systemctl status display-manager.service --no-pager || true")[1]) - machine.log(machine.execute("journalctl -u display-manager.service --no-pager | tail -n 200 || true")[1]) - machine.log(machine.execute("ls -la /tmp/.X11-unix/ || true")[1]) - machine.log(machine.execute("cat /var/log/X.0.log 2>/dev/null | tail -n 200 || true")[1]) - machine.log(machine.execute("find / -name 'Xorg.0.log' 2>/dev/null | head; cat $(find / -name 'Xorg.0.log' 2>/dev/null | head -1) 2>/dev/null | tail -n 120 || true")[1]) - raise - machine.log(machine.succeed("DISPLAY=:0 xinput list --short || true")) - - with subtest("Launch XI2 paint target + a control window that holds focus"): - machine.execute( - "sh -lc \"DISPLAY=:0 xi2paint /tmp/xi2paint-events.log >/tmp/paint.log 2>&1 & echo \\$! >/tmp/paint-pid.txt\"" - ) - machine.execute( - "sh -lc \"DISPLAY=:0 xterm -T 'Control' -fa Monospace -fs 14 -geometry 50x12+980+120 >/tmp/control.log 2>&1 & echo \\$! >/tmp/control-pid.txt\"" - ) - machine.wait_until_succeeds("DISPLAY=:0 xdotool search --sync --name 'XI2 MPX Paint' >/tmp/paint-xid.txt", timeout=20) - machine.wait_until_succeeds("DISPLAY=:0 xdotool search --sync --pid $(cat /tmp/control-pid.txt) >/tmp/control-xid.txt", timeout=20) - machine.succeed("DISPLAY=:0 xdotool windowactivate --sync $(head -1 /tmp/control-xid.txt)") - machine.succeed("DISPLAY=:0 xdotool windowfocus --sync $(head -1 /tmp/control-xid.txt)") - - with subtest("Record GIF and pilot cua-driver through parallel_mouse_drag"): - machine.copy_from_host("${mcpDragTest}", "/tmp/mcp-parallel-drag-test.py") - machine.execute( - "sh -lc '${recordGifScript} :0 /tmp/drag-frames /tmp/cua-driver-linux-parallel-drag-xserver.gif " - "/tmp/stop-drag-recorder /tmp/ffmpeg-drag.log 8 0.12 >/dev/null 2>&1 & echo $! >/tmp/ffmpeg-drag.pid'" - ) - result = machine.succeed("timeout 90 env DISPLAY=:0 python3 /tmp/mcp-parallel-drag-test.py 2>&1") - machine.log(result) - assert "parallel drag test complete" in result, result - machine.succeed("touch /tmp/stop-drag-recorder") - machine.wait_until_succeeds("! kill -0 $(cat /tmp/ffmpeg-drag.pid) 2>/dev/null", timeout=60) - - with subtest("Both cursors delivered cooked, window-targeted events"): - machine.log(machine.succeed("cat /tmp/xi2paint-events.log")) - machine.succeed("test \"$(grep -c '^PRESS ' /tmp/xi2paint-events.log)\" -eq 4") - distinct = machine.succeed( - "grep '^PRESS ' /tmp/xi2paint-events.log | sed -n 's/.*dev=\\([0-9]*\\).*/\\1/p' | sort -u | wc -l" - ).strip() - assert int(distinct) == 2, f"expected 2 distinct devices, got {distinct}" - machine.succeed("test \"$(grep -c '^MOTION ' /tmp/xi2paint-events.log)\" -ge 4") - - with subtest("The drags did not steal focus from the control window"): - active = machine.succeed("DISPLAY=:0 xdotool getactivewindow").strip() - control = machine.succeed("head -1 /tmp/control-xid.txt").strip() - assert active == control, f"focus stolen: active={active} control={control}" - - with subtest("GIF artifact exists"): - machine.succeed("test -s /tmp/cua-driver-linux-parallel-drag-xserver.gif") - - with subtest("Copy GIF out of the VM"): - machine.copy_from_machine("/tmp/cua-driver-linux-parallel-drag-xserver.gif", "") - ''; -} diff --git a/nix/cua-driver/tests/openbox-rc.nix b/nix/cua-driver/tests/openbox-rc.nix deleted file mode 100644 index 5ca8b439d1..0000000000 --- a/nix/cua-driver/tests/openbox-rc.nix +++ /dev/null @@ -1,29 +0,0 @@ -# Shared openbox config for the X11 GUI tests. -# -# Returns an openbox rc.xml whose only non-default setting is -# `no`: openbox must NOT auto-focus a -# window when it is first mapped. Everything else falls back to openbox's -# built-in defaults (missing nodes are defaulted by the parser). -# -# Why: the background-GUI tests launch a real app (chromium/electron) in the -# background while a control terminal stays focused, then assert focus never -# left the control terminal. Under the faster container backend, chromium-based -# apps finish loading and map/raise their toplevel mid-test; with the default -# `focusNew=yes` openbox would shift X input focus to the freshly-mapped window -# and break that invariant. Explicit `xdotool windowactivate` requests are user -# actions and are still honoured, so the control terminal is still focusable. -# -# Usage (in a test's `let`): -# openboxRc = import ./openbox-rc.nix { inherit pkgs; }; -# then launch openbox with it: -# openbox --config-file ${openboxRc} -{ pkgs }: - -pkgs.writeText "openbox-rc.xml" '' - - - - no - - -'' diff --git a/nix/cua-driver/tests/record-x11-gif.nix b/nix/cua-driver/tests/record-x11-gif.nix deleted file mode 100644 index 81a49a9001..0000000000 --- a/nix/cua-driver/tests/record-x11-gif.nix +++ /dev/null @@ -1,49 +0,0 @@ -# Shared X11 screen-recording helper for the Linux visual (GIF) tests. -# -# Returns a `pkgs.writeShellScript` that loops `import -display -# -window root frame-XXXX.png` until a stop-file appears, then stitches the -# frames into an animated GIF with `convert`. Both tools come from -# `pkgs.imagemagick`, which the importing test must add to -# `environment.systemPackages`. -# -# Usage (in a test's `let`): -# recordGifScript = import ./record-x11-gif.nix { inherit pkgs; }; -# then, inside the testScript, start it in the background before driving and -# `touch` the stop-file afterwards: -# ${recordGifScript} \ -# -{ pkgs }: - -pkgs.writeShellScript "record-x11-gif.sh" '' - set -eu - display="$1" - frames_dir="$2" - output_gif="$3" - stop_file="$4" - log_file="$5" - delay_cs="$6" - interval="$7" - - rm -f "$stop_file" "$output_gif" "$log_file" - rm -rf "$frames_dir" - mkdir -p "$frames_dir" - - # Cap the frame count: long driver runs (the skeleton budget is 300s) can - # otherwise pile up 1000+ frames and the final `convert` thrashes/OOMs the - # test container, wedging every later command in the job. 450 frames is a ~45s - # GIF at the default cadence — plenty. Each `import` and the final `convert` - # are also time-bounded so a wedged X grab or a slow stitch can't stall the - # job. - max_frames=450 - i=0 - while [ ! -f "$stop_file" ] && [ "$i" -lt "$max_frames" ]; do - frame=$(printf "%s/frame-%04d.png" "$frames_dir" "$i") - timeout 10 import -display "$display" -window root "$frame" >>"$log_file" 2>&1 || true - i=$((i + 1)) - sleep "$interval" - done - - if ls "$frames_dir"/frame-*.png >/dev/null 2>&1; then - timeout 120 convert -delay "$delay_cs" -loop 0 "$frames_dir"/frame-*.png "$output_gif" >>"$log_file" 2>&1 || true - fi -'' diff --git a/nix/cua-driver/tests/screenshot.nix b/nix/cua-driver/tests/screenshot.nix deleted file mode 100644 index 695983e407..0000000000 --- a/nix/cua-driver/tests/screenshot.nix +++ /dev/null @@ -1,265 +0,0 @@ -# CUA Driver Screenshot Test -# -# Runs the integration test and then uses cua-driver's own get_window_state -# tool to capture a screenshot of an xterm window via MCP. The screenshot -# is extracted as a PNG from the base64 MCP response and copied out of the -# container. -# -# To run: nix build .#checks.x86_64-linux.cua-driver-screenshot -# -{ - pkgs, - lib ? pkgs.lib, - cuaDriverModule, - ... -}: - -let - # Python MCP client that runs the integration tests and then uses - # cua-driver to screenshot an xterm window via get_window_state. - mcpScreenshotTest = pkgs.writeText "mcp-screenshot-test.py" '' - import subprocess - import json - import sys - import os - import threading - import time - import base64 - - DRIVER_BIN = os.environ.get("CUA_DRIVER_BIN", "cua-driver") - - def main(): - print("=== CUA Driver Screenshot Test ===", flush=True) - - proc = subprocess.Popen( - [DRIVER_BIN, "mcp", "--no-daemon-relaunch"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env={**os.environ}, - ) - - def drain_stderr(): - for line in proc.stderr: - sys.stderr.buffer.write(line) - sys.stderr.buffer.flush() - t = threading.Thread(target=drain_stderr, daemon=True) - t.start() - - next_id = [0] - def send_request(method, params=None, req_id=None): - msg = {"jsonrpc": "2.0", "method": method} - if params is not None: - msg["params"] = params - if req_id is not None: - msg["id"] = req_id - line = json.dumps(msg) + "\n" - print(f"[send] {method}", flush=True) - proc.stdin.write(line.encode()) - proc.stdin.flush() - - def read_response(timeout=30): - result = [None] - def reader(): - result[0] = proc.stdout.readline() - rt = threading.Thread(target=reader) - rt.start() - rt.join(timeout) - if rt.is_alive(): - raise TimeoutError("No response within timeout") - line = result[0].decode().strip() - if len(line) > 500: - print(f"[recv] {line[:200]}...({len(line)} bytes)", flush=True) - else: - print(f"[recv] {line}", flush=True) - return json.loads(line) - - try: - # Initialize - print("\n--- Initialize ---", flush=True) - send_request("initialize", { - "protocolVersion": "2024-11-05", - "capabilities": {}, - "clientInfo": {"name": "nixos-screenshot-test", "version": "1.0.0"}, - }, req_id=1) - resp = read_response() - assert "result" in resp, f"Initialize failed: {resp}" - print("PASS: initialize", flush=True) - - send_request("notifications/initialized", {}) - time.sleep(0.5) - - # List tools - print("\n--- tools/list ---", flush=True) - send_request("tools/list", {}, req_id=2) - resp = read_response() - tools = resp.get("result", {}).get("tools", []) - tool_names = [t["name"] for t in tools] - assert "get_window_state" in tool_names, f"get_window_state missing" - assert "list_windows" in tool_names, f"list_windows missing" - print(f"PASS: {len(tools)} tools", flush=True) - - # Read xterm window ID and PID from files saved by the test setup - pid = None - window_id = None - try: - with open("/tmp/xterm-xid.txt") as f: - window_id = int(f.read().strip().split("\n")[0]) - with open("/tmp/xterm-pid.txt") as f: - pid = int(f.read().strip().split("\n")[0]) - print(f"Found xterm window: pid={pid} window_id={window_id}", flush=True) - except Exception as e: - print(f"ERROR: Could not read window info: {e}", flush=True) - - if pid is not None and window_id is not None: - # Use get_window_state with capture_mode=vision to get screenshot - print(f"\n--- get_window_state (vision) pid={pid} window_id={window_id} ---", flush=True) - send_request("tools/call", { - "name": "get_window_state", - "arguments": { - "pid": pid, - "window_id": window_id, - "capture_mode": "vision", - }, - }, req_id=5) - resp = read_response(timeout=30) - - # Extract base64 image from response - content = resp.get("result", {}).get("content", []) - screenshot_saved = False - for item in content: - if item.get("type") == "image": - img_data = item.get("data", "") - if img_data: - img_bytes = base64.b64decode(img_data) - with open("/tmp/cua-driver-screenshot.png", "wb") as f: - f.write(img_bytes) - print(f"PASS: Screenshot saved ({len(img_bytes)} bytes)", flush=True) - screenshot_saved = True - break - elif item.get("type") == "text": - text = item.get("text", "") - # Check if text contains base64 image data - if "base64" in text.lower() or len(text) > 1000: - print(f"Text content (may contain image ref): {text[:200]}", flush=True) - - if not screenshot_saved: - print(f"WARN: No image in response, saving raw response", flush=True) - with open("/tmp/cua-driver-response.json", "w") as f: - json.dump(resp, f, indent=2) - else: - print("SKIP: No window found for screenshot", flush=True) - - print("\n=== Screenshot test complete ===", flush=True) - - finally: - proc.stdin.close() - proc.terminate() - proc.wait(timeout=5) - - if __name__ == "__main__": - main() - ''; - - # Shell script displayed in xterm for a visible window to screenshot - testPage = pkgs.writeText "test-page.sh" '' - #!/bin/sh - cat <<'HEREDOC' - - ╔══════════════════════════════════════════════╗ - ║ CUA Driver - NixOS Integration Test ║ - ║ ║ - ║ cua-driver v0.3.2 ║ - ║ MCP server running on Xvfb :99 ║ - ║ 34 tools registered ║ - ║ ║ - ║ All tests passed! ║ - ╚══════════════════════════════════════════════╝ - - HEREDOC - sleep infinity - ''; - - # openbox config with focusNew=no (see openbox-rc.nix) so the xterm under test - # keeps X focus when it maps. - openboxRc = import ./openbox-rc.nix { inherit pkgs; }; - -in - -pkgs.testers.nixosTest { - name = "cua-driver-screenshot-test"; - meta = { - maintainers = [ ]; - }; - - containers.machine = - { - config, - pkgs, - lib, - ... - }: - { - imports = [ cuaDriverModule ]; - services.cua-driver.enable = true; - environment.systemPackages = with pkgs; [ - xorg.xorgserver - xterm - openbox # lightweight WM needed for _NET_CLIENT_LIST - picom # compositing manager so X11 GetImage returns rendered pixels - xdotool # window ID lookup - python3 - jq - procps - ]; - }; - - testScript = '' - # cache-bust 2026-06-08.1: this comment is part of the test derivation, so - # bumping it changes the output path and forces the test to actually re-run - # instead of being substituted from the binary cache. Bump again to re-run. - machine.start() - machine.wait_for_unit("multi-user.target") - - with subtest("Binary exists and runs"): - machine.succeed("cua-driver --help") - - with subtest("list-tools prints available tools"): - result = machine.succeed("cua-driver list-tools") - assert "click" in result, f"click not in: {result}" - assert "get_window_state" in result, f"get_window_state not in: {result}" - - with subtest("Start Xvfb, window manager, and xterm"): - machine.execute("Xvfb :99 -screen 0 1280x1024x24 >/dev/null 2>&1 &") - machine.wait_until_succeeds("test -e /tmp/.X11-unix/X99", timeout=10) - # Start openbox WM so _NET_CLIENT_LIST is populated for list_windows - machine.execute("DISPLAY=:99 openbox --config-file ${openboxRc} >/dev/null 2>&1 &") - # Start compositing manager so X11 GetImage returns rendered pixels - machine.execute("DISPLAY=:99 picom --backend xrender >/dev/null 2>&1 &") - machine.copy_from_host("${testPage}", "/tmp/test-page.sh") - machine.succeed("chmod +x /tmp/test-page.sh") - machine.execute("DISPLAY=:99 xterm -T 'CUA Test' -fa Monospace -fs 14 -geometry 60x20+100+100 -e /tmp/test-page.sh >/dev/null 2>&1 &") - # Wait for xterm window to appear (poll via xdotool inside the container) - machine.wait_until_succeeds("DISPLAY=:99 xdotool search --class xterm", timeout=15) - # Save the xterm window ID and PID for the MCP screenshot test - machine.succeed("DISPLAY=:99 xdotool search --class xterm | head -1 > /tmp/xterm-xid.txt") - machine.succeed("pgrep -f 'xterm.*CUA Test' | head -1 > /tmp/xterm-pid.txt") - # Focus and raise the window so it's fully rendered - machine.succeed("DISPLAY=:99 xdotool windowactivate --sync $(cat /tmp/xterm-xid.txt)") - machine.succeed("DISPLAY=:99 xdotool windowfocus --sync $(cat /tmp/xterm-xid.txt)") - - with subtest("Screenshot via cua-driver MCP"): - machine.copy_from_host("${mcpScreenshotTest}", "/tmp/mcp-screenshot-test.py") - result = machine.succeed( - "timeout 60 env DISPLAY=:99 " - "python3 /tmp/mcp-screenshot-test.py 2>&1" - ) - machine.log(result) - assert "Screenshot test complete" in result, f"Test did not complete: {result}" - - with subtest("Extract screenshot"): - # Copy screenshot out of the container if it exists - machine.succeed("test -f /tmp/cua-driver-screenshot.png || test -f /tmp/cua-driver-response.json") - machine.copy_from_machine("/tmp/cua-driver-screenshot.png", "") - ''; -} diff --git a/nix/cua-driver/tests/set-config.nix b/nix/cua-driver/tests/set-config.nix deleted file mode 100644 index ef4cd84862..0000000000 --- a/nix/cua-driver/tests/set-config.nix +++ /dev/null @@ -1,185 +0,0 @@ -# CUA Driver set_config Persistence Test -# -# Regression test for #1923 (fixed in PR #1928): on Linux the `set_config` -# tool only read the legacy per-field keys, so a `{"key":..., "value":...}` -# write (the shape the Swift/macOS and Windows callers send) was silently -# dropped. This boots a NixOS container, drives the driver over MCP stdio, -# writes config via the `{key, value}` shape, then reads it back with -# `get_config` and asserts the new value persisted. -# -# To run: nix build .#checks.x86_64-linux.cua-driver-set-config -# -{ - pkgs, - lib ? pkgs.lib, - cuaDriverModule, - ... -}: - -let - # Python MCP client that drives cua-driver over stdio. - # Writes config via the {key, value} shape, reads it back, asserts. - setConfigTest = pkgs.writeText "set-config-test.py" '' - import subprocess - import json - import sys - import os - import threading - import time - - DRIVER_BIN = os.environ.get("CUA_DRIVER_BIN", "cua-driver") - - def main(): - print("=== CUA Driver set_config Persistence Test ===", flush=True) - - proc = subprocess.Popen( - [DRIVER_BIN, "mcp", "--no-daemon-relaunch"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env={**os.environ}, - ) - - # Drain stderr in background to prevent blocking - def drain_stderr(): - for line in proc.stderr: - sys.stderr.buffer.write(line) - sys.stderr.buffer.flush() - t = threading.Thread(target=drain_stderr, daemon=True) - t.start() - - def send_request(method, params=None, req_id=None): - msg = {"jsonrpc": "2.0", "method": method} - if params is not None: - msg["params"] = params - if req_id is not None: - msg["id"] = req_id - line = json.dumps(msg) + "\n" - print(f"[send] {line.strip()}", flush=True) - proc.stdin.write(line.encode()) - proc.stdin.flush() - - def read_response(timeout=30): - # Simple blocking read with timeout via thread - result = [None] - def reader(): - result[0] = proc.stdout.readline() - rt = threading.Thread(target=reader) - rt.start() - rt.join(timeout) - if rt.is_alive(): - raise TimeoutError("No response within timeout") - line = result[0].decode().strip() - print(f"[recv] {line}", flush=True) - return json.loads(line) - - def call_tool(name, arguments, req_id): - send_request("tools/call", {"name": name, "arguments": arguments}, req_id=req_id) - resp = read_response() - assert resp.get("id") == req_id, f"Expected id={req_id}, got {resp.get('id')}" - assert "result" in resp, f"Expected result in response: {resp}" - return resp["result"] - - def get_config(req_id): - result = call_tool("get_config", {}, req_id) - sc = result.get("structuredContent", {}) - assert sc, f"get_config returned no structuredContent: {result}" - return sc - - try: - # Initialize - send_request("initialize", { - "protocolVersion": "2024-11-05", - "capabilities": {}, - "clientInfo": {"name": "nixos-set-config-test", "version": "1.0.0"}, - }, req_id=1) - resp = read_response() - assert resp.get("id") == 1 and "result" in resp, f"initialize failed: {resp}" - send_request("notifications/initialized", {}) - time.sleep(0.5) - - # Baseline: confirm the value we will write differs from the default - # (max_image_dimension default is 1568), so a passing read-back can - # only mean the write took effect — not a coincidental default. - print("\n--- Baseline get_config ---", flush=True) - before = get_config(2) - print(f"before: max_image_dimension={before.get('max_image_dimension')} " - f"capture_mode={before.get('capture_mode')}", flush=True) - assert before.get("max_image_dimension") != 800, \ - f"baseline already 800; cannot prove persistence: {before}" - - # The regression: write via the {key, value} shape (#1923). This is - # the exact path that was silently dropped on Linux before #1928. - print("\n--- set_config {key, value} ---", flush=True) - call_tool("set_config", {"key": "max_image_dimension", "value": 800}, 3) - call_tool("set_config", {"key": "capture_mode", "value": "ax"}, 4) - - # Read back: get_config must reflect the {key, value} writes. - print("\n--- get_config read-back ---", flush=True) - after = get_config(5) - print(f"after: max_image_dimension={after.get('max_image_dimension')} " - f"capture_mode={after.get('capture_mode')}", flush=True) - assert after.get("max_image_dimension") == 800, \ - f"set_config {{key,value}} did not persist max_image_dimension: {after}" - assert after.get("capture_mode") == "ax", \ - f"set_config {{key,value}} did not persist capture_mode: {after}" - - print("\n=== set_config persistence test passed! ===", flush=True) - - finally: - proc.stdin.close() - proc.terminate() - proc.wait(timeout=5) - - if __name__ == "__main__": - main() - ''; - -in - -pkgs.testers.nixosTest { - name = "cua-driver-set-config-test"; - meta = { - maintainers = [ ]; - }; - - containers.machine = - { - config, - pkgs, - lib, - ... - }: - { - imports = [ cuaDriverModule ]; - services.cua-driver.enable = true; - environment.systemPackages = with pkgs; [ - xorg.xorgserver # Xvfb for headless X11 - python3 # MCP client test script - ]; - }; - - testScript = '' - # cache-bust 2026-06-08.1: this comment is part of the test derivation, so - # bumping it changes the output path and forces the test to actually re-run - # instead of being substituted from the binary cache. Bump again to re-run. - machine.start() - machine.wait_for_unit("multi-user.target") - - with subtest("Binary exists and runs"): - machine.succeed("cua-driver --help") - - with subtest("Start Xvfb"): - machine.execute("Xvfb :99 -screen 0 1280x1024x24 >/dev/null 2>&1 &") - machine.wait_until_succeeds("test -e /tmp/.X11-unix/X99", timeout=10) - - with subtest("set_config {key, value} persists and reads back via get_config"): - machine.copy_from_host("${setConfigTest}", "/tmp/set-config-test.py") - result = machine.succeed( - "timeout 60 env DISPLAY=:99 " - "python3 /tmp/set-config-test.py 2>&1" - ) - machine.log(result) - assert "set_config persistence test passed" in result, f"set_config test failed: {result}" - ''; -} diff --git a/nix/cua-driver/tests/wayland/README.md b/nix/cua-driver/tests/wayland/README.md deleted file mode 100644 index 33a01cd6d9..0000000000 --- a/nix/cua-driver/tests/wayland/README.md +++ /dev/null @@ -1,67 +0,0 @@ -# cua-driver native-Wayland TDD suite - -A reproduction of the cua-driver NixOS tests on **native Wayland**, across five -desktop sessions. These tests are a **TDD red suite**: the Linux backend is -X11-only today, so they are **expected to fail** until native Wayland support is -added. They exist to *specify* that work and to show, per compositor, exactly -what survives. - -## Why they fail today - -cua-driver enumerates windows with X11 `_NET_CLIENT_LIST`, captures with -`import`/`xwd`/XGetImage, and injects input with X11 `XSendEvent` (plus uinput + -XI2 for the agent cursor / parallel drags). On native Wayland: - -- apps are **Wayland clients with no X11 window id**, so `list_windows` returns - nothing and `find_window` times out; -- there is no X display to capture or `XSendEvent` into (the tests never set - `DISPLAY`, so there is no XWayland fallback either). - -To make a scenario green you implement the corresponding native-Wayland path in -`libs/cua-driver/rust/crates/platform-linux` (e.g. a Wayland window enumerator, -a `grim`/screencopy capture path, and a `wlr-virtual-pointer` / -`virtual-keyboard` / `libei` input path), then re-run the matching check. - -## Layout - -- `session.nix` — brings up a chosen desktop headless and exposes the Wayland - socket. The compositor is the only per-desktop difference. -- `driver-client.nix` — shared MCP client; window discovery goes **only** through - cua-driver's own `list_windows` (no xdotool/X11 cheat). -- `record-wayland-gif.nix` — `grim`-based GIF recorder (wlroots only; no-op - elsewhere). -- `integration.nix`, `screenshot.nix`, `cursor-click-gif.nix`, - `background-terminal-gif.nix`, `parallel-drag.nix`, `background-gui.nix` — the - scenarios, each parameterised by `desktop` (and `background-gui` also by `app`). - -## Matrix - -Desktops: `xfce-labwc`, `xfce-sway` (XFCE 4.20's Wayland-session compositors), -`kde` (kwin_wayland), `gnome` (mutter headless). (`xfce-wayfire` was dropped — -wayfire fails to build in the current nixpkgs pin, an upstream `wf-config`/ -`doctest` issue unrelated to cua-driver.) - -Checks (built on demand): - -``` -nix build .#checks.x86_64-linux.cua-driver-wayland-- -nix build .#checks.x86_64-linux.cua-driver-wayland--background-gui- -``` - -Scenarios: `integration`, `screenshot`, `cursor-click-gif`, -`background-terminal-gif`, `parallel-drag`. -Background-GUI apps: `foot`, `gtk3-gedit`, `qt6-kcalc`. - -`parallel-drag` is the hardest: it needs X11 MPX master pointers, for which there -is no native Wayland equivalent — keep it red, or redesign around multi-seat / -virtual-pointer protocols. - -## Running in CI - -The `.github/workflows/nix-wayland.yml` workflow runs the whole matrix on the -same triggers as the X11 `nix-build.yml` workflow — PRs and `main` pushes that -touch `nix/**`, `flake.{nix,lock}`, `libs/cua-driver/rust/**`, or the workflow -itself, plus manual `workflow_dispatch`. The jobs are BLOCKING: a red cell fails -the PR, so the suite only goes green once native Wayland support lands. Each job -uploads its screenshots / GIFs / logs as artifacts so you can see what each -compositor did. diff --git a/nix/cua-driver/tests/wayland/background-gui.nix b/nix/cua-driver/tests/wayland/background-gui.nix deleted file mode 100644 index 0df054f28b..0000000000 --- a/nix/cua-driver/tests/wayland/background-gui.nix +++ /dev/null @@ -1,159 +0,0 @@ -# CUA Driver native-Wayland background-GUI skeleton (per desktop, per app) — TDD red. -# -# Native-Wayland analogue of the READ-ONLY skeleton class in -# ../../tests/linux-background-gui.nix. Launches a real toolkit app as a NATIVE -# Wayland client (no GDK_BACKEND/QT_QPA forcing), finds it via cua-driver -# list_windows, and screenshots it via get_window_state (vision). Records a GIF -# via grim. Fails today: the driver can neither enumerate nor capture native -# Wayland surfaces — the spec for native Wayland read support across toolkits. -# -# To run: nix build .#checks.x86_64-linux.cua-driver-wayland--background-gui- -{ - pkgs, - lib ? pkgs.lib, - cuaDriverModule, - desktop, - app, - ... -}: - -let - session = import ./session.nix { inherit pkgs desktop; }; - driverClient = import ./driver-client.nix { inherit pkgs; }; - recordGifScript = import ./record-wayland-gif.nix { inherit pkgs; }; - - # Representative native-Wayland apps, one per major toolkit. `match` is the - # identity substring find_window looks for (title/app_id/app/class). - # - # NOTE: no per-app env prefix here. Apps are launched via cua-driver's - # `launch_app`, which execs the command's FIRST token as the program — an - # `ENV=val cmd` prefix would be (mis)read as the program name. The native - # Wayland backends (GDK_BACKEND/QT_QPA_PLATFORM) are exported by the session - # driver wrapper instead (session.nix appBackendEnv), so the launched toolkit - # apps inherit them. Do not reintroduce an `env` field on these. - apps = { - foot = { - packages = [ pkgs.foot ]; - launch = "foot --app-id=cua-wayland-foot-app --title=cua-wayland-foot-app"; - match = "cua-wayland-foot-app"; - }; - "gtk3-gedit" = { - packages = [ pkgs.gedit ]; - launch = "gedit"; - match = "gedit"; - }; - "qt6-kcalc" = { - packages = [ pkgs.kdePackages.kcalc ]; - launch = "kcalc"; - match = "kcalc"; - }; - }; - - cfg = apps.${app} or (throw "wayland/background-gui.nix: unknown app '${app}'"); - outputGif = "/tmp/cua-driver-wayland-${desktop}-background-gui-${app}.gif"; - outputPng = "/tmp/cua-driver-wayland-${desktop}-background-gui-${app}.png"; - - # GIF recorder env: native desktops point grim at the host socket; nested - # desktops leave WAYLAND_DISPLAY unset so the recorder self-resolves the - # driver's published nested socket ($XDG_RUNTIME_DIR/.cua-nested-display). - # No quotes around the command substitution: this string is interpolated into - # a python double-quoted string, and the socket path has no spaces. The inner - # `sh -c` re-parses and expands $(cat ...) itself. - recorderWlEnv = - if session.nested then - "env XDG_RUNTIME_DIR=/run/user/0" - else - "env XDG_RUNTIME_DIR=/run/user/0 WAYLAND_DISPLAY=$(cat /tmp/wl-display)"; - - testScriptPy = pkgs.writeText "wayland-bg-gui.py" '' - import base64, sys - sys.path.insert(0, "/tmp") - from driver_client import Driver - - MATCH = "${cfg.match}" - OUTPNG = "${outputPng}" - LAUNCH = "${cfg.launch}" - d = Driver() - try: - d.initialize("nixos-wayland-bg-gui") - # Launch through cua-driver so the app lands in the session it owns - # (host compositor on native desktops; nested labwc on kde/gnome). - d.launch_app(LAUNCH) - pid, wid = d.find_window(MATCH, timeout=50) - print(f"app window pid={pid} window_id={wid}", flush=True) - resp = d.call("get_window_state", { - "pid": pid, "window_id": wid, "capture_mode": "vision"}, timeout=45) - saved = False - for item in resp.get("result", {}).get("content", []): - if item.get("type") == "image" and item.get("data"): - with open(OUTPNG, "wb") as f: - f.write(base64.b64decode(item["data"])) - saved = True - break - assert saved, f"no capture for native Wayland window: {resp}" - print("background gui read test complete", flush=True) - finally: - d.close() - ''; -in - -pkgs.testers.nixosTest { - name = "cua-driver-wayland-${desktop}-background-gui-${app}-test"; - meta.maintainers = [ ]; - - nodes.machine = - { pkgs, ... }: - { - imports = [ cuaDriverModule ]; - virtualisation = { - cores = 2; - memorySize = 4096; - }; - services.cua-driver.enable = true; - boot.kernelModules = [ "uinput" ]; - hardware.graphics.enable = true; - services.udev.extraRules = '' - KERNEL=="uinput", MODE="0660", GROUP="input", OPTIONS+="static_node=uinput" - ''; - environment.systemPackages = - session.packages ++ cfg.packages ++ (with pkgs; [ python3 jq at-spi2-core ]); - }; - - testScript = '' - machine.start() - machine.wait_for_unit("multi-user.target") - - with subtest("Bring up ${session.label}"): - machine.execute("${session.start} >/tmp/session.log 2>&1 &") - try: - machine.wait_for_file("/tmp/wl-ready", timeout=120) - except Exception: - machine.log(machine.execute("cat /tmp/session.log || true")[1]) - machine.log(machine.execute("cat /tmp/compositor.log || true")[1]) - raise - - with subtest("Launch (via cua-driver) + record GIF + read inactive app window"): - machine.copy_from_host("${driverClient}", "/tmp/driver_client.py") - machine.copy_from_host("${testScriptPy}", "/tmp/wayland-bg-gui.py") - # Start the recorder first; it grabs frames while the driver launches - # and reads the app. (Native: host socket; nested: self-resolves.) - machine.execute( - "sh -lc '${recorderWlEnv} ${recordGifScript} /tmp/bg-gui-frames " - "${outputGif} /tmp/stop-bg-gui-recorder /tmp/rec-bg-gui.log 12 0.2 " - ">/dev/null 2>&1 & echo $! >/tmp/rec-bg-gui.pid'" - ) - result = machine.execute( - "timeout 150 env CUA_DRIVER_BIN=${session.driverWrapper} " - "XDG_RUNTIME_DIR=/run/user/0 python3 /tmp/wayland-bg-gui.py 2>&1" - )[1] - machine.log(result) - # Stop + copy the GIF BEFORE asserting so even failing jobs upload one. - machine.execute("touch /tmp/stop-bg-gui-recorder") - machine.execute("sh -lc 'for i in $(seq 1 60); do kill -0 $(cat /tmp/rec-bg-gui.pid) 2>/dev/null || break; sleep 1; done'") - machine.execute("test -e ${outputGif} || : > ${outputGif}") - machine.copy_from_machine("${outputGif}", "") - - with subtest("cua-driver read/captured the native Wayland app window"): - assert "background gui read test complete" in result, result - ''; -} diff --git a/nix/cua-driver/tests/wayland/background-terminal-gif.nix b/nix/cua-driver/tests/wayland/background-terminal-gif.nix deleted file mode 100644 index 871d62d18b..0000000000 --- a/nix/cua-driver/tests/wayland/background-terminal-gif.nix +++ /dev/null @@ -1,111 +0,0 @@ -# CUA Driver native-Wayland background-terminal GIF test (per desktop). -# -# The hard one: type into an UNFOCUSED window. Stock Wayland delivers keyboard -# only to the focused surface, so cua-driver nests its own cua-compositor (EIS -# mode) which injects wl_keyboard straight into a target surface by app_id, -# bypassing focus. A control terminal is launched AFTER the target so the target -# is NOT focused; cua-driver types a command into the inactive target and we -# verify it ran via a file side-effect. Records a GIF via grim. -# -# To run: nix build .#checks.x86_64-linux.cua-driver-wayland--background-terminal-gif -{ - pkgs, - lib ? pkgs.lib, - cuaDriverModule, - desktop, - ... -}: - -let - session = import ./session.nix { inherit pkgs desktop; eis = true; }; - driverClient = import ./driver-client.nix { inherit pkgs; }; - recordGifScript = import ./record-wayland-gif.nix { inherit pkgs; }; - - testScriptPy = pkgs.writeText "wayland-background-terminal.py" '' - import sys, time - sys.path.insert(0, "/tmp") - from driver_client import Driver - - d = Driver() - try: - d.initialize("nixos-wayland-bg-terminal") - # Launch the TARGET first, then the CONTROL — so control holds focus and - # the target is the inactive window we must type into focus-free. - d.launch_app("foot --app-id=cua-wayland-target --title=cua-wayland-target") - time.sleep(1.2) - d.launch_app("foot --app-id=cua-wayland-control --title=cua-wayland-control") - tpid, twid = d.find_window("cua-wayland-target", timeout=40) - cpid, cwid = d.find_window("cua-wayland-control", timeout=40) - print(f"target pid={tpid} window_id={twid} control pid={cpid} window_id={cwid}", flush=True) - d.call("set_agent_cursor_enabled", {"enabled": True}) - # Focus-free type into the INACTIVE target while control stays focused. - d.call("type_text", {"pid": tpid, "window_id": twid, - "text": "echo hello | tee /tmp/background-hello.txt"}) - d.call("press_key", {"pid": tpid, "window_id": twid, "key": "enter"}) - time.sleep(1.8) - print("background terminal GIF test complete", flush=True) - finally: - d.close() - ''; -in - -pkgs.testers.nixosTest { - name = "cua-driver-wayland-${desktop}-background-terminal-gif-test"; - meta.maintainers = [ ]; - - nodes.machine = - { pkgs, ... }: - { - imports = [ cuaDriverModule ]; - virtualisation = { - cores = 2; - memorySize = 3072; - }; - services.cua-driver.enable = true; - boot.kernelModules = [ "uinput" ]; - hardware.graphics.enable = true; - services.udev.extraRules = '' - KERNEL=="uinput", MODE="0660", GROUP="input", OPTIONS+="static_node=uinput" - ''; - environment.systemPackages = session.packages ++ (with pkgs; [ python3 jq ]); - }; - - testScript = '' - machine.start() - machine.wait_for_unit("multi-user.target") - - with subtest("Bring up ${session.label}"): - machine.execute("${session.start} >/tmp/session.log 2>&1 &") - try: - machine.wait_for_file("/tmp/wl-ready", timeout=120) - except Exception: - machine.log(machine.execute("cat /tmp/session.log || true")[1]) - machine.log(machine.execute("cat /tmp/compositor.log || true")[1]) - raise - - with subtest("Record GIF + driver types focus-free into the INACTIVE Wayland terminal"): - machine.copy_from_host("${driverClient}", "/tmp/driver_client.py") - machine.copy_from_host("${testScriptPy}", "/tmp/wayland-background-terminal.py") - # Recorder self-resolves the driver's nested cua-compositor socket. - machine.execute( - "sh -lc 'env XDG_RUNTIME_DIR=/run/user/0 ${recordGifScript} /tmp/background-frames " - "/tmp/cua-driver-wayland-${desktop}-background-terminal.gif /tmp/stop-background-recorder " - "/tmp/rec-background.log 10 0.15 >/dev/null 2>&1 & echo $! >/tmp/rec-background.pid'" - ) - result = machine.execute( - "timeout 120 env CUA_DRIVER_BIN=${session.driverWrapper} " - "XDG_RUNTIME_DIR=/run/user/0 python3 /tmp/wayland-background-terminal.py 2>&1" - )[1] - machine.log(result) - machine.execute("touch /tmp/stop-background-recorder") - machine.execute("sh -lc 'for i in $(seq 1 60); do kill -0 $(cat /tmp/rec-background.pid) 2>/dev/null || break; sleep 1; done'") - - with subtest("Copy GIF out of the VM (best-effort)"): - machine.execute("test -e /tmp/cua-driver-wayland-${desktop}-background-terminal.gif || : > /tmp/cua-driver-wayland-${desktop}-background-terminal.gif") - machine.copy_from_machine("/tmp/cua-driver-wayland-${desktop}-background-terminal.gif", "") - - with subtest("Driver typed into inactive terminal AND the command ran"): - assert "background terminal GIF test complete" in result, result - machine.wait_until_succeeds("grep -Fx 'hello' /tmp/background-hello.txt", timeout=20) - ''; -} diff --git a/nix/cua-driver/tests/wayland/cursor-click-gif.nix b/nix/cua-driver/tests/wayland/cursor-click-gif.nix deleted file mode 100644 index 3e05fbcaf6..0000000000 --- a/nix/cua-driver/tests/wayland/cursor-click-gif.nix +++ /dev/null @@ -1,129 +0,0 @@ -# CUA Driver native-Wayland cursor-click GIF test (per desktop). -# -# Launches two native Wayland terminals (foot) THROUGH cua-driver (launch_app), -# clicks the target, types a command through cua-driver's Wayland injection path, -# and verifies the command RAN by checking a file it writes -# (display-server-agnostic proof). -# Records a GIF of the composited output via grim. On wlroots desktops the driver -# can drive the host compositor, but compositor-specific focus heuristics make -# virtual-keyboard delivery after a click unreliable in headless CI. This visual -# test therefore uses the same EIS-backed nested compositor as the background -# terminal test so it remains a deterministic click+type regression guard across -# labwc, sway, KDE, and GNOME hosts. -# -# To run: nix build .#checks.x86_64-linux.cua-driver-wayland--cursor-click-gif -{ - pkgs, - lib ? pkgs.lib, - cuaDriverModule, - desktop, - ... -}: - -let - session = import ./session.nix { inherit pkgs desktop; eis = true; }; - driverClient = import ./driver-client.nix { inherit pkgs; }; - recordGifScript = import ./record-wayland-gif.nix { inherit pkgs; }; - - # GIF recorder env: native points grim at the host socket; nested self-resolves - # the driver's published nested socket ($XDG_RUNTIME_DIR/.cua-nested-display). - recorderWlEnv = - if session.nested then - "env XDG_RUNTIME_DIR=/run/user/0" - else - "env XDG_RUNTIME_DIR=/run/user/0 WAYLAND_DISPLAY=$(cat /tmp/wl-display)"; - - testScriptPy = pkgs.writeText "wayland-cursor-click.py" '' - import sys, time - sys.path.insert(0, "/tmp") - from driver_client import Driver - - d = Driver() - try: - d.initialize("nixos-wayland-click") - # Launch a control terminal then the target. The click still exercises - # the target pointer path, while type_text/press_key use the EIS-backed - # injection path so headless compositor focus policy cannot make the - # test flaky. - d.launch_app("foot --app-id=cua-wayland-control --title=cua-wayland-control") - time.sleep(1) - d.launch_app("foot --app-id=cua-wayland-target --title=cua-wayland-target") - pid, wid = d.find_window("cua-wayland-target", timeout=40) - print(f"target pid={pid} window_id={wid}", flush=True) - d.call("set_agent_cursor_enabled", {"enabled": True}) - d.call("move_cursor", {"x": 1100.0, "y": 900.0}) - time.sleep(0.6) - d.call("click", {"pid": pid, "window_id": wid, "x": 120.0, "y": 120.0}) - time.sleep(0.4) - d.call("type_text", {"pid": pid, "window_id": wid, - "text": "echo click-focus > /tmp/click-focus.txt"}) - d.call("press_key", {"pid": pid, "window_id": wid, "key": "enter"}) - time.sleep(1.5) - print("click GIF test complete", flush=True) - finally: - d.close() - ''; -in - -pkgs.testers.nixosTest { - name = "cua-driver-wayland-${desktop}-cursor-click-gif-test"; - meta.maintainers = [ ]; - - nodes.machine = - { pkgs, ... }: - { - imports = [ cuaDriverModule ]; - virtualisation = { - cores = 2; - memorySize = 3072; - }; - services.cua-driver.enable = true; - boot.kernelModules = [ "uinput" ]; - hardware.graphics.enable = true; - services.udev.extraRules = '' - KERNEL=="uinput", MODE="0660", GROUP="input", OPTIONS+="static_node=uinput" - ''; - environment.systemPackages = session.packages ++ (with pkgs; [ python3 jq ]); - }; - - testScript = '' - machine.start() - machine.wait_for_unit("multi-user.target") - machine.succeed("modprobe uinput && test -e /dev/uinput") - - with subtest("Bring up ${session.label}"): - machine.execute("${session.start} >/tmp/session.log 2>&1 &") - try: - machine.wait_for_file("/tmp/wl-ready", timeout=120) - except Exception: - machine.log(machine.execute("cat /tmp/session.log || true")[1]) - machine.log(machine.execute("cat /tmp/compositor.log || true")[1]) - raise - - with subtest("Record GIF + driver launches, clicks, and types into a native Wayland terminal"): - machine.copy_from_host("${driverClient}", "/tmp/driver_client.py") - machine.copy_from_host("${testScriptPy}", "/tmp/wayland-cursor-click.py") - machine.execute( - "sh -lc '${recorderWlEnv} ${recordGifScript} /tmp/click-frames " - "/tmp/cua-driver-wayland-${desktop}-cursor-click.gif /tmp/stop-click-recorder " - "/tmp/rec-click.log 10 0.15 >/dev/null 2>&1 & echo $! >/tmp/rec-click.pid'" - ) - result = machine.execute( - "timeout 120 env CUA_DRIVER_BIN=${session.driverWrapper} " - "XDG_RUNTIME_DIR=/run/user/0 python3 /tmp/wayland-cursor-click.py 2>&1" - )[1] - machine.log(result) - machine.execute("touch /tmp/stop-click-recorder") - machine.execute("sh -lc 'for i in $(seq 1 60); do kill -0 $(cat /tmp/rec-click.pid) 2>/dev/null || break; sleep 1; done'") - - with subtest("Copy GIF out of the VM (best-effort)"): - # Ensure the path exists so copy never errors before the real assertion; - # it may be empty on compositors where grim could not capture. - machine.execute("test -e /tmp/cua-driver-wayland-${desktop}-cursor-click.gif || : > /tmp/cua-driver-wayland-${desktop}-cursor-click.gif") - machine.copy_from_machine("/tmp/cua-driver-wayland-${desktop}-cursor-click.gif", "") - - with subtest("Driver completed click+type AND the command ran in the Wayland terminal"): - assert "click GIF test complete" in result, result - machine.wait_until_succeeds("test -f /tmp/click-focus.txt", timeout=20) - ''; -} diff --git a/nix/cua-driver/tests/wayland/driver-client.nix b/nix/cua-driver/tests/wayland/driver-client.nix deleted file mode 100644 index 5b20fe0c52..0000000000 --- a/nix/cua-driver/tests/wayland/driver-client.nix +++ /dev/null @@ -1,138 +0,0 @@ -# Shared MCP client used by the native-Wayland TDD tests. -# -# It is copied into the VM as /tmp/driver_client.py and imported by the small -# per-test scripts. Crucially, window discovery goes ONLY through cua-driver's -# own `list_windows` tool — there is no xdotool/X11 fallback — so on native -# Wayland (where the driver currently enumerates nothing) `find_window` times -# out and the test fails. That is the intended red state for TDD. -{ pkgs }: - -pkgs.writeText "driver_client.py" '' - import json, os, subprocess, sys, threading, time - - DRIVER_BIN = os.environ.get("CUA_DRIVER_BIN", "cua-driver") - - - class Driver: - def __init__(self): - self.proc = subprocess.Popen( - [DRIVER_BIN, "mcp", "--no-daemon-relaunch"], - stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env={**os.environ}, - ) - threading.Thread(target=self._drain, daemon=True).start() - self._id = 1 - - def _drain(self): - for line in self.proc.stderr: - sys.stderr.buffer.write(line) - sys.stderr.buffer.flush() - - def _send(self, method, params=None, req_id=None): - msg = {"jsonrpc": "2.0", "method": method} - if params is not None: - msg["params"] = params - if req_id is not None: - msg["id"] = req_id - self.proc.stdin.write((json.dumps(msg) + "\n").encode()) - self.proc.stdin.flush() - - def _recv(self, timeout=45): - result = [None] - - def reader(): - result[0] = self.proc.stdout.readline() - - th = threading.Thread(target=reader) - th.start() - th.join(timeout) - if th.is_alive(): - raise TimeoutError("no response from driver within timeout") - line = result[0].decode().strip() - if not line: - raise RuntimeError("driver returned an empty response") - return json.loads(line) - - def initialize(self, client="nixos-wayland"): - self._send("initialize", { - "protocolVersion": "2024-11-05", "capabilities": {}, - "clientInfo": {"name": client, "version": "1.0.0"}, - }, req_id=self._id) - resp = self._recv() - self._id += 1 - assert "result" in resp, f"initialize failed: {resp}" - self._send("notifications/initialized", {}) - time.sleep(0.3) - return resp - - def call(self, name, args, timeout=60): - self._send("tools/call", {"name": name, "arguments": args}, req_id=self._id) - self._id += 1 - resp = self._recv(timeout=timeout) - if resp.get("error"): - raise RuntimeError(f"{name} failed: {resp}") - if resp.get("result", {}).get("isError"): - raise RuntimeError(f"{name} returned isError: {resp}") - return resp - - def launch_app(self, command): - """Launch an app THROUGH cua-driver so it lands in whatever Wayland - session the driver owns — the host compositor (native desktops) or the - driver's own nested labwc (kde/gnome). Returns the launch response.""" - return self.call("launch_app", {"name": command}) - - def list_windows(self): - resp = self.call("list_windows", {}) - result = resp.get("result", {}) - # Preferred: MCP structuredContent = { "windows": [ {window_id, pid, - # title, ...}, ... ] }. - structured = result.get("structuredContent") - if isinstance(structured, dict) and isinstance(structured.get("windows"), list): - return structured["windows"] - if isinstance(structured, list): - return structured - # Fallback: a text item that happens to be JSON. - for item in result.get("content", []): - if item.get("type") == "text": - try: - data = json.loads(item.get("text", "")) - except Exception: - continue - if isinstance(data, list): - return data - if isinstance(data, dict) and "windows" in data: - return data["windows"] - return [] - - def find_window(self, title_substr, timeout=30, interval=1.0): - """Poll list_windows until a native Wayland toplevel whose title - contains `title_substr` appears. Raises on timeout — the red point on - a driver that cannot yet enumerate Wayland windows.""" - deadline = time.time() + timeout - needle = title_substr.lower() - last = [] - while time.time() < deadline: - last = self.list_windows() - for w in last: - # Match across whatever identity fields a Wayland-capable - # list_windows might surface (title today; app_id/app/class - # once native Wayland metadata is wired up). - hay = " ".join( - str(w.get(k, "")) for k in ("title", "app_id", "app", "class") - ).lower() - if needle in hay: - return int(w.get("pid") or 0), int(w["window_id"]) - time.sleep(interval) - raise AssertionError( - f"cua-driver never enumerated a Wayland window titled ~{title_substr!r}; " - f"last list_windows() = {last}" - ) - - def close(self): - try: - self.proc.stdin.close() - self.proc.terminate() - self.proc.wait(timeout=5) - except Exception: - pass -'' diff --git a/nix/cua-driver/tests/wayland/integration.nix b/nix/cua-driver/tests/wayland/integration.nix deleted file mode 100644 index e249141167..0000000000 --- a/nix/cua-driver/tests/wayland/integration.nix +++ /dev/null @@ -1,109 +0,0 @@ -# CUA Driver native-Wayland integration test (per desktop). -# -# Brings up the desktop session, then launches a native Wayland terminal (foot) -# THROUGH cua-driver (launch_app) and asserts cua-driver can ENUMERATE it via -# list_windows. On native (wlroots) desktops the driver talks to the host -# compositor; on kde/gnome it drives its own nested labwc (CUA_WAYLAND_NEST=1, -# set by the session driver wrapper). Either way the app lands in the session -# the driver owns. -# -# To run: nix build .#checks.x86_64-linux.cua-driver-wayland--integration -{ - pkgs, - lib ? pkgs.lib, - cuaDriverModule, - desktop, - ... -}: - -let - session = import ./session.nix { inherit pkgs desktop; }; - driverClient = import ./driver-client.nix { inherit pkgs; }; - - testScriptPy = pkgs.writeText "wayland-integration.py" '' - import sys, time - sys.path.insert(0, "/tmp") - from driver_client import Driver - - d = Driver() - try: - d.initialize("nixos-wayland-integration") - - # Handshake sanity: required tools are advertised. - d._send("tools/list", {}, req_id=999) - resp = d._recv() - names = [t["name"] for t in resp.get("result", {}).get("tools", [])] - for t in ("list_windows", "get_window_state", "click", "type_text", "launch_app"): - assert t in names, f"{t} not advertised: {names}" - print(f"tools advertised: {len(names)}", flush=True) - - # Launch foot through cua-driver so it lands in the session the driver - # owns (host compositor on native desktops; nested labwc on kde/gnome). - d.launch_app("foot --app-id=cua-wayland-foot --title=cua-wayland-foot") - - # list_windows must surface the foot terminal the driver just launched. - pid, wid = d.find_window("cua-wayland-foot", timeout=40) - print(f"RESOLVED native wayland window pid={pid} window_id={wid}", flush=True) - print("integration test complete", flush=True) - finally: - d.close() - ''; -in - -pkgs.testers.nixosTest { - name = "cua-driver-wayland-${desktop}-integration-test"; - meta.maintainers = [ ]; - - nodes.machine = - { pkgs, ... }: - { - imports = [ cuaDriverModule ]; - virtualisation = { - cores = 2; - memorySize = 3072; - }; - services.cua-driver.enable = true; - boot.kernelModules = [ "uinput" ]; - hardware.graphics.enable = true; - services.udev.extraRules = '' - KERNEL=="uinput", MODE="0660", GROUP="input", OPTIONS+="static_node=uinput" - ''; - environment.systemPackages = session.packages ++ (with pkgs; [ python3 jq ]); - }; - - testScript = '' - machine.start() - machine.wait_for_unit("multi-user.target") - - with subtest("Binary exists and lists tools"): - machine.succeed("cua-driver --help") - tools = machine.succeed("cua-driver list-tools") - for t in ("list_windows", "get_window_state", "click", "type_text"): - assert t in tools, f"{t} missing from list-tools" - - with subtest("Bring up ${session.label}"): - machine.execute("${session.start} >/tmp/session.log 2>&1 &") - try: - machine.wait_for_file("/tmp/wl-ready", timeout=120) - except Exception: - machine.log(machine.execute("cat /tmp/session.log || true")[1]) - machine.log(machine.execute("cat /tmp/compositor.log || true")[1]) - raise - machine.log("host WAYLAND_DISPLAY=" + machine.succeed("cat /tmp/wl-display").strip()) - - with subtest("doctor reports a Wayland status line"): - result = machine.succeed("timeout 30 ${session.driverWrapper} doctor 2>&1 || true") - machine.log(result) - assert "Wayland" in result, f"doctor did not report Wayland: {result}" - - with subtest("cua-driver launches + enumerates a native Wayland window"): - machine.copy_from_host("${driverClient}", "/tmp/driver_client.py") - machine.copy_from_host("${testScriptPy}", "/tmp/wayland-integration.py") - result = machine.succeed( - "timeout 120 env CUA_DRIVER_BIN=${session.driverWrapper} " - "XDG_RUNTIME_DIR=/run/user/0 python3 /tmp/wayland-integration.py 2>&1" - ) - machine.log(result) - assert "integration test complete" in result, result - ''; -} diff --git a/nix/cua-driver/tests/wayland/parallel-drag.nix b/nix/cua-driver/tests/wayland/parallel-drag.nix deleted file mode 100644 index 7d67ce50d0..0000000000 --- a/nix/cua-driver/tests/wayland/parallel-drag.nix +++ /dev/null @@ -1,104 +0,0 @@ -# CUA Driver native-Wayland parallel multi-cursor drag test (per desktop). -# -# Two simultaneous drag gestures on one window. Stock Wayland has a single seat -# cursor and no multi-pointer protocol, so cua-driver nests its own -# cua-compositor (EIS mode), which drives N independent logical cursors and -# injects wl_pointer per cursor by app_id. parallel_mouse_drag routes through -# the compositor's control socket (window-local coords, no X11 MPX). Records a -# GIF via grim. -# -# To run: nix build .#checks.x86_64-linux.cua-driver-wayland--parallel-drag -{ - pkgs, - lib ? pkgs.lib, - cuaDriverModule, - desktop, - ... -}: - -let - session = import ./session.nix { inherit pkgs desktop; eis = true; }; - driverClient = import ./driver-client.nix { inherit pkgs; }; - recordGifScript = import ./record-wayland-gif.nix { inherit pkgs; }; - - testScriptPy = pkgs.writeText "wayland-parallel-drag.py" '' - import sys, time - sys.path.insert(0, "/tmp") - from driver_client import Driver - - d = Driver() - try: - d.initialize("nixos-wayland-parallel-drag") - d.launch_app("foot --app-id=cua-wayland-paint --title=cua-wayland-paint") - pid, wid = d.find_window("cua-wayland-paint", timeout=40) - print(f"target pid={pid} window_id={wid}", flush=True) - d.call("parallel_mouse_drag", {"drags": [ - {"session": "agent-1", "window_id": wid, "from_x": 100.0, "from_y": 100.0, "to_x": 380.0, "to_y": 420.0, "duration_ms": 2000, "steps": 60}, - {"session": "agent-2", "window_id": wid, "from_x": 700.0, "from_y": 100.0, "to_x": 420.0, "to_y": 420.0, "duration_ms": 2000, "steps": 60}, - ]}) - time.sleep(0.6) - print("parallel drag test complete", flush=True) - finally: - d.close() - ''; -in - -pkgs.testers.nixosTest { - name = "cua-driver-wayland-${desktop}-parallel-drag-test"; - meta.maintainers = [ ]; - - nodes.machine = - { pkgs, ... }: - { - imports = [ cuaDriverModule ]; - virtualisation = { - cores = 2; - memorySize = 3072; - }; - services.cua-driver.enable = true; - boot.kernelModules = [ "uinput" ]; - hardware.graphics.enable = true; - services.udev.extraRules = '' - KERNEL=="uinput", MODE="0660", GROUP="input", OPTIONS+="static_node=uinput" - ''; - services.libinput.enable = true; - environment.systemPackages = session.packages ++ (with pkgs; [ python3 jq ]); - }; - - testScript = '' - machine.start() - machine.wait_for_unit("multi-user.target") - - with subtest("Bring up ${session.label}"): - machine.execute("${session.start} >/tmp/session.log 2>&1 &") - try: - machine.wait_for_file("/tmp/wl-ready", timeout=120) - except Exception: - machine.log(machine.execute("cat /tmp/session.log || true")[1]) - machine.log(machine.execute("cat /tmp/compositor.log || true")[1]) - raise - - with subtest("Record GIF + pilot cua-driver through parallel_mouse_drag (multi-cursor)"): - machine.copy_from_host("${driverClient}", "/tmp/driver_client.py") - machine.copy_from_host("${testScriptPy}", "/tmp/wayland-parallel-drag.py") - machine.execute( - "sh -lc 'env XDG_RUNTIME_DIR=/run/user/0 ${recordGifScript} /tmp/drag-frames " - "/tmp/cua-driver-wayland-${desktop}-parallel-drag.gif /tmp/stop-drag-recorder " - "/tmp/rec-drag.log 8 0.12 >/dev/null 2>&1 & echo $! >/tmp/rec-drag.pid'" - ) - result = machine.execute( - "timeout 120 env CUA_DRIVER_BIN=${session.driverWrapper} " - "XDG_RUNTIME_DIR=/run/user/0 python3 /tmp/wayland-parallel-drag.py 2>&1" - )[1] - machine.log(result) - machine.execute("touch /tmp/stop-drag-recorder") - machine.execute("sh -lc 'for i in $(seq 1 60); do kill -0 $(cat /tmp/rec-drag.pid) 2>/dev/null || break; sleep 1; done'") - - with subtest("Copy GIF out of the VM (best-effort)"): - machine.execute("test -e /tmp/cua-driver-wayland-${desktop}-parallel-drag.gif || : > /tmp/cua-driver-wayland-${desktop}-parallel-drag.gif") - machine.copy_from_machine("/tmp/cua-driver-wayland-${desktop}-parallel-drag.gif", "") - - with subtest("parallel_mouse_drag completed on native Wayland (multi-cursor via EIS compositor)"): - assert "parallel drag test complete" in result, result - ''; -} diff --git a/nix/cua-driver/tests/wayland/record-wayland-gif.nix b/nix/cua-driver/tests/wayland/record-wayland-gif.nix deleted file mode 100644 index bc870404ba..0000000000 --- a/nix/cua-driver/tests/wayland/record-wayland-gif.nix +++ /dev/null @@ -1,59 +0,0 @@ -# Native-Wayland screen-recording helper for the cua-driver Wayland TDD tests. -# -# Captures the composited Wayland output with `grim` (wlr-screencopy) in a loop -# and stitches the frames into an animated GIF. grim only works on wlroots -# compositors (labwc/wayfire/sway); on kwin/mutter it simply produces no frames -# and no GIF — which is fine, the GIF is a best-effort artifact and these tests -# are expected to fail before/around it anyway. -# -# Usage (in a test's `let`): -# recordGifScript = import ./record-wayland-gif.nix { inherit pkgs; }; -# then inside the testScript, start it in the background before driving and -# `touch` the stop-file afterwards: -# ${recordGifScript} \ -# -# The caller must export WAYLAND_DISPLAY and XDG_RUNTIME_DIR for grim. -{ pkgs }: - -pkgs.writeShellScript "record-wayland-gif.sh" '' - set -u - frames_dir="$1" - output_gif="$2" - stop_file="$3" - log_file="$4" - delay_cs="$5" - interval="$6" - - rm -f "$stop_file" "$output_gif" "$log_file" - rm -rf "$frames_dir" - mkdir -p "$frames_dir" - - # In nested mode the caller can't know cua-driver's private socket ahead of - # time; the driver publishes it to $XDG_RUNTIME_DIR/.cua-nested-display once - # up. If WAYLAND_DISPLAY isn't already set, wait for and adopt that socket so - # the GIF captures the SAME session the driver drives. - if [ -z "''${WAYLAND_DISPLAY:-}" ]; then - dfile="''${XDG_RUNTIME_DIR:-/run/user/0}/.cua-nested-display" - w=0 - while [ ! -s "$dfile" ] && [ "$w" -lt 60 ]; do sleep 0.5; w=$((w + 1)); done - if [ -s "$dfile" ]; then - WAYLAND_DISPLAY="$(cat "$dfile")" - export WAYLAND_DISPLAY - echo "recorder adopted nested WAYLAND_DISPLAY=$WAYLAND_DISPLAY" >>"$log_file" - fi - fi - - max_frames=450 - i=0 - while [ ! -f "$stop_file" ] && [ "$i" -lt "$max_frames" ]; do - frame=$(printf "%s/frame-%04d.png" "$frames_dir" "$i") - timeout 10 ${pkgs.grim}/bin/grim "$frame" >>"$log_file" 2>&1 || true - i=$((i + 1)) - sleep "$interval" - done - - if ls "$frames_dir"/frame-*.png >/dev/null 2>&1; then - timeout 120 ${pkgs.imagemagick}/bin/convert -delay "$delay_cs" -loop 0 \ - "$frames_dir"/frame-*.png "$output_gif" >>"$log_file" 2>&1 || true - fi -'' diff --git a/nix/cua-driver/tests/wayland/screenshot.nix b/nix/cua-driver/tests/wayland/screenshot.nix deleted file mode 100644 index 7a3cd31317..0000000000 --- a/nix/cua-driver/tests/wayland/screenshot.nix +++ /dev/null @@ -1,98 +0,0 @@ -# CUA Driver native-Wayland screenshot test (per desktop) — TDD red. -# -# Launches a native Wayland terminal (foot), finds it via cua-driver -# list_windows, and asserts get_window_state (capture_mode=vision) returns a -# real PNG of that native Wayland surface. The driver captures via X11 today, so -# both discovery and capture fail on native Wayland — the spec for native -# Wayland window capture. -# -# To run: nix build .#checks.x86_64-linux.cua-driver-wayland--screenshot -{ - pkgs, - lib ? pkgs.lib, - cuaDriverModule, - desktop, - ... -}: - -let - session = import ./session.nix { inherit pkgs desktop; }; - driverClient = import ./driver-client.nix { inherit pkgs; }; - - testScriptPy = pkgs.writeText "wayland-screenshot.py" '' - import base64, sys - sys.path.insert(0, "/tmp") - from driver_client import Driver - - d = Driver() - try: - d.initialize("nixos-wayland-screenshot") - d.launch_app("foot --app-id=cua-wayland-foot --title=cua-wayland-foot") - pid, wid = d.find_window("cua-wayland-foot", timeout=40) - print(f"window pid={pid} window_id={wid}", flush=True) - resp = d.call("get_window_state", { - "pid": pid, "window_id": wid, "capture_mode": "vision"}, timeout=30) - saved = False - for item in resp.get("result", {}).get("content", []): - if item.get("type") == "image" and item.get("data"): - data = base64.b64decode(item["data"]) - assert len(data) > 0, "empty image payload" - with open("/tmp/cua-driver-wayland-${desktop}-screenshot.png", "wb") as f: - f.write(data) - saved = True - break - assert saved, f"no image returned for native Wayland window: {resp}" - print("screenshot test complete", flush=True) - finally: - d.close() - ''; -in - -pkgs.testers.nixosTest { - name = "cua-driver-wayland-${desktop}-screenshot-test"; - meta.maintainers = [ ]; - - nodes.machine = - { pkgs, ... }: - { - imports = [ cuaDriverModule ]; - virtualisation = { - cores = 2; - memorySize = 3072; - }; - services.cua-driver.enable = true; - boot.kernelModules = [ "uinput" ]; - hardware.graphics.enable = true; - services.udev.extraRules = '' - KERNEL=="uinput", MODE="0660", GROUP="input", OPTIONS+="static_node=uinput" - ''; - environment.systemPackages = session.packages ++ (with pkgs; [ python3 jq ]); - }; - - testScript = '' - machine.start() - machine.wait_for_unit("multi-user.target") - - with subtest("Bring up ${session.label}"): - machine.execute("${session.start} >/tmp/session.log 2>&1 &") - try: - machine.wait_for_file("/tmp/wl-ready", timeout=120) - except Exception: - machine.log(machine.execute("cat /tmp/session.log || true")[1]) - machine.log(machine.execute("cat /tmp/compositor.log || true")[1]) - raise - - with subtest("Launch + screenshot a native Wayland window via cua-driver"): - machine.copy_from_host("${driverClient}", "/tmp/driver_client.py") - machine.copy_from_host("${testScriptPy}", "/tmp/wayland-screenshot.py") - result = machine.succeed( - "timeout 120 env CUA_DRIVER_BIN=${session.driverWrapper} " - "XDG_RUNTIME_DIR=/run/user/0 python3 /tmp/wayland-screenshot.py 2>&1" - ) - machine.log(result) - assert "screenshot test complete" in result, result - - with subtest("Extract screenshot"): - machine.copy_from_machine("/tmp/cua-driver-wayland-${desktop}-screenshot.png", "") - ''; -} diff --git a/nix/cua-driver/tests/wayland/session.nix b/nix/cua-driver/tests/wayland/session.nix deleted file mode 100644 index 8f1fa48533..0000000000 --- a/nix/cua-driver/tests/wayland/session.nix +++ /dev/null @@ -1,246 +0,0 @@ -# Shared Wayland desktop-session bring-up for the cua-driver Wayland suite. -# -# Two execution models, selected per desktop: -# -# * NATIVE (wlroots: labwc, sway) — apps run as real Wayland clients of the -# HOST compositor, and the driver talks to that same compositor's wlr -# protocols (foreign-toplevel + screencopy). The session `start` script -# launches the compositor headless, waits for its socket, and publishes it -# to /tmp/wl-display. -# -# * NESTED (kde: kwin, gnome: mutter) — kwin/mutter expose NO client protocols -# for cross-window enumeration/capture, so the driver cannot drive them -# directly. Instead cua-driver "brings its own compositor": with -# CUA_WAYLAND_NEST=1 it spawns a private headless labwc and points -# WAYLAND_DISPLAY at it, so every app launched via `launch_app` runs inside -# that nested session where the wlr protocols DO work. The host kwin/mutter -# is still booted (best-effort) to prove cua-driver coexists with a real -# KDE/GNOME host, but the test never depends on it. -# -# Given a `desktop` this returns: -# -# { packages; start; label; nested; driverWrapper; } -# -# * packages — extra `environment.systemPackages` the session needs. -# * start — a `writeShellScript` that brings the session up and -# `touch`es /tmp/wl-ready, then blocks. -# * label — human label for logs. -# * nested — true for kde/gnome (driver hosts its own compositor). -# * driverWrapper — a `writeShellScript` to use as CUA_DRIVER_BIN: it sets the -# correct env (host WAYLAND_DISPLAY for native; the nest env -# for nested) and execs `cua-driver "$@"`. Tests spawn the -# driver through this so the python harness stays -# desktop-agnostic. -{ - pkgs, - desktop, - # EIS mode: cua-driver nests its OWN cua-compositor (focus-free keyboard + - # multi-cursor injection over a control socket) instead of labwc, on ANY host. - # Used by the background-terminal-gif + parallel-drag cells. - eis ? false, -}: - -let - inherit (pkgs) lib; - cuaCompositor = pkgs.callPackage ../../compositor { }; - - # Native-Wayland tooling only. No xorg.* / xdotool / xterm here — those are - # X11 and would let the driver cheat via XWayland. `foot` is a Wayland-native - # terminal; `grim` captures the wlroots output for the GIF artifacts. - commonPkgs = with pkgs; [ - foot - grim - wtype # virtual-keyboard CLI: cua-driver shells out to it for type_text/press_key - dbus - procps - ]; - - # Minimal sway config: native Wayland only (XWayland off), stable output. - # `workspace_layout stacking` makes the focused window fill the output, so the - # driver's centre virtual-pointer click reliably lands on the activated target - # (sway needs a pointer interaction to route virtual-keyboard input on a - # headless seat with no physical keyboard). - swayConfig = pkgs.writeText "sway-config" '' - xwayland disable - output HEADLESS-1 resolution 1280x1024 - workspace_layout stacking - exec_always foot --server - ''; - - desktops = { - "xfce-labwc" = { - label = "XFCE on labwc (native Wayland)"; - packages = with pkgs; [ labwc ]; - wlroots = true; - nested = false; - launch = "labwc >/tmp/compositor.log 2>&1 &"; - }; - # NOTE: xfce-wayfire intentionally omitted — wayfire fails to build in the - # current nixpkgs pin (wf-config can't link -ldoctest). labwc + sway cover - # XFCE-on-wlroots. - "xfce-sway" = { - label = "XFCE on sway (native Wayland)"; - packages = with pkgs; [ sway ]; - wlroots = true; - nested = false; - launch = "sway -c ${swayConfig} >/tmp/compositor.log 2>&1 &"; - }; - "kde" = { - label = "KDE Plasma kwin_wayland host + cua-driver nested labwc"; - # labwc is what cua-driver nests; kwin is the host it coexists with. - packages = with pkgs; [ labwc kdePackages.kwin ]; - wlroots = false; - nested = true; - # No --xwayland: keep the host session native-Wayland only. - launch = "kwin_wayland --virtual --width 1280 --height 1024 --no-lockscreen >/tmp/compositor.log 2>&1 &"; - }; - "gnome" = { - label = "GNOME Shell / mutter host + cua-driver nested labwc"; - packages = with pkgs; [ labwc gnome-shell mutter ]; - wlroots = false; - nested = true; - launch = "gnome-shell --wayland --headless --virtual-monitor 1280x1024 --unsafe-mode >/tmp/compositor.log 2>&1 &"; - }; - }; - - cfg = desktops.${desktop} or (throw "wayland/session.nix: unknown desktop '${desktop}'"); - - wlrootsEnv = lib.optionalString cfg.wlroots '' - export WLR_BACKENDS=headless - export WLR_RENDERER=pixman - export WLR_RENDERER_ALLOW_SOFTWARE=1 - export WLR_LIBINPUT_NO_DEVICES=1 - ''; - - # Native model: block until the HOST compositor's socket appears, publish it. - nativeStart = '' - dbus-run-session -- sh -c ' - wlsock() { ls "$XDG_RUNTIME_DIR" 2>/dev/null | grep -E "^wayland-[0-9]+$" | head -1; } - - ${cfg.launch} - compositor_pid=$! - - n=0 - while [ -z "$(wlsock)" ] && [ "$n" -lt 120 ]; do - if ! kill -0 "$compositor_pid" 2>/dev/null; then - echo "compositor exited early" >&2 - break - fi - sleep 0.5 - n=$((n + 1)) - done - - sock="$(wlsock)" - if [ -z "$sock" ]; then - echo "no Wayland display socket appeared" >&2 - cat /tmp/compositor.log >&2 2>/dev/null || true - exit 1 - fi - echo "$sock" > /tmp/wl-display - - touch /tmp/wl-ready - exec sleep infinity - ' - ''; - - # Nested model: boot the host compositor BEST-EFFORT (log whether it came up), - # but never block the test on it — cua-driver provides its own session. We - # record the host socket (if any) for logs; the driver wrapper ignores it. - nestedStart = '' - dbus-run-session -- sh -c ' - wlsock() { ls "$XDG_RUNTIME_DIR" 2>/dev/null | grep -E "^wayland-[0-9]+$" | head -1; } - - ${cfg.launch} - - n=0 - while [ -z "$(wlsock)" ] && [ "$n" -lt 40 ]; do - sleep 0.5 - n=$((n + 1)) - done - sock="$(wlsock)" - if [ -n "$sock" ]; then - echo "host compositor up: $sock" >&2 - else - echo "host compositor did not come up; proceeding with nested session only" >&2 - cat /tmp/compositor.log >&2 2>/dev/null || true - fi - echo "''${sock:-none}" > /tmp/wl-display - - touch /tmp/wl-ready - exec sleep infinity - ' - ''; - - start = pkgs.writeShellScript "wayland-session-start-${desktop}.sh" '' - set -u - export PATH=${lib.makeBinPath (commonPkgs ++ cfg.packages)}:$PATH - - export XDG_RUNTIME_DIR=/run/user/0 - mkdir -p "$XDG_RUNTIME_DIR" - chmod 700 "$XDG_RUNTIME_DIR" - - ${wlrootsEnv} - export QT_QPA_PLATFORM=wayland - export GDK_BACKEND=wayland - export LIBGL_ALWAYS_SOFTWARE=1 - - rm -f /tmp/wl-ready /tmp/wl-display "$XDG_RUNTIME_DIR/.cua-nested-display" "$XDG_RUNTIME_DIR/cua-inject.sock" - - ${if (cfg.nested || eis) then nestedStart else nativeStart} - ''; - - # Driver launcher used as CUA_DRIVER_BIN. It owns the WAYLAND env so the python - # harness never has to. labwc/foot/grim are on PATH for the app/compositor - # processes cua-driver spawns. - # Toolkit apps cua-driver launches inherit its env, so force the NATIVE - # Wayland backends here (GTK auto-detects from WAYLAND_DISPLAY; Qt would - # otherwise default to xcb and fail with no X). Harmless for foot. - appBackendEnv = '' - export GDK_BACKEND=wayland - export QT_QPA_PLATFORM=wayland - export LIBGL_ALWAYS_SOFTWARE=1 - # The native-Wayland backend is opt-in (off by default); these tests exist - # to exercise it, so turn it on for every Wayland-session driver wrapper. - export CUA_DRIVER_RS_ENABLE_WAYLAND=1 - ''; - - driverWrapper = - if eis then - # EIS mode: nest cua-compositor (focus-free + multi-cursor injection over - # the control socket) on ANY host. Apps launched via launch_app land in it. - pkgs.writeShellScript "cua-driver-eis-${desktop}" '' - export PATH=${lib.makeBinPath (commonPkgs ++ cfg.packages ++ [ cuaCompositor ])}:$PATH - export XDG_RUNTIME_DIR=/run/user/0 - export CUA_WAYLAND_NEST=1 - export CUA_WAYLAND_NEST_COMPOSITOR=${cuaCompositor}/bin/cua-compositor - export CUA_INJECT_SOCKET=/run/user/0/cua-inject.sock - ${appBackendEnv} - unset DISPLAY WAYLAND_DISPLAY - exec cua-driver "$@" - '' - else if cfg.nested then - pkgs.writeShellScript "cua-driver-nested-${desktop}" '' - export PATH=${lib.makeBinPath (commonPkgs ++ cfg.packages)}:$PATH - export XDG_RUNTIME_DIR=/run/user/0 - export CUA_WAYLAND_NEST=1 - export CUA_WAYLAND_NEST_COMPOSITOR=labwc - ${appBackendEnv} - unset DISPLAY WAYLAND_DISPLAY - exec cua-driver "$@" - '' - else - pkgs.writeShellScript "cua-driver-native-${desktop}" '' - export PATH=${lib.makeBinPath (commonPkgs ++ cfg.packages)}:$PATH - export XDG_RUNTIME_DIR=/run/user/0 - export WAYLAND_DISPLAY="$(cat /tmp/wl-display)" - ${appBackendEnv} - unset DISPLAY - exec cua-driver "$@" - ''; -in -{ - label = if eis then "${cfg.label} + cua-compositor (EIS injection)" else cfg.label; - nested = cfg.nested || eis; - packages = commonPkgs ++ cfg.packages ++ lib.optional eis cuaCompositor; - inherit start driverWrapper; -} diff --git a/package-lock.json b/package-lock.json deleted file mode 100644 index 2df654cb9e..0000000000 --- a/package-lock.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "cua", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "devDependencies": { - "prettier": "^3.6.2" - } - }, - "node_modules/prettier": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.6.2.tgz", - "integrity": "sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==", - "dev": true, - "license": "MIT", - "bin": { - "prettier": "bin/prettier.cjs" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/prettier/prettier?sponsor=1" - } - } - } -} diff --git a/scripts/ci/README.md b/scripts/ci/README.md index 2e2bfa2a2a..ec6012752c 100644 --- a/scripts/ci/README.md +++ b/scripts/ci/README.md @@ -1,15 +1,33 @@ # Cua-driver CI runners These scripts are thin entrypoints around the Rust integration tests. They -build the repo-local fixture applications on demand, set the testkit path -overrides, and collect logs. They do not push code or alter branches. +build the repo-local fixture applications, run one strict Rust environment +preflight, set testkit paths, execute Rust targets, invoke the Rust report +validator, and collect artifacts. They do not define behavioral rows, push +code, or alter branches. + +For the test layout and the distinction between unit tests, shared harnesses, +and native harnesses, see +`libs/cua-driver/docs/test-harnesses-guide.md`. | Runner | Session | Canonical command | | --- | --- | --- | -| `linux/run-rust-e2e.sh` | Linux X11/Wayland desktop | `--suite shared` | -| `windows/run-rust-e2e.ps1` | Windows console/RDP user session | `-Suite shared -RequireGui` | +| `linux/run-rust-e2e.sh` | Existing Linux X11 or Wayland desktop | no selector | +| `linux/run-rust-e2e-wayland.sh` | Headless native Sway session | no selector | +| `windows/run-rust-e2e.ps1` | Windows console/RDP user session | `-RequireGui` | +| `macos/run-rust-e2e.sh` | Logged-in macOS session with TCC | no selector | + +Use the command without a selector for the canonical complete run. CI sets the +private `CUA_E2E_INTERNAL_LANE` partition to `shared`, `native`, or `capture` +when it fans the same matrix into independent jobs. Those values are not public +alternate suites. + +Run the Wayland wrapper through `nix develop .#cua-driver-wayland-e2e`. It +creates a pure Wayland session with Xwayland disabled and delegates every +scenario to `run-rust-e2e.sh`. -The Windows workflow accepts a runner label so the same command can execute on -the Azure VM's active-RDP runner when that self-hosted label is configured. A -GitHub-hosted Windows runner is useful for smoke validation, but it is not a -substitute for the Azure user-session run. +The GitHub-hosted Windows workflow is canonical when its strict preflight proves +an interactive desktop. The workflow also accepts a runner label so maintainers +can replay the same command on an Azure VM with an active RDP session for +environment parity; that replay is not a separate test definition or source of +behavioral truth. diff --git a/scripts/ci/link-e2e-evidence.sh b/scripts/ci/link-e2e-evidence.sh new file mode 100755 index 0000000000..a295e049b4 --- /dev/null +++ b/scripts/ci/link-e2e-evidence.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +if (($# != 2)); then + echo "usage: link-e2e-evidence.sh " >&2 + exit 2 +fi + +summary_path="$1" +artifact_url="$2" + +awk -F '|' -v artifact_url="${artifact_url}" ' + BEGIN { OFS = "|" } + function trim(value) { + sub(/^[[:space:]]+/, "", value) + sub(/[[:space:]]+$/, "", value) + return value + } + /^\|/ && NF >= 16 { + cell = trim($2) + evidence = trim($15) + if (cell != "Cell" && cell !~ /^-+$/ && evidence != "" && evidence != "-" && evidence !~ /^\[/) { + $15 = " [" evidence "](" artifact_url ") " + } + } + { print } +' "${summary_path}" diff --git a/scripts/ci/linux/run-rust-e2e-desktop.sh b/scripts/ci/linux/run-rust-e2e-desktop.sh new file mode 100755 index 0000000000..84c7ccc75c --- /dev/null +++ b/scripts/ci/linux/run-rust-e2e-desktop.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Validate the canonical Rust matrix in a representative maintainer desktop. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENVIRONMENT="${1:-}" +if [[ $# -gt 0 ]]; then + shift +fi + +case "${ENVIRONMENT}" in + gnome) + [[ "${XDG_SESSION_TYPE:-}" == wayland ]] || { + echo "GNOME validation requires an active Wayland user session" >&2 + exit 2 + } + [[ "${XDG_CURRENT_DESKTOP:-}" == *GNOME* ]] || { + echo "XDG_CURRENT_DESKTOP does not identify GNOME: ${XDG_CURRENT_DESKTOP:-}" >&2 + exit 2 + } + gdbus call --session \ + --dest org.cua.WinRects \ + --object-path /org/cua/WinRects \ + --method org.cua.WinRects.GetRects >/dev/null || { + echo "The GNOME WinRects helper is not available in this user session" >&2 + exit 2 + } + export CUA_E2E_COMPOSITOR=gnome-mutter + export CUA_E2E_INPUT_BACKENDS=atspi,libei-portal + export CUA_DRIVER_RS_ENABLE_WAYLAND=1 + ;; + kde) + [[ "${XDG_SESSION_TYPE:-}" == wayland ]] || { + echo "KDE validation requires an active Wayland user session" >&2 + exit 2 + } + [[ "${XDG_CURRENT_DESKTOP:-}" == *KDE* ]] || { + echo "XDG_CURRENT_DESKTOP does not identify KDE: ${XDG_CURRENT_DESKTOP:-}" >&2 + exit 2 + } + kwin_version="$(kwin_wayland --version 2>/dev/null | sed -n 's/^kwin \([0-9][0-9]*\).*/\1/p')" + [[ "${kwin_version:-0}" -ge 6 ]] || { + echo "Representative KDE validation requires Plasma/KWin 6; found ${kwin_version:-unknown}" >&2 + exit 2 + } + export CUA_E2E_COMPOSITOR=kwin + export CUA_E2E_INPUT_BACKENDS=atspi,libei-portal + export CUA_DRIVER_RS_ENABLE_WAYLAND=1 + ;; + xorg) + [[ -n "${DISPLAY:-}" && "${XDG_SESSION_TYPE:-x11}" != wayland ]] || { + echo "Real-Xorg validation requires DISPLAY in a non-Wayland session" >&2 + exit 2 + } + export CUA_E2E_COMPOSITOR=real-xorg + export CUA_E2E_INPUT_BACKENDS=atspi,xsend-event,xtest,mpx-uinput + ;; + *) + echo "Usage: run-rust-e2e-desktop.sh {gnome|kde|xorg} [--no-build]" >&2 + exit 2 + ;; +esac + +if [[ "${CUA_E2E_HARNESS_FILTER:-}" == *tauri* && ! -e /dev/dri/renderD128 ]]; then + echo "Tauri/WebKitGTK validation requires a representative DRM render node" >&2 + exit 2 +fi + +exec "${SCRIPT_DIR}/run-rust-e2e.sh" "$@" diff --git a/scripts/ci/linux/run-rust-e2e-inject.sh b/scripts/ci/linux/run-rust-e2e-inject.sh new file mode 100755 index 0000000000..c58a1df452 --- /dev/null +++ b/scripts/ci/linux/run-rust-e2e-inject.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Run the canonical Rust matrix inside the nested cua-compositor environment. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +export CUA_E2E_WAYLAND_SESSION=cua-compositor +exec "${SCRIPT_DIR}/run-rust-e2e-wayland.sh" "$@" diff --git a/scripts/ci/linux/run-rust-e2e-wayland.sh b/scripts/ci/linux/run-rust-e2e-wayland.sh new file mode 100755 index 0000000000..98ebd26e87 --- /dev/null +++ b/scripts/ci/linux/run-rust-e2e-wayland.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +# Start one native, headless Sway session and run the complete Rust E2E matrix. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +RUNTIME_DIR="$(mktemp -d)" +SWAY_CONFIG="$(mktemp)" +SESSION_KIND="${CUA_E2E_WAYLAND_SESSION:-sway}" +COMPOSITOR_LOG="${REPO_ROOT}/artifacts/cua-driver/linux/${SESSION_KIND}.log" +ATSPI_LOG="${REPO_ROOT}/artifacts/cua-driver/linux/at-spi-bus.log" +COMPOSITOR_PID="" +DBUS_PID="" +ATSPI_PID="" + +cleanup() { + if [[ -n "${COMPOSITOR_PID}" ]]; then + kill "${COMPOSITOR_PID}" 2>/dev/null || true + wait "${COMPOSITOR_PID}" 2>/dev/null || true + fi + if [[ -n "${DBUS_PID}" ]]; then + kill "${DBUS_PID}" 2>/dev/null || true + fi + if [[ -n "${ATSPI_PID}" ]]; then + kill "${ATSPI_PID}" 2>/dev/null || true + wait "${ATSPI_PID}" 2>/dev/null || true + fi + rm -rf "${RUNTIME_DIR}" + rm -f "${SWAY_CONFIG}" +} +trap cleanup EXIT + +mkdir -p "$(dirname "${COMPOSITOR_LOG}")" +chmod 700 "${RUNTIME_DIR}" +cat > "${SWAY_CONFIG}" <<'EOF' +xwayland disable +output HEADLESS-1 mode 1920x1080 +seat seat0 fallback true +focus_follows_mouse no +default_border none +default_floating_border none +for_window [title="^CuaTestHarness"] floating enable, resize set 940 780, move position 0 0 +for_window [title="CuaTestHarness Sentinel"] fullscreen enable +EOF + +unset DISPLAY +unset WAYLAND_DISPLAY +export XDG_RUNTIME_DIR="${RUNTIME_DIR}" +export XDG_SESSION_TYPE=wayland +export XDG_CURRENT_DESKTOP=sway +export XDG_SESSION_DESKTOP=sway +export WLR_BACKENDS=headless +export WLR_RENDERER=pixman +export WLR_RENDERER_ALLOW_SOFTWARE=1 +export WLR_LIBINPUT_NO_DEVICES=1 +export WLR_HEADLESS_OUTPUTS=1 +export CUA_DRIVER_RS_ENABLE_WAYLAND=1 +if [[ "${SESSION_KIND}" == cua-compositor ]]; then + export CUA_E2E_COMPOSITOR=cua-compositor-nested + export CUA_E2E_INPUT_BACKENDS=atspi,cua-compositor-inject + export CUA_E2E_HARNESS_FILTER=electron + export CUA_INJECT_SOCKET="${XDG_RUNTIME_DIR}/cua-inject.sock" +else + export CUA_E2E_COMPOSITOR=sway + export CUA_E2E_INPUT_BACKENDS=atspi,wlr-virtual-pointer +fi +export CUA_WAYLAND_RECORDING_OUTPUT=HEADLESS-1 +export ELECTRON_OZONE_PLATFORM_HINT=wayland +export GDK_BACKEND=wayland +export QT_QPA_PLATFORM=wayland +# WebKitGTK's accelerated compositor requires a DRM render node. The canonical +# hosted lane has none, so keep its best-effort software settings explicit; +# native-Wayland WebKit coverage runs on the representative GNOME/KDE VMs. +export WEBKIT_DISABLE_COMPOSITING_MODE=1 +export WEBKIT_DISABLE_DMABUF_RENDERER=1 +# This isolated CI session uses a private runtime directory and no user home +# namespaces. Modern WebKitGTK ignores WEBKIT_FORCE_SANDBOX; use its explicit +# test-only escape hatch so the WebProcess can publish its AT-SPI subtree. +export WEBKIT_DISABLE_SANDBOX_THIS_IS_DANGEROUS=1 +export NO_AT_BRIDGE=0 +export ACCESSIBILITY_ENABLED=1 + +if [[ -z "${DBUS_SESSION_BUS_ADDRESS:-}" ]]; then + dbus_daemon="$(command -v dbus-daemon)" + dbus_prefix="$(dirname "$(dirname "$(readlink -f "${dbus_daemon}")")")" + dbus_config="${dbus_prefix}/share/dbus-1/session.conf" + if [[ ! -f "${dbus_config}" ]]; then + echo "Nix DBus session config is missing: ${dbus_config}" >&2 + exit 1 + fi + dbus_info="$(dbus-daemon --config-file="${dbus_config}" --fork --print-address=1 --print-pid=1)" + export DBUS_SESSION_BUS_ADDRESS="$(sed -n '1p' <<< "${dbus_info}")" + DBUS_PID="$(sed -n '2p' <<< "${dbus_info}")" +fi + +# A private session bus does not activate the desktop accessibility stack by +# itself. Start the repo-pinned AT-SPI launcher, then require org.a11y.Bus to +# answer before any fixture or driver process inherits this session. +ATSPI_LAUNCHER="${CUA_AT_SPI_BUS_LAUNCHER:-$(command -v at-spi-bus-launcher || true)}" +if [[ ! -x "${ATSPI_LAUNCHER}" ]]; then + echo "AT-SPI bus launcher is unavailable: ${ATSPI_LAUNCHER:-}" >&2 + exit 1 +fi +"${ATSPI_LAUNCHER}" --launch-immediately > "${ATSPI_LOG}" 2>&1 & +ATSPI_PID=$! +deadline=$((SECONDS + 15)) +while ((SECONDS < deadline)); do + if ! kill -0 "${ATSPI_PID}" 2>/dev/null; then + echo "AT-SPI bus launcher exited before org.a11y.Bus became ready" >&2 + cat "${ATSPI_LOG}" >&2 + exit 1 + fi + if gdbus call --session \ + --dest org.a11y.Bus \ + --object-path /org/a11y/bus \ + --method org.a11y.Bus.GetAddress >/dev/null 2>&1; then + break + fi + sleep 0.2 +done +if ! gdbus call --session \ + --dest org.a11y.Bus \ + --object-path /org/a11y/bus \ + --method org.a11y.Bus.GetAddress >/dev/null 2>&1; then + echo "org.a11y.Bus did not become ready within 15 seconds" >&2 + cat "${ATSPI_LOG}" >&2 + exit 1 +fi + +# The bus launcher and the registry daemon are separate services. Resolve the +# private accessibility bus and require the registry to activate on it before +# any toolkit inherits this session. +a11y_address="$( + gdbus call --session \ + --dest org.a11y.Bus \ + --object-path /org/a11y/bus \ + --method org.a11y.Bus.GetAddress \ + | sed -e "s/^('//" -e "s/',)$//" +)" +export AT_SPI_BUS_ADDRESS="${a11y_address}" +if [[ -z "${a11y_address}" ]] || ! gdbus call \ + --address "${a11y_address}" \ + --dest org.a11y.atspi.Registry \ + --object-path /org/a11y/atspi/accessible/root \ + --method org.freedesktop.DBus.Properties.Get \ + org.a11y.atspi.Accessible ChildCount >/dev/null 2>&1; then + echo "AT-SPI registry did not activate on the accessibility bus" >&2 + cat "${ATSPI_LOG}" >&2 + exit 1 +fi +if ! gdbus call --session \ + --dest org.a11y.Bus \ + --object-path /org/a11y/bus \ + --method org.freedesktop.DBus.Properties.Set \ + org.a11y.Status IsEnabled '' >/dev/null 2>&1; then + echo "Could not enable accessibility on org.a11y.Bus" >&2 + cat "${ATSPI_LOG}" >&2 + exit 1 +fi + +if [[ "${SESSION_KIND}" == cua-compositor ]]; then + command -v cua-compositor >/dev/null || { + echo "cua-compositor is required for the nested injection lane" >&2 + exit 1 + } + cua-compositor > "${COMPOSITOR_LOG}" 2>&1 & +else + sway --unsupported-gpu --config "${SWAY_CONFIG}" > "${COMPOSITOR_LOG}" 2>&1 & +fi +COMPOSITOR_PID=$! + +deadline=$((SECONDS + 20)) +while ((SECONDS < deadline)); do + if ! kill -0 "${COMPOSITOR_PID}" 2>/dev/null; then + echo "${SESSION_KIND} exited before its Wayland socket became ready" >&2 + cat "${COMPOSITOR_LOG}" >&2 + exit 1 + fi + socket="$(find "${XDG_RUNTIME_DIR}" -maxdepth 1 -type s -name 'wayland-*' -print -quit)" + if [[ -n "${socket}" ]]; then + export WAYLAND_DISPLAY="$(basename "${socket}")" + break + fi + sleep 0.2 +done + +if [[ -z "${WAYLAND_DISPLAY:-}" ]]; then + echo "${SESSION_KIND} did not create a Wayland socket within 20 seconds" >&2 + cat "${COMPOSITOR_LOG}" >&2 + exit 1 +fi + +if [[ "${SESSION_KIND}" == cua-compositor ]]; then + deadline=$((SECONDS + 10)) + while [[ ! -S "${CUA_INJECT_SOCKET}" ]] && ((SECONDS < deadline)); do + sleep 0.2 + done + if [[ ! -S "${CUA_INJECT_SOCKET}" ]]; then + echo "cua-compositor did not expose its injection socket" >&2 + cat "${COMPOSITOR_LOG}" >&2 + exit 1 + fi +else + export SWAYSOCK="$(find "${XDG_RUNTIME_DIR}" -maxdepth 1 -type s -name 'sway-ipc.*.sock' -print -quit)" + if [[ -z "${SWAYSOCK}" ]]; then + echo "Sway did not expose its IPC socket" >&2 + cat "${COMPOSITOR_LOG}" >&2 + exit 1 + fi +fi + +echo "Native Wayland E2E session: ${SESSION_KIND} on ${WAYLAND_DISPLAY}" +set +e +"${SCRIPT_DIR}/run-rust-e2e.sh" "$@" +status=$? +set -e +if [[ "${status}" != 0 && "${SESSION_KIND}" == sway ]]; then + swaymsg -t get_tree > "${REPO_ROOT}/artifacts/cua-driver/linux/sway-tree.json" 2>/dev/null || true +fi +exit "${status}" diff --git a/scripts/ci/linux/run-rust-e2e.sh b/scripts/ci/linux/run-rust-e2e.sh index 12365c8431..e5e5e55975 100755 --- a/scripts/ci/linux/run-rust-e2e.sh +++ b/scripts/ci/linux/run-rust-e2e.sh @@ -8,21 +8,21 @@ REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" DRIVER_ROOT="${REPO_ROOT}/libs/cua-driver" RUST_ROOT="${DRIVER_ROOT}/rust" BUILD_FIXTURES=1 -SUITE="shared" +SUITE="${CUA_E2E_INTERNAL_LANE:-all}" usage() { cat <<'EOF' -Usage: run-rust-e2e.sh [--no-build] [--suite shared|modality|all] +Usage: run-rust-e2e.sh [--no-build] The caller must provide a real or virtual Linux desktop session. For a headless session, wrap this command in xvfb-run and dbus-run-session. +The contributor-facing command always runs the complete matrix. EOF } while (($#)); do case "$1" in --no-build) BUILD_FIXTURES=0 ;; - --suite) SUITE="${2:?missing suite}"; shift ;; -h|--help) usage; exit 0 ;; *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;; esac @@ -30,85 +30,150 @@ while (($#)); do done case "$SUITE" in - shared|modality|all) ;; - *) echo "unsupported suite: $SUITE" >&2; exit 2 ;; + shared|native|capture|all) ;; + *) echo "unsupported internal lane: $SUITE" >&2; exit 2 ;; esac -mkdir -p "${REPO_ROOT}/artifacts/cua-driver/linux" -RECORDING_ROOT="${REPO_ROOT}/artifacts/cua-driver/linux/recordings" +ARTIFACT_DIR="${REPO_ROOT}/artifacts/cua-driver/linux" +mkdir -p "${ARTIFACT_DIR}" +RECORDING_ROOT="${ARTIFACT_DIR}/recordings" rm -rf "${RECORDING_ROOT}" mkdir -p "${RECORDING_ROOT}" -RESULTS_FILE="${REPO_ROOT}/artifacts/cua-driver/linux/results.jsonl" -SUMMARY_FILE="${REPO_ROOT}/artifacts/cua-driver/linux/summary.md" +DECLARATIONS_FILE="${ARTIFACT_DIR}/cases.jsonl" +ENVIRONMENT_FILE="${ARTIFACT_DIR}/environment.jsonl" +RESULTS_FILE="${ARTIFACT_DIR}/results.jsonl" +SUMMARY_FILE="${ARTIFACT_DIR}/summary.md" +: > "${DECLARATIONS_FILE}" +: > "${ENVIRONMENT_FILE}" : > "${RESULTS_FILE}" -cat > "${SUMMARY_FILE}" <<'EOF' -# CUA Rust Linux E2E matrix - -| Platform | Host/lane | Scenario | Status | Duration | Details | -| --- | --- | --- | --- | --- | --- | -EOF +rm -f "${SUMMARY_FILE}" +export CUA_E2E_DECLARATIONS_FILE="${DECLARATIONS_FILE}" +export CUA_E2E_ENVIRONMENT_FILE="${ENVIRONMENT_FILE}" export CUA_E2E_RESULTS_FILE="${RESULTS_FILE}" -export CUA_E2E_SUMMARY_FILE="${SUMMARY_FILE}" export CUA_E2E_RECORDINGS_ROOT="${RECORDING_ROOT}" export CUA_TEST_WORKSPACE_ROOT="${RUST_ROOT}" export CUA_TEST_DRIVER_BIN="${RUST_ROOT}/target/release/cua-driver" export CUA_TEST_APPS_ROOT="${RUST_ROOT}/test-apps" export CUA_TEST_REQUIRE_FIXTURES=1 export CUA_TEST_DRIVER_STDERR=1 +export RUST_BACKTRACE="${RUST_BACKTRACE:-1}" +if [[ -z "${CUA_E2E_SOURCE_SHA:-}" && -f "${REPO_ROOT}/.cua-e2e-source-sha" ]]; then + export CUA_E2E_SOURCE_SHA="$(tr -d '[:space:]' < "${REPO_ROOT}/.cua-e2e-source-sha")" + export CUA_E2E_SOURCE_MARKER="${REPO_ROOT}/.cua-e2e-source-sha" +fi +if [[ -n "${WAYLAND_DISPLAY:-}" && -z "${DISPLAY:-}" ]]; then + export GDK_BACKEND="${GDK_BACKEND:-wayland}" + export CUA_E2E_COMPOSITOR="${CUA_E2E_COMPOSITOR:-wayland-unknown}" + export CUA_E2E_INPUT_BACKENDS="${CUA_E2E_INPUT_BACKENDS:-atspi}" +else + export CUA_E2E_COMPOSITOR="${CUA_E2E_COMPOSITOR:-openbox-x11}" + export CUA_E2E_INPUT_BACKENDS="${CUA_E2E_INPUT_BACKENDS:-atspi,xsend-event,xtest}" +fi +if [[ "${SUITE}" == shared || "${SUITE}" == all ]]; then + export CUA_ATSPI_DEBUG=1 +fi -command -v ffmpeg >/dev/null || { echo "ffmpeg is required for E2E trajectory videos" >&2; exit 1; } +if [[ -n "${WAYLAND_DISPLAY:-}" && -z "${DISPLAY:-}" ]]; then + command -v wf-recorder >/dev/null || { echo "wf-recorder is required for native Wayland E2E videos" >&2; exit 1; } + command -v grim >/dev/null || { echo "grim is required for native Wayland capture fallback" >&2; exit 1; } + command -v wtype >/dev/null || { echo "wtype is required for native Wayland keyboard input" >&2; exit 1; } +else + command -v ffmpeg >/dev/null || { echo "ffmpeg is required for X11 E2E trajectory videos" >&2; exit 1; } +fi command -v ffprobe >/dev/null || { echo "ffprobe is required for E2E trajectory validation" >&2; exit 1; } +command -v jq >/dev/null || { echo "jq is required for E2E ownership validation" >&2; exit 1; } if [[ "${BUILD_FIXTURES}" == 1 ]]; then cargo build --release -p cua-driver --manifest-path "${RUST_ROOT}/Cargo.toml" - bash "${DRIVER_ROOT}/tests/fixtures/build/linux.sh" + case "${SUITE}" in + shared) FIXTURE_TARGETS="${CUA_E2E_HARNESS_FILTER:-electron,tauri}" ;; + native|capture) FIXTURE_TARGETS="electron,gtk3" ;; + *) FIXTURE_TARGETS="${CUA_E2E_HARNESS_FILTER:-electron,tauri},gtk3" ;; + esac + bash "${DRIVER_ROOT}/tests/fixtures/build/linux.sh" --only "${FIXTURE_TARGETS}" fi if [[ ! -x "${CUA_TEST_DRIVER_BIN}" ]]; then echo "driver binary not found: ${CUA_TEST_DRIVER_BIN}" >&2 exit 1 fi -if [[ ! -x "${CUA_TEST_APPS_ROOT}/harness-electron/CuaTestHarness.Electron" ]]; then - echo "Electron fixture was not built: ${CUA_TEST_APPS_ROOT}/harness-electron" >&2 - exit 1 +required_fixtures=() +required_fixtures+=("${CUA_TEST_APPS_ROOT}/harness-electron/CuaTestHarness.Electron") +if [[ ("${SUITE}" == shared || "${SUITE}" == all) \ + && ",${CUA_E2E_HARNESS_FILTER:-electron,tauri}," == *,tauri,* ]]; then + required_fixtures+=( + "${CUA_TEST_APPS_ROOT}/harness-tauri/CuaTestHarness.Tauri" + ) +fi +if [[ "${SUITE}" == native || "${SUITE}" == all ]]; then + required_fixtures+=("${CUA_TEST_APPS_ROOT}/harness-gtk3/CuaTestHarness.Gtk3") fi +for fixture in "${required_fixtures[@]}"; do + if [[ ! -x "${fixture}" ]]; then + echo "Required fixture was not built: ${fixture}" >&2 + exit 1 + fi +done FAILURE_COUNT=0 +run_report() { + (cd "${RUST_ROOT}" && cargo run -p cua-driver-testkit --bin cua-e2e-report -- \ + --declarations "${DECLARATIONS_FILE}" \ + --environment "${ENVIRONMENT_FILE}" \ + --results "${RESULTS_FILE}" \ + --artifact-root "${ARTIFACT_DIR}" \ + --require-video \ + --output "${SUMMARY_FILE}") +} + +echo "[PREFLIGHT] Linux desktop, fixture, AX, capture, and video" +set +e +(cd "${RUST_ROOT}" && cargo test -p cua-driver --test e2e_environment_preflight_test -- \ + --ignored --exact canonical_e2e_environment_is_ready --nocapture --test-threads=1) \ + 2>&1 | tee "${ARTIFACT_DIR}/environment-preflight.log" +PREFLIGHT_EXIT=${PIPESTATUS[0]} +set -e +if [[ "${PREFLIGHT_EXIT}" != 0 ]]; then + set +e + run_report + set -e + echo "Linux E2E environment preflight failed" >&2 + exit 1 +fi + run_test() { local name="$1" shift echo "[RUN] ${name}" set +e - (cd "${RUST_ROOT}" && "$@") 2>&1 | tee "${REPO_ROOT}/artifacts/cua-driver/linux/${name}.log" + (cd "${RUST_ROOT}" && "$@") 2>&1 | tee "${ARTIFACT_DIR}/${name}.log" local exit_code=${PIPESTATUS[0]} set -e - local status="PASS" if [[ "${exit_code}" != 0 ]]; then - status="FAIL" FAILURE_COUNT=$((FAILURE_COUNT + 1)) fi - local details="-" - if [[ "${exit_code}" != 0 ]]; then - details="exit code ${exit_code}" - fi - printf '{"schema":"cua-e2e-result/v1","platform":"linux","host":"lane","scenario":"%s","status":"%s","message":"%s"}\n' \ - "${name}" "${status}" "${details}" >> "${RESULTS_FILE}" - printf '| Linux | lane | %s | %s | n/a | %s |\n' "${name}" "${status}" "${details}" >> "${SUMMARY_FILE}" } if [[ "${SUITE}" == shared || "${SUITE}" == all ]]; then run_test shared-behavior-matrix \ cargo test -p cua-driver --test cross_platform_behavior_test -- \ + --ignored --exact shared_web_action_matrix_is_state_verified \ + --nocapture --test-threads=1 +fi + +if [[ "${SUITE}" == native || "${SUITE}" == all ]]; then + run_test gtk3-native-harness \ + cargo test -p cua-driver --test harness_gtk3_test -- \ --ignored --nocapture --test-threads=1 fi -if [[ "${SUITE}" == modality || "${SUITE}" == all ]]; then - run_test modality-capture \ - cargo test -p cua-driver --test modality_capture_mode_test -- \ +if [[ "${SUITE}" == capture || "${SUITE}" == all ]]; then + run_test capture-contract \ + cargo test -p cua-driver --test capture_contract_test -- \ --ignored --nocapture --test-threads=1 - run_test modality-desktop-scope \ - cargo test -p cua-driver --test modality_desktop_scope_linux_test -- \ + run_test desktop-scope \ + cargo test -p cua-driver --test desktop_scope_linux_test -- \ --ignored --nocapture --test-threads=1 fi @@ -124,6 +189,20 @@ while IFS= read -r -d '' video; do fi done < <(find "${RECORDING_ROOT}" -type f -name recording.mp4 -print0) +OWNED_VIDEOS="$(mktemp)" +jq -r 'select(.evidence.video != null) | .evidence.video' "${RESULTS_FILE}" > "${OWNED_VIDEOS}" +while IFS= read -r -d '' video; do + relative="${video#${ARTIFACT_DIR}/}" + if [[ "${relative}" == recordings/environment-preflight-*/recording.mp4 ]]; then + continue + fi + if ! grep -Fxq -- "${relative}" "${OWNED_VIDEOS}"; then + echo "[VIDEO FAIL] Orphan trajectory has no typed result row: ${relative}" >&2 + FAILURE_COUNT=$((FAILURE_COUNT + 1)) + fi +done < <(find "${RECORDING_ROOT}" -type f -name recording.mp4 -print0) +rm -f "${OWNED_VIDEOS}" + while IFS= read -r -d '' recording_error; do echo "[VIDEO FAIL] ${recording_error}" >&2 cat "${recording_error}" >&2 @@ -135,6 +214,15 @@ if [[ "${video_count}" == 0 ]]; then FAILURE_COUNT=$((FAILURE_COUNT + 1)) fi +set +e +run_report +REPORT_EXIT=$? +set -e +if [[ "${REPORT_EXIT}" != 0 ]]; then + echo "Linux E2E result validation failed" >&2 + FAILURE_COUNT=$((FAILURE_COUNT + 1)) +fi + if [[ "${FAILURE_COUNT}" != 0 ]]; then echo "Linux Rust e2e suite had ${FAILURE_COUNT} failing lane(s)" >&2 exit 1 diff --git a/scripts/ci/macos/run-rust-e2e.sh b/scripts/ci/macos/run-rust-e2e.sh new file mode 100755 index 0000000000..f6eab291f1 --- /dev/null +++ b/scripts/ci/macos/run-rust-e2e.sh @@ -0,0 +1,236 @@ +#!/usr/bin/env bash +# Run the canonical Rust desktop matrix in a logged-in macOS user session. +# macOS harness tests use the installed, TCC-authorized cua-driver daemon path. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" +DRIVER_ROOT="${REPO_ROOT}/libs/cua-driver" +RUST_ROOT="${DRIVER_ROOT}/rust" +SUITE="${CUA_E2E_INTERNAL_LANE:-all}" +BUILD_FIXTURES=1 + +usage() { + cat <<'EOF' +Usage: run-rust-e2e.sh [--no-build] + +Run from a logged-in macOS desktop after install-local and TCC authorization. +The testkit proxies MCP calls through the installed CuaDriver daemon. +The contributor-facing command always runs the complete matrix. +EOF +} + +while (($#)); do + case "$1" in + --no-build) BUILD_FIXTURES=0 ;; + -h|--help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;; + esac + shift +done + +case "$SUITE" in + shared|native|capture|all) ;; + *) echo "unsupported internal lane: $SUITE" >&2; exit 2 ;; +esac + +if ! git -C "${REPO_ROOT}" diff --quiet || ! git -C "${REPO_ROOT}" diff --cached --quiet; then + echo "macOS canonical E2E requires a clean tracked working tree" >&2 + exit 2 +fi +if [[ -z "${CUA_E2E_SOURCE_SHA:-}" ]]; then + CUA_E2E_SOURCE_SHA="$(git -C "${REPO_ROOT}" rev-parse HEAD)" +fi +if [[ ! "${CUA_E2E_SOURCE_SHA}" =~ ^[0-9a-fA-F]{40}$ ]]; then + echo "CUA_E2E_SOURCE_SHA must be a full 40-character commit SHA" >&2 + exit 2 +fi +export CUA_E2E_SOURCE_SHA + +ARTIFACT_DIR="${REPO_ROOT}/artifacts/cua-driver/macos" +RECORDING_ROOT="${ARTIFACT_DIR}/recordings" +if [[ -e "${RECORDING_ROOT}" ]]; then + RECORDING_ARCHIVE="$(mktemp -d "${TMPDIR:-/tmp}/cua-macos-e2e-recordings.XXXXXX")" + mv "${RECORDING_ROOT}" "${RECORDING_ARCHIVE}/recordings" + echo "Previous recordings preserved at ${RECORDING_ARCHIVE}/recordings" +fi +mkdir -p "${RECORDING_ROOT}" +RESULTS_FILE="${ARTIFACT_DIR}/results.jsonl" +DECLARATIONS_FILE="${ARTIFACT_DIR}/cases.jsonl" +ENVIRONMENT_FILE="${ARTIFACT_DIR}/environment.jsonl" +SUMMARY_FILE="${ARTIFACT_DIR}/summary.md" +mkdir -p "${ARTIFACT_DIR}" +: > "${DECLARATIONS_FILE}" +: > "${ENVIRONMENT_FILE}" +: > "${RESULTS_FILE}" +rm -f "${SUMMARY_FILE}" + +export CUA_E2E_DECLARATIONS_FILE="${DECLARATIONS_FILE}" +export CUA_E2E_ENVIRONMENT_FILE="${ENVIRONMENT_FILE}" +export CUA_E2E_RESULTS_FILE="${RESULTS_FILE}" +export CUA_E2E_RECORDINGS_ROOT="${RECORDING_ROOT}" +export CUA_TEST_WORKSPACE_ROOT="${RUST_ROOT}" +export CUA_TEST_DRIVER_BIN="${RUST_ROOT}/target/release/cua-driver" +export CUA_TEST_APPS_ROOT="${RUST_ROOT}/test-apps" +export CUA_TEST_REQUIRE_FIXTURES=1 +export CUA_TEST_DRIVER_STDERR=1 + +command -v ffmpeg >/dev/null || { echo "ffmpeg is required for E2E trajectory videos" >&2; exit 1; } +command -v ffprobe >/dev/null || { echo "ffprobe is required for E2E trajectory validation" >&2; exit 1; } +command -v jq >/dev/null || { echo "jq is required for E2E ownership validation" >&2; exit 1; } + +if [[ "${BUILD_FIXTURES}" == 1 ]]; then + cargo build --release -p cua-driver --manifest-path "${RUST_ROOT}/Cargo.toml" + bash "${DRIVER_ROOT}/tests/fixtures/build/macos.sh" +fi + +if [[ ! -x "${CUA_TEST_DRIVER_BIN}" ]]; then + echo "Required driver binary was not built: ${CUA_TEST_DRIVER_BIN}" >&2 + exit 1 +fi + +required_fixtures=() +required_fixtures+=("${CUA_TEST_APPS_ROOT}/harness-electron/CuaTestHarness.Electron.app") +if [[ "${SUITE}" == shared || "${SUITE}" == all ]]; then + required_fixtures+=( + "${CUA_TEST_APPS_ROOT}/harness-tauri/CuaTestHarness.Tauri.app" + "${CUA_TEST_APPS_ROOT}/harness-wkwebview/CuaTestHarness.WKWebView.app" + ) +fi +if [[ "${SUITE}" == native || "${SUITE}" == all ]]; then + required_fixtures+=( + "${CUA_TEST_APPS_ROOT}/harness-appkit/CuaTestHarness.AppKit.app" + "${CUA_TEST_APPS_ROOT}/harness-swiftui/CuaTestHarness.SwiftUI.app" + ) +fi +for fixture in "${required_fixtures[@]}"; do + [[ -d "${fixture}" ]] || { echo "Required fixture missing: ${fixture}" >&2; exit 1; } +done + +FAILURE_COUNT=0 + +run_report() { + (cd "${RUST_ROOT}" && cargo run -p cua-driver-testkit --bin cua-e2e-report -- \ + --declarations "${DECLARATIONS_FILE}" \ + --environment "${ENVIRONMENT_FILE}" \ + --results "${RESULTS_FILE}" \ + --artifact-root "${ARTIFACT_DIR}" \ + --require-video \ + --output "${SUMMARY_FILE}") +} + +echo "[PREFLIGHT] macOS daemon identity, fixture, AX, capture, and video" +set +e +(cd "${RUST_ROOT}" && cargo test -p cua-driver --test e2e_environment_preflight_test -- \ + --ignored --exact canonical_e2e_environment_is_ready --nocapture --test-threads=1) \ + 2>&1 | tee "${ARTIFACT_DIR}/environment-preflight.log" +PREFLIGHT_EXIT=${PIPESTATUS[0]} +set -e +if [[ "${PREFLIGHT_EXIT}" != 0 ]]; then + set +e + run_report + set -e + echo "macOS E2E environment preflight failed" >&2 + exit 1 +fi + +run_test() { + local name="$1"; shift + echo "[RUN] ${name}" + set +e + (cd "${RUST_ROOT}" && "$@") 2>&1 | tee "${ARTIFACT_DIR}/${name}.log" + local exit_code=${PIPESTATUS[0]} + set -e + if [[ "${exit_code}" != 0 ]]; then + FAILURE_COUNT=$((FAILURE_COUNT + 1)) + fi +} + +if [[ "${SUITE}" == shared || "${SUITE}" == all ]]; then + run_test shared-app-matrix cargo test -p cua-driver --test cross_platform_behavior_test -- \ + --ignored --exact shared_web_action_matrix_is_state_verified \ + --nocapture --test-threads=1 +fi +if [[ "${SUITE}" == native || "${SUITE}" == all ]]; then + for appkit_test in \ + harness_appkit_smoke \ + harness_appkit_text_input \ + harness_appkit_type_text_background \ + harness_appkit_scroll_foreground \ + harness_appkit_scroll_background \ + harness_appkit_counter \ + harness_appkit_counter_px_background \ + harness_appkit_right_click_px_foreground \ + harness_appkit_right_click_px_background \ + harness_appkit_double_click_px_foreground \ + harness_appkit_double_click_px_background \ + harness_appkit_slider_drag_px_foreground \ + harness_appkit_slider_drag_px_background; do + run_test "appkit-${appkit_test}" cargo test -p cua-driver --test harness_appkit_test -- \ + --ignored --exact "${appkit_test}" --nocapture --test-threads=1 + done + for swiftui_test in \ + harness_swiftui_smoke \ + harness_swiftui_counter_background \ + harness_swiftui_set_value_background \ + harness_swiftui_popover_foreground; do + run_test "swiftui-${swiftui_test}" cargo test -p cua-driver --test harness_swiftui_test -- \ + --ignored --exact "${swiftui_test}" --nocapture --test-threads=1 + done +fi +if [[ "${SUITE}" == capture || "${SUITE}" == all ]]; then + run_test capture-contract cargo test -p cua-driver --test capture_contract_test -- \ + --ignored --nocapture --test-threads=1 + run_test desktop-scope cargo test -p cua-driver --test desktop_scope_macos_test -- \ + --ignored --nocapture --test-threads=1 +fi + +video_count=0 +while IFS= read -r -d '' video; do + video_count=$((video_count + 1)) + if ! ffprobe -v error -show_entries format=duration \ + -of default=noprint_wrappers=1:nokey=1 "${video}" >/dev/null; then + echo "[VIDEO FAIL] Unplayable trajectory: ${video}" >&2 + FAILURE_COUNT=$((FAILURE_COUNT + 1)) + fi +done < <(find "${RECORDING_ROOT}" -type f -name recording.mp4 -print0) + +OWNED_VIDEOS="$(mktemp)" +jq -r 'select(.evidence.video != null) | .evidence.video' "${RESULTS_FILE}" > "${OWNED_VIDEOS}" +while IFS= read -r -d '' video; do + relative="${video#${ARTIFACT_DIR}/}" + if [[ "${relative}" == recordings/environment-preflight-*/recording.mp4 ]]; then + continue + fi + if ! grep -Fxq -- "${relative}" "${OWNED_VIDEOS}"; then + echo "[VIDEO FAIL] Orphan trajectory has no typed result row: ${relative}" >&2 + FAILURE_COUNT=$((FAILURE_COUNT + 1)) + fi +done < <(find "${RECORDING_ROOT}" -type f -name recording.mp4 -print0) +rm -f "${OWNED_VIDEOS}" + +while IFS= read -r -d '' error_file; do + echo "[VIDEO FAIL] ${error_file}" >&2 + cat "${error_file}" >&2 + FAILURE_COUNT=$((FAILURE_COUNT + 1)) +done < <(find "${RECORDING_ROOT}" -type f -name recording-error.txt -print0) + +if [[ "${video_count}" == 0 ]]; then + echo "[VIDEO FAIL] No E2E trajectory videos were produced" >&2 + FAILURE_COUNT=$((FAILURE_COUNT + 1)) +fi + +set +e +run_report +REPORT_EXIT=$? +set -e +if [[ "${REPORT_EXIT}" != 0 ]]; then + echo "macOS E2E result validation failed" >&2 + FAILURE_COUNT=$((FAILURE_COUNT + 1)) +fi + +if [[ "${FAILURE_COUNT}" != 0 ]]; then + echo "macOS Rust E2E suite had ${FAILURE_COUNT} failing lane(s)" >&2 + exit 1 +fi +echo "macOS Rust E2E suite completed: ${SUITE}" diff --git a/scripts/ci/windows/build-harnesses.ps1 b/scripts/ci/windows/build-harnesses.ps1 index 0db8d1eb03..dcdbafef0e 100644 --- a/scripts/ci/windows/build-harnesses.ps1 +++ b/scripts/ci/windows/build-harnesses.ps1 @@ -1,7 +1,9 @@ # Build all repo-local Windows harness apps from source. param( [ValidateSet("none", "wpf", "winui3", "webview", "electron", "tauri")] - [string]$Skip = "none" + [string]$Skip = "none", + [ValidateSet("wpf", "winui3", "webview", "electron", "tauri")] + [string[]]$Targets = @("wpf", "winui3", "webview", "electron", "tauri") ) Set-StrictMode -Version Latest @@ -10,7 +12,7 @@ $scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Definition $repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..\..")).Path $fixtureBuild = Join-Path $repoRoot "libs\cua-driver\tests\fixtures\build\windows.ps1" -& $fixtureBuild -Skip $Skip +& $fixtureBuild -Skip $Skip -Targets $Targets if ($LASTEXITCODE -ne 0) { throw "Windows harness build failed with exit code $LASTEXITCODE" } diff --git a/scripts/ci/windows/run-rust-e2e.ps1 b/scripts/ci/windows/run-rust-e2e.ps1 index 11e8114d5b..e67533d3c8 100644 --- a/scripts/ci/windows/run-rust-e2e.ps1 +++ b/scripts/ci/windows/run-rust-e2e.ps1 @@ -2,8 +2,6 @@ # Scenario definitions and assertions stay in the Rust integration test. param( [switch]$NoBuild, - [ValidateSet("default", "guard", "shared", "native", "modality", "all")] - [string]$Suite = "shared", [switch]$RequireGui ) @@ -14,22 +12,27 @@ $scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Definition $repoRoot = (Resolve-Path (Join-Path $scriptDir "..\..\..")).Path $driverRoot = Join-Path $repoRoot "libs\cua-driver" $rustRoot = Join-Path $driverRoot "rust" +$suite = if ([string]::IsNullOrWhiteSpace($env:CUA_E2E_INTERNAL_LANE)) { "all" } else { $env:CUA_E2E_INTERNAL_LANE } +if ($suite -notin @("shared", "native", "capture", "all")) { + throw "Unsupported internal lane: $suite" +} $artifactDir = Join-Path $repoRoot "artifacts\cua-driver\windows" New-Item -ItemType Directory -Force $artifactDir | Out-Null $recordingRoot = Join-Path $artifactDir "recordings" Remove-Item -Path $recordingRoot -Recurse -Force -ErrorAction SilentlyContinue New-Item -ItemType Directory -Force $recordingRoot | Out-Null $resultsPath = Join-Path $artifactDir "results.jsonl" +$casesPath = Join-Path $artifactDir "cases.jsonl" +$environmentPath = Join-Path $artifactDir "environment.jsonl" $summaryPath = Join-Path $artifactDir "summary.md" -@( - "# CUA Rust Windows E2E matrix", - "", - "| Platform | Host/lane | Scenario | Status | Duration | Details |", - "| --- | --- | --- | --- | --- | --- |" -) | Set-Content -Path $summaryPath -Remove-Item -Force -ErrorAction SilentlyContinue $resultsPath +foreach ($path in @($casesPath, $environmentPath, $resultsPath)) { + New-Item -ItemType File -Force $path | Out-Null + Clear-Content $path +} +Remove-Item -Force -ErrorAction SilentlyContinue $summaryPath +$env:CUA_E2E_DECLARATIONS_FILE = $casesPath +$env:CUA_E2E_ENVIRONMENT_FILE = $environmentPath $env:CUA_E2E_RESULTS_FILE = $resultsPath -$env:CUA_E2E_SUMMARY_FILE = $summaryPath $env:CUA_E2E_RECORDINGS_ROOT = $recordingRoot $ffmpeg = Get-Command ffmpeg.exe -ErrorAction SilentlyContinue @@ -50,24 +53,69 @@ $env:CUA_TEST_DRIVER_STDERR = "1" if (-not $NoBuild) { & cargo build --release -p cua-driver --manifest-path (Join-Path $rustRoot "Cargo.toml") if ($LASTEXITCODE -ne 0) { throw "Rust driver build failed" } - & (Join-Path $scriptDir "build-harnesses.ps1") -} - -if ($Suite -in @("default", "guard", "modality", "all")) { - & cargo build -p focus-monitor-win --manifest-path (Join-Path $rustRoot "Cargo.toml") - if ($LASTEXITCODE -ne 0) { throw "Focus monitor build failed" } + $fixtureTargets = switch ($suite) { + "shared" { @("electron", "tauri") } + "native" { @("wpf", "winui3", "webview", "electron") } + "capture" { @("wpf", "electron") } + default { @("wpf", "winui3", "webview", "electron", "tauri") } + } + & (Join-Path $scriptDir "build-harnesses.ps1") -Targets $fixtureTargets } if (-not (Test-Path $env:CUA_TEST_DRIVER_BIN)) { throw "Driver binary not found: $($env:CUA_TEST_DRIVER_BIN)" } -foreach ($fixture in @( - (Join-Path $env:CUA_TEST_APPS_ROOT "harness-electron\CuaTestHarness.Electron.exe"), - (Join-Path $env:CUA_TEST_APPS_ROOT "harness-tauri\CuaTestHarness.Tauri.exe") -)) { +$requiredFixtures = @() +$requiredFixtures += Join-Path $env:CUA_TEST_APPS_ROOT "harness-electron\CuaTestHarness.Electron.exe" +if ($suite -in @("shared", "all")) { + $requiredFixtures += Join-Path $env:CUA_TEST_APPS_ROOT "harness-tauri\CuaTestHarness.Tauri.exe" +} +if ($suite -in @("native", "capture", "all")) { + $requiredFixtures += Join-Path $env:CUA_TEST_APPS_ROOT "harness-wpf\CuaTestHarness.Wpf.exe" +} +if ($suite -in @("native", "all")) { + $requiredFixtures += @( + (Join-Path $env:CUA_TEST_APPS_ROOT "harness-winui3\CuaTestHarness.WinUI3.exe"), + (Join-Path $env:CUA_TEST_APPS_ROOT "harness-webview\CuaTestHarness.WebView.exe") + ) +} +foreach ($fixture in $requiredFixtures) { if (-not (Test-Path $fixture)) { throw "Required fixture was not built: $fixture" } } +function Invoke-E2eReport { + Push-Location $rustRoot + try { + & cargo run -p cua-driver-testkit --bin cua-e2e-report -- ` + --declarations $casesPath ` + --environment $environmentPath ` + --results $resultsPath ` + --artifact-root $artifactDir ` + --require-video ` + --output $summaryPath | Out-Host + $exitCode = $LASTEXITCODE + return $exitCode + } finally { + Pop-Location + } +} + +Write-Host "[PREFLIGHT] Windows desktop, fixture, UIA, capture, and video" -ForegroundColor Yellow +Push-Location $rustRoot +try { + $preflightLog = Join-Path $artifactDir "environment-preflight.log" + $preflightOutput = & cargo test -p cua-driver --test e2e_environment_preflight_test -- ` + --ignored --exact canonical_e2e_environment_is_ready --nocapture --test-threads=1 2>&1 + $preflightExit = $LASTEXITCODE + $preflightOutput | Tee-Object -FilePath $preflightLog +} finally { + Pop-Location +} +if ($preflightExit -ne 0) { + Invoke-E2eReport | Out-Null + throw "Windows E2E environment preflight failed" +} + function Invoke-CargoTest { param([string]$Name, [string[]]$Arguments) Write-Host "[RUN] $Name" -ForegroundColor Yellow @@ -77,44 +125,6 @@ function Invoke-CargoTest { $output = & cargo @Arguments 2>&1 $exitCode = $LASTEXITCODE $output | Tee-Object -FilePath $logPath - foreach ($line in $output) { - $match = [regex]::Match( - [string]$line, - '^\s*test\s+(?\S+)\s+\.\.\.\s+(?ok|FAILED|ignored)\s*$' - ) - if (-not $match.Success) { continue } - - $testStatus = switch ($match.Groups["status"].Value) { - "ok" { "PASS" } - "FAILED" { "FAIL" } - default { "SKIP" } - } - $testName = $match.Groups["name"].Value - $testMessage = if ($testStatus -eq "FAIL") { "test case failed; see lane log" } else { "" } - $testRecord = [ordered]@{ - schema = "cua-e2e-result/v1" - platform = "windows" - host = "cargo" - scenario = $testName - status = $testStatus - message = $testMessage - } | ConvertTo-Json -Compress - Add-Content -Path $resultsPath -Value $testRecord - $testDetails = if ($testMessage) { $testMessage } else { "-" } - Add-Content -Path $summaryPath -Value "| Windows | cargo | $testName | $testStatus | n/a | $testDetails |" - } - $status = if ($exitCode -eq 0) { "PASS" } else { "FAIL" } - $record = [ordered]@{ - schema = "cua-e2e-result/v1" - platform = "windows" - host = "lane" - scenario = $Name - status = $status - message = if ($exitCode -eq 0) { "" } else { "exit code $exitCode" } - } | ConvertTo-Json -Compress - Add-Content -Path $resultsPath -Value $record - $details = if ($exitCode -eq 0) { "-" } else { "exit code $exitCode" } - Add-Content -Path $summaryPath -Value "| Windows | lane | $Name | $status | n/a | $details |" if ($exitCode -ne 0) { $script:FailureCount++ } @@ -146,33 +156,38 @@ function Test-E2eRecordings { Write-Host "[VIDEO PASS] $($video.FullName)" -ForegroundColor Green } } + + $ownedVideos = @{} + Get-Content $resultsPath | Where-Object { -not [string]::IsNullOrWhiteSpace($_) } | ForEach-Object { + $result = $_ | ConvertFrom-Json + if ($null -ne $result.evidence.video) { + $ownedVideos[$result.evidence.video.Replace("\", "/")] = $true + } + } + foreach ($video in $videos) { + $relative = [System.IO.Path]::GetRelativePath($artifactDir, $video.FullName).Replace("\", "/") + if ($relative -like "recordings/environment-preflight-*/recording.mp4") { + continue + } + if (-not $ownedVideos.ContainsKey($relative)) { + Write-Host "[VIDEO FAIL] Orphan trajectory has no typed result row: $relative" -ForegroundColor Red + $failureCount++ + } + } return $failureCount } $script:FailureCount = 0 -if ($Suite -in @("shared", "all")) { +if ($suite -in @("shared", "all")) { Invoke-CargoTest "shared behavior matrix" @( "test", "-p", "cua-driver", "--test", "cross_platform_behavior_test", "--", - "--ignored", "--nocapture", "--test-threads=1" - ) -} - -if ($Suite -in @("default", "all")) { - Invoke-CargoTest "default Rust tests" @( - "test", "-p", "cua-driver", "-p", "platform-windows", "--", - "--nocapture", "--test-threads=1" - ) -} - -if ($Suite -in @("guard", "all")) { - Invoke-CargoTest "guard UX" @( - "test", "-p", "cua-driver", "--test", "guard_ux_test", "--", + "--ignored", "--exact", "shared_web_action_matrix_is_state_verified", "--nocapture", "--test-threads=1" ) } -if ($Suite -in @("native", "all")) { +if ($suite -in @("native", "all")) { Invoke-CargoTest "Windows native harnesses" @( "test", "-p", "cua-driver", "--test", "harness_wpf_test", "--", "--ignored", "--nocapture", "--test-threads=1" @@ -185,19 +200,37 @@ if ($Suite -in @("native", "all")) { "test", "-p", "cua-driver", "--test", "harness_web_test", "--", "--ignored", "--nocapture", "--test-threads=1" ) + Invoke-CargoTest "Windows minimized launch" @( + "test", "-p", "cua-driver", "--test", "launch_windows_test", "--", + "--ignored", "--nocapture", "--test-threads=1" + ) + Invoke-CargoTest "Windows agent cursor" @( + "test", "-p", "cua-driver", "--test", "agent_cursor_windows_test", "--", + "--ignored", "--nocapture", "--test-threads=1" + ) } -if ($Suite -in @("modality", "all")) { - Invoke-CargoTest "Windows modality input e2e" @( - "test", "-p", "cua-driver", "--test", "modality_input_e2e_test", "--", +if ($suite -in @("capture", "all")) { + Invoke-CargoTest "capture contract" @( + "test", "-p", "cua-driver", "--test", "capture_contract_test", "--", + "--ignored", "--nocapture", "--test-threads=1" + ) + Invoke-CargoTest "Windows desktop scope" @( + "test", "-p", "cua-driver", "--test", "desktop_scope_windows_test", "--", "--ignored", "--nocapture", "--test-threads=1" ) } $script:FailureCount += (Test-E2eRecordings) +$reportExit = Invoke-E2eReport +if ($reportExit -ne 0) { + Write-Host "Windows E2E result validation failed" -ForegroundColor Red + $script:FailureCount++ +} + if ($script:FailureCount -ne 0) { throw "Windows Rust e2e suite had $($script:FailureCount) failing lane(s)" } -Write-Host "Windows Rust e2e suite completed: $Suite" -ForegroundColor Green +Write-Host "Windows Rust e2e matrix completed: $suite" -ForegroundColor Green