diff --git a/.github/workflows/cd-swift-cua-driver.yml b/.github/workflows/cd-swift-cua-driver.yml new file mode 100644 index 0000000000..cc7086b9b8 --- /dev/null +++ b/.github/workflows/cd-swift-cua-driver.yml @@ -0,0 +1,303 @@ +name: "CD: Cua Driver (macOS)" + +on: + push: + tags: + - "cua-driver-v*" + workflow_dispatch: + inputs: + version: + description: "Version to notarize (without v prefix)" + required: true + default: "0.0.1" + workflow_call: + inputs: + version: + description: "Version to notarize" + required: true + type: string + secrets: + APPLICATION_CERT_BASE64: + required: true + INSTALLER_CERT_BASE64: + required: true + CERT_PASSWORD: + required: true + APPLE_ID: + required: true + TEAM_ID: + required: true + APP_SPECIFIC_PASSWORD: + required: true + DEVELOPER_NAME: + required: true + +permissions: + contents: write + +env: + APPLICATION_CERT_BASE64: ${{ secrets.APPLICATION_CERT_BASE64 }} + INSTALLER_CERT_BASE64: ${{ secrets.INSTALLER_CERT_BASE64 }} + CERT_PASSWORD: ${{ secrets.CERT_PASSWORD }} + APPLE_ID: ${{ secrets.APPLE_ID }} + TEAM_ID: ${{ secrets.TEAM_ID }} + APP_SPECIFIC_PASSWORD: ${{ secrets.APP_SPECIFIC_PASSWORD }} + DEVELOPER_NAME: ${{ secrets.DEVELOPER_NAME }} + +jobs: + notarize: + runs-on: macos-15 + outputs: + sha256_checksums: ${{ steps.generate_checksums.outputs.checksums }} + version: ${{ steps.set_version.outputs.version }} + steps: + - uses: actions/checkout@v4 + + - name: Select Xcode 16.3 + run: | + sudo xcode-select -s /Applications/Xcode_16.3.app + xcodebuild -version + + - name: Install dependencies + run: | + brew install cpio + + - name: Create .release directory + run: mkdir -p .release + + - name: Set version + id: set_version + run: | + # Determine version from tag or input + if [[ "$GITHUB_REF" == refs/tags/cua-driver-v* ]]; then + VERSION="${GITHUB_REF#refs/tags/cua-driver-v}" + echo "Using version from tag: $VERSION" + elif [[ -n "${{ inputs.version }}" ]]; then + VERSION="${{ inputs.version }}" + echo "Using version from input: $VERSION" + elif [[ -n "${{ inputs.version }}" ]]; then + VERSION="${{ inputs.version }}" + echo "Using version from workflow_call input: $VERSION" + else + echo "Error: No version found in tag or input" + exit 1 + fi + + # Update version in CuaDriverCore.swift (powers `cua-driver --version`) + echo "Updating version in CuaDriverCore.swift to $VERSION" + sed -i '' "s/public static let version = \".*\"/public static let version = \"$VERSION\"/" libs/cua-driver/Sources/CuaDriverCore/CuaDriverCore.swift + + # Set output for later steps + echo "version=$VERSION" >> $GITHUB_OUTPUT + + - name: Import Certificates + env: + APPLICATION_CERT_BASE64: ${{ secrets.APPLICATION_CERT_BASE64 }} + INSTALLER_CERT_BASE64: ${{ secrets.INSTALLER_CERT_BASE64 }} + CERT_PASSWORD: ${{ secrets.CERT_PASSWORD }} + KEYCHAIN_PASSWORD: "temp_password" + run: | + # Create a temporary keychain + security create-keychain -p "$KEYCHAIN_PASSWORD" build.keychain + security default-keychain -s build.keychain + security unlock-keychain -p "$KEYCHAIN_PASSWORD" build.keychain + security set-keychain-settings -t 3600 -l build.keychain + + # Import certificates + echo $APPLICATION_CERT_BASE64 | base64 --decode > application.p12 + echo $INSTALLER_CERT_BASE64 | base64 --decode > installer.p12 + + # Import certificates silently (minimize output) + security import application.p12 -k build.keychain -P "$CERT_PASSWORD" -T /usr/bin/codesign -T /usr/bin/pkgbuild > /dev/null 2>&1 + security import installer.p12 -k build.keychain -P "$CERT_PASSWORD" -T /usr/bin/codesign -T /usr/bin/pkgbuild > /dev/null 2>&1 + + # Allow codesign to access the certificates (minimal output) + security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "$KEYCHAIN_PASSWORD" build.keychain > /dev/null 2>&1 + + # Verify certificates were imported + echo "Verifying signing identities..." + CERT_COUNT=$(security find-identity -v -p codesigning build.keychain | grep -c "Developer ID Application" || echo "0") + INSTALLER_COUNT=$(security find-identity -v build.keychain | grep -c "Developer ID Installer" || echo "0") + + if [ "$CERT_COUNT" -eq 0 ]; then + echo "Error: No Developer ID Application certificate found" + security find-identity -v -p codesigning build.keychain + exit 1 + fi + + if [ "$INSTALLER_COUNT" -eq 0 ]; then + echo "Error: No Developer ID Installer certificate found" + security find-identity -v build.keychain + exit 1 + fi + + echo "Found $CERT_COUNT Developer ID Application certificate(s) and $INSTALLER_COUNT Developer ID Installer certificate(s)" + echo "All required certificates verified successfully" + + # Clean up certificate files + rm application.p12 installer.p12 + + - name: Build and Notarize + id: build_notarize + env: + APPLE_ID: ${{ secrets.APPLE_ID }} + TEAM_ID: ${{ secrets.TEAM_ID }} + APP_SPECIFIC_PASSWORD: ${{ secrets.APP_SPECIFIC_PASSWORD }} + # These will now reference the imported certificates + CERT_APPLICATION_NAME: "Developer ID Application: ${{ secrets.DEVELOPER_NAME }} (${{ secrets.TEAM_ID }})" + CERT_INSTALLER_NAME: "Developer ID Installer: ${{ secrets.DEVELOPER_NAME }} (${{ secrets.TEAM_ID }})" + VERSION: ${{ steps.set_version.outputs.version }} + working-directory: ./libs/cua-driver + run: | + # Minimal debug information + echo "Starting build process..." + echo "Swift version: $(swift --version | head -n 1)" + echo "Building version: $VERSION" + + # Ensure .release directory exists + mkdir -p .release + chmod 755 .release + + # Build the project first (redirect verbose output) + echo "Building project..." + swift build --configuration release > build.log 2>&1 + echo "Build completed." + + # Run the notarization script with LOG_LEVEL env var + chmod +x scripts/build/build-release-notarized.sh + cd scripts/build + LOG_LEVEL=minimal ./build-release-notarized.sh + + # Return to the cua-driver directory + cd ../.. + + # Debug: List what files were actually created + echo "Files in .release directory:" + find .release -type f -name "*.tar.gz" -o -name "*.pkg.tar.gz" + + # Get architecture for output filename + ARCH=$(uname -m) + OS_IDENTIFIER="darwin-${ARCH}" + + # Output paths for later use + echo "tarball_path=.release/cua-driver-${VERSION}-${OS_IDENTIFIER}.tar.gz" >> $GITHUB_OUTPUT + echo "pkg_path=.release/cua-driver-${VERSION}-${OS_IDENTIFIER}.pkg.tar.gz" >> $GITHUB_OUTPUT + + - name: Upload build log on failure + if: failure() && steps.build_notarize.outcome == 'failure' + uses: actions/upload-artifact@v4 + with: + name: swift-build-log + path: ./libs/cua-driver/build.log + retention-days: 7 + + - name: Generate SHA256 Checksums + id: generate_checksums + working-directory: ./libs/cua-driver/.release + run: | + # Use existing checksums file if it exists, otherwise generate one + if [ -f "checksums.txt" ]; then + echo "Using existing checksums file" + cat checksums.txt + else + echo "## SHA256 Checksums" > checksums.txt + echo '```' >> checksums.txt + shasum -a 256 cua-driver-*.tar.gz >> checksums.txt + echo '```' >> checksums.txt + fi + + checksums=$(cat checksums.txt) + echo "checksums<> $GITHUB_OUTPUT + echo "$checksums" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + # Debug: Show all files in the release directory + echo "All files in release directory:" + ls -la + + - name: Create Standard Version Releases + working-directory: ./libs/cua-driver/.release + run: | + VERSION=${{ steps.set_version.outputs.version }} + ARCH=$(uname -m) + OS_IDENTIFIER="darwin-${ARCH}" + + # Create OS-tagged symlinks + ln -sf "cua-driver-${VERSION}-${OS_IDENTIFIER}.tar.gz" "cua-driver-darwin.tar.gz" + ln -sf "cua-driver-${VERSION}-${OS_IDENTIFIER}.pkg.tar.gz" "cua-driver-darwin.pkg.tar.gz" + + # Create simple symlinks + ln -sf "cua-driver-${VERSION}-${OS_IDENTIFIER}.tar.gz" "cua-driver.tar.gz" + ln -sf "cua-driver-${VERSION}-${OS_IDENTIFIER}.pkg.tar.gz" "cua-driver.pkg.tar.gz" + + # List all files (including symlinks) + echo "Files with symlinks in release directory:" + ls -la + + - name: Upload Notarized Package (Tarball) + uses: actions/upload-artifact@v4 + with: + name: cua-driver-notarized-tarball + path: ./libs/cua-driver/${{ steps.build_notarize.outputs.tarball_path }} + if-no-files-found: error + + - name: Upload Notarized Package (Installer) + uses: actions/upload-artifact@v4 + with: + name: cua-driver-notarized-installer + path: ./libs/cua-driver/${{ steps.build_notarize.outputs.pkg_path }} + if-no-files-found: error + + - name: Generate path-filtered release notes + if: startsWith(github.ref, 'refs/tags/cua-driver-v') + id: release-notes + run: | + # Find previous cua-driver tag + PREV_TAG=$(git tag -l "cua-driver-v*" --sort=-v:refname | grep -v "^${{ github.ref_name }}$" | head -n 1 || echo "") + + echo "Current tag: ${{ github.ref_name }}" + echo "Previous tag: $PREV_TAG" + + # Generate release notes filtered by libs/cua-driver path + if [ -n "$PREV_TAG" ]; then + echo "Generating notes for commits between $PREV_TAG and HEAD in libs/cua-driver" + NOTES=$(git log ${PREV_TAG}..HEAD --pretty=format:"* %s (%h) by @%an" -- "libs/cua-driver" | head -50) + else + echo "No previous tag found, generating notes for recent commits in libs/cua-driver" + NOTES=$(git log --pretty=format:"* %s (%h) by @%an" -- "libs/cua-driver" | head -50) + fi + + if [ -z "$NOTES" ]; then + NOTES="* Initial release or no path-specific changes found" + fi + + # Store notes in output + echo "RELEASE_NOTES<> $GITHUB_OUTPUT + echo "## What's Changed" >> $GITHUB_OUTPUT + echo "" >> $GITHUB_OUTPUT + echo "$NOTES" >> $GITHUB_OUTPUT + echo "EOF" >> $GITHUB_OUTPUT + + - name: Create Release + if: startsWith(github.ref, 'refs/tags/cua-driver-v') + uses: softprops/action-gh-release@v1 + with: + files: | + ./libs/cua-driver/${{ steps.build_notarize.outputs.tarball_path }} + ./libs/cua-driver/${{ steps.build_notarize.outputs.pkg_path }} + ./libs/cua-driver/.release/cua-driver-darwin.tar.gz + ./libs/cua-driver/.release/cua-driver-darwin.pkg.tar.gz + ./libs/cua-driver/.release/cua-driver.tar.gz + ./libs/cua-driver/.release/cua-driver.pkg.tar.gz + body: | + ${{ steps.release-notes.outputs.RELEASE_NOTES }} + + ${{ steps.generate_checksums.outputs.checksums }} + + ### Installation with script + + ```bash + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/cua-driver/scripts/install.sh)" + ``` + generate_release_notes: false + make_latest: true diff --git a/.github/workflows/ci-swift-cua-driver.yml b/.github/workflows/ci-swift-cua-driver.yml new file mode 100644 index 0000000000..7469b812e0 --- /dev/null +++ b/.github/workflows/ci-swift-cua-driver.yml @@ -0,0 +1,32 @@ +name: "CI: Cua Driver" +on: + pull_request: + paths: + - "libs/cua-driver/**" + - ".github/workflows/ci-swift-cua-driver.yml" + +concurrency: + group: cua-driver-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +# Runner images: https://github.com/actions/runner-images + +jobs: + test: + name: Test + runs-on: macos-15 + steps: + - uses: actions/checkout@v4 + - run: uname -a + - run: sudo xcode-select -s /Applications/Xcode_16.3.app # Swift 6.1 + - run: swift test + working-directory: ./libs/cua-driver + build: + name: Release build + runs-on: macos-15 + steps: + - uses: actions/checkout@v4 + - run: uname -a + - run: sudo xcode-select -s /Applications/Xcode_16.3.app # Swift 6.1 + - run: swift build --configuration release + working-directory: ./libs/cua-driver diff --git a/.lycheeignore b/.lycheeignore index 14ff721cb1..d1bd9790d2 100644 --- a/.lycheeignore +++ b/.lycheeignore @@ -4,3 +4,7 @@ https://api.cua.ai/ https://console.cloud.google.com/* # hud.ai rate-limits automated link checkers (429) https://www.hud.ai/* +# openai.com returns 403 to automated checkers but resolves fine in-browser +https://openai.com/* +# Self-referential install.sh URL: resolves once this PR merges to main +https://raw.githubusercontent.com/trycua/cua/main/libs/cua-driver/scripts/install.sh diff --git a/README.md b/README.md index 987289d43e..d5a3d4c32b 100644 --- a/README.md +++ b/README.md @@ -24,43 +24,27 @@
+ + + - - - @@ -69,6 +53,18 @@ --- +## Cua Driver - Background computer-use on macOS + +Drive any native macOS app **in the background** — agents click, type, and verify without stealing the cursor, focus, or Space, even on non-AX surfaces like Chromium web content and canvas-based tools (Blender, Figma, DAWs, game engines). Use with the CLI or MCP server for Claude Code, Cursor, and custom clients. Every session records as a replayable trajectory. + +```sh +/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/cua-driver/scripts/install.sh)" +``` + +Full tool reference, architecture notes, and the Claude Code skill ship with the package: [`libs/cua-driver/README.md`](libs/cua-driver/README.md). + +--- + ## Cua - Agent-Ready Sandboxes for Any OS Build agents that see screens, click buttons, and complete tasks autonomously. One API for any VM or container image — cloud or local. diff --git a/docs/content/docs/cua-driver/guide/getting-started/comparison.mdx b/docs/content/docs/cua-driver/guide/getting-started/comparison.mdx new file mode 100644 index 0000000000..aac71fadfc --- /dev/null +++ b/docs/content/docs/cua-driver/guide/getting-started/comparison.mdx @@ -0,0 +1,80 @@ +--- +title: Comparison +description: How Cua Driver compares to other macOS computer-use tools +--- + +import { Callout } from 'fumadocs-ui/components/callout'; + +This page compares Cua Driver with other macOS computer-use tools. All are quality projects. The best choice depends on your workload. + +## Quick comparison + +| Feature | Cua Driver | Codex Computer Use | Claude Computer Use | Lume | +| -------------------- | ----------------------- | -------------------- | -------------------- | ----------------------- | +| **License** | MIT | Closed source | Closed source | MIT | +| **Drives host Mac** | Yes | Yes | No (sandbox only) | No (hosts VMs) | +| **Sandbox / VM** | No | No | Yes (Cowork VM) | Yes (macOS + Linux VMs) | +| **Backgrounded** | Default | Default | N/A (sandbox) | N/A | +| **MCP server** | Yes (stdio) | No | No | Yes | +| **Agent-agnostic** | Yes (any MCP client) | Codex-only | Claude-only | Yes (HTTP API) | +| **Capture modes** | vision / ax / som | Vision | Vision | N/A | +| **Primary use case** | Agent automation | Agent automation | Safe agent execution | Ephemeral VMs | + +## Codex Computer Use + +[Codex](https://openai.com/codex) describes its macOS computer-use feature as: "With computer use on macOS, Codex can now use any app by seeing, clicking, and typing with its own cursor. It runs in the background without taking over your computer, working on tasks like frontend iteration, app testing, or any workflow that doesn't expose an API." + +That's a fair description of what Codex does on macOS. Cua Driver differs on four axes: + +- **Agent-agnostic.** Cua Driver works with any agent that speaks MCP or shells out. Codex's computer-use is Codex-only. +- **Open source, MIT licensed.** Codex is a closed product. +- **MCP-native.** Cua Driver speaks MCP over stdio. Paste `cua-driver mcp-config` into your client and it's wired up. Codex has no MCP surface. +- **Three capture modalities.** Codex is vision-only. Cua Driver ships `vision` (PNG only), `ax` (tree only), and `som` (both). AX mode skips Screen Recording entirely and gives deterministic element addressing; `som` gives both halves for disambiguation when labels repeat. + +**When to choose Codex**: if you're already inside Codex and the bundled computer-use covers your task. + +**When to choose Cua Driver**: if you want the same no-foreground contract across any agent, any modality, with an open source license. + +## Claude Computer Use (Cowork) + +[Claude Cowork](https://support.claude.com/en/articles/13345190-getting-started-with-cowork) runs Claude Code inside a sandboxed VM. The VM boots a Linux root filesystem where Claude can execute commands and drive a virtualized desktop without access to your host system. + +This is a different product shape. Cowork is sandbox-first: Claude operates an isolated environment. Cua Driver is host-first: it operates your real Mac with a background-drive contract. + +**When to choose Cowork**: if you want Claude to operate a disposable environment where destructive actions are contained by default. + +**When to choose Cua Driver**: if you want an agent to operate your real, running apps (the editor, the browser, Finder) without taking focus away from you. + + + The two aren't mutually exclusive. An agent running in Cowork could drive Cua Driver on a host via MCP + if you expose the stdio server through the sandbox boundary, but that's not a supported configuration + today. + + +## Lume + +[Lume](/lume/guide/getting-started/introduction) is a macOS VM runtime that spins up Apple Virtualization Framework guests. Cua Driver operates your host Mac; Lume hosts isolated VMs. + +**Key differences:** + +- Lume boots a macOS or Linux VM and hands it to you. Cua Driver does not host any VMs. +- Cua Driver drives apps on your current machine without changing which one is frontmost. Lume has nothing to do with the host's foreground state. +- Both are agent-useful, for different reasons. Lume gives you a disposable macOS environment for CI, sandboxing, or cross-version testing. Cua Driver gives you backgrounded control over the real thing. + +**When to choose Lume**: if you need an isolated macOS VM for automation, testing, or sandboxing untrusted workloads. + +**When to choose Cua Driver**: if you want an agent to drive real apps on your host without stealing focus. + +## Summary + +Cua Driver fits when all of these are true: + +- You want an agent to drive real apps on your own Mac (not a VM). +- The user needs to keep working in another app while the agent operates. +- You're building against any MCP-capable agent, not locked to one vendor. +- MIT licensing matters. + +The trade-offs you accept: + +- Destructive actions hit your real filesystem. Confirm user intent before deleting, overwriting, or sending. +- A handful of app classes (Chromium web-content right-click, canvas viewports like Blender or Unity) need known workarounds. See [Limits](/cua-driver/reference/limits). diff --git a/docs/content/docs/cua-driver/guide/getting-started/faq.mdx b/docs/content/docs/cua-driver/guide/getting-started/faq.mdx new file mode 100644 index 0000000000..7a0623d61c --- /dev/null +++ b/docs/content/docs/cua-driver/guide/getting-started/faq.mdx @@ -0,0 +1,190 @@ +--- +title: FAQ +description: Frequently asked questions about Cua Driver +--- + +import { Callout } from 'fumadocs-ui/components/callout'; + +Common gotchas and questions. For the full action loop and tool semantics, see the [CLI reference](/cua-driver/reference/cli-reference) and [MCP tools reference](/cua-driver/reference/mcp-tools). + +## The action loop + +### Why do I get `No cached AX state`? + +Element-indexed actions read an in-memory cache keyed on `(pid, window_id)`. The cache is populated by `get_window_state` and replaced on every snapshot. + +Two common causes: + +1. You didn't call `get_window_state` in the current turn. +2. You called it with a different `window_id` than the one in the action. + +```bash +# Populate the cache for this exact (pid, window_id). +cua-driver get_window_state '{"pid":844,"window_id":10725}' + +# Then act with the same window_id. +cua-driver click '{"pid":844,"window_id":10725,"element_index":14}' +``` + +If you're running one-shot CLI invocations without a daemon, the cache lives for one process lifetime. Start the daemon first: + +```bash +open -n -g -a CuaDriver --args serve +``` + +### Why did my screenshot come back empty (`has_screenshot: false`)? + +The window capture raced against a close, or the window has no backing store yet. Re-snapshot. If it persists, pick a different `window_id` via `list_windows`. + +### The AX tree is tiny. What's happening? + +Check the capture mode: + +```bash +cua-driver config get capture_mode +``` + +Default is `som` (tree + screenshot). If it reads `vision`, `get_window_state` omits the tree by design (PNG only). Switch back to `som` to get both: + +```bash +cua-driver config set capture_mode som +``` + +If the mode is `som` or `ax` and the tree is still small, the target uses custom rendering (Blender, Unity, Electron with AX disabled). For Chromium/Electron, retry `get_window_state` once — the tree populates on the second call. For canvas-backed apps, reach for pixel clicks instead. + +## Window state + +### My keyboard commit (Return, Space, Tab) on a minimized window silently no-ops. + +Minimized windows receive AX reads and AX-dispatched clicks normally, but keyboard commits fail because AX focus doesn't propagate to renderer focus on a minimized window. You hear the macOS system-alert beep, or nothing happens. + +Workarounds in order of preference: + +1. **Use `set_value` to write the field's entire value directly.** Bypasses keyboard commits. +2. **AX-click a commit-equivalent button** (Go, Submit, checkbox). Clicks route through `AXPress` and don't need renderer focus. +3. **Last resort: ask the user to un-minimize the window.** Don't deminiaturize programmatically — layout-disrupting on many apps. + +### My backgrounded SwiftUI app (System Settings) returns an almost-empty AX tree. + +Windows on another Space often strip their AX tree to the menu bar on SwiftUI apps. AppKit apps are usually fine. + +`get_window_state` returns `off_space: true` plus `window_space_ids` when this happens, so you can detect it. Solutions: + +- Ask the user to Mission-Control back to the Space that holds the target. +- Drive the app through in-window toolbar buttons (which often stay exposed) rather than deep nested controls. +- Accept the limitation for the current session. + +## Browsers and Electron + +### Right-click on Chromium web content fires as a left-click. + +A known Chromium renderer-IPC limit: the filter coerces synthetic right-click subtype to left on every non-HID-tap path. Use `right_click({pid, element_index})` on AX-addressable targets (links, buttons, toolbar items). For web content itself (right-clicking an image or selection), there is no backgrounded path today. See [Limits](/cua-driver/reference/limits) for the full note. + +### Pixel click on a YouTube video doesn't play or pause. + +HTML5's click-to-play handler rejects some synthetic click paths. Use keyboard instead: + +```bash +cua-driver press_key '{"pid":,"key":"k"}' # YouTube play/pause +cua-driver press_key '{"pid":,"key":"space"}' # generic video play/pause +``` + +Keyboard events travel through a different auth envelope and reach the page. + +### How do I navigate to a URL in Chrome without stealing focus? + +Pass the URL to `launch_app`: + +```bash +cua-driver launch_app '{"bundle_id":"com.google.Chrome","urls":["https://trycua.com"]}' +``` + +The URL opens in a new window via Chrome's `application(_:open:)` delegate. The driver's focus-restore guard catches Chrome's internal activation and clobbers the frontmost back to what it was before the call. + + + Don't use `hotkey ⌘L` to focus the omnibox. Even when delivered to a backgrounded pid, `⌘L` steals + focus because the receiving app interprets "user wants to type here" as activation intent. + + +## The agent cursor + +### How do I disable the visual cursor overlay? + +```bash +cua-driver set_agent_cursor_enabled '{"enabled":false}' +``` + +Or via config: + +```bash +cua-driver config set agent_cursor.enabled false +``` + +The overlay only renders when the driver has an AppKit run loop (inside `cua-driver serve` or `cua-driver mcp`). One-shot CLI invocations skip it entirely. + +### Can I make the cursor move faster? + +Tune the motion knobs: + +```bash +cua-driver set_agent_cursor_motion '{"glide_duration_ms":300}' +``` + +See `set_agent_cursor_motion` in the [MCP tools reference](/cua-driver/reference/mcp-tools) for every knob. + +## Permissions + +### `check_permissions` says `NOT granted` but I granted both. + +TCC checks the calling process, not `CuaDriver.app`. Inside IDE terminals (Claude Code, Cursor, VS Code, Conductor), the shell inherits the IDE's TCC responsibility chain. So running `cua-driver check_permissions` in one of those shells reads against the IDE's bundle, not `com.trycua.driver`. + +Start the daemon first, which runs through LaunchServices under the CuaDriver bundle: + +```bash +open -n -g -a CuaDriver --args serve +cua-driver check_permissions # forwards to the daemon — authoritative answer +``` + +### I keep seeing the permissions dialog on every launch. + +macOS is attributing the process to a different bundle id than the one you granted. Run `cua-driver diagnose` and share the output when filing an issue. It reports cdhash, team id, and which bundle TCC matched against. + +## Config and telemetry + +### Where does config live? + +``` +~/Library/Application Support/Cua Driver/config.json +``` + +Read and write via `cua-driver config`: + +```bash +cua-driver config # show full config +cua-driver config get capture_mode +cua-driver config set capture_mode som +cua-driver config reset # overwrite with defaults +``` + +### How do I opt out of telemetry? + +```bash +cua-driver config telemetry disable +``` + +Or set `CUA_DRIVER_TELEMETRY_ENABLED=0` in the environment for a one-off override. + +Telemetry records anonymous subcommand usage (`cua_driver_api_click`, `cua_driver_serve`, etc). No command arguments, file paths, or personal information are collected. + +## Testing + +### Are there tests I can run against my install? + +The project includes Python integration tests under `libs/cua-driver/tests/` that exercise the real `cua-driver` stdio server against `unittest`. They run as part of `scripts/test.sh`: + +```bash +cd libs/cua-driver +./scripts/test.sh +``` + +For a quick manual smoke check, the Calculator test from `libs/cua-driver/Skills/cua-driver/TESTS.md` is a good five-minute run: launch Calculator hidden, snapshot, click 17 × 23 by element index, re-snapshot, verify the display reads `391` and Calculator never came to the foreground. diff --git a/docs/content/docs/cua-driver/guide/getting-started/installation.mdx b/docs/content/docs/cua-driver/guide/getting-started/installation.mdx new file mode 100644 index 0000000000..ed81d911ad --- /dev/null +++ b/docs/content/docs/cua-driver/guide/getting-started/installation.mdx @@ -0,0 +1,140 @@ +--- +title: Installation +description: Install Cua Driver on your Mac +--- + +import { Callout } from 'fumadocs-ui/components/callout'; + +Install Cua Driver with a single command: + +```bash +/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/cua-driver/scripts/install.sh)" +``` + +The installer drops `CuaDriver.app` into `/Applications` and symlinks the binary at `/usr/local/bin/cua-driver`. Both are signed with the same bundle id (`com.trycua.driver`) so TCC grants survive rebuilds. + + +**First-time install?** If `/usr/local/bin` isn't in your PATH, add it and reload: + +```bash +echo 'export PATH="$PATH:/usr/local/bin"' >> ~/.zshrc +source ~/.zshrc +``` + +Or restart your terminal after the first command. + + +### Verify it worked + +```bash +cua-driver --version +# cua-driver 0.1.0 + +cua-driver --help +# OVERVIEW: macOS Accessibility-driven computer-use agent — MCP stdio server. +``` + +## Grant TCC permissions + +Cua Driver needs two permissions: + +- **Accessibility** — to walk AX trees and dispatch `AXUIElementPerformAction`. +- **Screen Recording** — to capture per-window screenshots via ScreenCaptureKit. + +The first time you run `cua-driver serve` or `cua-driver mcp`, a permissions gate opens and walks you through granting both. Once the green checkmarks appear, close the window. + +To check status from the shell: + +```bash +cua-driver check_permissions +# ✅ Accessibility: granted. +# ✅ Screen Recording: granted. +``` + + + `check_permissions` reports the TCC status of the *calling* process. Inside an IDE terminal (Claude + Code, Cursor, VS Code, Conductor) the shell inherits the IDE's TCC responsibility chain, so results + can read "NOT granted" even when you've granted both to `CuaDriver.app`. Start the daemon first + (`cua-driver serve &`) and the CLI forwards through it for authoritative answers. + + +If a grant reads `NOT granted`, open **System Settings → Privacy & Security**, find `CuaDriver.app` under Accessibility and Screen Recording, and flip the toggle. + +## Requirements + +- macOS 14 (Sonoma) or later +- Apple Silicon (M1/M2/M3/M4) or Intel Mac +- 50 MB free disk space for the app bundle + +## Run the daemon + +Most workflows benefit from a persistent daemon. Element-indexed workflows require one: the per-pid element cache lives in-process, so one-shot CLI invocations lose it between calls. + +```bash +# Start the daemon in the background. +open -n -g -a CuaDriver --args serve + +# Confirm it's up. +cua-driver status +# cua-driver daemon is running +# socket: /Users/you/Library/Caches/cua-driver/cua-driver.sock +# pid: 12345 + +# Stop it cleanly when done. +cua-driver stop +``` + +`open -n -g -a CuaDriver --args serve` is the recommended form because LaunchServices attributes the process to `CuaDriver.app`'s bundle id, which is what the user actually granted TCC against. `cua-driver serve &` also works and auto-relaunches itself via `open` when it detects the wrong TCC context. + +## Register with an MCP client + +Cua Driver speaks MCP over stdio. Generate a client config snippet: + +```bash +cua-driver mcp-config +``` + +Output: + +```json +{ + "mcpServers": { + "cua-driver": { + "command": "/Applications/CuaDriver.app/Contents/MacOS/cua-driver", + "args": ["mcp"] + } + } +} +``` + +Paste into `~/.claude/mcp.json` (Claude Code) or the equivalent config for your client. The client spawns `cua-driver mcp` on demand. + +## Uninstall + +```bash +# Stop the daemon if running. +cua-driver stop 2>/dev/null + +# Remove the app and the CLI symlink. +rm -rf /Applications/CuaDriver.app +rm -f /usr/local/bin/cua-driver + +# Optional: remove config + telemetry state. +rm -rf ~/.cua-driver +rm -rf ~/Library/Application\ Support/Cua\ Driver +rm -rf ~/Library/Caches/cua-driver + +# Optional: remove the updater LaunchAgent. +launchctl unload ~/Library/LaunchAgents/com.trycua.cua_driver_updater.plist 2>/dev/null +rm -f ~/Library/LaunchAgents/com.trycua.cua_driver_updater.plist +``` + +## Troubleshooting + +**`cua-driver: command not found`** — `/usr/local/bin` isn't on your PATH. Add it and reload (see the first-time install callout above). + +**Permissions dialogs reappear after every launch** — macOS is attributing the process to a different bundle id than the one you granted. Run `cua-driver diagnose` and paste the output when filing an issue; it reports cdhash, team id, and which bundle TCC is checking against. + +**Daemon won't start** — another daemon may already be bound to the socket. Check with `cua-driver status` and stop it with `cua-driver stop`. For stale lock files after a crash, the daemon's own probe detects those and proceeds. + +Ready to drive an app? Head to the [Quickstart](/cua-driver/guide/getting-started/quickstart). diff --git a/docs/content/docs/cua-driver/guide/getting-started/introduction.mdx b/docs/content/docs/cua-driver/guide/getting-started/introduction.mdx new file mode 100644 index 0000000000..2c14126674 --- /dev/null +++ b/docs/content/docs/cua-driver/guide/getting-started/introduction.mdx @@ -0,0 +1,68 @@ +--- +title: What is Cua Driver? +description: Background computer-use driver for any agent on macOS +--- + +import { Callout } from 'fumadocs-ui/components/callout'; + +Cua Driver is a macOS computer-use driver that speaks MCP over stdio. It lets any agent (Claude, GPT, Gemini, Codex, custom loops) click, type, scroll, and snapshot a native macOS app without bringing the target to the foreground. Your frontmost app stays where it is; the user keeps typing in their editor while the agent drives something else in the background. + + + Cua Driver is open-source and MIT licensed. If you find it useful, we'd appreciate a [star on + GitHub](https://github.com/trycua/cua)! + + +```bash +cua-driver launch_app '{"bundle_id":"com.apple.calculator"}' +cua-driver get_window_state '{"pid":844,"window_id":10725}' +cua-driver click '{"pid":844,"window_id":10725,"element_index":14}' +``` + +A single binary. Launch it as an MCP stdio server, a long-running daemon, or invoke any tool directly from the shell. + +## The no-foreground contract + +One rule: the user's frontmost app does not change. Not during launch, not during a click, not during a keystroke, not during a re-snapshot. Three corollaries follow: + +- The real cursor stays where the user left it. No warp. +- The target window stays at its current z-rank. No raise. +- The user's Space does not follow the target. No bounce. + +Every dispatch path inside the driver honors those four invariants. `launch_app` runs hidden. Keyboard tools post via `CGEvent.postToPid` scoped to a named pid, so a driver-backgrounded app cannot leak keystrokes into the user's foreground app. Pixel clicks route through an auth-signed `SLEventPostToPid` recipe that borrows yabai's focus-without-raise pattern. Element-indexed clicks go through `AXUIElementPerformAction` directly and skip event synthesis entirely. + +## Three modalities + +`capture_mode` controls what `get_window_state` returns. Pick based on what the agent needs: + +- **`som`** (default) — set-of-mark. Both the AX tree and the screenshot. Tree for dispatch, screenshot for visual disambiguation when labels repeat or stay empty. Works out of the box for element-indexed clicks. +- **`ax`** — accessibility tree only. No Screen Recording cost, deterministic element addressing. Best for structured loops over apps with real AX coverage. +- **`vision`** — window PNG only. No AX walk. Best for vision-first models that ground on pixels and don't use element_index. Pair with pixel-addressed clicks. + +Backgrounded drive is the default across all three, not a mode you toggle. Switch modalities with `cua-driver config set capture_mode som`. + +## How it works + +Three dispatch paths, one per modality the target exposes. + +- **Accessibility elements, via public AX.** Where the target has a real AX tree, the driver walks it, tags every actionable node with an index, and caches the `AXUIElement` ref against `(pid, window_id)`. Clicks go through `AXUIElementPerformAction` directly. +- **Chromium and Electron trees, via an AX observer SPI.** Chrome, Slack, VS Code, Discord, and every Electron app pause AX tree updates when occluded unless the observer is registered via a private SPI variant. The driver uses that variant so the tree stays populated through the full launch-snapshot-act loop without bringing the target forward. +- **Non-AX surfaces, via `SLEventPostToPid`.** Canvas, WebView, HTML5 video, custom-drawn controls. A backgrounded click recipe stamps events through SkyLight's auth-signed per-pid path. The cursor never moves, the window never rises, Spaces never follows. + +Keyboard is simpler. Every key goes through `CGEvent.postToPid` scoped to the named pid. There is no frontmost-routed variant in the API surface. + +## Who it's for + +- **Agent builders** who want their agent to operate macOS without stealing the user's focus. Any MCP-capable client works: Claude Code, Cursor, Codex, Gemini, custom loops. +- **Dev-loop automation.** An agent drives an app, reads the pixels or the AX tree, edits source, rebuilds, verifies. The editor stays frontmost the entire time. +- **Demo capture.** Record a trajectory while the user works in another app. Because the clicks are backgrounded, the overlay cursor the driver paints is the only cursor in the final video. + +## What it doesn't do + +- Requires macOS 14 (Sonoma) or later. Works on Apple Silicon and Intel. +- Not a VM. Cua Driver operates the real host, so grant Accessibility and Screen Recording with intent. +- No right-click on Chromium web content through pixel synthesis: the renderer-IPC filter drops right-click subtype on non-HID-tap paths. Use `right_click({pid, element_index})` on AX-addressable targets. See [Limits](/cua-driver/reference/limits). +- Canvas apps (Blender, Unity, games) need a brief frontmost activation because their event loops filter per-pid-routed events. Everything else stays backgrounded. + +## Get started + +Ready to try it? [Install Cua Driver](/cua-driver/guide/getting-started/installation) and drive your first app in the [Quickstart](/cua-driver/guide/getting-started/quickstart). diff --git a/docs/content/docs/cua-driver/guide/getting-started/meta.json b/docs/content/docs/cua-driver/guide/getting-started/meta.json new file mode 100644 index 0000000000..3c9c8b84e0 --- /dev/null +++ b/docs/content/docs/cua-driver/guide/getting-started/meta.json @@ -0,0 +1,7 @@ +{ + "title": "Getting Started", + "description": "Get up and running with Cua Driver", + "icon": "Rocket", + "defaultOpen": true, + "pages": ["introduction", "installation", "quickstart", "comparison", "faq"] +} diff --git a/docs/content/docs/cua-driver/guide/getting-started/quickstart.mdx b/docs/content/docs/cua-driver/guide/getting-started/quickstart.mdx new file mode 100644 index 0000000000..912cbcda03 --- /dev/null +++ b/docs/content/docs/cua-driver/guide/getting-started/quickstart.mdx @@ -0,0 +1,159 @@ +--- +title: Quickstart +description: Drive your first macOS app with Cua Driver +--- + +import { Callout } from 'fumadocs-ui/components/callout'; + +In under 5 minutes, you'll launch a macOS app in the background, snapshot its AX tree, click a button by its element index, and verify the action landed. + +## Start the daemon + +Element-indexed workflows require a persistent daemon. The per-pid element cache lives in-process, so one-shot CLI invocations lose it between calls. + +```bash +open -n -g -a CuaDriver --args serve +cua-driver status +# cua-driver daemon is running +# socket: /Users/you/Library/Caches/cua-driver/cua-driver.sock +# pid: 12345 +``` + + + `cua-driver serve &` also works. The CLI auto-relaunches itself via LaunchServices when your + shell's TCC context is wrong (any IDE terminal). + + +Switch to `som` mode so snapshots carry both the AX tree and a screenshot: + +```bash +cua-driver config set capture_mode som +``` + +## Launch an app (hidden) + +```bash +cua-driver launch_app '{"bundle_id":"com.apple.calculator"}' +``` + +Output: + +``` +✅ Launched Calculator (pid 844) in background. + +Windows: +- "Calculator" [window_id: 10725] +→ Call get_window_state(pid: 844, window_id) to inspect. +``` + +The app's pid and a `windows` array come back in one call. `launch_app` is idempotent: relaunching a running app returns the existing pid with no side effects. The window's AX tree is fully populated (clickable via `element_index`) but not drawn on screen. + +## Snapshot the window + +```bash +cua-driver get_window_state '{"pid":844,"window_id":10725}' +``` + +Output (trimmed): + +``` +✅ Calculator — 34 elements, turn 1 + screenshot + +- AXApplication "Calculator" + - [0] AXWindow "Calculator" actions=[AXRaise] + - [1] AXButton "All Clear" + - [2] AXButton "Plus/Minus" + - [3] AXButton "Percent" + - [4] AXButton "Divide" + - [5] AXButton "Seven" + ... + - [14] AXButton "Three" + ... +``` + +Every actionable element is tagged with `[N]` — that's the `element_index` you pass to `click`, `type_text`, and friends. The index map is replaced on every snapshot, keyed on `(pid, window_id)`, so always snapshot before acting. + + + Large trees (Finder is ~1600 elements) exceed most LLM context limits. Pass a `query` field to filter + the Markdown to matching lines plus their ancestors: `get_window_state '{"pid":844,"window_id":10725,"query":"Three"}'`. + + +## Click by element_index + +```bash +cua-driver click '{"pid":844,"window_id":10725,"element_index":14}' +# ✅ Performed AXPress on [14] AXButton "Three". +``` + +No cursor moved. Calculator never came to the foreground. The click went through `AXUIElementPerformAction` directly. + +## Verify the action landed + +Re-snapshot and check the AX tree diff: + +```bash +cua-driver get_window_state '{"pid":844,"window_id":10725}' +``` + +Look for the display `AXStaticText` reading `3`. If the tree didn't change, the action failed silently, and you should say so rather than assume success. + +## The pixel-click variant + +Pixel clicks are for surfaces the AX tree doesn't reach: canvases, video players, WebGL, custom controls. Coordinates are window-local screenshot pixels (same space as the PNG `get_window_state` returns). + +```bash +# Write the screenshot to disk. Works in every capture mode. +cua-driver get_window_state '{"pid":844,"window_id":10725}' --image-out /tmp/shot.png + +# Look at /tmp/shot.png, pick a target pixel, then: +cua-driver click '{"pid":844,"window_id":10725,"x":120,"y":240}' +# ✅ Posted click to pid 844. +``` + +The PNG is capped at 1568 px long-side by default (matching Anthropic's multimodal-vision downsampling limit), so the image you reason over and the coordinate space the click tool expects are the same resolution. No scaling math. + + + The `window_id` field is optional on pixel clicks but recommended. It pins the coordinate conversion + to the window whose screenshot produced the pixel, rather than letting the driver pick heuristically. + + +## Cleanup + +```bash +# Quit the target app via a hotkey to its pid. +cua-driver hotkey '{"pid":844,"keys":["cmd","q"]}' + +# Stop the daemon. +cua-driver stop +``` + +## The canonical loop + +Every multi-step workflow follows the same shape: + +```bash +open -n -g -a CuaDriver --args serve + +# 1. Launch the target (idempotent; returns pid + windows). +cua-driver launch_app '{"bundle_id":"..."}' + +# 2. Snapshot the window you care about (populates the element cache). +cua-driver get_window_state '{"pid":,"window_id":}' + +# 3. Dispatch an action by element_index or pixel coordinates. +cua-driver click '{"pid":,"window_id":,"element_index":}' + +# 4. Re-snapshot to verify the action landed. +cua-driver get_window_state '{"pid":,"window_id":}' + +cua-driver stop +``` + +The snapshot-before-AND-after invariant is not optional. Indices are stale across turns. Actions that silently no-op (disabled buttons, minimized windows that reject keyboard commits, Chromium right-clicks) are indistinguishable from successes without the post-action diff. + +## What's next + +- Full CLI reference: every subcommand and its flags. See [CLI reference](/cua-driver/reference/cli-reference). +- MCP tool schemas: the authoritative input shape for every tool. See [MCP tools](/cua-driver/reference/mcp-tools). +- Known limits: Chromium right-click coercion, canvas apps, off-Space SwiftUI. See [Limits](/cua-driver/reference/limits). +- Common gotchas: "No cached AX state", minimized windows + keyboard commit, disabling the agent cursor. See the [FAQ](/cua-driver/guide/getting-started/faq). diff --git a/docs/content/docs/cua-driver/guide/meta.json b/docs/content/docs/cua-driver/guide/meta.json new file mode 100644 index 0000000000..7febd12285 --- /dev/null +++ b/docs/content/docs/cua-driver/guide/meta.json @@ -0,0 +1,6 @@ +{ + "title": "Guide", + "description": "Learn how to use Cua Driver", + "icon": "Book", + "pages": ["getting-started"] +} diff --git a/docs/content/docs/cua-driver/meta.json b/docs/content/docs/cua-driver/meta.json new file mode 100644 index 0000000000..c781e7aa4d --- /dev/null +++ b/docs/content/docs/cua-driver/meta.json @@ -0,0 +1,5 @@ +{ + "title": "Cua Driver", + "description": "Background computer-use driver for any agents", + "pages": ["guide", "reference"] +} diff --git a/docs/content/docs/cua-driver/reference/cli-reference.mdx b/docs/content/docs/cua-driver/reference/cli-reference.mdx new file mode 100644 index 0000000000..13809f61b4 --- /dev/null +++ b/docs/content/docs/cua-driver/reference/cli-reference.mdx @@ -0,0 +1,367 @@ +--- +title: CLI Reference +description: Command Line Interface reference for Cua Driver +--- + +import { Callout } from 'fumadocs-ui/components/callout'; +import { VersionHeader } from '@/components/version-selector'; + + + +`cua-driver` is a single-binary CLI. Two naming conventions divide its surface: + +- **Tool names are `snake_case`** (`launch_app`, `get_window_state`, `click`). Invoke them as `cua-driver ''` — the CLI routes through the same `ToolRegistry` the MCP server uses. +- **Management subcommands are `kebab-case`** (`list-tools`, `describe`, `mcp-config`). These never take JSON args. + +Different separators mean no ambiguity. Unknown first-positional args dispatch to the `call` subcommand automatically, so `cua-driver list_apps` is shorthand for `cua-driver call list_apps`. + +## Quick start + +```bash +# Start the persistent daemon (required for element_index workflows). +open -n -g -a CuaDriver --args serve + +# Drive an app. +cua-driver launch_app '{"bundle_id":"com.apple.calculator"}' +cua-driver get_window_state '{"pid":844,"window_id":10725}' +cua-driver click '{"pid":844,"window_id":10725,"element_index":14}' + +# Stop the daemon. +cua-driver stop +``` + +## Tool dispatch + +### cua-driver call + +Invoke any MCP tool from the shell. + +```bash +cua-driver call '' +# Shorthand — any unknown first positional arg auto-prefixes `call`: +cua-driver '' +``` + +**Arguments:** + +- `` — Name of the tool to invoke. Run `cua-driver list-tools` for the full list. +- `` — JSON object matching the tool's `inputSchema`. Omit when stdin is a pipe (JSON is read from stdin) or when the tool takes no arguments. + +**Flags:** + +- `--raw` — Print the raw `CallTool.Result` JSON (content + structuredContent + isError) instead of unwrapping `structuredContent`. +- `--image-out ` — Write the first image content block from the response to `path` (PNG bytes). The default text formatter would drop image content otherwise. +- `--compact` — Emit minified JSON instead of pretty-printed. +- `--no-daemon` — Skip the running daemon and run the tool in-process. Element-indexed workflows fail without a daemon because the per-pid cache dies between CLI invocations. +- `--socket ` — Override the daemon Unix socket path. + +**Examples:** + +```bash +cua-driver call list_apps +cua-driver call launch_app '{"bundle_id":"com.apple.finder"}' +echo '{"pid":844,"window_id":1234}' | cua-driver call get_window_state +cua-driver get_window_state '{"pid":844,"window_id":1234}' --image-out /tmp/shot.png +``` + +### cua-driver list-tools + +List every MCP tool exposed by the driver with a one-line summary. + +```bash +cua-driver list-tools +``` + +**Flags:** `--no-daemon`, `--socket `. + +### cua-driver describe + +Print a tool's full description and JSON input schema. + +```bash +cua-driver describe +``` + +**Flags:** `--compact`, `--no-daemon`, `--socket `. + +## Daemon management + +### cua-driver serve + +Run cua-driver as a long-running daemon on a Unix domain socket. Required for any workflow that uses `element_index` dispatch: the per-pid element cache lives in-process and survives only between CLI calls routed to the same daemon. + +```bash +# Recommended form — routes through LaunchServices for correct TCC context. +open -n -g -a CuaDriver --args serve + +# Alternate — auto-relaunches via open when TCC context is wrong. +cua-driver serve & +``` + +**Flags:** + +- `--socket ` — Override the Unix socket path. Default: `~/Library/Caches/cua-driver/cua-driver.sock`. +- `--no-relaunch` — Stay in the current process instead of relaunching via LaunchServices. Also toggleable via `CUA_DRIVER_NO_RELAUNCH=1`. Use when the calling context already has the right TCC responsibility. + +### cua-driver stop + +Ask the running daemon to exit gracefully. Polls for the socket file to vanish (up to 2s) as proof of clean shutdown. + +```bash +cua-driver stop +``` + +**Flags:** `--socket `. + +### cua-driver status + +Report whether a daemon is currently reachable. Probes by sending a trivial `list` request — connecting alone doesn't prove the peer speaks the protocol. + +```bash +cua-driver status +# cua-driver daemon is running +# socket: /Users/you/Library/Caches/cua-driver/cua-driver.sock +# pid: 12345 +``` + +**Flags:** `--socket `, `--pid-file `. + +### cua-driver mcp + +Run the stdio MCP server. MCP clients (Claude Code, Cursor, custom SDK clients) spawn this on demand. + +```bash +cua-driver mcp +``` + +No flags. Use `cua-driver mcp-config` to generate a paste-able client config snippet. + +### cua-driver mcp-config + +Print an MCP client config snippet pointing at this binary. + +```bash +cua-driver mcp-config +# { +# "mcpServers": { +# "cua-driver": { +# "command": "/Applications/CuaDriver.app/Contents/MacOS/cua-driver", +# "args": ["mcp"] +# } +# } +# } +``` + +Paste into `~/.claude/mcp.json` (Claude Code) or the equivalent for your MCP client. + +## Trajectory recording + +### cua-driver recording start + +Enable the trajectory recorder. Every subsequent action-tool call (`click`, `right_click`, `scroll`, `type_text`, `type_text_chars`, `press_key`, `hotkey`, `set_value`) writes a numbered turn folder under ``. + +```bash +cua-driver recording start ~/cua-trajectories/demo1 +``` + +**Arguments:** + +- `` — Directory to write turn folders into. Expands `~`; created if missing. + +**Flags:** + +- `--video-experimental` — Also capture the main display to `/recording.mp4` via SCStream (H.264, 30fps, no audio, no cursor). Experimental. +- `--socket ` — Override the daemon socket path. + +Requires a running daemon. + +### cua-driver recording stop + +Disable recording. Prints the captured turn count and directory. + +```bash +cua-driver recording stop +# Recording disabled (23 turns captured in /Users/you/cua-trajectories/demo1) +``` + +### cua-driver recording status + +Report whether recording is currently enabled. + +```bash +cua-driver recording status +# Recording: enabled +# Output dir: /Users/you/cua-trajectories/demo1 +# Next turn: 24 +``` + +### cua-driver recording render + +Render a recording directory to a zoomed-on-click MP4. Post-processes the captured `recording.mp4`, `cursor.jsonl`, and `turn-*/action.json` files. + +```bash +cua-driver recording render ~/cua-rec --output /tmp/out.mp4 +cua-driver recording render ~/cua-rec --output /tmp/baseline.mp4 --no-zoom +cua-driver recording render ~/cua-rec --output /tmp/out.mp4 --scale 2.5 +``` + +**Arguments:** + +- `` — Recording directory (contains `session.json`, `recording.mp4`, `cursor.jsonl`, `turn-*/`). + +**Flags:** + +- `--output ` — Destination MP4 path. Overwrites any existing file. +- `--no-zoom` — Skip the zoom curve and re-encode the input as-is. Useful as a baseline check. +- `--scale ` — Zoom factor applied to each click event. Default `2.0`. Set to `1.0` to disable zoom; `2.0` is 2× magnification. + + + `recording render` runs in the CLI process directly — it does not require a running daemon. + + +## Configuration + +### cua-driver config + +Read and write persistent settings at `~/Library/Application Support/Cua Driver/config.json`. + +```bash +cua-driver config # print full config +cua-driver config get +cua-driver config set +cua-driver config reset # overwrite with defaults +``` + +**Supported keys:** + +- `capture_mode` — `som` | `ax` | `vision`. Default `som`. +- `max_image_dimension` — integer. PNG long-side cap. Default 1568. +- `agent_cursor.enabled` — boolean. +- `agent_cursor.motion.start_handle` — number in [0, 1]. +- `agent_cursor.motion.end_handle` — number in [0, 1]. +- `agent_cursor.motion.arc_size` — number (fraction of path length). +- `agent_cursor.motion.arc_flow` — number in [-1, 1]. +- `agent_cursor.motion.spring` — number in [0.3, 1.0]. + +**Flags on `set`:** `--socket `. + +Writes route through the daemon when one's reachable so live state (e.g. `AgentCursor.shared`) picks up the change without a restart. + +### cua-driver config telemetry + +Manage anonymous telemetry. + +```bash +cua-driver config telemetry status +cua-driver config telemetry enable +cua-driver config telemetry disable +``` + +Environment override: `CUA_DRIVER_TELEMETRY_ENABLED=0|1`. + +### cua-driver config updates + +Manage automatic updates. + +```bash +cua-driver config updates status +cua-driver config updates enable +cua-driver config updates disable +``` + +Environment override: `CUA_DRIVER_AUTO_UPDATE_ENABLED=0|1`. + +## Diagnostics + +### cua-driver diagnose + +Print a paste-able state report for support. Covers: running-process identity (path, bundle id, pid, cdhash), TCC probe results, install layout (`/Applications/CuaDriver.app` + codesign info, `/usr/local/bin/cua-driver` symlink resolution), TCC database rows for `com.trycua.driver`, and config + state paths. + +```bash +cua-driver diagnose +``` + +Use this when filing an issue about permissions or install problems. + +### cua-driver update + +Trigger a manual update check and optionally apply updates. Refuses to run when auto-update is disabled. + +```bash +cua-driver update +cua-driver update --silent +``` + +**Flags:** `--silent` — apply updates silently without prompting. + +## Global options + +Available on all commands: + +- `--help` — Show help information. +- `--version` — Show version number. + +## Tool inventory + +The following MCP tools are callable via `cua-driver `. For input schemas and response shapes, see the [MCP tools reference](/cua-driver/reference/mcp-tools). + +**Discovery and permissions** + +- `list_apps` — running + installed apps with pid / bundle id / active state. +- `list_windows` — every layer-0 top-level window (including off-screen). +- `check_permissions` — Accessibility + Screen Recording TCC status. +- `get_screen_size` — main display size in points + scale factor. +- `get_cursor_position` — current mouse cursor position. +- `get_accessibility_tree` — lightweight desktop snapshot (apps + visible windows). + +**App lifecycle** + +- `launch_app` — launch hidden; returns pid + windows array. + +**Snapshot** + +- `get_window_state` — per-window AX tree + screenshot. Populates the element_index cache. +- `screenshot` — raw ScreenCaptureKit capture. Full display or single window. +- `zoom` — native-resolution crop of a previously captured window region. + +**Mouse** + +- `click` — left-click by `element_index` or `(x, y)`. +- `double_click` — double-click by `element_index` (AXOpen) or `(x, y)`. +- `right_click` — right-click by `element_index` (AXShowMenu) or `(x, y)`. +- `move_cursor` — warp the real cursor to `(x, y)`. + +**Keyboard** + +- `type_text` — insert text via `AXSelectedText`. Pid-scoped. +- `type_text_chars` — character-by-character via `CGEvent.postToPid`. Reaches Chromium/Electron inputs. +- `press_key` — single key press. Pid-scoped. +- `hotkey` — modifier combo (e.g. `["cmd","c"]`). Pid-scoped. + +**Element attributes** + +- `set_value` — write an element's `AXValue` directly. For sliders, steppers, and text fields. +- `scroll` — synthesize PageUp/PageDown/arrow keystrokes against the target pid. + +**Agent cursor overlay** + +- `set_agent_cursor_enabled` — toggle the visual overlay. +- `set_agent_cursor_motion` — tune the Bezier-arc + spring motion knobs. +- `get_agent_cursor_state` — read the current overlay configuration. + +**Config** + +- `get_config` — report persistent config as JSON. +- `set_config` — write a single dotted-path config key. + +**Recording** + +- `set_recording` — toggle the trajectory recorder. +- `get_recording_state` — report recorder state. +- `replay_trajectory` — re-invoke every turn's tool call in lexical order. diff --git a/docs/content/docs/cua-driver/reference/limits.mdx b/docs/content/docs/cua-driver/reference/limits.mdx new file mode 100644 index 0000000000..55845dd4e9 --- /dev/null +++ b/docs/content/docs/cua-driver/reference/limits.mdx @@ -0,0 +1,78 @@ +--- +title: Known limits +description: What cua-driver can't do in v0.1, and the workarounds +--- + +import { Callout } from 'fumadocs-ui/components/callout'; + +cua-driver's no-foreground contract holds for every app the driver reaches via AX or via the SkyLight-routed pixel click. Four categories of target fall outside that envelope. Each is documented here with the workaround. + +## Chromium coerces synthetic right-clicks on web content + +**Symptom:** `right_click({pid, x, y})` on a Chrome, Edge, Brave, or Arc tab's web content fires a left-click instead of opening the context menu. + +**Cause:** Chromium's renderer-IPC filter drops the right-click subtype bit on events that don't come through the HID tap. Every synthesized-event path on macOS hits this wall, not just ours. + +**Workarounds, in order of preference:** + +1. Use `right_click({pid, element_index})` on AX-addressable targets (links, buttons, toolbar items). AX dispatch sidesteps the renderer filter entirely. +2. For context menus on pure web content (nothing in the AX tree), activate Chrome briefly and fall back to a HID-tap right-click. Breaks the no-foreground-steal promise for that one click. + + +Element-indexed right-click (`right_click({pid, element_index})`) works fine. The limit is specifically pixel right-click on non-AX Chromium web content. + + +## Canvas apps need brief frontmost activation + +**Affected:** Blender (GHOST event source), Unity editor / Unity games, most native games, some WebGL-heavy Electron apps. + +**Symptom:** `click({pid, x, y})` on a Blender viewport silently no-ops. `launch_app` works, the window is visible, the tree says nothing actionable is there. Clicks vanish. + +**Cause:** These apps only accept events from `cghidEventTap` with a leading `mouseMoved`. They explicitly filter out per-pid-routed events, which is the path cua-driver uses for its backgrounded dispatch. There's no per-pid recipe that reaches them. + +**Workaround:** The driver auto-detects these targets and falls back to a brief activation + HID-tap click. The cursor visibly warps. The activation is the shortest possible and the click fires within the same event loop turn, so focus returns to the user's previous app within a frame or two, but the warp is noticeable. + + +If you're automating Blender or a game, the no-foreground-steal contract does not apply. The driver will tell you: the `click` response includes `dispatch: "hid_tap"` when the HID fallback fired. + + +## Off-Space SwiftUI windows strip their AX tree + +**Symptom:** `get_window_state({pid, window_id})` on a window that's on a different Space (e.g. System Settings parked on Space 2 while you're on Space 1) returns a tiny tree that only contains the menu bar, or just the AXApplication root. + +**Cause:** macOS 14+ strips AX detail from non-current-Space SwiftUI windows as a privacy / performance tradeoff. AppKit apps are not affected. This is an Apple design decision; no workaround exists that keeps the window off-Space. + +**Response shape:** cua-driver surfaces this explicitly. Every `get_window_state` response on an off-current-Space window carries `off_space: true` plus the `window_space_ids` array, so callers can decide to switch Space, pick a different window, or skip the turn. + +**Workarounds:** + +1. Switch the user to the target's Space first (`SpaceMigrator.migrate`, exposed via the forthcoming `migrate_space` tool). Breaks the no-Space-bounce promise. +2. Target an AppKit equivalent of the app if one exists. +3. Limit off-Space automation to AppKit apps where the tree stays populated. + +## Minimized windows silently drop keyboard commits + +**Symptom:** `press_key({pid, element_index, key: "return"})` on a text field in a minimized window returns success, but the field doesn't commit. You hear the macOS system-alert beep, or nothing happens. + +**Cause:** AX reads and AX-dispatched clicks propagate through to minimized windows normally, but keyboard-commit events (Return, Space, Tab) require renderer focus, which AX focus does not confer on a minimized window. This is a macOS-wide behavior; every automation tool hits it. + +**Workarounds:** + +1. Use `set_value({pid, element_index, value: "..."})` to write the field's value directly. No keyboard event involved; no focus handoff required. +2. AX-click a commit-equivalent button (Go, Submit, Send, OK) rather than relying on Return. +3. Un-minimize the window (`hotkey({pid, keys: ["cmd", "m"]})` or click the Dock icon). Breaks the background contract for that window. + + +`set_value` is the right answer 90% of the time. It sidesteps both the minimized-focus issue and the general "which event commits this field" ambiguity. + + +## Permission boundaries + +cua-driver is constrained by the usual macOS permission model. Two relevant grants: + +- **Accessibility** (System Settings → Privacy & Security → Accessibility) is required for every AX read, every element-indexed click, every keyboard / text primitive. Without it, `check_permissions` returns `accessibility: false` and every tool returns a structured error. +- **Screen Recording** is required for screenshots and for the `som` / `vision` capture modes. Without it, `get_window_state` in those modes returns a tree but no PNG; pure `ax` mode still works. + +Grants are tied to the `CuaDriver.app` bundle identity. Rebuilding cua-driver from source preserves the grants because `build-app.sh` pins a stable bundle id (`com.trycua.driver`). + +See [Installation](/cua-driver/guide/getting-started/installation) for the grant flow. diff --git a/docs/content/docs/cua-driver/reference/mcp-tools.mdx b/docs/content/docs/cua-driver/reference/mcp-tools.mdx new file mode 100644 index 0000000000..dc19f3a509 --- /dev/null +++ b/docs/content/docs/cua-driver/reference/mcp-tools.mdx @@ -0,0 +1,432 @@ +--- +title: MCP Tools +description: Reference for every MCP tool cua-driver exposes +--- + +import { Callout } from 'fumadocs-ui/components/callout'; + +`cua-driver` exposes 28 MCP tools through a single stdio server (`cua-driver mcp`). Every tool is also callable from the shell as `cua-driver ''`. + +Tool names are `snake_case`. Responses are MCP `CallTool.Result` envelopes: a text content block prefixed with a `✅` summary (or the error reason on failure), plus optional image or structured-content blocks on tools that produce them. See the [CLI reference](/cua-driver/reference/cli-reference) for CLI-specific flags like `--raw` and `--image-out`. + + + Tool names here match the CLI form exactly. `cua-driver list_apps` and the MCP `list_apps` tool run the same code path. + + +## Discovery + +### list_apps + +List macOS apps, both running and installed-but-not-running, with pid / bundle id / active state. + +**Arguments:** none. + +```json +{} +``` + +### list_windows + +List every layer-0 top-level window currently known to WindowServer, including off-screen ones (hidden-launched, minimized, on another Space). + +**Arguments:** + +- `pid` (integer, optional): Pid filter. When set, only this pid's windows are returned. +- `on_screen_only` (boolean, optional): When true, drop windows that aren't currently on the user's Space. Default false. + +```json +{"pid": 844, "on_screen_only": true} +``` + +### get_screen_size + +Return the main display's logical size in points plus the backing scale factor. Retina displays report `2.0`. + +**Arguments:** none. + +```json +{} +``` + +### get_cursor_position + +Return the current mouse cursor position in screen points (top-left origin). + +**Arguments:** none. + +```json +{} +``` + +### get_accessibility_tree + +Lightweight desktop snapshot: running regular apps and on-screen visible windows with bounds, z-order, and owner pid. For a single window's internal UI, use `get_window_state`. + +**Arguments:** none. + +```json +{} +``` + +### screenshot + +Raw ScreenCaptureKit capture. Full main display, or a single window when `window_id` is set. Returns an image content block plus a text summary listing on-screen windows. + +**Arguments:** + +- `format` (string, optional): `"png"` or `"jpeg"`. Default `"png"`. +- `quality` (integer, optional): JPEG quality 1-95; ignored for png. +- `window_id` (integer, optional): CGWindowID / `kCGWindowNumber` to capture just that window. + +```json +{"format": "jpeg", "quality": 80, "window_id": 10725} +``` + +### get_window_state + +Snapshot a single window: AX element tree plus a screenshot. Populates the per-pid, per-window `element_index` cache that mouse and keyboard tools consume. + +**Arguments:** + +- `pid` (integer, required): Process ID from `list_apps`. +- `window_id` (integer, required): CGWindowID of the target window. Must belong to `pid`. Enumerate via `list_windows` or read from `launch_app`'s `windows` array. +- `query` (string, optional): Case-insensitive substring. When set, `tree_markdown` only contains matching lines plus their ancestor chain; element indices and `element_count` are unchanged. + +```json +{"pid": 844, "window_id": 10725, "query": "save"} +``` + + + Response shape varies by `capture_mode` (`som` / `ax` / `vision`). Default is `som`. See the [CLI reference](/cua-driver/reference/cli-reference) for the full matrix. + + +## App lifecycle + +### launch_app + +Launch an app hidden (no focus steal) and return its pid plus the initial `windows` array. Either `bundle_id` or `name` must be provided. + +**Arguments:** + +- `bundle_id` (string, optional): App bundle identifier, e.g. `com.apple.calculator`. +- `name` (string, optional): App display name. Used only when `bundle_id` is absent. +- `urls` (array of string, optional): `file://` / `http(s)://` URLs (or plain paths with `~` expansion) handed to the launched app via `application(_:open:)`. For Finder, a folder URL opens a backgrounded window rooted there. Apps that don't implement `application(_:open:)` launch normally and ignore these. + +```json +{"bundle_id": "com.apple.finder", "urls": ["~/Documents"]} +``` + +### check_permissions + +Report TCC permission status for Accessibility and Screen Recording. + +**Arguments:** + +- `prompt` (boolean, optional): If true, raise the system permission prompts for any missing grants. Otherwise the call is purely read-only. + +```json +{"prompt": true} +``` + +## Mouse + +All mouse tools accept either the `element_index` path (preferred; works on backgrounded windows) or a pixel `(x, y)` path. Pixel coordinates are in window-local screenshot pixels, the same space as the PNG `get_window_state` returns. + +### click + +Left-click by element or pixel. + +**Arguments:** + +- `pid` (integer, required): Target process ID. +- `element_index` (integer, optional): Element index from the last `get_window_state` for the same `(pid, window_id)`. Routes through the AX action path. Requires `window_id`. +- `window_id` (integer, optional): CGWindowID for the window whose `get_window_state` produced `element_index`. Required in the element path; ignored in the pixel path. +- `x`, `y` (number, optional): Pixel coordinates (top-left origin). Must be provided together. +- `action` (string, optional): AX action: `press` | `show_menu` | `pick` | `confirm` | `cancel` | `open`. Default `press`. Element path only. +- `modifier` (array of string, optional): Modifiers held during the click: `cmd` / `shift` / `option` / `ctrl`. Pixel path only. +- `count` (integer, optional): Click count 1-3 (single, double, triple). Default 1. Pixel path only. +- `from_zoom` (boolean, optional): When true, `x`, `y` are pixel coordinates in the last `zoom` image for this pid; the driver maps them back to window coordinates automatically. +- `debug_image_out` (string, optional): Absolute path. On a pixel-addressed click, the tool captures the window at current `max_image_dimension`, draws a red crosshair at the received `(x, y)`, and writes the PNG here. Use to verify coordinate-space correctness. Incompatible with `from_zoom`. + +```json +{"pid": 844, "window_id": 10725, "element_index": 14} +``` + +### double_click + +Double-click by element (routes through `AXOpen` when advertised, else pixel double-click at the element's center) or by pixel. + +**Arguments:** + +- `pid` (integer, required): Target process ID. +- `element_index` (integer, optional): Requires `window_id`. +- `window_id` (integer, optional): Required when `element_index` is used. +- `x`, `y` (number, optional): Pixel coordinates (top-left origin). Provided together. +- `modifier` (array of string, optional): `cmd` / `shift` / `option` / `ctrl`. Pixel path only. + +```json +{"pid": 844, "x": 320, "y": 180} +``` + +### right_click + +Right-click by element (routes through `AXShowMenu`) or by pixel. + +**Arguments:** + +- `pid` (integer, required): Target process ID. +- `element_index` (integer, optional): Requires `window_id`. +- `window_id` (integer, optional): Required when `element_index` is used. +- `x`, `y` (number, optional): Pixel coordinates (top-left origin). Provided together. +- `modifier` (array of string, optional): `cmd` / `shift` / `option` / `ctrl`. Pixel path only. + +```json +{"pid": 844, "window_id": 10725, "element_index": 7} +``` + +### move_cursor + +Warp the real mouse cursor to a screen-point coordinate. Does not click. + +**Arguments:** + +- `x` (integer, required): X in screen points. +- `y` (integer, required): Y in screen points. + +```json +{"x": 640, "y": 400} +``` + +### scroll + +Synthesize PageUp/PageDown/arrow keystrokes against the target pid. When `element_index` is provided, the element is focused before the keystrokes fire. + +**Arguments:** + +- `pid` (integer, required): Target process ID. +- `direction` (string, required): `up` | `down` | `left` | `right`. +- `amount` (integer, optional): Keystroke repetitions, 1-50. Default 3. +- `by` (string, optional): `line` | `page`. Default `line`. +- `element_index` (integer, optional): Requires `window_id`. +- `window_id` (integer, optional): Required when `element_index` is used. + +```json +{"pid": 844, "direction": "down", "amount": 5, "by": "page"} +``` + +## Keyboard and text + +All keyboard tools are pid-scoped: the event is delivered to the target process regardless of current frontmost app. + +### press_key + +Single key press, optionally with modifiers. Delivered via `CGEvent.postToPid`. + +**Arguments:** + +- `pid` (integer, required): Target process ID. +- `key` (string, required): Key name: `return`, `tab`, `escape`, `up`, `down`, `left`, `right`, `space`, `delete`, `home`, `end`, `pageup`, `pagedown`, `f1`-`f12`, letter, digit. +- `modifiers` (array of string, optional): `cmd` / `shift` / `option` / `ctrl` / `fn` held while the key is pressed. +- `element_index` (integer, optional): When present, the element is focused before the key fires. Requires `window_id`. +- `window_id` (integer, optional): Required when `element_index` is used. + +```json +{"pid": 844, "key": "return"} +``` + +### hotkey + +Modifier combo as a single array, e.g. `["cmd", "c"]`. Requires at least two entries (one or more modifiers plus one non-modifier key). + +**Arguments:** + +- `pid` (integer, required): Target process ID. +- `keys` (array of string, required): Modifier(s) and one non-modifier key. + +```json +{"pid": 844, "keys": ["cmd", "shift", "s"]} +``` + +### type_text + +Insert text at the target's current cursor via `AXSelectedText`. Fast (single AX write) but skipped by apps with custom text layers; for Chromium / Electron inputs use `type_text_chars`. + +**Arguments:** + +- `pid` (integer, required): Target process ID. +- `text` (string, required): Text to insert at the target's cursor. +- `element_index` (integer, optional): When present, the element is focused before the write. Requires `window_id`. +- `window_id` (integer, optional): Required when `element_index` is used. + +```json +{"pid": 844, "window_id": 10725, "element_index": 12, "text": "hello"} +``` + +### type_text_chars + +Character-by-character input via `CGEvent.postToPid`. Slower than `type_text` but reaches Chromium and Electron inputs that ignore AX writes. + +**Arguments:** + +- `pid` (integer, required): Target process ID. +- `text` (string, required): Text to type into the target's focused element. +- `delay_ms` (integer, optional): Milliseconds between characters, 0-200. Default 30. + +```json +{"pid": 844, "text": "hello world", "delay_ms": 40} +``` + +## Element values + +### set_value + +Write an element's `AXValue` directly. For sliders, steppers, text fields, and similar controls where AX coerces the string to the native type. + +**Arguments:** + +- `pid` (integer, required): Target process ID. +- `window_id` (integer, required): CGWindowID for the window whose `get_window_state` produced `element_index`. +- `element_index` (integer, required): Element index from the last `get_window_state` for the same `(pid, window_id)`. +- `value` (string, required): New value. AX coerces it to the element's native type. + +```json +{"pid": 844, "window_id": 10725, "element_index": 9, "value": "42"} +``` + +## Zoom + +### zoom + +Native-resolution crop of a previously captured window region. Pass the region in resized-image pixel coordinates (same space `get_window_state` reports); the tool scales back to the source resolution, pads by 20% on each side, captures the frontmost window of `pid`, and returns the crop. + +**Arguments:** + +- `pid` (integer, required): Target process ID. +- `x1` (number, required): Left edge of the region (resized-image pixels). +- `y1` (number, required): Top edge of the region (resized-image pixels). +- `x2` (number, required): Right edge of the region (resized-image pixels). +- `y2` (number, required): Bottom edge of the region (resized-image pixels). + +```json +{"pid": 844, "x1": 200, "y1": 100, "x2": 400, "y2": 250} +``` + +After `zoom`, pass `from_zoom: true` to `click` with pixel coordinates in the zoomed image; the driver maps them back automatically. + +## Agent cursor overlay + +The agent cursor is an optional visual overlay (Bezier-arc glide + click ripple + dwell) drawn on top of every synthetic click. Motion parameters persist across restarts via the config file. + +### get_agent_cursor_state + +Read the current overlay configuration: enabled flag, motion options, glide / dwell / idle-hide timings. + +**Arguments:** none. + +```json +{} +``` + +### set_agent_cursor_enabled + +Toggle the overlay. Persists to config. + +**Arguments:** + +- `enabled` (boolean, required): True to show; false to hide. + +```json +{"enabled": true} +``` + +### set_agent_cursor_motion + +Tune the Bezier-arc + spring motion knobs. All fields are optional; omitted fields keep their current value. + +**Arguments:** + +- `start_handle` (number, optional): Start-handle fraction in [0, 1]. Default 0.3. +- `end_handle` (number, optional): End-handle fraction in [0, 1]. Default 0.3. +- `arc_size` (number, optional): Arc deflection as fraction of path length. Default 0.25. +- `arc_flow` (number, optional): Asymmetry bias in [-1, 1]. Default 0. +- `spring` (number, optional): Settle damping in [0.3, 1]. Default 0.72. +- `glide_duration_ms` (number, optional): Flight duration per click, 50-5000. Default 750. +- `dwell_after_click_ms` (number, optional): Pause after the click ripple, 0-5000. Default 400. +- `idle_hide_ms` (number, optional): Overlay linger after last click before auto-hide, 100-60000. Default 3000. + +```json +{"arc_size": 0.3, "glide_duration_ms": 900} +``` + +## Config + +Persistent settings live at `~/Library/Application Support/Cua Driver/config.json`. Writes route through the running daemon when reachable so live state (e.g. `AgentCursor.shared`) picks up changes without a restart. + +### get_config + +Return the current config as pretty-printed JSON, identical to the on-disk shape. + +**Arguments:** none. + +```json +{} +``` + +### set_config + +Write a single leaf field identified by a dotted `snake_case` path. + +**Arguments:** + +- `key` (string, required): Dotted `snake_case` path to a leaf config field, e.g. `agent_cursor.enabled`. +- `value` (required): New value. JSON type depends on the key. + +```json +{"key": "capture_mode", "value": "ax"} +``` + +Supported keys and ranges: see the [CLI reference](/cua-driver/reference/cli-reference#cua-driver-config). + +## Recording and replay + +The trajectory recorder captures every action-tool call (`click`, `right_click`, `scroll`, `type_text`, `type_text_chars`, `press_key`, `hotkey`, `set_value`) into numbered turn folders. Recordings can be replayed turn-by-turn. + +### get_recording_state + +Report whether the recorder is currently enabled, the output directory, and the next turn number. + +**Arguments:** none. + +```json +{} +``` + +Structured-content response: `{"enabled": bool, "next_turn": int, "output_dir"?: string}`. + +### set_recording + +Toggle the recorder. When enabling, `output_dir` is required. + +**Arguments:** + +- `enabled` (boolean, required): True to start; false to stop. +- `output_dir` (string, optional): Absolute or `~`-rooted directory for turn folders. Required when `enabled=true`. +- `video_experimental` (boolean, optional): Experimental: also capture the main display to `/recording.mp4` via SCStream (H.264, 30fps, no audio, no cursor). Off by default. Ignored when `enabled=false`. + +```json +{"enabled": true, "output_dir": "~/cua-trajectories/demo1"} +``` + +### replay_trajectory + +Re-invoke every turn's tool call in lexical order against the live system. + +**Arguments:** + +- `dir` (string, required): Trajectory directory previously written by `set_recording`. Absolute or `~`-rooted. +- `delay_ms` (integer, optional): Milliseconds to sleep between turns, 0-10000. Default 500. +- `stop_on_error` (boolean, optional): Stop replay on the first tool-call error. Default true; set false to best-effort through the full trajectory. + +```json +{"dir": "~/cua-trajectories/demo1", "delay_ms": 800, "stop_on_error": false} +``` diff --git a/docs/content/docs/cua-driver/reference/meta.json b/docs/content/docs/cua-driver/reference/meta.json new file mode 100644 index 0000000000..6a86740522 --- /dev/null +++ b/docs/content/docs/cua-driver/reference/meta.json @@ -0,0 +1,6 @@ +{ + "title": "Reference", + "description": "CLI and MCP reference documentation", + "icon": "FileText", + "pages": ["cli-reference", "mcp-tools", "limits"] +} diff --git a/docs/content/docs/meta.json b/docs/content/docs/meta.json index f4116a1c13..c3849ca9b9 100644 --- a/docs/content/docs/meta.json +++ b/docs/content/docs/meta.json @@ -1,5 +1,5 @@ { "title": "Home", "description": "Documentation Home", - "pages": ["cua", "cuabench", "cuabot", "lume"] + "pages": ["cua", "cua-driver", "cuabench", "cuabot", "lume"] } diff --git a/docs/src/assets/cua-driver-icon-black.png b/docs/src/assets/cua-driver-icon-black.png new file mode 100644 index 0000000000..f76149acba Binary files /dev/null and b/docs/src/assets/cua-driver-icon-black.png differ diff --git a/docs/src/assets/cua-driver-icon-white.png b/docs/src/assets/cua-driver-icon-white.png new file mode 100644 index 0000000000..ba5e6d6aef Binary files /dev/null and b/docs/src/assets/cua-driver-icon-white.png differ diff --git a/docs/src/components/custom-header.tsx b/docs/src/components/custom-header.tsx index ab248cc1b2..f1715fe813 100644 --- a/docs/src/components/custom-header.tsx +++ b/docs/src/components/custom-header.tsx @@ -18,6 +18,8 @@ import LumeIconBlack from '@/assets/lume-icon-black.svg'; import LumeIconWhite from '@/assets/lume-icon-white.svg'; import CuaBotLogoBlack from '@/assets/cuabot-logo-black.svg'; import CuaBotLogoWhite from '@/assets/cuabot-logo-white.svg'; +import CuaDriverIconBlack from '@/assets/cua-driver-icon-black.png'; +import CuaDriverIconWhite from '@/assets/cua-driver-icon-white.png'; const docsSites = [ { @@ -39,6 +41,32 @@ const docsSites = [ { name: 'Reference', href: '/cua/reference/computer-sdk', prefix: '/cua/reference' }, ], }, + { + name: 'Cua Driver', + label: 'Docs', + href: '/cua-driver/guide/getting-started/introduction', + prefix: '/cua-driver', + isDefault: false, + description: 'Background computer-use', + logoBlack: CuaDriverIconBlack, + logoWhite: CuaDriverIconWhite, + iconWidth: 24, + iconHeight: 24, + dropdownIconWidth: 28, + dropdownIconHeight: 28, + navTabs: [ + { + name: 'Guide', + href: '/cua-driver/guide/getting-started/introduction', + prefix: '/cua-driver/guide', + }, + { + name: 'Reference', + href: '/cua-driver/reference/cli-reference', + prefix: '/cua-driver/reference', + }, + ], + }, { name: 'Cua Bench', label: 'Docs', diff --git a/docs/src/middleware.ts b/docs/src/middleware.ts index a96448a589..281b805c6e 100644 --- a/docs/src/middleware.ts +++ b/docs/src/middleware.ts @@ -9,6 +9,9 @@ const redirects: Record = { '/cua/guide': '/docs/cua/guide/get-started/what-is-cua', '/cua/examples': '/docs/cua/examples/automation/form-filling', '/cua/reference': '/docs/cua/reference/computer-sdk', + '/cua-driver': '/docs/cua-driver/guide/getting-started/introduction', + '/cua-driver/guide': '/docs/cua-driver/guide/getting-started/introduction', + '/cua-driver/reference': '/docs/cua-driver/reference/cli-reference', '/cuabench': '/docs/cuabench/guide/getting-started/introduction', '/cuabench/guide': '/docs/cuabench/guide/getting-started/introduction', '/cuabench/reference': '/docs/cuabench/reference/cli-reference', diff --git a/img/card-cua-bench.png b/img/card-cua-bench.png new file mode 100644 index 0000000000..0973c3b871 Binary files /dev/null and b/img/card-cua-bench.png differ diff --git a/img/card-cua-driver.png b/img/card-cua-driver.png new file mode 100644 index 0000000000..c84b1e4ce8 Binary files /dev/null and b/img/card-cua-driver.png differ diff --git a/img/card-cua-lume.png b/img/card-cua-lume.png new file mode 100644 index 0000000000..ca9bdb2cc4 Binary files /dev/null and b/img/card-cua-lume.png differ diff --git a/img/card-cua-sandbox.png b/img/card-cua-sandbox.png new file mode 100644 index 0000000000..e63de3871d Binary files /dev/null and b/img/card-cua-sandbox.png differ diff --git a/libs/cua-driver/.gitignore b/libs/cua-driver/.gitignore new file mode 100644 index 0000000000..5653bbd52d --- /dev/null +++ b/libs/cua-driver/.gitignore @@ -0,0 +1,79 @@ +# macOS +.DS_Store + +# Local tooling artifacts +.gumbranch/ + +# Local-only notes, never committed +docs/_local-research/ +docs/_local/ + +# (historical research capture directories; kept out of tree by the +# docs/_local/ rule above) + +# Xcode +# +# gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore + +## User settings +xcuserdata/ +.swiftpm/ + +## Obj-C/Swift specific +*.hmap + +## App packaging +*.ipa +*.dSYM.zip +*.dSYM + +## Playgrounds +timeline.xctimeline +playground.xcworkspace + +# Swift Package Manager +# +# Add this line if you want to avoid checking in source code from Swift Package Manager dependencies. +# Packages/ +# Package.pins +# Package.resolved +# *.xcodeproj +# +# Xcode automatically generates this directory with a .xcworkspacedata file and xcuserdata +# hence it is not needed unless you have added a package configuration file to your project +# .swiftpm + +.build/ + +# Test harness build artifacts — rebuilt by each test run. +Tests/FocusMonitorApp/FocusMonitorApp.app/ + +# CocoaPods +# +# We recommend against adding the Pods directory to your .gitignore. However +# you should judge for yourself, the pros and cons are mentioned at: +# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control +# +# Pods/ +# +# Add this line if you want to avoid checking in source code from the Xcode workspace +# *.xcworkspace + +# Carthage +# +# Add this line if you want to avoid checking in source code from Carthage dependencies. +# Carthage/Checkouts + +Carthage/Build/ + +# fastlane +# +# It is recommended to not store the screenshots in the git repo. +# Instead, use fastlane to re-generate the screenshots whenever they are needed. +# For more information about the recommended setup visit: +# https://docs.fastlane.tools/best-practices/source-control/#source-control + +fastlane/report.xml +fastlane/Preview.html +fastlane/screenshots/**/*.png +fastlane/test_output diff --git a/libs/cua-driver/App/CuaDriver/AppIcon.icns b/libs/cua-driver/App/CuaDriver/AppIcon.icns new file mode 100644 index 0000000000..55a556168b Binary files /dev/null and b/libs/cua-driver/App/CuaDriver/AppIcon.icns differ diff --git a/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_128x128.png b/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_128x128.png new file mode 100644 index 0000000000..9daf313885 Binary files /dev/null and b/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_128x128.png differ diff --git a/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_128x128@2x.png b/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_128x128@2x.png new file mode 100644 index 0000000000..d20591a235 Binary files /dev/null and b/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_128x128@2x.png differ diff --git a/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_16x16.png b/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_16x16.png new file mode 100644 index 0000000000..cbebe7b9c6 Binary files /dev/null and b/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_16x16.png differ diff --git a/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_16x16@2x.png b/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_16x16@2x.png new file mode 100644 index 0000000000..683d749e59 Binary files /dev/null and b/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_16x16@2x.png differ diff --git a/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_256x256.png b/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_256x256.png new file mode 100644 index 0000000000..d20591a235 Binary files /dev/null and b/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_256x256.png differ diff --git a/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_256x256@2x.png b/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_256x256@2x.png new file mode 100644 index 0000000000..6e59abb8fe Binary files /dev/null and b/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_256x256@2x.png differ diff --git a/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_32x32.png b/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_32x32.png new file mode 100644 index 0000000000..683d749e59 Binary files /dev/null and b/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_32x32.png differ diff --git a/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_32x32@2x.png b/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_32x32@2x.png new file mode 100644 index 0000000000..956cab164f Binary files /dev/null and b/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_32x32@2x.png differ diff --git a/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_512x512.png b/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_512x512.png new file mode 100644 index 0000000000..6e59abb8fe Binary files /dev/null and b/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_512x512.png differ diff --git a/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_512x512@2x.png b/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_512x512@2x.png new file mode 100644 index 0000000000..de28840ff8 Binary files /dev/null and b/libs/cua-driver/App/CuaDriver/AppIcon.iconset/icon_512x512@2x.png differ diff --git a/libs/cua-driver/App/CuaDriver/Info.plist b/libs/cua-driver/App/CuaDriver/Info.plist new file mode 100644 index 0000000000..589ea71d0c --- /dev/null +++ b/libs/cua-driver/App/CuaDriver/Info.plist @@ -0,0 +1,32 @@ + + + + + CFBundleIdentifier + com.trycua.driver + CFBundleName + Cua Driver + CFBundleDisplayName + Cua Driver + CFBundleExecutable + cua-driver + CFBundleIconFile + AppIcon + CFBundleIconName + AppIcon + CFBundlePackageType + APPL + CFBundleShortVersionString + 0.0.1 + CFBundleVersion + 1 + LSMinimumSystemVersion + 14.0 + LSUIElement + + NSHighResolutionCapable + + NSSupportsAutomaticTermination + + + diff --git a/libs/cua-driver/Package.resolved b/libs/cua-driver/Package.resolved new file mode 100644 index 0000000000..0d2869c224 --- /dev/null +++ b/libs/cua-driver/Package.resolved @@ -0,0 +1,78 @@ +{ + "originHash" : "5967ac3dc4b99f934ac42d2c3bd6f52f187f92607c34cce5b2be0ebdcffe11be", + "pins" : [ + { + "identity" : "eventsource", + "kind" : "remoteSourceControl", + "location" : "https://github.com/mattt/eventsource.git", + "state" : { + "revision" : "a3a85a85214caf642abaa96ae664e4c772a59f6e", + "version" : "1.4.1" + } + }, + { + "identity" : "swift-argument-parser", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-argument-parser.git", + "state" : { + "revision" : "626b5b7b2f45e1b0b1c6f4a309296d1d21d7311b", + "version" : "1.7.1" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "b601256eab081c0f92f059e12818ac1d4f178ff7", + "version" : "1.3.0" + } + }, + { + "identity" : "swift-collections", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-collections.git", + "state" : { + "revision" : "6675bc0ff86e61436e615df6fc5174e043e57924", + "version" : "1.4.1" + } + }, + { + "identity" : "swift-log", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-log.git", + "state" : { + "revision" : "5073617dac96330a486245e4c0179cb0a6fd2256", + "version" : "1.12.0" + } + }, + { + "identity" : "swift-nio", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-nio.git", + "state" : { + "revision" : "cd6710454f25733900e133c6caf5188952763c36", + "version" : "2.98.0" + } + }, + { + "identity" : "swift-sdk", + "kind" : "remoteSourceControl", + "location" : "https://github.com/modelcontextprotocol/swift-sdk.git", + "state" : { + "revision" : "6132fd4b5b4217ce4717c4775e4607f5c3120129", + "version" : "0.12.0" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system.git", + "state" : { + "revision" : "7c6ad0fc39d0763e0b699210e4124afd5041c5df", + "version" : "1.6.4" + } + } + ], + "version" : 3 +} diff --git a/libs/cua-driver/Package.swift b/libs/cua-driver/Package.swift new file mode 100644 index 0000000000..ce51c9652a --- /dev/null +++ b/libs/cua-driver/Package.swift @@ -0,0 +1,43 @@ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "CuaDriver", + platforms: [ + .macOS(.v14) + ], + products: [ + .executable(name: "cua-driver", targets: ["CuaDriverCLI"]), + .library(name: "CuaDriverCore", targets: ["CuaDriverCore"]), + .library(name: "CuaDriverServer", targets: ["CuaDriverServer"]), + ], + dependencies: [ + .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.5.0"), + .package(url: "https://github.com/modelcontextprotocol/swift-sdk.git", from: "0.9.0"), + ], + targets: [ + .target( + name: "CuaDriverCore" + ), + .target( + name: "CuaDriverServer", + dependencies: [ + "CuaDriverCore", + .product(name: "MCP", package: "swift-sdk"), + ] + ), + .executableTarget( + name: "CuaDriverCLI", + dependencies: [ + "CuaDriverCore", + "CuaDriverServer", + .product(name: "ArgumentParser", package: "swift-argument-parser"), + .product(name: "MCP", package: "swift-sdk"), + ] + ), + .testTarget( + name: "ZoomMathTests", + dependencies: ["CuaDriverCore"] + ), + ] +) diff --git a/libs/cua-driver/README.md b/libs/cua-driver/README.md new file mode 100644 index 0000000000..b6bbe950ab --- /dev/null +++ b/libs/cua-driver/README.md @@ -0,0 +1,5 @@ +# Cua Driver + +Background computer-use driver for any agents. Speaks MCP over stdio; drives native macOS apps without stealing focus. + +**[Documentation](https://cua.ai/docs/cua-driver)** - Installation, guides, and API reference. diff --git a/libs/cua-driver/Skills/cua-driver/README.md b/libs/cua-driver/Skills/cua-driver/README.md new file mode 100644 index 0000000000..a60996b1e8 --- /dev/null +++ b/libs/cua-driver/Skills/cua-driver/README.md @@ -0,0 +1,128 @@ +# cua-driver — Claude Code skill + +A [Claude Code](https://code.claude.com) skill that teaches Claude to +drive native macOS apps via the +[`cua-driver`](https://github.com/trycua/cua/tree/main/libs/cua-driver) +CLI — snapshot an app's accessibility tree, click/type/scroll by +`element_index`, and verify via re-snapshot. Backgrounded-first: no +focus steal, no cursor warp, no Space follow. + +## What the skill covers + +- The snapshot-before-AND-after invariant that keeps the agent honest + about whether an action actually landed. +- The backgrounded-click recipe (yabai focus-without-raise + stamped + SLEventPostToPid) that lets synthetic clicks land on Chrome web + content without raising the window or pulling the user across Spaces. +- Web-app quirks (`WEB_APPS.md`) — Chromium/WebKit/Electron/Tauri, + including the minimized-Chrome keyboard-commit caveat and the + `set_value` workaround. +- Trajectory recording (`RECORDING.md`) — optional per-session + recording + replay for demos and regressions. +- Canvas/viewport apps (Blender, Unity, GHOST, Qt, wxWidgets) — + HID-tap fallback when AX is empty. + +See `SKILL.md` for the main body. + +## Prerequisites + +1. **macOS 14 or newer** — the driver depends on SkyLight private SPIs + that were stabilized in Sonoma. +2. **`cua-driver` CLI + `CuaDriver.app`** — installable one-liner: + ```bash + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/cua-driver/scripts/install.sh)" + ``` + Or from a clone of `trycua/cua`: + ```bash + cd libs/cua-driver + scripts/install-local.sh # builds + installs + symlinks for dev use + ``` + The driver runs as an `.app` bundle because macOS TCC grants are + tied to a stable bundle id (`com.trycua.driver`). The CLI symlink + lets Claude invoke tools via plain shell. +3. **TCC grants on `CuaDriver.app`** — **Accessibility** and + **Screen Recording** in System Settings → Privacy & Security. + Verify with: + ```bash + cua-driver check_permissions + ``` + Both fields must be `true`. If not, the app appears in the + relevant panes of System Settings after first use; toggle it on + there. + +## Install + +The skill is two drop-in directories. + +**Personal scope** (all Claude Code sessions on your machine): + +```bash +mkdir -p ~/.claude/skills +cp -R Skills/cua-driver ~/.claude/skills/ +``` + +Or symlink if you want edits-in-place: + +```bash +ln -s "$PWD/Skills/cua-driver" ~/.claude/skills/cua-driver +``` + +**Project scope** (committed alongside a specific repo): + +```bash +mkdir -p .claude/skills +cp -R /path/to/cua/libs/cua-driver/Skills/cua-driver .claude/skills/ +``` + +## Invoking the skill + +Claude Code auto-invokes the skill when you ask for macOS GUI +automation — e.g. "open the Downloads folder in Finder", "click the +Save button in Numbers", "navigate to trycua.com in Chrome". You can +also invoke it explicitly: + +``` +/cua-driver +``` + +## Files + +- `SKILL.md` — the main skill body (~500 lines). Loaded on first + invocation; stays in context for the session. +- `WEB_APPS.md` — browsers, Electron, Tauri (Chromium + WebKit). Loaded + on demand when SKILL.md's pointer is followed. +- `RECORDING.md` — trajectory recording / replay. Loaded on demand. +- `TESTS.md` — manual test scripts for end-to-end skill verification. + +## Troubleshooting + +- `cua-driver: command not found` → re-run the installer or add + `.build/CuaDriver.app/Contents/MacOS/` to `$PATH`. +- `No cached AX state for pid X window_id W` → element_index was + reused across turns, or across different windows of the same app. + Call `get_window_state({pid, window_id})` first in the same turn, + with the same window_id you're about to act against. +- Empty `tree_markdown` → `capture_mode` is set to `vision`, which + skips the AX walk by design. Flip back to the default `som` + (`cua-driver config set capture_mode som`) to get the tree. + Tiny screenshot → likely a stale window capture. See "Behavior + matrix" in SKILL.md for the full mode table. +- System-alert beep when pressing Return on a minimized Chrome + omnibox → the keyboard-commit-on-minimized limitation. Use + `set_value` on the field instead, or AX-click a Go/Submit button. + See `WEB_APPS.md`. + +## Updates + +The skill evolves alongside the driver. To update: + +```bash +cd /path/to/cua && git pull +# if you copied: re-copy +cp -R libs/cua-driver/Skills/cua-driver ~/.claude/skills/ +# if you symlinked: nothing needed +``` + +## License + +MIT. Same license as the parent `trycua/cua` repo. diff --git a/libs/cua-driver/Skills/cua-driver/RECORDING.md b/libs/cua-driver/Skills/cua-driver/RECORDING.md new file mode 100644 index 0000000000..79fc526398 --- /dev/null +++ b/libs/cua-driver/Skills/cua-driver/RECORDING.md @@ -0,0 +1,114 @@ +# Recording & replaying trajectories + +Session-scoped capture of action sequences + pre/post state, suitable +for demos, regression diffs, and training data. Invoked only when the +user explicitly asks to record — the skill does not auto-enable this. + +`set_recording` turns on a session-scoped trajectory recorder. While +enabled, every action-tool call (`click`, `right_click`, `scroll`, +`type_text`, `type_text_chars`, `press_key`, `hotkey`, `set_value`) +writes a numbered turn folder under a caller-chosen output +directory. Read-only tools (`get_window_state`, `list_windows`, +`screenshot`, `list_apps`, permission probes, agent-cursor getters / +setters, and `set_recording` itself) are not recorded. + +## Enable / disable + +Two equivalent surfaces: the `set_recording` MCP tool, or the +friendlier `cua-driver recording` subcommand group (wraps +`set_recording` + `get_recording_state` with human-readable output). + +``` +cua-driver recording start ~/cua-trajectories/run-1 +# … run the workflow … +cua-driver recording status # -> enabled / disabled, next_turn, output_dir +cua-driver recording stop # -> "Recording disabled (N turns captured in …)" +``` + +Raw-tool equivalent: + +``` +cua-driver set_recording '{"enabled":true,"output_dir":"~/cua-trajectories/run-1"}' +cua-driver get_recording_state +cua-driver set_recording '{"enabled":false}' +``` + +The `recording` subcommands require a running daemon (`cua-driver +serve &`) because recording state is per-process. `output_dir` expands +`~` and is created (with intermediates) if missing. Turn numbering +starts at `1` every time recording is (re-)enabled, regardless of any +existing contents in the directory. State lives in memory only — a +daemon restart resets to disabled. + +## What each turn folder contains + +Each action writes to `turn-NNNNN/` (five-digit zero-padded counter): + +- `app_state.json` — post-action AX snapshot for the target pid, same + shape `get_window_state` returns (tree_markdown, element_count, + turn_id, etc.) minus the screenshot fields. The recorder resolves a + frontmost window internally (visible + on-current-Space preferred, + max-area fallback) since individual action tools carry a + window_id but the recorder has no caller-supplied anchor. +- `screenshot.png` — post-action capture of the same window the + recorder just snapshotted. Omitted when the pid has no visible + window. +- `action.json` — the tool name, full input arguments, result + summary, pid, click point (when applicable), ISO-8601 timestamp. +- `click.png` — only for click-family actions (`click`, + `right_click`): a copy of `screenshot.png` with a red dot drawn at + the click point (screen-absolute point → window-local pixels via + the screenshot's `scale_factor`). Absent for other tools and for + clicks whose point falls outside the captured window. + +## When to use it + +- Demos and screen recordings — play the turn folder back to show + exactly what the agent saw and what it did. +- Replay for regression — re-run the same sequence against a future + build and diff the new trajectory against the saved one. +- Training data collection — each turn is a + `(state, action, next_state)` triple ready for offline learning. + +## When to invoke it + +This skill does **not** auto-enable recording. The client invokes +`set_recording` explicitly when the user asks to capture a session. +If the user says "record this session" or similar, call +`set_recording({enabled:true, output_dir:…})` before the first +action, and `set_recording({enabled:false})` when done. + +## Replaying a recorded trajectory + +`replay_trajectory({dir})` walks `/turn-NNNNN/` folders in +lexical order, reads each `action.json`, and re-invokes the recorded +tool with its recorded `arguments`. Optional knobs: `delay_ms` +(pacing between turns, default 500) and `stop_on_error` (halt on +first failure, default true). + +``` +cua-driver recording start ~/cua-trajectories/demo1 +# … run the workflow … +cua-driver recording stop +# Later: replay against a new build. +cua-driver replay_trajectory '{"dir":"~/cua-trajectories/demo1","delay_ms":500}' +``` + +Important caveat: **element_index doesn't survive across sessions**. +Indices are assigned fresh on every `get_window_state` snapshot, +keyed on `(pid, window_id)`, so a recorded +`click({pid, window_id, element_index: 14})` from yesterday won't +resolve today — the pid is usually different, the window_id always +is. The call returns `Invalid element_index` or `No cached AX +state`. Pixel clicks (`click({pid, x, y})`) and keyboard tools +(`press_key`, `type_text_chars`, `hotkey`, `type_text` without +element_index) replay cleanly; element-indexed actions require a +live snapshot that replay doesn't currently re-emit (read-only tools +like `get_window_state` aren't recorded). For a reliable replay, either +compose the trajectory from pixel + keyboard primitives, or capture +it as a regression artifact (compare the failure/success pattern +across builds) rather than a re-driving script. + +If recording is still enabled while replay runs, the replay is +itself recorded into the current output directory — that's the +intended regression-diff workflow. diff --git a/libs/cua-driver/Skills/cua-driver/SKILL.md b/libs/cua-driver/Skills/cua-driver/SKILL.md new file mode 100644 index 0000000000..69885cc1f6 --- /dev/null +++ b/libs/cua-driver/Skills/cua-driver/SKILL.md @@ -0,0 +1,816 @@ +--- +name: cua-driver +description: Drive a native macOS app via the cua-driver CLI (default) or MCP server — snapshot its AX tree, click/type/scroll by element_index, verify via re-snapshot. Use when the user asks you to operate, drive, automate, or perform a GUI task in a real macOS application on the host (e.g. "open a file in TextEdit", "navigate to /Applications in Finder", "click the Save button in Numbers"). +--- + +# cua-driver + +Orchestrates macOS app automation via `cua-driver`. Whenever a user +asks to drive a native macOS app, follow the loop in this skill rather +than calling tools ad-hoc — the snapshot-before-action invariant is not +optional and silently breaks if you skip it. + +## The no-foreground contract — read this first + +**The user's frontmost app MUST NOT change.** This is the whole +reason cua-driver exists. Users pay for the right to keep typing in +their editor while an agent drives another app in the background. +Violate this rule and every other nice property the driver gives +you (no cursor warp, no Space switch, no window raise) stops +mattering — you just shipped the Accessibility Inspector with extra +steps. + +Before running any shell command, ask: **"does this raise, +activate, foreground, or make-key any app?"** If yes, don't run it. +Every one of the commands below activates the target on macOS and +is therefore forbidden unless the user **explicitly** asked for +frontmost state: + +- **Every form of the `open` CLI — `open -a `, `open -b + `, `open `, `open `, `open + ` — always activates.** macOS routes all forms through + LaunchServices, which unhides and foregrounds the target + regardless of whether you passed an app name, a bundle id, a + document, a URL, or the bundle path itself. The activation + happens even when the only intent was "start the process." + **Never use `open` for any app launch.** This includes launching + a just-built .app from a local build dir (e.g. `open + build/Build/Products/Debug/MyApp.app`) — resolve the + `CFBundleIdentifier` from `Info.plist` and use `launch_app` + with that id. See "The narrow carve-out" below for why + `launch_app` is safe even when the app internally calls + `NSApp.activate`. +- `osascript -e 'tell application "X" to activate'` — + activates by design. Same for `... to open `, + `... to launch`, and anything with `activate` in the tell block. +- `osascript -e 'tell application "System Events" to ... frontmost'` + in a mutating form (setting `frontmost` rather than reading it). +- AppleScript files that invoke `activate`, `launch`, or `open` + against the target app. +- `cliclick` (moves the user's real cursor to the target coords + before clicking — a focus-steal-equivalent even if the app's + window state is unchanged). +- `CGEventPost` with `cghidEventTap` targeting a coordinate over + a different app's window (warps the cursor, possibly activates + on hit). +- `AppleScriptTask`, `NSAppleScript`, `Process` wrapping `osascript` + that contains any of the above. +- `NSRunningApplication.activate(options:)` called from your own + helper binary — same class. +- Dock clicks and any `open` invocation (see the first bullet — + every form of `open` goes through LaunchServices which + activates, full stop). +- **Keyboard shortcuts that semantically mean "focus here" — + most notably Chrome / Safari / Arc's `⌘L` (focus omnibox) and + Finder's `⌘⇧G` (Go to Folder).** These aren't pure key events — + the receiving app interprets "user wants to type here" as + activation intent and raises its window to be key. Even when + delivered to a backgrounded pid via `hotkey`, the downstream app + pulls focus. **For omnibox navigation specifically**, don't + `hotkey ⌘L`; instead find the omnibox AX element via `som` snapshot + (`AXTextField` id=something like `location_bar` / `omnibox`) and + either AX-click it by `element_index` or dispatch `set_value` with + the URL directly — both stay backgrounded. The general principle: + a shortcut that says "put my cursor inside this app" is a + focus-steal; a shortcut that says "do this thing" (copy, save, + quit) is fine. +- **Tab-switching shortcuts in browsers (`⌘1..⌘9`, `⌘]`, `⌘[`, + `⌘⇧[`, `⌘⇧]`) are visibly disruptive even when delivered to a + backgrounded pid.** The app's key handler processes the shortcut, + the window re-renders the new tab's content, the user sees their + tabs flipping. There is no AX-only workaround: page content (HTML, + form state, `AXWebArea`) populates only for the focused tab; + inspecting a background tab requires activating it, which is the + visible flip. Observed with Dia; the same mechanic applies to every + Chromium-family browser (Chrome, Arc, Brave, Edge). + + **Prefer the windows-over-tabs pattern**: for each URL you need to + drive backgrounded, use `launch_app({bundle_id, urls: [url]})` — + browsers open each URL in a new **window**. Each window has its own + `window_id`, its own AX tree, and can be inspected / interacted with + via `element_index` without activating or switching anything. Tabs + are a UX grouping for humans; cua-driver workflows should default to + windows. See `WEB_APPS.md` → "Tabs vs windows" for the full pattern. + + Tab-title enumeration (read-only) IS safe — walk a window's toolbar + AX tree for `AXTab` / `AXRadioButton` children and read their + `AXTitle`s. Tab switching (activating one) is not. + +Reading frontmost state is fine (`osascript -e 'tell application +"System Events" to get name of first application process whose +frontmost is true'`). Mutating it is not. + +**Corollary — the AXMenuBar rule.** `AXMenuBarItem` + AXPick +dispatches at the AX layer regardless of which app is frontmost, +but macOS's on-screen menu bar always belongs to the frontmost +app. If you drive a *backgrounded* app's menu bar, the AX call +succeeds but the viewer sees the dispatch rendered over the +*frontmost* app's menu bar — confusing in any observed session and +routinely a silent no-op too, because action menu items go +`DISABLED` when their owning app isn't the key window. **So: only +use menu-bar navigation when the target is already frontmost.** For +backgrounded targets, read state via in-window AX (window title, +toolbar `AXStaticText`) and dispatch via in-window `element_index` +or pixel clicks — both paths are frontmost-insensitive. Full +rationale in "Navigating native menu bars" below. + +**"Open \" in user speech means launch, not activate.** +`cua-driver launch_app` is the one correct path for process +startup — it's idempotent (no-op on a running app), returns the +pid, and has an internal `FocusRestoreGuard` that catches +`NSApp.activate(ignoringOtherApps:)` calls the target makes during +`application(_:open:)` and clobbers the frontmost back to what it +was before the launch. That guard is why `launch_app` with `urls` +(e.g. `{"bundle_id": "com.colliderli.iina", "urls": ["~/video.mp4"]}`) +is safe even for apps that normally foreground on media-load +(Chrome, Electron, media players). + +## Defaults — always prefer cua-driver over shell shims + +**Default transport is the `cua-driver` CLI** — `Bash` shelling out +to `cua-driver ''`. MCP tools (prefix +`mcp__cua-driver__*`) only when the user explicitly asks for them. +CLI wins because it picks up rebuilds instantly, failures are +easier to diagnose, and there's no per-tool schema-load overhead. + +Every reference to `click(...)`, `get_window_state(...)` etc. in this +skill means `cua-driver click '{...}'` — translate to MCP form only +when MCP is requested. + +Intent → tool mapping. If you find yourself reaching for the right +column, something has gone wrong — re-read "The no-foreground +contract" above: + +| Intent | Use | Don't use | +|---|---|---| +| Open / launch an app | `launch_app({bundle_id})` or `launch_app({bundle_id, urls:[...]})` | `open -a`, `osascript 'tell app … to launch/activate/open'` | +| Find a pid | `list_apps` or `launch_app`'s return | `pgrep`, `ps`, `osascript frontmost` | +| Enumerate an app's windows | `list_windows({pid})` — or read the `windows` array `launch_app` already returns | `osascript 'every window of app …'` | +| Click / type / scroll / keys | `click`, `type_text`, `scroll`, `press_key`, `hotkey` | `osascript`, `cliclick`, raw `CGEvent`, `open ` | +| Screenshot | `screenshot` or the PNG in `get_window_state` | `screencapture` | +| Quit an app | ask the user first, then `hotkey({pid, keys:["cmd","q"]})` | `kill`, `killall`, `pkill` | +| Hand a file/URL to an app | `launch_app({bundle_id, urls:[]})` | `open -a `, `open ` | + +### The narrow carve-out + +The **only** legitimate use of `osascript -e 'tell app X to +activate'` is when the user **explicitly** asked for frontmost +state ("bring Chrome to the front", "make it frontmost", "I want +to see X"). Reaching for it because a tool call returned something +confusing is wrong — that's the skill's classic foot-in-the-door +failure mode and it steals focus every time. + +When a cua-driver call surprises you, diagnose cua-driver first: + +- **Tiny screenshot / empty `tree_markdown`?** Check + `cua-driver get_config` → `capture_mode`. Default `"vision"` omits + the AX tree (PNG only), `"ax"` omits the PNG, `"som"` returns + both. If a snapshot lacks a tree, `capture_mode` is almost + certainly `"vision"` — either reason purely from the PNG or flip + to `"som"` / `"ax"` via `set_config`. +- **`has_screenshot: false`?** The window capture failed (transient + race against a close, or the window has no backing store yet). + Re-snapshot; if persistent, pick a different `window_id` via + `list_windows`. +- **`Invalid element_index` / `No cached AX state`?** You either + skipped `get_window_state` this turn or passed a different + `window_id` than the one the snapshot cached against. The cache + is keyed on `(pid, window_id)` — indices don't carry across + windows of the same app. Re-snapshot with the same window_id + you're about to click in. +- **Sparse Chromium AX tree?** Retry `get_window_state` once — the + tree populates on second call. + +Only after those are ruled out, and only if the user's action +genuinely needs frontmost state, fall through to the activate +fallback. Always name the focus steal in your response ("I'll +briefly bring Chrome to the front because …"). + +### Self-check pattern + +Before every `Bash` call whose command line touches any macOS app +(launching, opening, clicking, typing, scripting, screenshotting), +run the self-check: + +1. **Does this command foreground the target?** If yes — stop and + translate to the cua-driver equivalent from the mapping table. +2. **Does this command move the user's real cursor?** (`cliclick`, + any `CGEventPost` at `cghidEventTap` over another app's window). + If yes — stop; use `click({pid, x, y})` which routes per-pid + via SkyLight and never warps the cursor. +3. **Does this command bypass cua-driver entirely?** (`osascript` + mutating GUI state, AppleScript files, external helpers.) If + yes — stop; find the cua-driver tool that does the intent. + +If all three are "no," the command is safe. If you can't answer, +default to stop and ask rather than proceed. A single `open -a` +run by accident kills the demo, the trust, and the user's in-flight +editor state. + +## Prerequisites — check before starting + +1. `cua-driver` is on `$PATH` (`which cua-driver`). If not, point the + user at `scripts/install-local.sh` and stop. +2. Run `cua-driver check_permissions`. If either grant is `false`, tell + the user to open System Settings → Privacy & Security and grant + Accessibility and Screen Recording to `CuaDriver.app`, then stop. + (`cua-driver check_permissions '{"prompt":true}'` raises the system + dialogs, but only do that if the user asks — it steals focus.) +3. Start the daemon with `open -n -g -a CuaDriver --args serve` (the + recommended form — goes through LaunchServices so TCC attributes + the process to CuaDriver.app). `cua-driver serve &` also works; + the CLI auto-relaunches through `open -n -g -a CuaDriver` when it + detects a wrong-TCC context (any IDE-spawned shell: Claude Code, + Cursor, VS Code, Conductor). Verify with `cua-driver status`. + +## Using cua-driver from the shell + +Tool names are `snake_case`, management subcommands are +`kebab-case` — no ambiguity. Tools invoked as `cua-driver + ''`. Management subcommands: + +- `open -n -g -a CuaDriver --args serve` — start persistent daemon + (**required** for `element_index` workflows; without it each CLI + invocation spawns a fresh process and the per-pid element cache + dies between calls). `cua-driver serve &` also works — the CLI + auto-relaunches via `open` when the shell's TCC context is wrong. + Pass `--no-relaunch` / `CUA_DRIVER_NO_RELAUNCH=1` to opt out. +- `cua-driver stop` / `status` +- `cua-driver list-tools`, `describe ` +- `cua-driver recording start|stop|status` — see `RECORDING.md` + +Canonical multi-step workflow: + +``` +open -n -g -a CuaDriver --args serve +cua-driver launch_app '{"bundle_id":"com.apple.calculator"}' +# → {pid: 844, windows: [{window_id: 10725, ...}]} +cua-driver get_window_state '{"pid":844,"window_id":10725}' +cua-driver click '{"pid":844,"window_id":10725,"element_index":14}' +cua-driver stop +``` + +## Agent cursor overlay + +Visual cursor overlay for demos and screen recordings. Default: +enabled. Toggle with `cua-driver set_agent_cursor_enabled +'{"enabled":true|false}'`. A triangle pointer Bezier-glides to each +click target, ring-ripples on landing, idle-hides after ~1.5s. +Motion knobs: `set_agent_cursor_motion` takes any subset of +`start_handle`, `end_handle`, `arc_size`, `arc_flow`, `spring` — +tuneable at runtime, persisted to config. + +Requires an AppKit runloop, which `cua-driver serve` / `mcp` +bootstraps. One-shot CLI invocations skip the overlay entirely. + +## The core invariant — snapshot before AND after every action + +**Every action MUST be bracketed by `get_window_state(pid, window_id)`**: + +- **Before** — the pre-action snapshot resolves the `element_index` + you're about to use. Indices from previous turns are stale; the + server replaces the element index map on every snapshot, keyed + on `(pid, window_id)`. Indices from turn N don't resolve in turn + N+1, and indices from window A don't resolve against window B of + the same app. Skip this and element-indexed actions fail with + `No cached AX state`. +- **After** — the post-action snapshot verifies the action actually + landed. Without it you can't tell a silent no-op from a real + effect. The AX tree change (new value, new window, disappeared + menu, disabled button, etc.) is your evidence that the action + fired. If nothing changed, the action probably failed silently — + say so, don't assume success. + +This applies to pixel clicks too — re-snapshot after to confirm the +click landed on the intended target. + +### Why window selection is the caller's job now + +`get_app_state` used to pick a window for you via a max-area heuristic +that returned the wrong surface on apps with large off-screen utility +panels. Concrete reproducer: IINA's OpenSubtitles helper (600×432 +off-screen) out-area'd the visible 320×240 player window, so +`get_app_state(pid)` screenshot'd the invisible panel and clicks landed +there silently. The new `get_window_state(pid, window_id)` makes the +caller name the window explicitly — the driver validates that the +window belongs to the pid and is on the current Space, then snapshots +exactly what was asked for. Enumerate candidates via `list_windows` or +read the `windows` array `launch_app` already returns. + +## Behavior matrix + +Two orthogonal axes shape what the agent can do. + +**capture_mode → addressing mode** + +| `capture_mode` | `get_window_state` returns | Use for actions | +|---|---|---| +| **`som`** (default) | tree + screenshot | `element_index` preferred; pixel fallback | +| **`ax`** | tree only (no PNG) | `element_index` only | +| **`vision`** | PNG only (no tree) | pixel only — see [SCREENSHOT.md](./SCREENSHOT.md) | + +`vision` was renamed from `screenshot` — the old name still decodes +as a deprecated alias, so an on-disk `"capture_mode": "screenshot"` +keeps working. Default is `som` so element_index clicks work the +first time a user calls `get_window_state`; the other modes are +opt-in when the caller specifically doesn't want one half of the +work. Note the tool named `screenshot` is separate (raw PNG, no AX +walk) and unrelated to the capture mode. + +When a snapshot looks wrong (tiny screenshot / empty tree), check +`cua-driver get_config` for `capture_mode` before anything else. + +Pure-vision mode has its own caveats — Claude Code's vision +pipeline downsamples dense text aggressively, so pixel grounding +takes multiple correction cycles on text-heavy UIs. Read +[SCREENSHOT.md](./SCREENSHOT.md) before driving anything in that +mode; it documents the iterate/annotate/verify recipe plus the +JPEG-over-PNG finding. + +**Window state → what works** + +| state | `get_window_state` | `click`/`set_value` (AX) | `press_key` commit (Return/Space/Tab) | pixel click | +|---|---|---|---|---| +| frontmost | ✅ | ✅ | ✅ | ✅ | +| backgrounded / visible | ✅ | ✅ | ✅ | ✅ | +| **minimized** (Dock genie) | ✅ | ✅ (no deminiaturize — AX actions fire on the minimized window in place) | ❌ silent no-op / system beep — use `set_value` or click equivalent | ❌ no on-screen bounds | +| hidden (`hides=true` / `NSApp.hide`) | ✅ | ✅ | depends | ❌ | +| on another Space | ⚠️ AX tree often stripped to menu-bar-only on SwiftUI apps (System Settings) — AppKit apps usually fine. Response carries `off_space: true` + `window_space_ids` so you can detect it | ✅ | ✅ | ❌ window not in current-Space list | + +**Critical cell — minimized + keyboard commit.** The keystroke +reaches the app but AX focus doesn't propagate to renderer focus on +a minimized window. Workarounds in order of preference: +`set_value` to write the field's entire value directly, or AX-click +a commit-equivalent button (Go, Submit, checkbox). Tell the user +the window needs to un-minimize only as a last resort. + +## The canonical loop + +``` +launch_app(target) + → pick window_id from the returned `windows` array + (or call list_windows(pid) separately) + → get_window_state(pid, window_id) + → [act] # every action also takes (pid, window_id) + → get_window_state(pid, window_id) → verify +``` + +`launch_app` now returns a `windows` array alongside the pid, so the +common case collapses to two calls (`launch_app` → `get_window_state`) +without a separate `list_windows` hop. + +### 1. Resolve target pid — always via `launch_app` + +**Always start with `launch_app`**, whether or not the target is already +running. It's idempotent (relaunching returns the existing pid with no +side effects) and gives you the pid in one call — no `list_apps` hop. + +- `launch_app({bundle_id: "com.apple.finder"})` — preferred, unambiguous. +- `launch_app({name: "Calculator"})` — when bundle_id isn't known. + +`launch_app` is a **hidden-launch primitive by design** — that's the +entire point of cua-driver: agents drive apps in the background while +the user keeps typing in their real foreground app. The target's +window is initialized (AX tree fully populated, clickable via +`element_index`, the pid appears in `list_apps`) but not drawn on +screen. The driver never activates or unhides apps on its own; that +would violate the no-foreground contract the whole driver exists to +protect. + +If the user explicitly wants the window visible (usually for a demo +or recording), they unhide it themselves — Dock click, Cmd-Tab, or +Spotlight. Do not reach for `open` / `osascript activate` as a +shortcut to make the window visible; those paths break the backgrounded +invariant on every call, not just the call that "needed" the +foreground. Say out loud what the user needs to do ("click the +Todo app in your Dock to bring it forward") and let them do it. + +Never shell out to **any** form of `open` (including `open +` for a just-built binary — resolve the bundle id +from `Info.plist` and use `launch_app` with that), `osascript 'tell +app … to launch/open'`, or similar. Those paths activate the target, +bypass the driver's focus-restore guard, and require a Bash +permission prompt the agent loop shouldn't be burning on app launch. +See "Prefer cua-driver tools over shell shims" above for the full +intent → tool mapping. + +`list_apps` is for app-level discovery (answering "what's installed / +running / frontmost?") — not part of the core action loop. Skip it in +the loop. For **window-level** questions — "does this app have a +visible window?", "which Space is this window on?", "which of this +pid's windows is the main one?" — call `list_windows` instead; the +app record doesn't carry window state on purpose. In the common +single-window case you can skip `list_windows` entirely and read the +`windows` array that `launch_app` already returned. + +### 2. Snapshot and act by element_index + +Call `get_window_state({pid, window_id})` with the `window_id` from +`launch_app`'s `windows` array (or a fresh `list_windows({pid})` if +you're interacting with a long-lived process). In the default +`vision` capture_mode the response carries **only the screenshot** +— no AX tree — so the canonical loop is `list_windows → +get_window_state → reason over PNG → pixel click`. When you need +`element_index` dispatch (AX-addressable elements, backgrounded +clicks), flip to `som` first: `cua-driver set_config '{"key": +"capture_mode", "value": "som"}'`, or call `get_accessibility_tree` +directly. The rest of this section walks through `som` mode, which +is what you want once you've decided element-indexed addressing is +required. + +In `som` mode the response carries: + +- `tree_markdown` — every actionable element tagged `[N]`. That `N` + is the `element_index`. The tree can be very large (Finder is + ~1600 elements, ~190 KB); when it exceeds token limits the MCP + harness saves it to a file and returns the path. Use `Bash` + + `jq -r '.tree_markdown'` + `grep` to pull the section you need. +- `screenshot_png_b64` + `screenshot_width` / `_height` / + `_scale_factor` — the window screenshot (actually JPEG-85 despite + the `_png_` field name, hard-coded in + `WindowCapture.captureFrontmostWindow`). Present in `som` mode + (spliced into the structured JSON alongside the tree). In `vision` + mode the image arrives as a native MCP image content block with no + structured wrapper. Omitted when the target has no on-screen + window. +- `has_screenshot: bool` — **gate on this before piping the PNG**. + Otherwise `jq -r '.screenshot_png_b64'` emits the literal + `"null"`, base64-decodes into 3 bytes of garbage, and downstream + vision APIs reject it with an opaque "Could not process image" + error. + +``` +# canonical, works in every capture mode — writes the image bytes +# wherever you point, stdout stays readable (tree in som, summary +# in vision). stderr warns (exit 0) if the response had no image. +cua-driver get_window_state '{"pid":N,"window_id":W}' --image-out /tmp/shot.png + +# som-only legacy path: pull the spliced base64 out of structuredContent. +# Prefer --image-out above — it's one flag vs a probe + pipe. +if [ "$(cua-driver get_window_state '{"pid":N,"window_id":W}' | jq -r '.has_screenshot')" = "true" ]; then + cua-driver get_window_state '{"pid":N,"window_id":W}' | jq -r '.screenshot_png_b64' | base64 -d > shot.png +fi +``` + +**Reason over both the tree AND the screenshot — they're +complementary, not redundant.** In `som` mode every +turn's `get_window_state` gives you both halves and you should pull +signal from each: + +- The **AX tree** tells you *what's clickable* — roles, labels, + `element_index` handles, advertised actions, parent-child + structure. This is the ground truth for dispatching. +- The **screenshot** tells you *which one* — the tree often has + many buttons with similar or empty labels ("Delete", "OK", + anonymous UUID-labeled buttons, five `AXStaticText = " "`), and + visual context disambiguates. Captions, colors, layout relationships + visible in pixels often don't show up in the AX tree at all + (especially in Chromium / Electron / web content). + +Canonical pattern: look at the screenshot to decide "the blue +Subscribe button on the top-right of the video card", then walk the +tree to find the matching `AXButton` and dispatch by its +`element_index`. Don't try to do it from just the tree — you'll +pick the wrong element when labels repeat. Don't try to do it from +just the screenshot — you lose the reliable AX-action path and the +safe backgrounded-dispatch. + +Reach for pixel coordinates only when the target is a canvas / +video / WebGL / custom-drawn surface that isn't in the AX tree +(see Pixel-coordinate clicks below). + +The `actions=[...]` list on each element is **advisory**, not +authoritative. cua-driver does not pre-flight check against it — +`click({pid, element_index})` always attempts `AXPress` (or the +action you pass) and surfaces whatever the target returns. Many +apps accept `AXPress` on elements that don't advertise it — Chrome's +omnibox suggestion `AXMenuItem` is a live example. **Try the click +first** — pivot only on the returned AX error code. + +Dispatch table (every row assumes a `(pid, window_id)` pair from the +last `get_window_state`; `window_id` is required alongside +`element_index`, ignored on pixel-only forms unless you want to +anchor the conversion against a specific window): + +| Intent | Tool | Notes | +|---|---|---| +| List an app's windows | `list_windows({pid})` | returns `window_id`, `title`, `bounds`, `z_index`, `is_on_screen`, `on_current_space`. Already included in `launch_app`'s response — only call this for long-lived pids | +| Snapshot a window | `get_window_state({pid, window_id})` | returns `tree_markdown` + `screenshot_*`; populates the `(pid, window_id)` element_index cache | +| Left click | `click({pid, window_id, element_index})` | default `action: "press"`. Pixel form: `click({pid, x, y})` (window_id optional — when supplied, pinpoints the anchor window) — `modifier: ["cmd"]` | +| Double-click / open | `double_click({pid, window_id, element_index})` | AXOpen when advertised (Finder items, openable rows); else stamped pixel double-click at the element's center. Pixel form: `double_click({pid, x, y})` — primer-gated recipe lands on backgrounded Chromium web content (YouTube fullscreen, Finder open-on-dbl). `click({..., count: 2})` still works and routes through the same recipe; `double_click` is the intent-first spelling | +| Right click / context menu | `right_click({pid, window_id, element_index})` or `click({pid, window_id, element_index, action: "show_menu"})` | Chromium web-content coerces pixel right-click to left — see `WEB_APPS.md` | +| Type at cursor | `type_text({pid, text, window_id, element_index})` | `AXSelectedText` write; focuses first | +| Set whole field value | `set_value({pid, window_id, element_index, value})` | sliders, steppers, text fields; **use for keyboard-commit workarounds on minimized windows** | +| Scroll | `scroll({pid, direction, amount, by, window_id, element_index})` | synthesizes PageUp/PageDown/arrows via SLEventPostToPid | +| Focus + send key | `press_key({pid, key, window_id, element_index, modifiers})` | element_index sets AXFocused, then posts key | +| Send key to pid | `press_key({pid, key, modifiers})` | no focus change; key goes to pid's current focus | +| Modifier combo | `hotkey({pid, keys})` | e.g. `["cmd","c"]`; posted per-pid, not HID tap | +| Unicode keystrokes | `type_text_chars({pid, text, delay_ms})` | CGEvent-to-pid; reaches Chromium/Electron inputs | + +**All keyboard/text primitives require `pid`.** There is no +frontmost-routed variant — every key goes to the named target via +`CGEvent.postToPid`, so the driver cannot leak keystrokes into the +user's foreground app. + +**Why `element_index` is the primary path:** works on hidden / +occluded / off-Space windows, no focus steal, stable across +rebuilds, labels tell you what you're clicking. Reach for pixel +coordinates only when AX can't. + +### Pixel-coordinate clicks + +The pixel path (`click({pid, x, y})`) is for surfaces the AX tree +doesn't reach — canvases, video players, WebGL, custom-drawn controls. +Coords are **window-local screenshot pixels** (same space as the PNG +`get_window_state` returns). Top-left origin, y-down. The driver +handles screen-point conversion internally. Passing `window_id` +alongside `x, y` is optional but recommended — it pins the +coordinate conversion to the window whose screenshot produced the +pixel, rather than the driver's heuristic choice. + +#### Reading coordinates from the PNG + +PNGs returned by `get_window_state` are capped at **1568 px +long-side by default** (`max_image_dimension` config), matching +Anthropic's multimodal-vision downsampling limit. That means the +image the model reasons over and the image the click tool's +coordinate system lives in are the **same resolution** — just look +at the PNG, pick a pixel, click at that pixel. No scaling math. + +This is the default because the mismatch between "rendered +thumbnail" and "native PNG" was a recurring coord-estimation +footgun. If you opt out (explicit `max_image_dimension=0` for +pixel-perfect verification flows), the old rule applies: don't +eyeball coords from whatever your client renders — it may be +2-4× smaller than the PNG on disk, and a 2% error in thumbnail +space becomes ~80 px in the real image. Use the crosshair recipe +below against the full-resolution file in that case. + +1. `get_window_state({pid, window_id})` returns an image capped + at 1568 long-side (default) plus its dimensions + (`screenshot_width` / `screenshot_height`). Write the bytes to + disk with `--image-out ` in any capture mode — works + identically in `vision` (where it's the only way) and `som` + (where it sidesteps the jq + base64 dance on the spliced + `screenshot_png_b64` field). +2. You are a multimodal model — look at the PNG. Since the PNG + matches what you see, pick the target pixel directly. No + fractional math needed. +3. When precision matters (small targets, dense UIs), draw a + crosshair on the image (do **not** crop — cropping loses the + coordinate system and requires error-prone offset math) and + show it before clicking: + +```python +from PIL import Image, ImageDraw +img = Image.open('/tmp/shot.png') +draw = ImageDraw.Draw(img) +x, y = +r = 18 +draw.ellipse([x-r, y-r, x+r, y+r], outline='red', width=4) +draw.line([x-30, y, x+30, y], fill='red', width=3) +draw.line([x, y-30, x, y+30], fill='red', width=3) +img.save('/tmp/shot_annotated.png') +``` + +4. Only dispatch the click after the user (or your own re-read of + the annotated image) confirms the crosshair is on target. + +#### Addressing variants + +- `click({pid, x, y})` — single left-click. +- `click({pid, x, y, count: 2})` — double-click. +- `click({pid, x, y, modifier: ["cmd"]})` — cmd-click. Accepts any + subset of `cmd/shift/option/ctrl`. +- `right_click({pid, x, y})` — also takes `modifier`. + +The pixel path animates the agent cursor overlay but never warps +the real cursor. If the pid has no on-screen window the call errors +with `pid X has no on-screen window` — you need a visible window to +anchor the conversion. + +#### How the pixel click is dispatched + +The recipe is the backgrounded "noraise" sequence: yabai's +focus-without-raise SLPS event records followed by an off-screen +user-activation primer and the real click, all stamped via +`SLEventPostToPid`. The target app becomes AppKit-active for event +routing but its window does **not** rise to the front of the +z-stack, and macOS's "switch to Space with windows for app" follow +is suppressed. Full mechanics in +`Sources/CuaDriverCore/Input/MouseInput.swift` (`clickViaAuthSignedPost`) +and the companion `FocusWithoutRaise.swift`. + +#### Known limits + +- **Chromium `
+ + Cua Driver — The background computer-use agent + +
- - - - Cua - + Cua & Cua Sandbox - - - - Cua-Bench - + Cua Bench - - - - Lume - - -
- - - - - Cua Bot - + Lume