Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ rm -rf ~/.cua-driver
rm -rf ~/Library/Application\ Support/Cua\ Driver
rm -rf ~/Library/Caches/cua-driver

# Optional: remove the updater LaunchAgent.
# Optional: remove legacy LaunchAgent from older installs (≤ v0.0.5).
launchctl unload ~/Library/LaunchAgents/com.trycua.cua_driver_updater.plist 2>/dev/null
rm -f ~/Library/LaunchAgents/com.trycua.cua_driver_updater.plist
```
Expand Down
27 changes: 26 additions & 1 deletion docs/content/docs/cua-driver/reference/mcp-tools.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -111,9 +111,11 @@ Launch an app hidden (no focus steal) and return its pid plus the initial `windo
- `bundle_id` (string, optional): App bundle identifier, e.g. `com.apple.calculator`.
- `name` (string, optional): App display name. Used only when `bundle_id` is absent.
- `urls` (array of string, optional): `file://` / `http(s)://` URLs (or plain paths with `~` expansion) handed to the launched app via `application(_:open:)`. For Finder, a folder URL opens a backgrounded window rooted there. Apps that don't implement `application(_:open:)` launch normally and ignore these.
- `creates_new_application_instance` (boolean, optional): When true, force-launches a separate process even if the app is already running. Useful for isolated browser sessions. Default false.
- `additional_arguments` (array of string, optional): Extra command-line arguments passed to the launched process, e.g. `["--user-data-dir=/tmp/session-a"]` for an isolated Chrome profile.

```json
{"bundle_id": "com.apple.finder", "urls": ["~/Documents"]}
{"bundle_id": "com.google.Chrome", "urls": ["https://example.com"], "creates_new_application_instance": true, "additional_arguments": ["--user-data-dir=/tmp/cua-session"]}
```

### check_permissions
Expand Down Expand Up @@ -293,6 +295,29 @@ Write an element's `AXValue` directly. For sliders, steppers, text fields, and s
{"pid": 844, "window_id": 10725, "element_index": 9, "value": "42"}
```

## Browser

### page

Browser page primitives — execute JavaScript, extract page text, or query DOM elements. Supports Chrome, Brave, Edge, and Safari (requires "Allow JavaScript from Apple Events"). For WKWebView/Tauri apps where the remote inspector is blocked, `get_text` and `query_dom` automatically fall back to the accessibility tree.

**Arguments:**

- `pid` (integer, required): Browser process ID.
- `window_id` (integer, required): CGWindowID of the target browser window.
- `action` (string, required): One of `execute_javascript`, `get_text`, `query_dom`, `enable_javascript_apple_events`.
- `javascript` (string): JS to evaluate — action `execute_javascript` only. Wrap in an IIFE with try-catch for safety.
- `css_selector` (string): CSS selector — action `query_dom` only.
- `attributes` (array of string, optional): Attributes to include per element — action `query_dom` only. `tag` and `text` are always included.
- `bundle_id` (string): Browser bundle ID — action `enable_javascript_apple_events` only.
- `user_has_confirmed_enabling` (boolean): Must be `true` — action `enable_javascript_apple_events` only. **You must ask the user for explicit permission before passing this.**

```json
{"pid": 1234, "window_id": 5678, "action": "get_text"}
{"pid": 1234, "window_id": 5678, "action": "execute_javascript", "javascript": "document.title"}
{"pid": 1234, "window_id": 5678, "action": "query_dom", "css_selector": "a[href]", "attributes": ["href"]}
```

## Zoom

### zoom
Expand Down
200 changes: 200 additions & 0 deletions libs/cua-driver/Sources/CuaDriverCore/Browser/AXPageReader.swift
Original file line number Diff line number Diff line change
@@ -0,0 +1,200 @@
import Foundation

/// Extract page content from a WKWebView / Tauri app's AX tree markdown.
///
/// Used by `PageTool` as a JS-free fallback for apps where the WebKit remote
/// inspector is blocked (macOS 15+, Tauri without `TAURI_WEBVIEW_AUTOMATION`).
///
/// ## Input format (treeMarkdown)
///
/// Each line is an AX element rendered as:
/// ```
/// - [idx] AXRole "title" = "value" (description) help="..." id=identifier
/// ```
/// Indentation (two spaces per depth level) encodes the tree structure.
/// Interactive elements have `[idx]`; non-interactive elements omit it.
public enum AXPageReader {

// MARK: - Parsed element

public struct Element {
public let role: String
public let title: String
public let value: String
public let description: String
public let index: Int? // element_index if interactive, nil otherwise
}

// MARK: - Text extraction

/// Return the visible text content of the page by collecting all
/// `AXStaticText`, `AXHeading`, and value-bearing elements from the tree.
///
/// - Parameter treeMarkdown: The `treeMarkdown` string from an `AppStateSnapshot`.
/// - Returns: Concatenated text content, newline-separated. Empty if none found.
public static func extractText(from treeMarkdown: String) -> String {
var lines: [String] = []
for line in treeMarkdown.split(separator: "\n", omittingEmptySubsequences: false) {
let str = String(line)
guard let parsed = parseLine(str) else { continue }
switch parsed.role {
case "AXStaticText", "AXHeading", "AXWebArea":
// title first, then value, then description as fallback.
// WebKit/Tauri AX trees sometimes store text content in description.
let text = !parsed.title.isEmpty ? parsed.title
: !parsed.value.isEmpty ? parsed.value
: parsed.description
if !text.isEmpty { lines.append(text) }
default:
// Include text for inputs, links, buttons with meaningful content.
let text = !parsed.title.isEmpty ? parsed.title
: !parsed.value.isEmpty ? parsed.value
: parsed.description
if !text.isEmpty, parsed.role != "AXWindow", parsed.role != "AXApplication",
parsed.role != "AXGroup", parsed.role != "AXScrollArea",
parsed.role != "AXSplitGroup", parsed.role != "AXSplitter",
parsed.role != "AXMenuBar", parsed.role != "AXMenu",
parsed.role != "AXMenuBarItem", parsed.role != "AXUnknown" {
lines.append(text)
}
}
}
// De-duplicate consecutive identical lines (AX trees often repeat titles).
var deduped: [String] = []
for line in lines {
if deduped.last != line { deduped.append(line) }
}
return deduped.joined(separator: "\n")
}

// MARK: - DOM query (CSS selector → AX role mapping)

/// Query the AX tree using a CSS selector, mapping element types to AX roles.
///
/// Supported selector forms:
/// - Tag selectors: `a`, `button`, `input`, `h1`–`h6`, `p`, `img`, `select`, `li`
/// - Class/id selectors: ignored (AX tree has no class/id concept); returns all
/// elements of the mapped role if a class/id selector is combined with a tag,
/// or all interactive elements if only a class/id is given.
/// - `*` — returns all elements.
///
/// - Parameter selector: CSS selector string.
/// - Parameter treeMarkdown: The `treeMarkdown` string from an `AppStateSnapshot`.
/// - Returns: Array of matched elements (may be empty).
public static func query(selector: String, from treeMarkdown: String) -> [Element] {
let roles = cssToAXRoles(selector)
let matchAll = roles.isEmpty // empty means wildcard

var results: [Element] = []
for line in treeMarkdown.split(separator: "\n", omittingEmptySubsequences: false) {
guard let parsed = parseLine(String(line)) else { continue }
if matchAll || roles.contains(parsed.role) {
results.append(parsed)
}
}
return results
}

// MARK: - CSS → AX role mapping

/// Map a simplified CSS selector to a set of matching AX role strings.
/// Returns an empty set for wildcards or unrecognised selectors (caller
/// should treat empty as "match everything").
private static func cssToAXRoles(_ selector: String) -> Set<String> {
// Strip pseudo-classes, attribute selectors, combinators for simplicity.
let cleaned = selector
.components(separatedBy: CharacterSet(charactersIn: ":>+~["))
.first?
.trimmingCharacters(in: .whitespaces) ?? selector

// If it's purely a class (.foo) or id (#foo) selector we can't map it.
if cleaned.hasPrefix(".") || cleaned.hasPrefix("#") || cleaned == "*" || cleaned.isEmpty {
return []
}

// Extract the tag portion (before any class/id modifier).
let tag = cleaned
.components(separatedBy: CharacterSet(charactersIn: ".#"))
.first?
.lowercased() ?? cleaned.lowercased()

switch tag {
case "a", "link": return ["AXLink"]
case "button": return ["AXButton"]
case "input": return ["AXTextField", "AXCheckBox", "AXRadioButton",
"AXSlider", "AXComboBox", "AXSearchField",
"AXSecureTextField"]
case "select": return ["AXComboBox", "AXPopUpButton"]
case "textarea": return ["AXTextArea"]
case "img", "image": return ["AXImage"]
case "h1", "h2", "h3",
"h4", "h5", "h6": return ["AXHeading"]
case "p", "span", "div",
"section", "article",
"main", "header", "footer": return ["AXStaticText", "AXGroup"]
case "li": return ["AXCell", "AXStaticText"]
case "table": return ["AXTable"]
case "tr": return ["AXRow"]
case "td", "th": return ["AXCell"]
case "nav": return ["AXToolbar"]
case "form": return ["AXGroup"]
default: return []
}
}

// MARK: - Line parser

/// Parse one treeMarkdown line into an `Element`.
/// Returns `nil` for blank lines or lines that don't match the format.
///
/// Format: `<indent>- [idx] AXRole "title" = "value" (description) ...`
private static func parseLine(_ line: String) -> Element? {
// Must start with "- " (possibly indented).
guard let dashRange = line.range(of: "- ") else { return nil }
var rest = String(line[dashRange.upperBound...])

// Optional element index: [42]
var index: Int? = nil
if rest.hasPrefix("[") {
if let close = rest.firstIndex(of: "]") {
let idxStr = String(rest[rest.index(after: rest.startIndex)..<close])
index = Int(idxStr)
rest = String(rest[rest.index(after: close)...]).trimmingCharacters(in: .init(charactersIn: " "))
}
}

// Role: first whitespace-delimited token.
let tokens = rest.components(separatedBy: " ")
guard let role = tokens.first, !role.isEmpty else { return nil }
rest = rest.dropFirst(role.count).trimmingCharacters(in: .whitespaces)

// Title: optional quoted string.
var title = ""
if rest.hasPrefix("\"") {
if let end = rest.dropFirst().firstIndex(of: "\"") {
title = String(rest[rest.index(after: rest.startIndex)..<end])
rest = String(rest[rest.index(after: end)...]).trimmingCharacters(in: .whitespaces)
}
}

// Value: optional = "..."
var value = ""
if rest.hasPrefix("= \"") {
let afterEq = rest.dropFirst(3) // drop `= "`
if let end = afterEq.firstIndex(of: "\"") {
value = String(afterEq[afterEq.startIndex..<end])
rest = String(afterEq[afterEq.index(after: end)...]).trimmingCharacters(in: .whitespaces)
}
}

// Description: optional (...)
var description = ""
if rest.hasPrefix("(") {
if let end = rest.firstIndex(of: ")") {
description = String(rest[rest.index(after: rest.startIndex)..<end])
}
}

return Element(role: role, title: title, value: value, description: description, index: index)
}
}
Loading
Loading