Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docs/content/docs/cua-driver/reference/cli-reference.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ description: Command Line Interface reference for Cua Driver
AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY
Generated by: npx tsx scripts/docs-generators/cua-driver.ts
Source: recursive Swift sources under libs/cua-driver/Sources
Version: 0.1.6
Version: 0.1.7
*/}

import { Callout } from 'fumadocs-ui/components/callout';
Expand All @@ -16,7 +16,7 @@ import { VersionHeader } from '@/components/version-selector';
<VersionHeader
versions={[{"version":"0.1","href":"/cua-driver/reference/cli-reference","isCurrent":true}]}
currentVersion="0.1"
fullVersion="0.1.6"
fullVersion="0.1.7"
packageName="cua-driver"
installCommand="curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/cua-driver/scripts/install.sh | bash"
/>
Expand Down
45 changes: 23 additions & 22 deletions docs/content/docs/cua-driver/reference/mcp-tools.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ description: Reference for every MCP tool cua-driver exposes
AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY
Generated by: npx tsx scripts/docs-generators/cua-driver.ts
Source: recursive Swift sources under libs/cua-driver/Sources
Version: 0.1.6
Version: 0.1.7
*/}

import { Callout } from 'fumadocs-ui/components/callout';
Expand Down Expand Up @@ -322,24 +322,23 @@ keeps its name:
Change with `cua-driver config set capture_mode <mode>` or
the `set_config` tool.

Screenshot capture failures in `som` / `vision` modes are
non-fatal for `som`: the AX tree still ships in the
response and the summary line carries a hint. The macOS
26.4.x SCK regression (SCStreamError -3801, "Could not
start streaming") is handled this way — agents can keep
doing element-indexed clicks against the same window even
when the screenshot is unavailable. Switching to
`capture_mode: ax` skips the capture attempt entirely on
subsequent turns.
Screenshot capture failures behave differently per mode:
in `som`, they're non-fatal — the AX tree still ships in
the response and the summary line carries a hint, so
agents can keep doing element-indexed clicks against the
same window even when the screenshot is unavailable. In
`vision`, the screenshot IS the deliverable, so the same
failure returns `isError: true` with an actionable hint
(try another window, retry later, or switch to
`capture_mode: ax`). The macOS 26.4.x SCK regression
(SCStreamError -3801, "Could not start streaming") is
surfaced this way. Switching to `capture_mode: ax` skips
the capture attempt entirely on subsequent turns.

Requires Accessibility and Screen Recording permissions.

**Arguments:**

<Callout type="info">
**Sticky (pid, window_id) context.** Each `get_window_state(pid, window_id)` call refreshes the element-index cache for that specific window. All element-indexed actions (`click`, `type_text`, `scroll`, etc.) in the same turn resolve against the *most recent snapshot for that (pid, window_id) pair* — indices from a different window or an older snapshot are stale. Always re-call `get_window_state` after any action that might shift focus to a different window or app, and always pass the same `(pid, window_id)` you intend to act against.
</Callout>

- `javascript` (string, optional): Optional JavaScript to execute in the browser tab and return alongside the AX snapshot — one round-trip instead of two. Only works for Chromium-family browsers (Chrome, Brave, Edge) and Safari; requires 'Allow JavaScript from Apple Events' to be enabled first (see WEB_APPS.md). The result is appended to the response as a `## JavaScript result` section. Use for read-only queries (document.title, innerText, querySelectorAll, etc.). For mutations or side effects use the `page` tool instead.
- `pid` (integer, required): Process ID from `list_apps`.
- `query` (string, optional): Optional case-insensitive substring. When set, `tree_markdown` only contains lines that match plus their ancestor chain; element indices and `element_count` are unchanged.
Expand Down Expand Up @@ -424,6 +423,16 @@ the background. Works with any app that implements
`application(_:open:)`; apps that ignore the delegate simply
launch without side effects.

⚠️ BROWSER WINDOW REQUIREMENT: Browsers (Safari, Chrome,
Firefox, Arc, Brave, Edge) require at least one URL in
`urls` — without it NSWorkspace starts the process but
never creates a window, so subsequent `get_window_state`,
`click`, and `screenshot` calls will fail with "no window
found." Use `urls=["about:blank"]` for a blank window.
Example: `{"bundle_id": "com.apple.Safari", "urls":
["about:blank"]}`. Electron apps also follow this contract
when their entry point depends on a URL argument.

Optional `electron_debugging_port` launches an Electron app
with `--remote-debugging-port=<N>`, activating its Chrome
DevTools Protocol (CDP) on that port. This gives the `page`
Expand Down Expand Up @@ -460,14 +469,6 @@ later to resolve a target.

**Arguments:**

<Callout type="warn">
**Browsers require at least one URL.** Launching Safari, Chrome, Firefox, Arc, Brave, or Edge without a `urls` argument starts the process but no window is ever created — subsequent `get_window_state`, `click`, and `screenshot` calls will fail with "no window found." Always pass `urls=["about:blank"]` for a blank window, or a real URL to open.

```json
{"bundle_id": "com.apple.Safari", "urls": ["about:blank"]}
```
</Callout>

- `additional_arguments` (array of string, optional): Extra command-line arguments passed to the launched process. Passed directly as argv entries — no shell expansion. Example: ["--user-data-dir=/tmp/cua-session", "--no-first-run"] for an isolated Chrome session.
- `bundle_id` (string, optional): App bundle identifier, e.g. com.apple.calculator.
- `creates_new_application_instance` (boolean, optional): Force a brand-new process even if the app is already running. Useful for isolated browser sessions: pass creates_new_application_instance=true together with additional_arguments=["--user-data-dir=/tmp/session-a", "--no-first-run", "--no-default-browser-check"] to launch a sandboxed Chrome that cannot see the user's real profile, cookies, or extensions. Each session gets its own pid and window identity and can be controlled independently.
Expand Down
58 changes: 58 additions & 0 deletions libs/cua-driver/Sources/CuaDriverCore/Apps/AppLauncher.swift
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,8 @@ public enum AppLauncher {
throw LaunchError.notFound("bundle_id '\(bundleId)'")
}
if let name, !name.isEmpty {
// Pass 1 — filesystem lookup by bundle filename (fastest; locale-independent
// for English app names whose on-disk bundle name matches the display name).
let appName = name.hasSuffix(".app") ? name : "\(name).app"
// System roots first — they're canonical. User-local paths come
// after so an app present in /Applications wins over a same-name
Expand All @@ -250,6 +252,62 @@ public enum AppLauncher {
return URL(fileURLWithPath: path)
}
}

// Pass 2 — LaunchServices bundle-ID lookup, in case the caller
// passed a bundle identifier string as `name` rather than using
// the `bundle_id` parameter (e.g. "com.apple.calculator").
if let url = NSWorkspace.shared.urlForApplication(
withBundleIdentifier: name)
{
return url
}

// Pass 3 — scan all candidate directories and match against each
// bundle's metadata, in priority order:
// a) localizedName from NSRunningApplication (locale-aware; works
// on non-English systems, e.g. "計算機" on JP macOS)
// b) CFBundleDisplayName / CFBundleName (English; from Info.plist)
// c) bundle URL stem (filename minus .app)
//
// Matching is case-insensitive throughout so "calculator" and
// "Calculator" both resolve.
let needle = name.lowercased()

// Check running apps first — NSRunningApplication.localizedName
// gives the OS-locale display name without touching the disk.
for app in NSWorkspace.shared.runningApplications {
guard let url = app.bundleURL else { continue }
if (app.localizedName?.lowercased() == needle) {
return url
}
}

// Fall back to scanning installed bundles in the same roots.
let fm = FileManager.default
for root in roots {
guard let children = try? fm.contentsOfDirectory(atPath: root)
else { continue }
for child in children where child.hasSuffix(".app") {
let path = "\(root)/\(child)"
guard let bundle = Bundle(path: path) else { continue }
// CFBundleDisplayName > CFBundleName > stem
let displayName =
(bundle.infoDictionary?["CFBundleDisplayName"] as? String)
?? (bundle.infoDictionary?["CFBundleName"] as? String)
?? URL(fileURLWithPath: path)
.deletingPathExtension().lastPathComponent
if displayName.lowercased() == needle {
Comment on lines +293 to +299

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major | ⚡ Quick win

CFBundle fallback order is implemented incorrectly

On Line 294-Line 299, CFBundleDisplayName ?? CFBundleName ?? stem only tests one value. If CFBundleDisplayName exists but doesn’t match, CFBundleName is never compared, so valid names can still fail resolution.

Suggested fix
-                    // CFBundleDisplayName > CFBundleName > stem
-                    let displayName =
-                        (bundle.infoDictionary?["CFBundleDisplayName"] as? String)
-                        ?? (bundle.infoDictionary?["CFBundleName"] as? String)
-                        ?? URL(fileURLWithPath: path)
-                            .deletingPathExtension().lastPathComponent
-                    if displayName.lowercased() == needle {
-                        return URL(fileURLWithPath: path)
-                    }
-                    // Also match against the raw stem ("Calculator" → "Calculator.app")
-                    let stem = URL(fileURLWithPath: path)
-                        .deletingPathExtension().lastPathComponent
-                    if stem.lowercased() == needle {
+                    let displayName =
+                        bundle.infoDictionary?["CFBundleDisplayName"] as? String
+                    let bundleName =
+                        bundle.infoDictionary?["CFBundleName"] as? String
+                    let stem = URL(fileURLWithPath: path)
+                        .deletingPathExtension().lastPathComponent
+
+                    if displayName?.lowercased() == needle
+                        || bundleName?.lowercased() == needle
+                        || stem.lowercased() == needle
+                    {
                         return URL(fileURLWithPath: path)
                     }
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@libs/cua-driver/Sources/CuaDriverCore/Apps/AppLauncher.swift` around lines
293 - 299, The current logic builds a single displayName by picking the first
non-nil of CFBundleDisplayName, CFBundleName, or the file stem and then compares
it to needle, which skips checking subsequent candidates if the first exists but
doesn't match; update the check in AppLauncher.swift so you compare needle
against each candidate separately (CFBundleDisplayName, CFBundleName, and the
stem from URL(fileURLWithPath: path).deletingPathExtension().lastPathComponent)
— e.g. gather the three candidate strings and test if any.lowercased() == needle
(or otherwise normalize and compare each) instead of using the single
displayName variable for the match.

return URL(fileURLWithPath: path)
}
// Also match against the raw stem ("Calculator" → "Calculator.app")
let stem = URL(fileURLWithPath: path)
.deletingPathExtension().lastPathComponent
if stem.lowercased() == needle {
return URL(fileURLWithPath: path)
}
}
}

throw LaunchError.notFound("name '\(name)'")
}
throw LaunchError.nothingSpecified
Expand Down
110 changes: 110 additions & 0 deletions libs/cua-driver/Tests/integration/test_app_name_locale_fallback.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
"""Integration test: app name resolution fallback (#1481).

Verifies that launch_app accepts more than just the exact bundle-file name:

1. Bundle ID passed as `name` (e.g. "com.apple.calculator") resolves via
LaunchServices — callers don't need to use the `bundle_id` parameter.

2. Case-insensitive match on CFBundleName / display name works so
"calculator" and "CALCULATOR" both resolve to Calculator.app.

3. Exact name still works (regression guard).

The JP-locale localizedName path ("計算機") cannot be exercised on an EN
locale machine; that path is exercised by the same NSRunningApplication
localizedName lookup and would pass on a JP-locale host.

Run:
scripts/test.sh test_app_name_locale_fallback
"""

from __future__ import annotations

import os
import subprocess
import sys
import time
import unittest

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from driver_client import DriverClient, default_binary_path, reset_calculator

CALCULATOR_BUNDLE = "com.apple.calculator"


def _kill_calc() -> None:
subprocess.run(["pkill", "-x", "Calculator"], check=False)
time.sleep(0.4)


class AppNameLocaleFallbackTests(unittest.TestCase):
"""launch_app name= accepts bundle IDs and case-insensitive display names."""

def setUp(self) -> None:
reset_calculator()
self.client = DriverClient(default_binary_path()).__enter__()

def tearDown(self) -> None:
self.client.__exit__(None, None, None)
_kill_calc()

def _launch_by_name(self, name: str) -> dict:
result = self.client.call_tool("launch_app", {"name": name})
return result

def test_bundle_id_as_name_resolves(self) -> None:
"""Bundle ID string passed as name= launches the correct app."""
result = self._launch_by_name(CALCULATOR_BUNDLE)
self.assertFalse(
result.get("isError"),
f"launch_app(name='{CALCULATOR_BUNDLE}') failed: {result}",
)
sc = result.get("structuredContent", {})
self.assertEqual(
sc.get("bundle_id"),
CALCULATOR_BUNDLE,
f"Unexpected bundle_id in response: {sc}",
)
self.assertGreater(sc.get("pid", 0), 0)

def test_case_insensitive_name_lowercase(self) -> None:
"""Lowercase name= ('calculator') matches Calculator.app."""
result = self._launch_by_name("calculator")
self.assertFalse(
result.get("isError"),
f"launch_app(name='calculator') failed: {result}",
)
sc = result.get("structuredContent", {})
self.assertEqual(sc.get("bundle_id"), CALCULATOR_BUNDLE)

def test_case_insensitive_name_uppercase(self) -> None:
"""All-caps name= ('CALCULATOR') matches Calculator.app."""
result = self._launch_by_name("CALCULATOR")
self.assertFalse(
result.get("isError"),
f"launch_app(name='CALCULATOR') failed: {result}",
)
sc = result.get("structuredContent", {})
self.assertEqual(sc.get("bundle_id"), CALCULATOR_BUNDLE)

def test_exact_name_still_works(self) -> None:
"""Exact canonical name= ('Calculator') still works (regression guard)."""
result = self._launch_by_name("Calculator")
self.assertFalse(
result.get("isError"),
f"launch_app(name='Calculator') failed: {result}",
)
sc = result.get("structuredContent", {})
self.assertEqual(sc.get("bundle_id"), CALCULATOR_BUNDLE)

def test_unknown_name_returns_error(self) -> None:
"""Completely unknown name returns isError (not a silent empty result)."""
result = self._launch_by_name("ThisAppDefinitelyDoesNotExist_xyzzy")
self.assertTrue(
result.get("isError"),
f"Expected isError for unknown app name, got: {result}",
)


if __name__ == "__main__":
unittest.main()
Loading