diff --git a/docs/content/docs/cua-driver/reference/cli-reference.mdx b/docs/content/docs/cua-driver/reference/cli-reference.mdx index d7fd2bc7e8..9bba78d283 100644 --- a/docs/content/docs/cua-driver/reference/cli-reference.mdx +++ b/docs/content/docs/cua-driver/reference/cli-reference.mdx @@ -7,7 +7,7 @@ description: Command Line Interface reference for Cua Driver AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY Generated by: npx tsx scripts/docs-generators/cua-driver.ts Source: recursive Swift sources under libs/cua-driver/Sources - Version: 0.1.6 + Version: 0.1.7 */} import { Callout } from 'fumadocs-ui/components/callout'; @@ -16,7 +16,7 @@ import { VersionHeader } from '@/components/version-selector'; diff --git a/docs/content/docs/cua-driver/reference/mcp-tools.mdx b/docs/content/docs/cua-driver/reference/mcp-tools.mdx index eae786c00b..c0982f220b 100644 --- a/docs/content/docs/cua-driver/reference/mcp-tools.mdx +++ b/docs/content/docs/cua-driver/reference/mcp-tools.mdx @@ -7,7 +7,7 @@ description: Reference for every MCP tool cua-driver exposes AUTO-GENERATED FILE - DO NOT EDIT DIRECTLY Generated by: npx tsx scripts/docs-generators/cua-driver.ts Source: recursive Swift sources under libs/cua-driver/Sources - Version: 0.1.6 + Version: 0.1.7 */} import { Callout } from 'fumadocs-ui/components/callout'; @@ -322,24 +322,23 @@ keeps its name: Change with `cua-driver config set capture_mode ` or the `set_config` tool. -Screenshot capture failures in `som` / `vision` modes are -non-fatal for `som`: the AX tree still ships in the -response and the summary line carries a hint. The macOS -26.4.x SCK regression (SCStreamError -3801, "Could not -start streaming") is handled this way — agents can keep -doing element-indexed clicks against the same window even -when the screenshot is unavailable. Switching to -`capture_mode: ax` skips the capture attempt entirely on -subsequent turns. +Screenshot capture failures behave differently per mode: +in `som`, they're non-fatal — the AX tree still ships in +the response and the summary line carries a hint, so +agents can keep doing element-indexed clicks against the +same window even when the screenshot is unavailable. In +`vision`, the screenshot IS the deliverable, so the same +failure returns `isError: true` with an actionable hint +(try another window, retry later, or switch to +`capture_mode: ax`). The macOS 26.4.x SCK regression +(SCStreamError -3801, "Could not start streaming") is +surfaced this way. Switching to `capture_mode: ax` skips +the capture attempt entirely on subsequent turns. Requires Accessibility and Screen Recording permissions. **Arguments:** - - **Sticky (pid, window_id) context.** Each `get_window_state(pid, window_id)` call refreshes the element-index cache for that specific window. All element-indexed actions (`click`, `type_text`, `scroll`, etc.) in the same turn resolve against the *most recent snapshot for that (pid, window_id) pair* — indices from a different window or an older snapshot are stale. Always re-call `get_window_state` after any action that might shift focus to a different window or app, and always pass the same `(pid, window_id)` you intend to act against. - - - `javascript` (string, optional): Optional JavaScript to execute in the browser tab and return alongside the AX snapshot — one round-trip instead of two. Only works for Chromium-family browsers (Chrome, Brave, Edge) and Safari; requires 'Allow JavaScript from Apple Events' to be enabled first (see WEB_APPS.md). The result is appended to the response as a `## JavaScript result` section. Use for read-only queries (document.title, innerText, querySelectorAll, etc.). For mutations or side effects use the `page` tool instead. - `pid` (integer, required): Process ID from `list_apps`. - `query` (string, optional): Optional case-insensitive substring. When set, `tree_markdown` only contains lines that match plus their ancestor chain; element indices and `element_count` are unchanged. @@ -424,6 +423,16 @@ the background. Works with any app that implements `application(_:open:)`; apps that ignore the delegate simply launch without side effects. +⚠️ BROWSER WINDOW REQUIREMENT: Browsers (Safari, Chrome, +Firefox, Arc, Brave, Edge) require at least one URL in +`urls` — without it NSWorkspace starts the process but +never creates a window, so subsequent `get_window_state`, +`click`, and `screenshot` calls will fail with "no window +found." Use `urls=["about:blank"]` for a blank window. +Example: `{"bundle_id": "com.apple.Safari", "urls": +["about:blank"]}`. Electron apps also follow this contract +when their entry point depends on a URL argument. + Optional `electron_debugging_port` launches an Electron app with `--remote-debugging-port=`, activating its Chrome DevTools Protocol (CDP) on that port. This gives the `page` @@ -460,14 +469,6 @@ later to resolve a target. **Arguments:** - - **Browsers require at least one URL.** Launching Safari, Chrome, Firefox, Arc, Brave, or Edge without a `urls` argument starts the process but no window is ever created — subsequent `get_window_state`, `click`, and `screenshot` calls will fail with "no window found." Always pass `urls=["about:blank"]` for a blank window, or a real URL to open. - - ```json - {"bundle_id": "com.apple.Safari", "urls": ["about:blank"]} - ``` - - - `additional_arguments` (array of string, optional): Extra command-line arguments passed to the launched process. Passed directly as argv entries — no shell expansion. Example: ["--user-data-dir=/tmp/cua-session", "--no-first-run"] for an isolated Chrome session. - `bundle_id` (string, optional): App bundle identifier, e.g. com.apple.calculator. - `creates_new_application_instance` (boolean, optional): Force a brand-new process even if the app is already running. Useful for isolated browser sessions: pass creates_new_application_instance=true together with additional_arguments=["--user-data-dir=/tmp/session-a", "--no-first-run", "--no-default-browser-check"] to launch a sandboxed Chrome that cannot see the user's real profile, cookies, or extensions. Each session gets its own pid and window identity and can be controlled independently. diff --git a/libs/cua-driver/Sources/CuaDriverCore/Apps/AppLauncher.swift b/libs/cua-driver/Sources/CuaDriverCore/Apps/AppLauncher.swift index 60343ca1b1..8e2701404f 100644 --- a/libs/cua-driver/Sources/CuaDriverCore/Apps/AppLauncher.swift +++ b/libs/cua-driver/Sources/CuaDriverCore/Apps/AppLauncher.swift @@ -228,6 +228,8 @@ public enum AppLauncher { throw LaunchError.notFound("bundle_id '\(bundleId)'") } if let name, !name.isEmpty { + // Pass 1 — filesystem lookup by bundle filename (fastest; locale-independent + // for English app names whose on-disk bundle name matches the display name). let appName = name.hasSuffix(".app") ? name : "\(name).app" // System roots first — they're canonical. User-local paths come // after so an app present in /Applications wins over a same-name @@ -250,6 +252,62 @@ public enum AppLauncher { return URL(fileURLWithPath: path) } } + + // Pass 2 — LaunchServices bundle-ID lookup, in case the caller + // passed a bundle identifier string as `name` rather than using + // the `bundle_id` parameter (e.g. "com.apple.calculator"). + if let url = NSWorkspace.shared.urlForApplication( + withBundleIdentifier: name) + { + return url + } + + // Pass 3 — scan all candidate directories and match against each + // bundle's metadata, in priority order: + // a) localizedName from NSRunningApplication (locale-aware; works + // on non-English systems, e.g. "計算機" on JP macOS) + // b) CFBundleDisplayName / CFBundleName (English; from Info.plist) + // c) bundle URL stem (filename minus .app) + // + // Matching is case-insensitive throughout so "calculator" and + // "Calculator" both resolve. + let needle = name.lowercased() + + // Check running apps first — NSRunningApplication.localizedName + // gives the OS-locale display name without touching the disk. + for app in NSWorkspace.shared.runningApplications { + guard let url = app.bundleURL else { continue } + if (app.localizedName?.lowercased() == needle) { + return url + } + } + + // Fall back to scanning installed bundles in the same roots. + let fm = FileManager.default + for root in roots { + guard let children = try? fm.contentsOfDirectory(atPath: root) + else { continue } + for child in children where child.hasSuffix(".app") { + let path = "\(root)/\(child)" + guard let bundle = Bundle(path: path) else { continue } + // CFBundleDisplayName > CFBundleName > stem + let displayName = + (bundle.infoDictionary?["CFBundleDisplayName"] as? String) + ?? (bundle.infoDictionary?["CFBundleName"] as? String) + ?? URL(fileURLWithPath: path) + .deletingPathExtension().lastPathComponent + if displayName.lowercased() == needle { + return URL(fileURLWithPath: path) + } + // Also match against the raw stem ("Calculator" → "Calculator.app") + let stem = URL(fileURLWithPath: path) + .deletingPathExtension().lastPathComponent + if stem.lowercased() == needle { + return URL(fileURLWithPath: path) + } + } + } + throw LaunchError.notFound("name '\(name)'") } throw LaunchError.nothingSpecified diff --git a/libs/cua-driver/Tests/integration/test_app_name_locale_fallback.py b/libs/cua-driver/Tests/integration/test_app_name_locale_fallback.py new file mode 100644 index 0000000000..e348581bb3 --- /dev/null +++ b/libs/cua-driver/Tests/integration/test_app_name_locale_fallback.py @@ -0,0 +1,110 @@ +"""Integration test: app name resolution fallback (#1481). + +Verifies that launch_app accepts more than just the exact bundle-file name: + + 1. Bundle ID passed as `name` (e.g. "com.apple.calculator") resolves via + LaunchServices — callers don't need to use the `bundle_id` parameter. + + 2. Case-insensitive match on CFBundleName / display name works so + "calculator" and "CALCULATOR" both resolve to Calculator.app. + + 3. Exact name still works (regression guard). + +The JP-locale localizedName path ("計算機") cannot be exercised on an EN +locale machine; that path is exercised by the same NSRunningApplication +localizedName lookup and would pass on a JP-locale host. + +Run: + scripts/test.sh test_app_name_locale_fallback +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import time +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from driver_client import DriverClient, default_binary_path, reset_calculator + +CALCULATOR_BUNDLE = "com.apple.calculator" + + +def _kill_calc() -> None: + subprocess.run(["pkill", "-x", "Calculator"], check=False) + time.sleep(0.4) + + +class AppNameLocaleFallbackTests(unittest.TestCase): + """launch_app name= accepts bundle IDs and case-insensitive display names.""" + + def setUp(self) -> None: + reset_calculator() + self.client = DriverClient(default_binary_path()).__enter__() + + def tearDown(self) -> None: + self.client.__exit__(None, None, None) + _kill_calc() + + def _launch_by_name(self, name: str) -> dict: + result = self.client.call_tool("launch_app", {"name": name}) + return result + + def test_bundle_id_as_name_resolves(self) -> None: + """Bundle ID string passed as name= launches the correct app.""" + result = self._launch_by_name(CALCULATOR_BUNDLE) + self.assertFalse( + result.get("isError"), + f"launch_app(name='{CALCULATOR_BUNDLE}') failed: {result}", + ) + sc = result.get("structuredContent", {}) + self.assertEqual( + sc.get("bundle_id"), + CALCULATOR_BUNDLE, + f"Unexpected bundle_id in response: {sc}", + ) + self.assertGreater(sc.get("pid", 0), 0) + + def test_case_insensitive_name_lowercase(self) -> None: + """Lowercase name= ('calculator') matches Calculator.app.""" + result = self._launch_by_name("calculator") + self.assertFalse( + result.get("isError"), + f"launch_app(name='calculator') failed: {result}", + ) + sc = result.get("structuredContent", {}) + self.assertEqual(sc.get("bundle_id"), CALCULATOR_BUNDLE) + + def test_case_insensitive_name_uppercase(self) -> None: + """All-caps name= ('CALCULATOR') matches Calculator.app.""" + result = self._launch_by_name("CALCULATOR") + self.assertFalse( + result.get("isError"), + f"launch_app(name='CALCULATOR') failed: {result}", + ) + sc = result.get("structuredContent", {}) + self.assertEqual(sc.get("bundle_id"), CALCULATOR_BUNDLE) + + def test_exact_name_still_works(self) -> None: + """Exact canonical name= ('Calculator') still works (regression guard).""" + result = self._launch_by_name("Calculator") + self.assertFalse( + result.get("isError"), + f"launch_app(name='Calculator') failed: {result}", + ) + sc = result.get("structuredContent", {}) + self.assertEqual(sc.get("bundle_id"), CALCULATOR_BUNDLE) + + def test_unknown_name_returns_error(self) -> None: + """Completely unknown name returns isError (not a silent empty result).""" + result = self._launch_by_name("ThisAppDefinitelyDoesNotExist_xyzzy") + self.assertTrue( + result.get("isError"), + f"Expected isError for unknown app name, got: {result}", + ) + + +if __name__ == "__main__": + unittest.main()