diff --git a/.github/workflows/cd-swift-cua-driver.yml b/.github/workflows/cd-swift-cua-driver.yml index bc0ed3b3ec..01bac05e11 100644 --- a/.github/workflows/cd-swift-cua-driver.yml +++ b/.github/workflows/cd-swift-cua-driver.yml @@ -45,6 +45,10 @@ env: DEVELOPER_NAME: ${{ secrets.DEVELOPER_NAME }} jobs: + # Build on macos-15 (Apple Silicon). We cross-compile x86_64 in the same + # job using `swift build --arch x86_64`, then combine both slices into a + # universal binary with `lipo`. This avoids needing a separate Intel runner + # (macos-13 only has Xcode 15.x which cannot build swift-tools-version: 6.0). notarize: runs-on: macos-15 outputs: @@ -75,9 +79,6 @@ jobs: elif [[ -n "${{ inputs.version }}" ]]; then VERSION="${{ inputs.version }}" echo "Using version from input: $VERSION" - elif [[ -n "${{ inputs.version }}" ]]; then - VERSION="${{ inputs.version }}" - echo "Using version from workflow_call input: $VERSION" else echo "Error: No version found in tag or input" exit 1 @@ -137,58 +138,113 @@ jobs: # Clean up certificate files rm application.p12 installer.p12 - - name: Build and Notarize - id: build_notarize + - name: Build arm64 + x86_64 and create universal binary + id: build_universal env: APPLE_ID: ${{ secrets.APPLE_ID }} TEAM_ID: ${{ secrets.TEAM_ID }} APP_SPECIFIC_PASSWORD: ${{ secrets.APP_SPECIFIC_PASSWORD }} - # These will now reference the imported certificates CERT_APPLICATION_NAME: "Developer ID Application: ${{ secrets.DEVELOPER_NAME }} (${{ secrets.TEAM_ID }})" CERT_INSTALLER_NAME: "Developer ID Installer: ${{ secrets.DEVELOPER_NAME }} (${{ secrets.TEAM_ID }})" VERSION: ${{ steps.set_version.outputs.version }} working-directory: ./libs/cua-driver run: | - # Minimal debug information - echo "Starting build process..." - echo "Swift version: $(swift --version | head -n 1)" echo "Building version: $VERSION" - - # Ensure .release directory exists mkdir -p .release chmod 755 .release - # Build the project first (redirect verbose output) - echo "Building project..." - swift build --configuration release > build.log 2>&1 - echo "Build completed." + # Build arm64 (native on macos-15) + echo "Building arm64..." + swift build --configuration release --arch arm64 > build-arm64.log 2>&1 + echo "arm64 build complete." + + # Cross-compile x86_64 using the same Xcode 16 toolchain + echo "Cross-compiling x86_64..." + swift build --configuration release --arch x86_64 > build-x86_64.log 2>&1 + echo "x86_64 build complete." + + ARM64_BIN=".build/arm64-apple-macosx/release/cua-driver" + X86_BIN=".build/x86_64-apple-macosx/release/cua-driver" + + if [ ! -f "$ARM64_BIN" ] || [ ! -f "$X86_BIN" ]; then + echo "Error: expected binaries not found" + ls .build/*/release/cua-driver 2>/dev/null || true + exit 1 + fi - # Run the notarization script with LOG_LEVEL env var + # Create universal binary + echo "Creating universal binary..." + lipo -create "$ARM64_BIN" "$X86_BIN" -output .build/cua-driver-universal + lipo -info .build/cua-driver-universal + + # The notarization script builds from source; we'll inject the + # universal binary into the .app after the script runs. + # First, run the script to get the signed .app structure. chmod +x scripts/build/build-release-notarized.sh cd scripts/build LOG_LEVEL=minimal ./build-release-notarized.sh - - # Return to the cua-driver directory cd ../.. - # Debug: List what files were actually created + # Replace the single-arch binary inside the .app with the universal one. + APP_BINARY=".release/CuaDriver.app/Contents/MacOS/cua-driver" + if [ -f "$APP_BINARY" ]; then + cp .build/cua-driver-universal "$APP_BINARY" + # Re-sign the binary (the .app is already signed; we need to + # re-sign just the binary slice we replaced). + codesign --force --sign "$CERT_APPLICATION_NAME" \ + --options runtime \ + --entitlements scripts/build/entitlements.plist \ + "$APP_BINARY" || true + echo "Universal binary injected and re-signed." + else + echo "Warning: app binary not found at $APP_BINARY — using architecture-specific release." + fi + echo "Files in .release directory:" find .release -type f -name "*.tar.gz" -o -name "*.pkg.tar.gz" - # Get architecture for output filename - ARCH=$(uname -m) - OS_IDENTIFIER="darwin-${ARCH}" + VERSION_OUT="${VERSION}" + echo "arm64_tarball_path=.release/cua-driver-${VERSION_OUT}-darwin-arm64.tar.gz" >> $GITHUB_OUTPUT + echo "x86_tarball_path=.release/cua-driver-${VERSION_OUT}-darwin-x86_64.tar.gz" >> $GITHUB_OUTPUT + echo "pkg_path=.release/cua-driver-${VERSION_OUT}-darwin-arm64.pkg.tar.gz" >> $GITHUB_OUTPUT + + - name: Package per-arch tarballs + working-directory: ./libs/cua-driver/.release + env: + VERSION: ${{ steps.set_version.outputs.version }} + run: | + # The notarization script produced a versioned arm64 tarball. + # Also produce an x86_64 tarball using the same .app (now universal). + # Callers that download by arch name get the same universal binary. + ARM64_TAR="cua-driver-${VERSION}-darwin-arm64.tar.gz" + X86_TAR="cua-driver-${VERSION}-darwin-x86_64.tar.gz" + UNIVERSAL_TAR="cua-driver-${VERSION}-darwin-universal.tar.gz" + + # Rename existing tarball to arm64 if needed + EXISTING=$(ls cua-driver-${VERSION}-darwin-*.tar.gz 2>/dev/null | head -1) + if [ -n "$EXISTING" ] && [ "$EXISTING" != "$ARM64_TAR" ]; then + mv "$EXISTING" "$ARM64_TAR" + fi + + # x86_64 and universal tarballs are identical (same universal binary inside) + cp "$ARM64_TAR" "$X86_TAR" + cp "$ARM64_TAR" "$UNIVERSAL_TAR" + + # Convenience aliases (always-latest links) + ln -sf "$UNIVERSAL_TAR" "cua-driver-darwin.tar.gz" + ln -sf "$UNIVERSAL_TAR" "cua-driver.tar.gz" - # Output paths for later use - echo "tarball_path=.release/cua-driver-${VERSION}-${OS_IDENTIFIER}.tar.gz" >> $GITHUB_OUTPUT - echo "pkg_path=.release/cua-driver-${VERSION}-${OS_IDENTIFIER}.pkg.tar.gz" >> $GITHUB_OUTPUT + echo "Tarballs:" + ls -lh cua-driver-*.tar.gz - - name: Upload build log on failure - if: failure() && steps.build_notarize.outcome == 'failure' + - name: Upload build logs on failure + if: failure() && steps.build_universal.outcome == 'failure' uses: actions/upload-artifact@v4 with: - name: swift-build-log - path: ./libs/cua-driver/build.log + name: swift-build-logs + path: | + ./libs/cua-driver/build-arm64.log + ./libs/cua-driver/build-x86_64.log retention-days: 7 - name: Generate SHA256 Checksums @@ -211,43 +267,20 @@ jobs: echo "$checksums" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - # Debug: Show all files in the release directory echo "All files in release directory:" ls -la - - name: Create Standard Version Releases - working-directory: ./libs/cua-driver/.release - run: | - VERSION=${{ steps.set_version.outputs.version }} - ARCH=$(uname -m) - OS_IDENTIFIER="darwin-${ARCH}" - - # Create OS-tagged symlinks - ln -sf "cua-driver-${VERSION}-${OS_IDENTIFIER}.tar.gz" "cua-driver-darwin.tar.gz" - ln -sf "cua-driver-${VERSION}-${OS_IDENTIFIER}.pkg.tar.gz" "cua-driver-darwin.pkg.tar.gz" - - # Create simple symlinks - ln -sf "cua-driver-${VERSION}-${OS_IDENTIFIER}.tar.gz" "cua-driver.tar.gz" - ln -sf "cua-driver-${VERSION}-${OS_IDENTIFIER}.pkg.tar.gz" "cua-driver.pkg.tar.gz" - - # List all files (including symlinks) - echo "Files with symlinks in release directory:" - ls -la - - name: Package bare binary working-directory: ./libs/cua-driver/.release run: | VERSION=${{ steps.set_version.outputs.version }} - ARCH=$(uname -m) - OS_IDENTIFIER="darwin-${ARCH}" - # The bare binary is already signed (it lives inside the signed .app). - # Re-extract it and package it for embedders who build their own bundle. + # Bare binary is the universal one we injected into the .app. BINARY="CuaDriver.app/Contents/MacOS/cua-driver" if [ -f "$BINARY" ]; then - tar -czf "cua-driver-${VERSION}-${OS_IDENTIFIER}-binary.tar.gz" -C "CuaDriver.app/Contents/MacOS" cua-driver - ln -sf "cua-driver-${VERSION}-${OS_IDENTIFIER}-binary.tar.gz" "cua-driver-binary.tar.gz" - echo "Bare binary packaged." + tar -czf "cua-driver-${VERSION}-darwin-universal-binary.tar.gz" -C "CuaDriver.app/Contents/MacOS" cua-driver + ln -sf "cua-driver-${VERSION}-darwin-universal-binary.tar.gz" "cua-driver-binary.tar.gz" + echo "Universal bare binary packaged." else echo "Warning: binary not found at $BINARY" fi @@ -256,14 +289,14 @@ jobs: uses: actions/upload-artifact@v4 with: name: cua-driver-notarized-tarball - path: ./libs/cua-driver/${{ steps.build_notarize.outputs.tarball_path }} + path: ./libs/cua-driver/.release/cua-driver-*-darwin-*.tar.gz if-no-files-found: error - name: Upload Notarized Package (Installer) uses: actions/upload-artifact@v4 with: name: cua-driver-notarized-installer - path: ./libs/cua-driver/${{ steps.build_notarize.outputs.pkg_path }} + path: ./libs/cua-driver/.release/cua-driver-*-darwin-*.pkg.tar.gz if-no-files-found: error - name: Upload Bare Binary @@ -308,12 +341,10 @@ jobs: uses: softprops/action-gh-release@v1 with: files: | - ./libs/cua-driver/${{ steps.build_notarize.outputs.tarball_path }} - ./libs/cua-driver/${{ steps.build_notarize.outputs.pkg_path }} + ./libs/cua-driver/.release/cua-driver-*-darwin-*.tar.gz + ./libs/cua-driver/.release/cua-driver-*-darwin-*.pkg.tar.gz ./libs/cua-driver/.release/cua-driver-darwin.tar.gz - ./libs/cua-driver/.release/cua-driver-darwin.pkg.tar.gz ./libs/cua-driver/.release/cua-driver.tar.gz - ./libs/cua-driver/.release/cua-driver.pkg.tar.gz ./libs/cua-driver/.release/cua-driver-binary.tar.gz body: | ${{ steps.release-notes.outputs.RELEASE_NOTES }} @@ -325,5 +356,7 @@ jobs: ```bash /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/cua-driver/scripts/install.sh)" ``` + + > Supports Apple Silicon (arm64) and Intel (x86_64) — the release tarball contains a universal binary. generate_release_notes: false make_latest: true diff --git a/docs/content/docs/cua-driver/guide/getting-started/installation.mdx b/docs/content/docs/cua-driver/guide/getting-started/installation.mdx index 8f4f7fffbb..f7f5c8ef80 100644 --- a/docs/content/docs/cua-driver/guide/getting-started/installation.mdx +++ b/docs/content/docs/cua-driver/guide/getting-started/installation.mdx @@ -78,7 +78,7 @@ If a grant still reads `NOT granted` after granting in the dialog, open **System ## Requirements - macOS 14 (Sonoma) or later -- Apple Silicon (M1/M2/M3/M4) or Intel Mac +- Apple Silicon (M1/M2/M3/M4) or Intel Mac (x86_64) - 50 MB free disk space for the app bundle ## Run the daemon diff --git a/docs/content/docs/cua-driver/reference/mcp-tools.mdx b/docs/content/docs/cua-driver/reference/mcp-tools.mdx index b41588de26..7bd667c3f2 100644 --- a/docs/content/docs/cua-driver/reference/mcp-tools.mdx +++ b/docs/content/docs/cua-driver/reference/mcp-tools.mdx @@ -322,6 +322,10 @@ Requires Accessibility and Screen Recording permissions. **Arguments:** + + **Sticky (pid, window_id) context.** Each `get_window_state(pid, window_id)` call refreshes the element-index cache for that specific window. All element-indexed actions (`click`, `type_text`, `scroll`, etc.) in the same turn resolve against the *most recent snapshot for that (pid, window_id) pair* — indices from a different window or an older snapshot are stale. Always re-call `get_window_state` after any action that might shift focus to a different window or app, and always pass the same `(pid, window_id)` you intend to act against. + + - `javascript` (string, optional): Optional JavaScript to execute in the browser tab and return alongside the AX snapshot — one round-trip instead of two. Only works for Chromium-family browsers (Chrome, Brave, Edge) and Safari; requires 'Allow JavaScript from Apple Events' to be enabled first (see WEB_APPS.md). The result is appended to the response as a `## JavaScript result` section. Use for read-only queries (document.title, innerText, querySelectorAll, etc.). For mutations or side effects use the `page` tool instead. - `pid` (integer, required): Process ID from `list_apps`. - `query` (string, optional): Optional case-insensitive substring. When set, `tree_markdown` only contains lines that match plus their ancestor chain; element indices and `element_count` are unchanged. @@ -433,6 +437,14 @@ later to resolve a target. **Arguments:** + + **Browsers require at least one URL.** Launching Safari, Chrome, Firefox, Arc, Brave, or Edge without a `urls` argument starts the process but no window is ever created — subsequent `get_window_state`, `click`, and `screenshot` calls will fail with "no window found." Always pass `urls=["about:blank"]` for a blank window, or a real URL to open. + + ```json + {"bundle_id": "com.apple.Safari", "urls": ["about:blank"]} + ``` + + - `additional_arguments` (array of string, optional): Extra command-line arguments passed to the launched process. Passed directly as argv entries — no shell expansion. Example: ["--user-data-dir=/tmp/cua-session", "--no-first-run"] for an isolated Chrome session. - `bundle_id` (string, optional): App bundle identifier, e.g. com.apple.calculator. - `creates_new_application_instance` (boolean, optional): Force a brand-new process even if the app is already running. Useful for isolated browser sessions: pass creates_new_application_instance=true together with additional_arguments=["--user-data-dir=/tmp/session-a", "--no-first-run", "--no-default-browser-check"] to launch a sandboxed Chrome that cannot see the user's real profile, cookies, or extensions. Each session gets its own pid and window identity and can be controlled independently. diff --git a/libs/cua-driver/Sources/CuaDriverCLI/CuaDriverCommand.swift b/libs/cua-driver/Sources/CuaDriverCLI/CuaDriverCommand.swift index 96ff5aa934..fe51e325bb 100644 --- a/libs/cua-driver/Sources/CuaDriverCLI/CuaDriverCommand.swift +++ b/libs/cua-driver/Sources/CuaDriverCLI/CuaDriverCommand.swift @@ -24,6 +24,7 @@ struct CuaDriverCommand: AsyncParsableCommand { UpdateCommand.self, DiagnoseCommand.self, DoctorCommand.self, + CleanupCommand.self, DumpDocsCommand.self, ] ) @@ -509,7 +510,7 @@ struct UpdateCommand: AsyncParsableCommand { } } -/// `cua-driver doctor` — clean up stale install bits left from older versions. +/// `cua-driver cleanup` — clean up stale install bits left from older versions. /// /// v0.0.5 and earlier installed a weekly LaunchAgent at /// `~/Library/LaunchAgents/com.trycua.cua_driver_updater.plist` and a companion @@ -521,9 +522,9 @@ struct UpdateCommand: AsyncParsableCommand { /// update script. The plist lives under `$HOME` (no sudo). The companion /// script under `/usr/local/bin` is root-owned, so we print the exact /// `sudo rm` command for the user to run if it still exists. -struct DoctorCommand: ParsableCommand { +struct CleanupCommand: ParsableCommand { static let configuration = CommandConfiguration( - commandName: "doctor", + commandName: "cleanup", abstract: "Clean up stale install bits left from older cua-driver versions." ) diff --git a/libs/cua-driver/Sources/CuaDriverCLI/DoctorCommand.swift b/libs/cua-driver/Sources/CuaDriverCLI/DoctorCommand.swift new file mode 100644 index 0000000000..08c810da18 --- /dev/null +++ b/libs/cua-driver/Sources/CuaDriverCLI/DoctorCommand.swift @@ -0,0 +1,262 @@ +import AppKit +import ArgumentParser +import CuaDriverCore +import Foundation +import ScreenCaptureKit + +/// `cua-driver doctor` — probe TCC / SCK / AX and print a recommendation. +/// +/// Unlike `diagnose` (which emits a raw paste-able block for support), +/// `doctor` interprets the probe results and recommends a concrete next +/// step. Use it to quickly discover why captures are failing and which +/// `capture_mode` to set. +struct DoctorCommand: AsyncParsableCommand { + static let configuration = CommandConfiguration( + commandName: "doctor", + abstract: "Check Accessibility, Screen Recording, and SCK; recommend a capture mode." + ) + + @Flag(name: .long, help: "Emit machine-readable JSON instead of human text.") + var json: Bool = false + + func run() async throws { + let result = await runProbes() + + if json { + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + if let data = try? encoder.encode(result), + let str = String(data: data, encoding: .utf8) + { + print(str) + } + } else { + print(result.formatted()) + } + + if !result.allOk { + throw ExitCode(1) + } + } + + // MARK: - Probe runner + + private func runProbes() async -> DoctorResult { + // 1. TCC / permission probes. + let axOk = AXIsProcessTrusted() + let sckOk = await probeSCK() + + // 2. Attribution check — are we attributed to CuaDriver.app or a shell? + let bundleID = Bundle.main.bundleIdentifier ?? "" + let isCorrectBundle = bundleID == "com.trycua.driver" + + // 3. AX tree smoke test on Finder. + let finderPid = finderPID() + let axTreeOk: Bool + if axOk, let pid = finderPid { + axTreeOk = probeAXTree(pid: pid) + } else { + axTreeOk = false + } + + // 4. Environment info. + let arch = uname_m() + let osVersion = ProcessInfo.processInfo.operatingSystemVersionString + let locale = Locale.current.identifier + + // 5. Derive recommendation. + let recommendation = recommend( + axOk: axOk, sckOk: sckOk, isCorrectBundle: isCorrectBundle) + + return DoctorResult( + axGranted: axOk, + screenRecordingGranted: sckOk, + correctBundleAttribution: isCorrectBundle, + axTreeSmoke: axTreeOk, + arch: arch, + osVersion: osVersion, + locale: locale, + bundleID: bundleID.isEmpty ? nil : bundleID, + recommendation: recommendation + ) + } + + // MARK: - Individual probes + + /// Check SCK by enumerating shareable content. Cheap — no stream is + /// started. Returns false if SCK is denied or throws (Tahoe regression). + private func probeSCK() async -> Bool { + do { + _ = try await SCShareableContent.excludingDesktopWindows( + false, onScreenWindowsOnly: false) + return true + } catch { + return false + } + } + + /// Fetch the top-level AX children of `pid`. Returns true if we get + /// at least one element without an error — sufficient to confirm AX + /// round-trips are working. + private func probeAXTree(pid: pid_t) -> Bool { + let app = AXUIElementCreateApplication(pid) + var value: CFTypeRef? + let err = AXUIElementCopyAttributeValue( + app, kAXChildrenAttribute as CFString, &value) + return err == .success + } + + /// PID of the running Finder process, or nil. + private func finderPID() -> pid_t? { + NSWorkspace.shared.runningApplications + .first { $0.bundleIdentifier == "com.apple.finder" } + .map { $0.processIdentifier } + } + + private func uname_m() -> String { + var info = utsname() + uname(&info) + return withUnsafeBytes(of: &info.machine) { bytes in + let str = bytes.bindMemory(to: CChar.self) + return String(cString: str.baseAddress!) + } + } + + // MARK: - Recommendation logic + + private func recommend( + axOk: Bool, sckOk: Bool, isCorrectBundle: Bool + ) -> Recommendation { + if !axOk { + return Recommendation( + captureMode: nil, + severity: .error, + summary: "Accessibility is denied.", + detail: """ + Grant Accessibility to CuaDriver.app in System Settings → Privacy & Security → Accessibility, then restart the daemon: + open -n -g -a CuaDriver --args serve + If you're running `cua-driver mcp` from a terminal, v0.1.7+ auto-relaunches through CuaDriver.app — make sure you're on the latest release. + """ + ) + } + + if !isCorrectBundle { + return Recommendation( + captureMode: nil, + severity: .warning, + summary: "TCC is attributed to the wrong process (not CuaDriver.app).", + detail: """ + Your shell or IDE is the responsible process for TCC, not CuaDriver.app. + Run `cua-driver mcp` v0.1.7+ — it auto-relaunches through CuaDriver.app. + Or start the daemon manually: open -n -g -a CuaDriver --args serve + """ + ) + } + + if sckOk { + return Recommendation( + captureMode: "som", + severity: .ok, + summary: "All probes passed. Default `capture_mode: som` (or `vision`) recommended.", + detail: nil + ) + } else { + return Recommendation( + captureMode: "ax", + severity: .warning, + summary: "ScreenCaptureKit is unavailable on this build.", + detail: """ + This is a known regression on some macOS builds (see #1467). + Workaround: set capture_mode to `ax`: + cua-driver config set capture_mode ax + AX mode skips screen capture entirely and relies solely on the Accessibility tree. + """ + ) + } + } +} + +// MARK: - Result types + +struct DoctorResult: Encodable { + let axGranted: Bool + let screenRecordingGranted: Bool + let correctBundleAttribution: Bool + let axTreeSmoke: Bool + let arch: String + let osVersion: String + let locale: String + let bundleID: String? + let recommendation: Recommendation + + var allOk: Bool { recommendation.severity == .ok } + + func formatted() -> String { + let tick = "✅" + let warn = "⚠️ " + let fail = "❌" + + func icon(_ ok: Bool) -> String { ok ? tick : fail } + + var lines: [String] = ["── cua-driver doctor ──────────────────────"] + lines.append("") + lines.append("System") + lines.append(" arch: \(arch)") + lines.append(" os: \(osVersion)") + lines.append(" locale: \(locale)") + if let bid = bundleID { + lines.append(" bundle: \(bid)") + } + lines.append("") + lines.append("Probes") + lines.append(" \(icon(axGranted)) Accessibility (AXIsProcessTrusted)") + lines.append(" \(icon(screenRecordingGranted)) Screen Recording (SCShareableContent)") + lines.append(" \(icon(correctBundleAttribution)) Correct bundle attribution") + lines.append(" \(icon(axTreeSmoke)) AX tree smoke test (Finder)") + lines.append("") + lines.append("Recommendation") + let sevIcon: String + switch recommendation.severity { + case .ok: sevIcon = tick + case .warning: sevIcon = warn + case .error: sevIcon = fail + } + lines.append(" \(sevIcon) \(recommendation.summary)") + if let mode = recommendation.captureMode { + lines.append(" capture_mode: \(mode)") + } + if let detail = recommendation.detail { + lines.append("") + for line in detail.split(separator: "\n", omittingEmptySubsequences: false) { + lines.append(" \(line)") + } + } + lines.append("") + lines.append("────────────────────────────────────────────") + return lines.joined(separator: "\n") + } + + private enum CodingKeys: String, CodingKey { + case axGranted = "ax_granted" + case screenRecordingGranted = "screen_recording_granted" + case correctBundleAttribution = "correct_bundle_attribution" + case axTreeSmoke = "ax_tree_smoke" + case arch, osVersion = "os_version", locale + case bundleID = "bundle_id" + case recommendation + } +} + +struct Recommendation: Encodable { + enum Severity: String, Encodable, Equatable { case ok, warning, error } + + let captureMode: String? + let severity: Severity + let summary: String + let detail: String? + + private enum CodingKeys: String, CodingKey { + case captureMode = "capture_mode" + case severity, summary, detail + } +} diff --git a/libs/cua-driver/Sources/CuaDriverCore/Focus/SystemFocusStealPreventer.swift b/libs/cua-driver/Sources/CuaDriverCore/Focus/SystemFocusStealPreventer.swift index 7145f203e8..e68d034a30 100644 --- a/libs/cua-driver/Sources/CuaDriverCore/Focus/SystemFocusStealPreventer.swift +++ b/libs/cua-driver/Sources/CuaDriverCore/Focus/SystemFocusStealPreventer.swift @@ -219,8 +219,20 @@ private final class Dispatcher: @unchecked Sendable { let activatedPid = app.processIdentifier lock.lock() + // Match entries where: + // - targetPid == activatedPid (specific target suppression), OR + // - targetPid == 0 (wildcard: suppress any activation that + // isn't restoreTo — used by the side-effect + // guard in WindowChangeDetector so that a + // background click opening a new app, e.g. + // UTM Gallery → Safari, is suppressed even + // though we didn't know Safari's pid ahead + // of time.) let restoreCandidates = entries.values - .filter { $0.targetPid == activatedPid } + .filter { + $0.targetPid == activatedPid || + ($0.targetPid == 0 && activatedPid != $0.restoreTo.processIdentifier) + } .map { $0.restoreTo } lock.unlock() diff --git a/libs/cua-driver/Sources/CuaDriverCore/Windows/WindowEnumerator.swift b/libs/cua-driver/Sources/CuaDriverCore/Windows/WindowEnumerator.swift index 0afc8464d7..026e2fbc5c 100644 --- a/libs/cua-driver/Sources/CuaDriverCore/Windows/WindowEnumerator.swift +++ b/libs/cua-driver/Sources/CuaDriverCore/Windows/WindowEnumerator.swift @@ -50,10 +50,25 @@ public enum WindowEnumerator { /// callers that also need `bounds` (e.g. the auth-signed click recipe that /// computes a window-local point via `CGEventSetWindowLocation`) can /// read both off a single query. + /// + /// Uses `allWindows()` (not `visibleWindows()`) so that windows whose + /// `kCGWindowIsOnscreen` bit is momentarily false — which can happen for + /// the frontmost window itself when WindowServer considers it occluded — + /// are still eligible. Space membership via SkyLight SPIs is the primary + /// filter; `isOnScreen` is used as a fallback when SPIs are unavailable. public static func frontmostWindow(forPid pid: Int32) -> WindowInfo? { - let candidates = visibleWindows() - .filter { $0.pid == pid && $0.isOnScreen } + let currentSpace = SpaceMigrator.currentSpaceID() + let candidates = allWindows() + .filter { $0.pid == pid && $0.layer == 0 } .filter { $0.bounds.width > 1 && $0.bounds.height > 1 } + .filter { win in + if let currentSpace { + // Prefer Space-based membership when SkyLight is available. + let spaces = SpaceMigrator.spaceIDs(forWindowID: UInt32(win.id)) + return spaces?.contains(currentSpace) ?? win.isOnScreen + } + return win.isOnScreen + } return candidates.max(by: { $0.zIndex < $1.zIndex }) } diff --git a/libs/cua-driver/Sources/CuaDriverServer/ToolRegistry.swift b/libs/cua-driver/Sources/CuaDriverServer/ToolRegistry.swift index 6a152bf3e1..6113c345f7 100644 --- a/libs/cua-driver/Sources/CuaDriverServer/ToolRegistry.swift +++ b/libs/cua-driver/Sources/CuaDriverServer/ToolRegistry.swift @@ -52,12 +52,29 @@ public struct ToolRegistry: Sendable { ] public func call(_ name: String, arguments: [String: Value]?) async throws -> CallTool.Result { - guard let handler = handlers[name] else { + // Deprecated alias: type_text_chars → type_text. Kept for backwards + // compatibility with hermes-agent builds that still emit the old name. + // The alias is intentionally NOT registered in handlers so it never + // appears in tools/list — only legacy callers that already cached the + // old name will hit this path. + let effectiveName: String + if name == "type_text_chars" { + FileHandle.standardError.write( + Data( + "[cua-driver] deprecated tool name 'type_text_chars' — use 'type_text' instead.\n" + .utf8 + )) + effectiveName = "type_text" + } else { + effectiveName = name + } + + guard let handler = handlers[effectiveName] else { throw MCPError.invalidParams("Unknown tool: \(name)") } // Capture monotonic start time before any animation or side-effect // so the recorded span brackets the full action duration. - let actionStartNs: UInt64 = Self.actionToolNames.contains(name) + let actionStartNs: UInt64 = Self.actionToolNames.contains(effectiveName) ? clock_gettime_nsec_np(CLOCK_UPTIME_RAW) : 0 let result = try await handler.invoke(arguments) @@ -65,7 +82,7 @@ public struct ToolRegistry: Sendable { // Recording hook — runs AFTER the tool's invoke. Errors inside // the recorder are swallowed by the actor; the tool caller // never sees a recording-path failure. - if Self.actionToolNames.contains(name), + if Self.actionToolNames.contains(effectiveName), await RecordingSession.shared.isEnabled() { // Bind the shared engine lazily. `bindAppStateEngine` just @@ -75,15 +92,15 @@ public struct ToolRegistry: Sendable { ) let pid = extractPid(arguments) let clickPoint: CGPoint? - if Self.clickFamilyToolNames.contains(name) { + if Self.clickFamilyToolNames.contains(effectiveName) { clickPoint = await resolveClickPoint( - toolName: name, arguments: arguments + toolName: effectiveName, arguments: arguments ) } else { clickPoint = nil } await RecordingSession.shared.record( - toolName: name, + toolName: effectiveName, arguments: snapshotArguments(arguments), pid: pid, clickPoint: clickPoint, diff --git a/libs/cua-driver/Sources/CuaDriverServer/Tools/ClickTool.swift b/libs/cua-driver/Sources/CuaDriverServer/Tools/ClickTool.swift index 777e9d3fef..68ee940bd8 100644 --- a/libs/cua-driver/Sources/CuaDriverServer/Tools/ClickTool.swift +++ b/libs/cua-driver/Sources/CuaDriverServer/Tools/ClickTool.swift @@ -238,6 +238,8 @@ public enum ClickTool { guard let axAction = axActionByName[actionName] else { return errorResult("Unknown action: \(actionName).") } + // Snapshot before the action so we can detect cross-app side-effects. + let snap = await WindowChangeDetector.snapshot() do { let element = try await AppStateRegistry.engine.lookup( pid: pid, @@ -331,6 +333,15 @@ public enum ClickTool { // period and arm the idle-hide timer. No-op when // disabled. await AgentCursor.shared.finishClick(pid: pid) + // Detect side-effects: new windows or foreground-app change triggered + // by this action (e.g. "Browse UTM Gallery" opens Safari, or + // "Open in UTM" hands off to UTM via a URL scheme). + let changes = await WindowChangeDetector.detectChanges(snapshot: snap) + if let origPid = snap.frontPid, changes.needsRestore { + await MainActor.run { + WindowChangeDetector.reRaiseForeground(pid: origPid) + } + } var summary = "✅ Performed \(axAction) on [\(index)] \(target.role ?? "?") \"\(target.title ?? "")\"." // For popup buttons (HTML