diff --git a/docs/content/docs/cua-driver/guide/getting-started/installation.mdx b/docs/content/docs/cua-driver/guide/getting-started/installation.mdx index 9d7069c83d..297ccda6fc 100644 --- a/docs/content/docs/cua-driver/guide/getting-started/installation.mdx +++ b/docs/content/docs/cua-driver/guide/getting-started/installation.mdx @@ -77,7 +77,7 @@ If a grant still reads `NOT granted` after granting in the dialog, open **System ## Requirements -- macOS 14 (Sonoma) or later +- macOS 13 (Ventura) or later - Apple Silicon (M1/M2/M3/M4) or Intel Mac (x86_64) - 50 MB free disk space for the app bundle diff --git a/docs/content/docs/cua-driver/guide/getting-started/introduction.mdx b/docs/content/docs/cua-driver/guide/getting-started/introduction.mdx index 2c14126674..f5b560c6f0 100644 --- a/docs/content/docs/cua-driver/guide/getting-started/introduction.mdx +++ b/docs/content/docs/cua-driver/guide/getting-started/introduction.mdx @@ -58,7 +58,7 @@ Keyboard is simpler. Every key goes through `CGEvent.postToPid` scoped to the na ## What it doesn't do -- Requires macOS 14 (Sonoma) or later. Works on Apple Silicon and Intel. +- Requires macOS 13 (Ventura) or later. Works on Apple Silicon and Intel. - Not a VM. Cua Driver operates the real host, so grant Accessibility and Screen Recording with intent. - No right-click on Chromium web content through pixel synthesis: the renderer-IPC filter drops right-click subtype on non-HID-tap paths. Use `right_click({pid, element_index})` on AX-addressable targets. See [Limits](/cua-driver/reference/limits). - Canvas apps (Blender, Unity, games) need a brief frontmost activation because their event loops filter per-pid-routed events. Everything else stays backgrounded. diff --git a/docs/content/docs/cua-driver/guide/getting-started/swift-integration.mdx b/docs/content/docs/cua-driver/guide/getting-started/swift-integration.mdx index e5e981ddfe..23a1e912a5 100644 --- a/docs/content/docs/cua-driver/guide/getting-started/swift-integration.mdx +++ b/docs/content/docs/cua-driver/guide/getting-started/swift-integration.mdx @@ -85,7 +85,7 @@ let server = MCPServer(tools: ToolRegistry.default.allTools) { name, args in ## Minimum requirements -- macOS 14 (Sonoma) or later +- macOS 13 (Ventura) or later - Swift 6.0+ ## TCC permissions diff --git a/libs/cua-driver/App/CuaDriver/Info.plist b/libs/cua-driver/App/CuaDriver/Info.plist index 589ea71d0c..d31167a8ee 100644 --- a/libs/cua-driver/App/CuaDriver/Info.plist +++ b/libs/cua-driver/App/CuaDriver/Info.plist @@ -21,7 +21,7 @@ CFBundleVersion 1 LSMinimumSystemVersion - 14.0 + 13.0 LSUIElement NSHighResolutionCapable diff --git a/libs/cua-driver/Package.swift b/libs/cua-driver/Package.swift index ce51c9652a..5656ceb5d9 100644 --- a/libs/cua-driver/Package.swift +++ b/libs/cua-driver/Package.swift @@ -4,7 +4,7 @@ import PackageDescription let package = Package( name: "CuaDriver", platforms: [ - .macOS(.v14) + .macOS(.v13) ], products: [ .executable(name: "cua-driver", targets: ["CuaDriverCLI"]), diff --git a/libs/cua-driver/Skills/cua-driver/README.md b/libs/cua-driver/Skills/cua-driver/README.md index 88d0c9f8ed..aff4866d0a 100644 --- a/libs/cua-driver/Skills/cua-driver/README.md +++ b/libs/cua-driver/Skills/cua-driver/README.md @@ -26,8 +26,9 @@ See `SKILL.md` for the main body. ## Prerequisites -1. **macOS 14 or newer** — the driver depends on SkyLight private SPIs - that were stabilized in Sonoma. +1. **macOS 13 or newer** — the driver depends on Accessibility, + ScreenCaptureKit, and SkyLight private SPIs available on Ventura + or later. 2. **`cua-driver` CLI + `CuaDriver.app`** — installable one-liner: ```bash /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/trycua/cua/main/libs/cua-driver/scripts/install.sh)" diff --git a/libs/cua-driver/Sources/CuaDriverCLI/BundleHelpers.swift b/libs/cua-driver/Sources/CuaDriverCLI/BundleHelpers.swift index 11f97c5aed..6aed45ae08 100644 --- a/libs/cua-driver/Sources/CuaDriverCLI/BundleHelpers.swift +++ b/libs/cua-driver/Sources/CuaDriverCLI/BundleHelpers.swift @@ -18,6 +18,14 @@ import Foundation /// Subcommands may wrap this with additional gating (env vars, flags, /// parent-pid checks, etc.) when their relaunch heuristics diverge. func isExecutableInsideCuaDriverApp() -> Bool { + resolvedCuaDriverAppExecutablePath() != nil +} + +/// Resolve the currently-running binary through symlinks and return the +/// installed `CuaDriver.app/Contents/MacOS/cua-driver` executable path when +/// the symlink points into a CuaDriver.app bundle. Returns nil for raw +/// `.build` executables and other non-app layouts. +func resolvedCuaDriverAppExecutablePath() -> String? { // Prefer Foundation's executablePath (stable, absolute). // Fall back to argv[0] when unset, which realpath() still // resolves via $PATH lookup at the shell level — good enough @@ -25,10 +33,10 @@ func isExecutableInsideCuaDriverApp() -> Bool { let candidate = Bundle.main.executablePath ?? CommandLine.arguments.first ?? "" - guard !candidate.isEmpty else { return false } + guard !candidate.isEmpty else { return nil } var buffer = [CChar](repeating: 0, count: Int(PATH_MAX)) - guard realpath(candidate, &buffer) != nil else { return false } + guard realpath(candidate, &buffer) != nil else { return nil } let resolved = String(cString: buffer) - return resolved.contains("/CuaDriver.app/Contents/MacOS/") + return resolved.contains("/CuaDriver.app/Contents/MacOS/") ? resolved : nil } diff --git a/libs/cua-driver/Sources/CuaDriverCLI/CuaDriverCommand.swift b/libs/cua-driver/Sources/CuaDriverCLI/CuaDriverCommand.swift index c2b628ff9e..013b0233c3 100644 --- a/libs/cua-driver/Sources/CuaDriverCLI/CuaDriverCommand.swift +++ b/libs/cua-driver/Sources/CuaDriverCLI/CuaDriverCommand.swift @@ -350,8 +350,8 @@ struct MCPCommand: ParsableCommand { terminal — not to CuaDriver.app — so AX probes silently fail \ against the wrong bundle id. To sidestep this without breaking \ the stdio MCP transport, `mcp` detects the context, ensures a \ - `cua-driver serve` daemon is running under LaunchServices \ - (relaunching via `open -n -g -a CuaDriver --args serve` if not), \ + `cua-driver serve` daemon is running under CuaDriver.app's bundle \ + context (relaunching through the installed app executable if not), \ and proxies every MCP tool call through the daemon's Unix \ socket. Tool semantics are identical to the in-process path. \ Pass `--no-daemon-relaunch` (or set CUA_DRIVER_MCP_NO_RELAUNCH=1) \ @@ -410,27 +410,33 @@ struct MCPCommand: ParsableCommand { // live NSApplication event loop to draw. When the cursor's // never enabled, this costs us one idle run-loop. AppKitBootstrap.runBlockingAppKitWith { + // Warm config before the TCC preflight: AX-only mode does not + // need Screen Recording, which matters on Ventura-compatible + // builds where pixel capture may be intentionally unavailable. + let config = await ConfigStore.shared.load() + // Preflight TCC grants. When both are already active this // returns immediately; otherwise a small panel guides the // user through granting them and we resume once everything // flips green. User closing the panel without granting -> // exit with a clear message. - let granted = await MainActor.run { - PermissionsGate.shared - }.ensureGranted() + let granted: Bool + if config.captureMode == .ax { + let status = await Permissions.currentStatus() + granted = status.accessibility + } else { + granted = await MainActor.run { + PermissionsGate.shared + }.ensureGranted() + } if !granted { FileHandle.standardError.write( Data( - "cua-driver: required permissions (Accessibility + Screen Recording) not granted; MCP server exiting.\n" + "cua-driver: required permissions not granted; MCP server exiting.\n" .utf8)) throw AppKitBootstrapError.permissionsDenied } - // Same startup-warm as `serve`: surface any config decode - // warnings on the host's stderr before the first tool call - // hits the disk-read path. - let config = await ConfigStore.shared.load() - // Apply persisted agent-cursor preferences to the live // singleton so stdio MCP sessions also honor the user's // last-written state. @@ -490,7 +496,7 @@ extension MCPCommand { if !DaemonClient.isDaemonListening(socketPath: socketPath) { FileHandle.standardError.write( Data( - "cua-driver: mcp launched without CuaDriver.app's TCC grants; auto-launching the daemon via `open -n -g -a CuaDriver --args serve` and proxying MCP requests through it. Pass --no-daemon-relaunch to stay in-process.\n" + "cua-driver: mcp launched without CuaDriver.app's TCC grants; auto-launching the daemon through CuaDriver.app and proxying MCP requests through it. Pass --no-daemon-relaunch to stay in-process.\n" .utf8)) try launchDaemonViaOpen() try waitForDaemon(socketPath: socketPath, timeout: 10.0) @@ -519,20 +525,22 @@ extension MCPCommand { } } - /// Spawn `/usr/bin/open -n -g -a CuaDriver --args serve`. Mirror of - /// `ServeCommand.relaunchViaOpen` minus the post-launch probe (we - /// poll separately via `waitForDaemon`, since the timeout there is + /// Spawn the installed CuaDriver.app executable with `serve --no-relaunch`. + /// Mirror of `ServeCommand.relaunchViaOpen` minus the post-launch probe + /// (we poll separately via `waitForDaemon`, since the timeout there is /// MCP-specific). fileprivate func launchDaemonViaOpen() throws { + guard let executablePath = resolvedCuaDriverAppExecutablePath() else { + FileHandle.standardError.write( + Data( + "cua-driver: installed CuaDriver.app executable not found. Check that `/Applications/CuaDriver.app` is installed, or pass --no-daemon-relaunch to bypass.\n" + .utf8)) + throw ExitCode(1) + } + let process = Process() - process.executableURL = URL(fileURLWithPath: "/usr/bin/open") - // -n: force a new instance. CuaDriver.app may already be - // running from a previous `mcp` (different MCP client - // session); without -n, `open -a` would re-use it and - // drop our `--args serve`, leaving no daemon up. - // -g: keep the new instance backgrounded. CuaDriver.app is - // LSUIElement=true anyway, but this makes that explicit. - process.arguments = ["-n", "-g", "-a", "CuaDriver", "--args", "serve"] + process.executableURL = URL(fileURLWithPath: executablePath) + process.arguments = ["serve", "--no-relaunch"] process.standardOutput = FileHandle.nullDevice process.standardError = FileHandle.nullDevice do { @@ -540,15 +548,7 @@ extension MCPCommand { } catch { FileHandle.standardError.write( Data( - "cua-driver: failed to exec `/usr/bin/open`: \(error). Pass --no-daemon-relaunch to bypass.\n" - .utf8)) - throw ExitCode(1) - } - process.waitUntilExit() - if process.terminationStatus != 0 { - FileHandle.standardError.write( - Data( - "cua-driver: `open -n -g -a CuaDriver --args serve` exited \(process.terminationStatus). Check that `/Applications/CuaDriver.app` is installed, or pass --no-daemon-relaunch to bypass.\n" + "cua-driver: failed to exec CuaDriver.app daemon: \(error). Pass --no-daemon-relaunch to bypass.\n" .utf8)) throw ExitCode(1) } @@ -569,7 +569,7 @@ extension MCPCommand { } FileHandle.standardError.write( Data( - "cua-driver: daemon did not appear on \(socketPath) within \(Int(timeout))s. If this is the first launch, grant Accessibility + Screen Recording to CuaDriver.app in System Settings and retry. Pass --no-daemon-relaunch to stay in-process.\n" + "cua-driver: daemon did not appear on \(socketPath) within \(Int(timeout))s. If this is the first launch, grant Accessibility to CuaDriver.app in System Settings and retry. Pass --no-daemon-relaunch to stay in-process.\n" .utf8)) throw ExitCode(1) } diff --git a/libs/cua-driver/Sources/CuaDriverCLI/ServeCommand.swift b/libs/cua-driver/Sources/CuaDriverCLI/ServeCommand.swift index 533b6e3235..bf709491fe 100644 --- a/libs/cua-driver/Sources/CuaDriverCLI/ServeCommand.swift +++ b/libs/cua-driver/Sources/CuaDriverCLI/ServeCommand.swift @@ -33,8 +33,8 @@ struct ServeCommand: ParsableCommand { macOS attributes the serve process to the parent shell/IDE, not to CuaDriver.app. AX probes no-op silently and the daemon never becomes useful. To sidestep, `serve` detects that context and re-execs itself - via `open -n -g -a CuaDriver --args serve`, which relaunches under - LaunchServices so TCC attributes the process to com.trycua.driver. + via the installed CuaDriver.app executable so TCC attributes the + process to com.trycua.driver. Pass `--no-relaunch` (or set `CUA_DRIVER_NO_RELAUNCH=1`) to opt out and stay in the current process — useful when you know the caller already has the right TCC context or you're deliberately testing the @@ -103,19 +103,30 @@ struct ServeCommand: ParsableCommand { let useDefaultSocket = (socket == nil) AppKitBootstrap.runBlockingAppKitWith { + // Warm config before the TCC preflight: AX-only mode does not + // need Screen Recording, which matters on Ventura-compatible + // builds where pixel capture may be intentionally unavailable. + let config = await ConfigStore.shared.load() + // Preflight TCC grants BEFORE we acquire the daemon lock — // otherwise a first-run user who needs to grant perms would // be blocked by "another daemon starting" if they ran // `serve` once, saw the permissions panel, and triggered // any sibling probe. Panel flow is idempotent and cheap // (<50ms) when grants are already live. - let granted = await MainActor.run { - PermissionsGate.shared - }.ensureGranted() + let granted: Bool + if config.captureMode == .ax { + let status = await Permissions.currentStatus() + granted = status.accessibility + } else { + granted = await MainActor.run { + PermissionsGate.shared + }.ensureGranted() + } if !granted { FileHandle.standardError.write( Data( - "cua-driver: required permissions (Accessibility + Screen Recording) not granted; daemon exiting.\n" + "cua-driver: required permissions not granted; daemon exiting.\n" .utf8)) throw AppKitBootstrapError.permissionsDenied } @@ -137,12 +148,6 @@ struct ServeCommand: ParsableCommand { } _ = _lockFD // silence "variable never used" in release - // Warm the persistent config so any decode warnings surface - // in the daemon's stderr at startup rather than on the first - // tool call. A missing or malformed file falls through to - // defaults inside `ConfigStore.load()` — no failure here. - let config = await ConfigStore.shared.load() - // Non-blocking version hint — prints to stderr if a newer release // exists on GitHub. Fails silently when offline. VersionCheck.warnIfOutdated() @@ -165,7 +170,7 @@ struct ServeCommand: ParsableCommand { extension ServeCommand { /// Decide whether the current `serve` invocation should re-exec itself - /// via `/usr/bin/open -n -g -a CuaDriver --args serve`. True when all of + /// through the installed CuaDriver.app executable. True when all of /// the following hold: /// /// - `--no-relaunch` is NOT set and `CUA_DRIVER_NO_RELAUNCH` is not @@ -174,10 +179,10 @@ extension ServeCommand { /// signal we were invoked as a bare binary — almost always the /// `~/.local/bin/cua-driver` symlink from a shell — rather /// than as the main executable of a loaded `.app` bundle. The - /// `open -n -g -a` path always lands in the second form - /// (bundlePath ends in `/CuaDriver.app`), so checking for its - /// absence distinguishes "shell-spawned via symlink" from - /// "already relaunched by LaunchServices" without a loop risk. + /// direct app-executable path lands in the second form (bundlePath + /// ends in `/CuaDriver.app`), so checking for its absence + /// distinguishes "shell-spawned via symlink" from "already running + /// inside the app bundle" without a loop risk. /// - The symlink / argv path resolves (via `realpath`) to a file /// living inside some `CuaDriver.app/Contents/MacOS/`. This /// rules out raw `swift run cua-driver serve` dev invocations, @@ -185,17 +190,15 @@ extension ServeCommand { /// no `.app` to relaunch into. /// - `getppid() != 1`: we were spawned by a regular process /// (shell, IDE, another daemon), not by `launchd` / LaunchServices. - /// `open -n -g -a` always reparents the launched app to launchd, - /// so a process with ppid == 1 already came in through the - /// LaunchServices path. This is a belt-and-suspenders check on - /// top of the bundlePath heuristic. + /// This is a belt-and-suspenders check on top of the bundlePath + /// heuristic. /// /// The point: shell-spawned subprocesses inherit the parent shell / /// IDE's TCC responsibility chain, which means AX + Screen Recording /// checks are evaluated against the IDE's bundle id — not /// com.trycua.driver — and the daemon's AppKitBootstrap silently - /// no-ops. Bouncing through `open` relaunches under LaunchServices so - /// TCC attributes the fresh process to CuaDriver.app. + /// no-ops. Re-executing the resolved app-bundle executable gives the + /// fresh process the CuaDriver.app bundle context. fileprivate func shouldRelaunchViaOpen() -> Bool { if noRelaunch { return false } if isEnvTruthy(ProcessInfo.processInfo.environment["CUA_DRIVER_NO_RELAUNCH"]) { @@ -220,11 +223,9 @@ extension ServeCommand { return true } - /// Spawn `/usr/bin/open -n -g -a CuaDriver --args serve [--socket …]`, + /// Spawn the installed CuaDriver.app executable with `serve --no-relaunch`, /// then wait (up to 5s) for the canonical daemon socket to accept a - /// protocol-speaking probe. The `open` CLI returns immediately once - /// LaunchServices accepts the request, which is well before the - /// daemon has bound its socket — hence the poll. + /// protocol-speaking probe. /// /// On success: prints a confirmation to stdout and returns so the /// CLI exits 0. On failure: writes a diagnostic and throws @@ -233,7 +234,7 @@ extension ServeCommand { fileprivate func relaunchViaOpen(socketPath: String) throws { FileHandle.standardError.write( Data( - "cua-driver: relaunching via `open -n -g -a CuaDriver --args serve` for correct TCC context. Pass --no-relaunch to stay in this process.\n" + "cua-driver: relaunching through CuaDriver.app for correct TCC context. Pass --no-relaunch to stay in this process.\n" .utf8)) // If --socket was ever passed through to `serve`, forward it to @@ -246,19 +247,17 @@ extension ServeCommand { extraArgs += ["--socket", socket] } + guard let executablePath = resolvedCuaDriverAppExecutablePath() else { + FileHandle.standardError.write( + Data("cua-driver: installed CuaDriver.app executable not found.\n".utf8)) + throw ExitCode(1) + } + let process = Process() - process.executableURL = URL(fileURLWithPath: "/usr/bin/open") - // -n: force a new instance. Critical — `open -a CuaDriver` - // against an already-running CuaDriver process (e.g. a - // `cua-driver mcp` started by an MCP client) would re-use - // that instance and drop our `--args serve`, leaving us - // with "launched something but no serve daemon appeared". - // -g: keep the new instance in the background. CuaDriver.app is - // LSUIElement=true so it wouldn't take focus anyway, but this - // makes that explicit. - process.arguments = ["-n", "-g", "-a", "CuaDriver", "--args", "serve"] + extraArgs - // Discard `open`'s own stdout/stderr — on success it's silent, - // on failure the exit code is what we care about. + process.executableURL = URL(fileURLWithPath: executablePath) + process.arguments = ["serve", "--no-relaunch"] + extraArgs + // Discard the child daemon's stdout/stderr; readiness is verified + // by polling the socket below. process.standardOutput = FileHandle.nullDevice process.standardError = FileHandle.nullDevice @@ -267,19 +266,10 @@ extension ServeCommand { } catch { FileHandle.standardError.write( Data( - "cua-driver: failed to exec `/usr/bin/open`: \(error)\n" + "cua-driver: failed to exec CuaDriver.app daemon: \(error)\n" .utf8)) throw ExitCode(1) } - process.waitUntilExit() - if process.terminationStatus != 0 { - FileHandle.standardError.write( - Data( - "cua-driver: `open -n -g -a CuaDriver --args serve` exited \(process.terminationStatus). Check that `/Applications/CuaDriver.app` is installed, or pass --no-relaunch to bypass.\n" - .utf8)) - throw ExitCode(1) - } - // Poll for the daemon to come up. 5s covers a cold launch // including the first-run PermissionsGate preflight; on a warm // machine the socket usually appears in <500ms. @@ -295,15 +285,13 @@ extension ServeCommand { if probeReachable { FileHandle.standardOutput.write( - Data( - "cua-driver daemon is running (relaunched via CuaDriver.app)\n socket: \(socketPath)\n" - .utf8)) + Data("cua-driver daemon is running\n socket: \(socketPath)\n".utf8)) return } FileHandle.standardError.write( Data( - "cua-driver: relaunched CuaDriver.app but no daemon appeared on \(socketPath) within 5s. Check Accessibility + Screen Recording grants for CuaDriver.app, or re-run with --no-relaunch to see in-process errors.\n" + "cua-driver: launched CuaDriver.app daemon but no socket appeared on \(socketPath) within 5s. Check Accessibility grants for CuaDriver.app, or re-run with --no-relaunch to see in-process errors.\n" .utf8)) throw ExitCode(1) } diff --git a/libs/cua-driver/Sources/CuaDriverCore/Capture/WindowCapture.swift b/libs/cua-driver/Sources/CuaDriverCore/Capture/WindowCapture.swift index 2137cda868..6ffc7d9df3 100644 --- a/libs/cua-driver/Sources/CuaDriverCore/Capture/WindowCapture.swift +++ b/libs/cua-driver/Sources/CuaDriverCore/Capture/WindowCapture.swift @@ -78,20 +78,8 @@ public actor WindowCapture { throw CaptureError.noDisplay } - let filter = SCContentFilter(display: display, excludingWindows: []) - let config = SCStreamConfiguration() - config.width = display.width - config.height = display.height - config.showsCursor = true - - let cgImage: CGImage - do { - cgImage = try await SCScreenshotManager.captureImage( - contentFilter: filter, - configuration: config - ) - } catch { - throw classify(error) + guard let cgImage = CGDisplayCreateImage(display.displayID) else { + throw CaptureError.captureFailed("CGDisplayCreateImage returned no image") } let data = try encode(cgImage, format: format, quality: quality) @@ -127,57 +115,17 @@ public actor WindowCapture { throw CaptureError.windowNotFound(windowID) } - let filter = SCContentFilter(desktopIndependentWindow: window) - let config = SCStreamConfiguration() // Output pixel size ≈ window point size × the target display's scale // factor. Locating the display by maximal frame-intersection (rather // than defaulting to `NSScreen.main`) is what keeps multi-display // setups correct: a window on a 1x external monitor captured against // a 2x main display's scale would otherwise come out oversized. let scale = scaleFactor(for: window.frame) - config.width = max(1, Int(window.frame.width * scale)) - config.height = max(1, Int(window.frame.height * scale)) - config.showsCursor = false - - // One-shot SCK call with a single retry on streaming-start failure. - // macOS 26.4.x has a regression where `SCScreenshotManager.captureImage` - // intermittently returns "Could not start streaming because audio/video - // capture failed" (SCStreamError code -3801) on physical Macs, often - // recovering on a second attempt a moment later. We retry once with a - // brief back-off; if it still fails, we surface `.streamingFailed` so - // the tool layer can hint the caller toward `capture_mode: ax` for - // `get_window_state` workflows. - let cgImage: CGImage - do { - cgImage = try await captureSCKWithRetry(filter: filter, config: config) - } catch let error as CaptureError { - // Already classified — re-throw without wrapping. CGWindowList - // is intentionally NOT tried for permission errors (it'd just - // fail the same way and confuse the user-facing message). - if case .permissionDenied = error { throw error } - // For streaming / generic SCK failures, try the legacy - // CGWindowListCreateImage path. It's deprecated on macOS 15+ - // but still works in many cases where SCK refuses — particularly - // useful as a last-ditch fallback for the 26.4 SCK regression. - if let fallback = legacyCaptureWindow(windowID: windowID) { - let origW = fallback.width - let origH = fallback.height - let resized = resizeIfNeeded(fallback, maxDim: maxImageDimension) - let didResize = resized.width != origW || resized.height != origH - let data = try encode(resized, format: format, quality: quality) - return Screenshot( - imageData: data, - format: format, - width: resized.width, - height: resized.height, - scaleFactor: Double(scale), - originalWidth: didResize ? origW : nil, - originalHeight: didResize ? origH : nil - ) - } - throw error - } catch { - throw classify(error) + + guard let cgImage = legacyCaptureWindow(windowID: windowID) else { + throw CaptureError.captureFailed( + "CGWindowListCreateImage returned no usable image for window \(windowID)" + ) } let origW = cgImage.width @@ -246,56 +194,13 @@ public actor WindowCapture { return (best ?? NSScreen.main)?.backingScaleFactor ?? 1.0 } - /// Attempt `SCScreenshotManager.captureImage` once; on a streaming-start - /// failure, wait briefly and retry once more. Returns a classified - /// `CaptureError` on persistent failure so the caller can branch on the - /// kind (permission vs. streaming vs. generic) without string-matching. - /// - /// The retry covers the macOS 26.4.x SCK regression where the very first - /// call after the SCK daemon has been idle returns -3801 ("Could not - /// start streaming because audio/video capture failed") but a second - /// call ~250ms later succeeds. A second failure isn't transient and we - /// stop retrying — the caller falls back to CGWindowList or surfaces - /// the error. - private func captureSCKWithRetry( - filter: SCContentFilter, - config: SCStreamConfiguration - ) async throws -> CGImage { - do { - return try await SCScreenshotManager.captureImage( - contentFilter: filter, - configuration: config - ) - } catch { - let classified = classify(error) - // Only retry on streaming-start failures; permission errors and - // not-found errors won't change on a second attempt. - guard case .streamingFailed = classified else { throw classified } - try? await Task.sleep(nanoseconds: 250_000_000) - do { - return try await SCScreenshotManager.captureImage( - contentFilter: filter, - configuration: config - ) - } catch { - throw classify(error) - } - } - } - - /// Legacy `CGWindowListCreateImage` fallback for the SCK 26.4 regression. - /// Deprecated by Apple in macOS 15 but still functional on most windows, - /// and frequently works where SCK refuses. Returns nil on failure — the - /// caller surfaces the original SCK error in that case so the user knows - /// the real cause. - /// - /// Marked with `@available(*, deprecated)` suppression because the API - /// is the entire point: we *want* the legacy path here. + /// `CGWindowListCreateImage` capture path used when building against the + /// Ventura SDK, where `SCScreenshotManager` is not declared. Deprecated + /// by Apple in macOS 15 but still functional on most windows. private func legacyCaptureWindow(windowID: UInt32) -> CGImage? { - // CGWindowListCreateImage is deprecated on macOS 15+. The deprecation - // diagnostic is silenced with the @available pragma. Apple has not - // (yet) removed the symbol, and this path is the only practical - // fallback when SCK's streaming-start is broken for a given window. + // CGWindowListCreateImage is deprecated on macOS 15+. Apple has not + // (yet) removed the symbol, and it remains the practical capture path + // for a binary that can be built with the macOS 13 SDK. let opts: CGWindowImageOption = [.boundsIgnoreFraming, .bestResolution] let listOption: CGWindowListOption = .optionIncludingWindow // Wrap the deprecated call so we keep the unsafePointer-style @@ -348,11 +253,8 @@ public actor WindowCapture { /// deprecation-warning suppression is isolated to one place. Returns nil /// if the legacy path also refuses to produce an image. /// - /// Marking the wrapper itself deprecated downgrades the call-site - /// warning to a no-op — we *want* this legacy path because SCK has a - /// well-known regression on macOS 26.4.x where streaming-start fails - /// for specific windows on physical Macs. - @available(*, deprecated, message: "Intentional fallback for SCK streaming-start failures.") + /// Keep the deprecated call in one place so any SDK-specific warning is + /// isolated to this compatibility shim. private func legacyCGWindowImage( windowID: UInt32, listOption: CGWindowListOption, diff --git a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorRenderer.swift b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorRenderer.swift index db24f285e7..bbb5e34262 100644 --- a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorRenderer.swift +++ b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorRenderer.swift @@ -1,6 +1,5 @@ import CoreGraphics import Foundation -import Observation // MARK: - Public API -------------------------------------------------------- @@ -13,7 +12,6 @@ import Observation /// `AgentCursor` is the public facade; this type owns the math and /// per-frame state. Call `AgentCursor.shared.animate(to:)` from tool /// invocation sites — do not call `AgentCursorRenderer.shared` directly. -@Observable @MainActor public final class AgentCursorRenderer { public static let shared = AgentCursorRenderer() @@ -57,8 +55,9 @@ public final class AgentCursorRenderer { public var focusRect: CGRect? = nil /// Visual style applied to the cursor overlay. Changing this property - /// takes effect on the next rendered frame — `AgentCursorView` reads - /// it via `@Observable`. Set via `AgentCursor.shared.setStyle(_:)`. + /// takes effect on the next rendered frame. `AgentCursorView` is driven + /// by `TimelineView`, so it samples renderer state every frame without + /// requiring the macOS 14 Observation framework. public var style: AgentCursorStyle = .default // -------- Internal state --------------------------------------------- diff --git a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorView.swift b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorView.swift index b02d972c16..c05d94c50c 100644 --- a/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorView.swift +++ b/libs/cua-driver/Sources/CuaDriverCore/Cursor/AgentCursorView.swift @@ -9,12 +9,18 @@ import SwiftUI /// matches the existing gradient-arrow design: a classic pointer with /// the tip at upper-left, scaled for legibility at any display density. public struct AgentCursorView: View { - @Bindable var renderer: AgentCursorRenderer + let renderer: AgentCursorRenderer - public init(renderer: AgentCursorRenderer = .shared) { + @MainActor + public init(renderer: AgentCursorRenderer) { self.renderer = renderer } + @MainActor + public init() { + self.renderer = .shared + } + public var body: some View { TimelineView(.animation(minimumInterval: 1.0 / 120.0)) { ctx in Canvas { gctx, size in @@ -31,6 +37,7 @@ public struct AgentCursorView: View { /// element (if `renderer.focusRect` is set). Color is derived from the /// current cursor style's bloom color so the focus rect always matches /// the cursor's visual identity. + @MainActor private func drawFocusRect(in ctx: GraphicsContext, canvasSize: CGSize) { guard let screenRect = renderer.focusRect else { return } let r = CGRect( @@ -51,6 +58,7 @@ public struct AgentCursorView: View { /// Draw the cursor centered on `renderer.position`, rotated to /// `renderer.heading`. When `renderer.style.image` is set, draws that /// image instead of the default gradient arrow. + @MainActor private func drawCursor(in ctx: GraphicsContext) { let p = renderer.position guard p.x > -100 else { return } // skip until first moveTo diff --git a/libs/cua-driver/Sources/CuaDriverServer/Tools/ScreenshotTool.swift b/libs/cua-driver/Sources/CuaDriverServer/Tools/ScreenshotTool.swift index 0c43f48631..7b3ed99e7a 100644 --- a/libs/cua-driver/Sources/CuaDriverServer/Tools/ScreenshotTool.swift +++ b/libs/cua-driver/Sources/CuaDriverServer/Tools/ScreenshotTool.swift @@ -10,8 +10,8 @@ public enum ScreenshotTool { tool: Tool( name: "screenshot", description: """ - Capture a screenshot using ScreenCaptureKit. Returns base64-encoded - image data for a single window in the requested format (default png). + Capture a screenshot of a single window. Returns base64-encoded + image data in the requested format (default png). `window_id` is required. Get window ids from `list_windows`. @@ -19,13 +19,9 @@ public enum ScreenshotTool { Requires the Screen Recording TCC grant — call `check_permissions` first if unsure. - On macOS 26.4.x, ScreenCaptureKit can refuse specific windows on - physical Macs (SCStreamError -3801, "Could not start streaming"). - The driver retries once and falls back to the legacy - CGWindowList path before failing; if both refuse, the error - response includes a hint to try a different `window_id` or - switch to `capture_mode: ax` for `get_window_state` (the - element-indexed flow doesn't need pixels). + On macOS builds where pixel capture refuses a specific window, + try a different `window_id` or switch to `capture_mode: ax` for + `get_window_state` (the element-indexed flow doesn't need pixels). """, inputSchema: [ "type": "object", @@ -121,21 +117,17 @@ public enum ScreenshotTool { isError: true ) } catch CaptureError.streamingFailed(let msg) { - // SCK streaming-start regression on macOS 26.4.x — the - // legacy CGWindowList fallback also refused this specific - // window. There's nothing we can do at the pixel layer; - // surface an actionable hint pointing at `get_window_state` - // (which can fall back to AX-only via `capture_mode: ax`) - // or trying a different window. + // Pixel capture refused this specific window. There's nothing + // else to do at this layer; surface an actionable hint pointing + // at AX-only `get_window_state` or trying a different window. return CallTool.Result( content: [ .text( text: """ - ScreenCaptureKit refused this window: \(msg) + Pixel capture refused this window: \(msg) - This is a known macOS 26.4.x SCK regression that hits \ - specific windows on physical Macs. The legacy \ - CGWindowList fallback also returned no image. + Some macOS builds refuse specific windows even when \ + Screen Recording is granted. Workarounds: • Try a different `window_id` on the same app — \ diff --git a/libs/cua-driver/scripts/install.sh b/libs/cua-driver/scripts/install.sh index 0cf50eb8d2..915ded83bf 100755 --- a/libs/cua-driver/scripts/install.sh +++ b/libs/cua-driver/scripts/install.sh @@ -54,6 +54,17 @@ if [[ "$(uname -s)" != "Darwin" ]]; then exit 1 fi +MACOS_VERSION="$(sw_vers -productVersion 2>/dev/null || true)" +MACOS_MAJOR="${MACOS_VERSION%%.*}" +if [[ -z "$MACOS_VERSION" ]] || ! [[ "$MACOS_MAJOR" =~ ^[0-9]+$ ]]; then + err "could not determine macOS version via sw_vers" + exit 1 +fi +if (( MACOS_MAJOR < 13 )); then + err "cua-driver requires macOS 13 (Ventura) or later; this Mac is running macOS ${MACOS_VERSION}" + exit 1 +fi + for cmd in curl tar; do if ! command -v "$cmd" >/dev/null 2>&1; then err "$cmd not found on PATH"